Compare commits
4
Commits
519f433534
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6229c4eca0 | ||
|
|
e35cd4a045 | ||
|
|
c0b099cc48 | ||
|
|
dce5e050c3 |
@@ -9,8 +9,9 @@ downloading, streaming, and mirroring. It was first published in 2022. The
|
||||
manifest files' data is serialized with Google's
|
||||
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
||||
The structure of these files can be found
|
||||
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
||||
which is included in the [project repository](https://git.eeqj.de/sneak/mfer).
|
||||
[in the format specification](docs/FORMAT.md), which refers to the protobuf
|
||||
schema `mfer/mf.proto` for exact field numbers and types. Both are included in
|
||||
the [project repository](https://git.eeqj.de/sneak/mfer).
|
||||
|
||||
The current version is pre-1.0 and while the repo was published in 2022, there
|
||||
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
||||
@@ -268,13 +269,33 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
- recurses under current directory and writes out an `index.mf`
|
||||
- records every file's mode as `0000` unless given `--include-permissions`,
|
||||
which records each file's permission bits (`0777` at most)
|
||||
- `mfer gen /media/drive`
|
||||
- writes `/media/drive/index.mf`, listing each file by its path under
|
||||
`/media/drive`, so `mfer check /media/drive` verifies it. Given a file,
|
||||
gen writes `index.mf` beside it and lists the file by its name; given
|
||||
several paths, it writes `index.mf` in the current directory
|
||||
- `--output` names another file to write instead. What gen lists depends
|
||||
only on the paths it is given and the files under them, so with the same
|
||||
`--seed` and an unchanged tree it writes the same bytes wherever the
|
||||
manifest goes. The file it writes to is never listed
|
||||
- `mfer check` / `mfer check .`
|
||||
- verifies checksums of all files in manifest, displaying error and exiting
|
||||
nonzero if any files are missing or corrupted, or have permission bits
|
||||
other than the mode the manifest records, unless that is `0000`
|
||||
- looks for those files under the base directory: the one `--base` names, or
|
||||
else the directory holding the manifest, or the current directory for a
|
||||
manifest given by URL. So `mfer check /media/drive` checks a drive against
|
||||
the `index.mf` at its root, from any directory
|
||||
- warns about each file under the base directory that the manifest does not
|
||||
list, hidden files included; with `--no-extra-files` each one is a failure
|
||||
instead
|
||||
- `mfer freshen` / `mfer freshen .`
|
||||
- rewrites `index.mf` to list the files now under the directory holding it,
|
||||
or under the one `--base` names, hashing only the files that are new or
|
||||
changed
|
||||
- leaves out hidden files unless given `--include-dotfiles`, and symlinks
|
||||
unless given `--follow-symlinks`, which lists each symlink to a file under
|
||||
its own name with the contents of the file it points to
|
||||
- `mfer fetch https://example.com/stuff/`
|
||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest into
|
||||
the current directory, or the one given with `--dest`, and assures
|
||||
|
||||
+12
-9
@@ -108,7 +108,7 @@ func (mfa *CLIApp) fetchManifestToTemp(
|
||||
if tmpErr != nil {
|
||||
_ = rc.Close()
|
||||
|
||||
return "", fmt.Errorf("failed to create temp file: %w", tmpErr)
|
||||
return "", tmpErr
|
||||
}
|
||||
|
||||
tmpPath := tmpFile.Name()
|
||||
@@ -126,7 +126,7 @@ func (mfa *CLIApp) fetchManifestToTemp(
|
||||
if cpErr != nil {
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
|
||||
return "", fmt.Errorf("failed to download manifest: %w", cpErr)
|
||||
return "", fmt.Errorf("download manifest: %w", cpErr)
|
||||
}
|
||||
|
||||
return tmpPath, nil
|
||||
@@ -142,7 +142,7 @@ func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
|
||||
|
||||
_, err := hex.DecodeString(requiredSigner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid fingerprint: must be valid hex: %w", err)
|
||||
return fmt.Errorf("invalid fingerprint: %w", err)
|
||||
}
|
||||
|
||||
if !chk.IsSigned() {
|
||||
@@ -232,7 +232,7 @@ func findExtraFiles(
|
||||
|
||||
err := chk.FindExtraFiles(ctx, extraResults)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check for extra files: %w", err)
|
||||
return fmt.Errorf("find extra files: %w", err)
|
||||
}
|
||||
|
||||
<-extraDone
|
||||
@@ -275,7 +275,7 @@ func runCheck(
|
||||
progressWg.Wait()
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("check failed: %w", err)
|
||||
return 0, fmt.Errorf("check files: %w", err)
|
||||
}
|
||||
|
||||
// Wait for results processing to complete
|
||||
@@ -296,14 +296,18 @@ func (mfa *CLIApp) checkManifestOperation(
|
||||
|
||||
manifestPath, err := mfa.resolveManifestArg(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Done before a URL is swapped for the temp file it is downloaded to,
|
||||
// whose directory is not the base.
|
||||
basePath := resolveBasePath(cmd, manifestPath)
|
||||
|
||||
// URL manifests need to be downloaded to a temp file for the checker
|
||||
if isHTTPURL(manifestPath) {
|
||||
tmpPath, tmpErr := mfa.fetchManifestToTemp(ctx, manifestPath)
|
||||
if tmpErr != nil {
|
||||
return fmt.Errorf("check: %w", tmpErr)
|
||||
return tmpErr
|
||||
}
|
||||
|
||||
defer func() { _ = mfa.Fs.Remove(tmpPath) }()
|
||||
@@ -311,7 +315,6 @@ func (mfa *CLIApp) checkManifestOperation(
|
||||
manifestPath = tmpPath
|
||||
}
|
||||
|
||||
basePath := cmd.String("base")
|
||||
showProgress := cmd.Bool("progress")
|
||||
|
||||
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
||||
@@ -324,7 +327,7 @@ func (mfa *CLIApp) checkManifestOperation(
|
||||
Fs: mfa.Fs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
return fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
|
||||
// Check signature requirement
|
||||
|
||||
+261
-2
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -382,7 +383,7 @@ func TestGenerateRefusesTwoFilesAtOnePath(t *testing.T) {
|
||||
}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
`generate: failed to enumerate paths: duplicate path "a.txt": `+
|
||||
`enumerate files: duplicate path "a.txt": `+
|
||||
tc.first+"/a.txt and "+tc.second+"/a.txt")
|
||||
|
||||
exists, err := afero.Exists(fs, testOutput)
|
||||
@@ -674,7 +675,7 @@ func TestCheckRequireSignatureRefusesOtherSigningKey(t *testing.T) {
|
||||
}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
"failed to load manifest: signature verification failed: "+
|
||||
"load manifest: "+
|
||||
"embedded public key block must hold exactly one key, found 2")
|
||||
}
|
||||
|
||||
@@ -979,6 +980,90 @@ func TestCheckNeverReportsManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The directory setupManifestInSubdir makes and the manifest it writes there,
|
||||
// relative to the working directory it sets.
|
||||
const (
|
||||
testSubdir = "sub"
|
||||
testSubdirManifest = testSubdir + "/" + defaultManifestName
|
||||
)
|
||||
|
||||
// setupManifestInSubdir makes a temp dir holding file.txt and sub/b.txt,
|
||||
// where sub/index.mf is the manifest gen writes for sub, and makes it the
|
||||
// working directory, so a test calling it cannot run in parallel. It returns
|
||||
// the temp dir.
|
||||
func setupManifestInSubdir(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
root := t.TempDir()
|
||||
sub := filepath.Join(root, testSubdir)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(sub, 0o750))
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "not in the manifest")
|
||||
writeTestFile(t, fs, filepath.Join(sub, "b.txt"), "in the manifest")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "-o", filepath.Join(sub, defaultManifestName), sub,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
t.Chdir(root)
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
// TestCheckResolvesEntriesAgainstManifestDirectory runs check from the
|
||||
// directory above sub, on the manifest in sub. Without --base, the
|
||||
// manifest's entries are looked for in sub, whether check is given the
|
||||
// manifest or sub, and the files above sub are not reported. --base names
|
||||
// the directory to look in instead, the current one included.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestCheckResolvesEntriesAgainstManifestDirectory(t *testing.T) {
|
||||
root := setupManifestInSubdir(t)
|
||||
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
exitCode int
|
||||
failure string // a line check must print, if any
|
||||
}{
|
||||
{[]string{testSubdir}, 0, ""},
|
||||
{[]string{testSubdirManifest}, 0, ""},
|
||||
{[]string{filepath.Join(root, testSubdir)}, 0, ""},
|
||||
{[]string{testFlagBase, testSubdir, testSubdirManifest}, 0, ""},
|
||||
{[]string{testFlagBase, ".", testSubdirManifest}, 1, "MISSING: b.txt"},
|
||||
} {
|
||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdCheck, testFlagNoExtra}, tc.args,
|
||||
), afero.NewOsFs())
|
||||
assert.Equal(t, tc.exitCode, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStderr(t, opts), tc.failure)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory runs check on
|
||||
// a manifest given by URL, from a directory holding the file it lists: the
|
||||
// file is looked for there.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory(t *testing.T) {
|
||||
files := map[string][]byte{testFileTxt: []byte("hello")}
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
|
||||
defer server.Close()
|
||||
|
||||
cwd := chdirTemp(t)
|
||||
require.NoError(t,
|
||||
os.WriteFile(filepath.Join(cwd, testFileTxt), files[testFileTxt], 0o600))
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdCheck, testFlagNoExtra, server.URL + "/" + defaultManifestName,
|
||||
}, afero.NewOsFs())
|
||||
assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
}
|
||||
|
||||
// unlistableDirFs is a filesystem on which one directory cannot be listed.
|
||||
type unlistableDirFs struct {
|
||||
afero.Fs
|
||||
@@ -1204,6 +1289,180 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
|
||||
}
|
||||
|
||||
// writeTestTree writes file.txt and sub/nested.txt under dir.
|
||||
func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(dir, testSubdir), 0o750))
|
||||
writeTestFile(t, fs, filepath.Join(dir, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, filepath.Join(dir, testSubdir, "nested.txt"), "in sub")
|
||||
}
|
||||
|
||||
// TestGenerateDefaultOutput runs gen without --output on one directory or
|
||||
// one file: it writes index.mf in that directory, or beside that file,
|
||||
// listing each file by its path under the directory index.mf is in, and
|
||||
// check given that directory passes.
|
||||
func TestGenerateDefaultOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Paths are relative to a temp dir holding file.txt and sub/nested.txt.
|
||||
for name, tc := range map[string]struct {
|
||||
input, output string
|
||||
listed []string
|
||||
}{
|
||||
"directory": {
|
||||
".", defaultManifestName, []string{testFileTxt, "sub/nested.txt"},
|
||||
},
|
||||
"subdirectory": {testSubdir, testSubdirManifest, []string{"nested.txt"}},
|
||||
"file": {testFileTxt, defaultManifestName, []string{testFileTxt}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
fs := afero.NewOsFs()
|
||||
writeTestTree(t, fs, root)
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", filepath.Join(root, tc.input),
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
output := filepath.Join(root, tc.output)
|
||||
assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q", filepath.Dir(output),
|
||||
}, fs)
|
||||
assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateSeveralPathsDefaultOutput runs gen without --output on two
|
||||
// directories: it writes index.mf in the current directory, listing both
|
||||
// directories' files, and writes no index.mf in either directory.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestGenerateSeveralPathsDefaultOutput(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fs := afero.NewOsFs()
|
||||
dirs := []string{"first", "second"}
|
||||
|
||||
for _, dir := range dirs {
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(root, dir), 0o750))
|
||||
writeTestFile(t, fs, filepath.Join(root, dir, dir+".txt"), dir)
|
||||
}
|
||||
|
||||
t.Chdir(root)
|
||||
|
||||
opts := testOpts(append([]string{testApp, cmdGenerate, "-q"}, dirs...), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{"first.txt", "second.txt"},
|
||||
manifestPaths(t, fs, filepath.Join(root, defaultManifestName)))
|
||||
|
||||
for _, dir := range dirs {
|
||||
exists, err := afero.Exists(fs, filepath.Join(root, dir, defaultManifestName))
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists, "index.mf written in %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateBytesDoNotDependOnOutput runs gen --seed on one tree, each
|
||||
// time writing to another file, over a file already there and beside an
|
||||
// earlier run's temp file: neither is listed, and what is listed depends
|
||||
// only on the tree, so every manifest has the same bytes.
|
||||
func TestGenerateBytesDoNotDependOnOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
tree := filepath.Join(root, "tree")
|
||||
defaultOutput := filepath.Join(tree, defaultManifestName)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestTree(t, fs, tree)
|
||||
|
||||
var first []byte
|
||||
|
||||
for _, output := range []string{
|
||||
defaultOutput,
|
||||
filepath.Join(tree, "listing.mf"),
|
||||
filepath.Join(tree, testSubdir, "listing.mf"),
|
||||
filepath.Join(root, "outside.mf"),
|
||||
} {
|
||||
writeTestFile(t, fs, output, "previous manifest")
|
||||
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
||||
|
||||
args := []string{testApp, cmdGenerate, "-q", "-f", "--seed", "mfer"}
|
||||
if output != defaultOutput {
|
||||
args = append(args, "-o", output)
|
||||
}
|
||||
|
||||
args = append(args, tree)
|
||||
|
||||
opts := testOpts(args, fs)
|
||||
require.Equal(t, 0, runCLI(opts),
|
||||
"output %s, stderr: %s", output, testStderr(t, opts))
|
||||
|
||||
got, err := afero.ReadFile(fs, output)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fs.Remove(output))
|
||||
|
||||
if first == nil {
|
||||
first = got
|
||||
}
|
||||
|
||||
assert.Equal(t, first, got, "output %s", output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateRefusesExistingDefaultOutput runs gen without --output or
|
||||
// --force on a directory already holding index.mf: gen fails, naming that
|
||||
// file, and leaves it as it was.
|
||||
func TestGenerateRefusesExistingDefaultOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
output := filepath.Join(testDir, defaultManifestName)
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello")
|
||||
writeTestFile(t, fs, output, "previous manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", testDir}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
"output file "+output+" already exists (use --force to overwrite)")
|
||||
|
||||
content, err := afero.ReadFile(fs, output)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "previous manifest", string(content))
|
||||
}
|
||||
|
||||
// TestGenerateRefusesEmptyOutput runs gen with --force and an --output
|
||||
// given an empty value, as an unset shell variable gives it, on a
|
||||
// directory already holding index.mf: gen fails and leaves that file as it
|
||||
// was.
|
||||
func TestGenerateRefusesEmptyOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
output := filepath.Join(testDir, defaultManifestName)
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello")
|
||||
writeTestFile(t, fs, output, "previous manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-f", "-o", "", testDir}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), errEmptyOutput.Error())
|
||||
|
||||
content, err := afero.ReadFile(fs, output)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "previous manifest", string(content))
|
||||
}
|
||||
|
||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+104
-3
@@ -303,7 +303,7 @@ func TestManifestLoaderHTTPStatusMessage(t *testing.T) {
|
||||
_, err := mfa.openManifestReader(context.Background(), server.URL+"/foo.mf")
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
assert.EqualError(t, err,
|
||||
"failed to fetch "+server.URL+"/foo.mf: HTTP 404")
|
||||
"download manifest "+server.URL+"/foo.mf: unexpected HTTP status 404")
|
||||
}
|
||||
|
||||
func TestFetchManifestHTTPStatusMessage(t *testing.T) {
|
||||
@@ -325,7 +325,7 @@ func TestFetchManifestHTTPStatusMessage(t *testing.T) {
|
||||
return cmd.Run(context.Background(), []string{cmdFetch, server.URL})
|
||||
})
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
assert.EqualError(t, err, "failed to fetch manifest: HTTP 404")
|
||||
assert.EqualError(t, err, "download manifest: unexpected HTTP status 404")
|
||||
}
|
||||
|
||||
func TestFetchFileHTTPStatusMessage(t *testing.T) {
|
||||
@@ -343,7 +343,108 @@ func TestFetchFileHTTPStatusMessage(t *testing.T) {
|
||||
&mfer.MFFilePath{}, nil)
|
||||
})
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
assert.EqualError(t, err, "HTTP 500")
|
||||
assert.EqualError(t, err, "unexpected HTTP status 500")
|
||||
}
|
||||
|
||||
// TestCheckCorruptManifestMessage runs check on a file that is not a
|
||||
// manifest.
|
||||
func TestCheckCorruptManifestMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
||||
|
||||
mfa := &CLIApp{Fs: fs}
|
||||
cmd := mfa.checkCommand()
|
||||
cmd.Action = mfa.checkManifestOperation
|
||||
|
||||
// checkManifestOperation logs to the process-global logger.
|
||||
err := runLocked(func() error {
|
||||
return cmd.Run(context.Background(), []string{cmdCheck, "/bad.mf"})
|
||||
})
|
||||
assert.EqualError(t, err, "load manifest: invalid file format")
|
||||
}
|
||||
|
||||
// TestListMissingManifestMessage runs list on a manifest file that does not
|
||||
// exist.
|
||||
func TestListMissingManifestMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
|
||||
cmd := mfa.listCommand()
|
||||
|
||||
// listManifestOperation sets the process-global log level.
|
||||
err := runLocked(func() error {
|
||||
return cmd.Run(context.Background(), []string{cmdList, "/nope.mf"})
|
||||
})
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
assert.EqualError(t, err, "open /nope.mf: file does not exist")
|
||||
}
|
||||
|
||||
// TestFetchHashMismatchMessage runs fetch against a server that sends a
|
||||
// listed file with other content of the same size.
|
||||
func TestFetchHashMismatchMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := builtManifest(t, map[string][]byte{testFileTxt: []byte("listed")})
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest,
|
||||
map[string][]byte{testFileTxt: []byte("served")}))
|
||||
defer server.Close()
|
||||
|
||||
mfa := &CLIApp{Fs: afero.NewMemMapFs(), maxManifestSize: mfer.MaxManifestSize}
|
||||
cmd := mfa.fetchCommand()
|
||||
cmd.Action = mfa.fetchManifestOperation
|
||||
|
||||
// fetchManifestOperation logs to the process-global logger.
|
||||
err := runLocked(func() error {
|
||||
return cmd.Run(context.Background(),
|
||||
[]string{cmdFetch, "--" + flagDest, t.TempDir(), server.URL})
|
||||
})
|
||||
require.ErrorIs(t, err, errHashMismatch)
|
||||
assert.EqualError(t, err, "download "+testFileTxt+": hash mismatch")
|
||||
}
|
||||
|
||||
// TestFreshenBackslashPathMessage runs freshen on a tree that has gained a
|
||||
// file whose name holds a backslash, which a manifest path may not contain.
|
||||
func TestFreshenBackslashPathMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
root, manifestPath := setupFreshenDir(t, fs,
|
||||
map[string]string{testFileTxt: "content"})
|
||||
writeTestFile(t, fs, filepath.Join(root, `a\b.txt`), "new")
|
||||
|
||||
mfa := &CLIApp{Fs: fs}
|
||||
cmd := mfa.freshenCommand()
|
||||
cmd.Action = mfa.freshenManifestOperation
|
||||
|
||||
// freshenManifestOperation logs to the process-global logger.
|
||||
err := runLocked(func() error {
|
||||
return cmd.Run(context.Background(),
|
||||
[]string{cmdFreshen, testFlagBase, root, manifestPath})
|
||||
})
|
||||
assert.EqualError(t, err,
|
||||
`path "a\\b.txt" contains backslash; use forward slashes only`)
|
||||
}
|
||||
|
||||
// TestFreshenReadErrorMessage has freshen hash a directory as though it
|
||||
// were a file, so reading it fails.
|
||||
func TestFreshenReadErrorMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.Mkdir(filepath.Join(root, "sub"), 0o750))
|
||||
|
||||
hasher := &freshenHasher{
|
||||
fs: afero.NewOsFs(),
|
||||
absBase: root,
|
||||
builder: mfer.NewBuilder(),
|
||||
}
|
||||
|
||||
err := hasher.processEntry(&freshenEntry{path: "sub", needsHash: true})
|
||||
assert.EqualError(t, err,
|
||||
"read "+filepath.Join(root, "sub")+": is a directory")
|
||||
}
|
||||
|
||||
func TestURLRequiredMessage(t *testing.T) {
|
||||
|
||||
@@ -26,12 +26,12 @@ func (mfa *CLIApp) exportManifestOperation(
|
||||
) error {
|
||||
pathOrURL, err := mfa.resolveManifestArg(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
rc, err := mfa.openManifestReader(ctx, pathOrURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = rc.Close() }()
|
||||
@@ -39,7 +39,7 @@ func (mfa *CLIApp) exportManifestOperation(
|
||||
//nolint:contextcheck // mfer loads a manifest without a context
|
||||
manifest, err := mfer.NewManifestFromReader(rc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: failed to parse manifest: %w", err)
|
||||
return fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
|
||||
files := manifest.Files()
|
||||
@@ -76,7 +76,7 @@ func (mfa *CLIApp) exportManifestOperation(
|
||||
|
||||
err = enc.Encode(entries)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: failed to encode JSON: %w", err)
|
||||
return fmt.Errorf("encode JSON: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
+16
-21
@@ -239,7 +239,7 @@ func reportDownloadProgress(progress <-chan DownloadProgress, done chan<- struct
|
||||
func manifestBaseURL(manifestURL string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(manifestURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch: invalid manifest URL: %w", err)
|
||||
return nil, fmt.Errorf("invalid manifest URL: %w", err)
|
||||
}
|
||||
|
||||
// JoinPath cleans the path it builds, so ".." drops the manifest's
|
||||
@@ -268,7 +268,7 @@ func downloadManifestFiles(
|
||||
// Sanitize the path to prevent path traversal attacks
|
||||
localPath, err := sanitizePath(f.GetPath())
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid path in manifest: %w", err)
|
||||
return 0, 0, fmt.Errorf("invalid file entry: %w", err)
|
||||
}
|
||||
|
||||
if alreadyPresent(dest, localPath, f) {
|
||||
@@ -284,7 +284,7 @@ func downloadManifestFiles(
|
||||
|
||||
err = downloadFile(ctx, client, fileURL, dest, localPath, f, progress)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
|
||||
return 0, 0, fmt.Errorf("download %s: %w", f.GetPath(), err)
|
||||
}
|
||||
|
||||
downloaded++
|
||||
@@ -376,7 +376,7 @@ func (mfa *CLIApp) fetchManifestOperation(
|
||||
|
||||
err = os.MkdirAll(dest, dirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create destination directory %s: %w", dest, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Create progress channel and start progress reporter goroutine
|
||||
@@ -403,7 +403,7 @@ func (mfa *CLIApp) fetchManifestOperation(
|
||||
// "mfer check" can verify the tree later.
|
||||
err = saveManifest(dest, manifestData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save manifest: %w", err)
|
||||
return fmt.Errorf("save manifest: %w", err)
|
||||
}
|
||||
|
||||
// Print summary
|
||||
@@ -446,11 +446,11 @@ func (mfa *CLIApp) fetchManifest(
|
||||
return readErr
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
|
||||
return nil, nil, fmt.Errorf("download manifest: %w", err)
|
||||
}
|
||||
|
||||
if int64(len(manifestData)) > mfa.maxManifestSize {
|
||||
return nil, nil, fmt.Errorf("failed to fetch manifest: %w of %d bytes",
|
||||
return nil, nil, fmt.Errorf("download manifest: %w of %d bytes",
|
||||
errManifestTooLarge, mfa.maxManifestSize)
|
||||
}
|
||||
|
||||
@@ -458,7 +458,7 @@ func (mfa *CLIApp) fetchManifest(
|
||||
//nolint:contextcheck // mfer loads a manifest without a context
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
|
||||
return nil, nil, fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
|
||||
requiredSigner := cmd.String(flagRequireSignature)
|
||||
@@ -561,7 +561,7 @@ func verifyFetchedSigner(manifestData []byte, requiredSigner string) error {
|
||||
Fs: memFs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
return fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
|
||||
return verifyRequiredSigner(chk, requiredSigner)
|
||||
@@ -653,7 +653,7 @@ func checkNoSymlinks(dest, p string) error {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check %s for a symlink: %w", current, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
@@ -770,7 +770,7 @@ func tempPathFor(localPath string) string {
|
||||
func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error {
|
||||
computed, err := multihash.Encode(digest, multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encode hash: %w", err)
|
||||
return fmt.Errorf("encode hash: %w", err)
|
||||
}
|
||||
|
||||
for _, hash := range entry.GetHashes() {
|
||||
@@ -797,7 +797,7 @@ func downloadFile(
|
||||
// so every entry point to downloadFile gets the same treatment.
|
||||
localPath, err := sanitizePath(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid path: %w", err)
|
||||
return fmt.Errorf("invalid file entry: %w", err)
|
||||
}
|
||||
|
||||
// Create parent directories if needed
|
||||
@@ -812,7 +812,7 @@ func downloadFile(
|
||||
|
||||
err = os.MkdirAll(dir, dirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -846,7 +846,7 @@ func createTempFile(dest, tmpPath string) (*os.File, error) {
|
||||
out, err := os.OpenFile( //nolint:gosec // G304: see comment above
|
||||
path, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, nil
|
||||
@@ -859,12 +859,7 @@ func moveIntoPlace(dest, tmpPath, localPath string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Rename(filepath.Join(dest, tmpPath), filepath.Join(dest, localPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return os.Rename(filepath.Join(dest, tmpPath), filepath.Join(dest, localPath))
|
||||
}
|
||||
|
||||
// saveResponse writes resp's body to tmpPath, verifies it against entry,
|
||||
@@ -899,7 +894,7 @@ func saveResponse(
|
||||
_ = out.Close()
|
||||
_ = os.Remove(filepath.Join(dest, tmpPath))
|
||||
|
||||
return fmt.Errorf("failed to set mode: %w", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+13
-11
@@ -502,8 +502,9 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
|
||||
[]string{cmd.Name, server.URL + "/index.mf"})
|
||||
})
|
||||
require.ErrorIs(t, err, errManifestTooLarge, cmd.Name)
|
||||
require.ErrorContains(t, err,
|
||||
"maximum allowed size of 65536 bytes", cmd.Name)
|
||||
require.EqualError(t, err,
|
||||
"download manifest: file exceeds maximum allowed size of 65536 bytes",
|
||||
cmd.Name)
|
||||
}
|
||||
|
||||
leftover, err := os.ReadDir(tmpDir)
|
||||
@@ -605,27 +606,27 @@ func TestFetchRefusesSymlinks(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
"parent directory", "sub/deeper/file.txt", "sub", ".",
|
||||
"failed to download sub/deeper/file.txt",
|
||||
"download sub/deeper/file.txt",
|
||||
},
|
||||
{
|
||||
"directory inside a plain directory", "docs/data/passwd", "docs/data", ".",
|
||||
"failed to download docs/data/passwd",
|
||||
"download docs/data/passwd",
|
||||
},
|
||||
{
|
||||
"temp file", testFileTxt, ".file.txt.tmp", newFile,
|
||||
"failed to download " + testFileTxt,
|
||||
"download " + testFileTxt,
|
||||
},
|
||||
{
|
||||
"file", testFileTxt, testFileTxt, newFile,
|
||||
"failed to download " + testFileTxt,
|
||||
"download " + testFileTxt,
|
||||
},
|
||||
{
|
||||
"manifest temp file", testFileTxt, tempPathFor(defaultManifestName), newFile,
|
||||
"failed to save manifest",
|
||||
"save manifest",
|
||||
},
|
||||
{
|
||||
"manifest", testFileTxt, defaultManifestName, newFile,
|
||||
"failed to save manifest",
|
||||
"save manifest",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -692,7 +693,7 @@ func TestFetchDoesNotSkipThroughSymlink(t *testing.T) {
|
||||
}, afero.NewOsFs())
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
"failed to download sub/"+testFileTxt+": symlink in path not allowed: "+link)
|
||||
"download sub/"+testFileTxt+": symlink in path not allowed: "+link)
|
||||
assert.Equal(t, map[string][]byte{testFileTxt: content}, filesUnder(t, outside))
|
||||
}
|
||||
|
||||
@@ -762,7 +763,8 @@ func TestGetRetriesTransientStatusesOnly(t *testing.T) {
|
||||
|
||||
err := getNothing(testClient(), server.URL, 10*time.Second)
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
require.EqualError(t, err, fmt.Sprintf("HTTP %d", tt.status))
|
||||
require.EqualError(t, err,
|
||||
fmt.Sprintf("unexpected HTTP status %d", tt.status))
|
||||
assert.Equal(t, tt.requests, requests.Load())
|
||||
})
|
||||
}
|
||||
@@ -1228,7 +1230,7 @@ func TestFetchRequireSignature(t *testing.T) {
|
||||
manifest, required := manifestSignedByAnotherKey(t, files)
|
||||
|
||||
assertFetchRefused(t, manifest, files,
|
||||
"failed to parse manifest: signature verification failed: "+
|
||||
"parse manifest: "+
|
||||
"embedded public key block must hold exactly one key, found 2",
|
||||
"--"+flagRequireSignature, required)
|
||||
})
|
||||
|
||||
+16
-30
@@ -165,8 +165,7 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(s.absBase, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Handle dotfiles
|
||||
@@ -280,13 +279,8 @@ func (h *freshenHasher) reportProgress(n int64) {
|
||||
// processEntry hashes the entry if needed and adds it to the builder.
|
||||
func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
if !e.needsHash {
|
||||
// Use existing entry
|
||||
err := addExistingToBuilder(h.builder, e.existing)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
// Use existing entry; the error names the entry
|
||||
return addExistingToBuilder(h.builder, e.existing)
|
||||
}
|
||||
|
||||
// Need to read and hash the file
|
||||
@@ -294,26 +288,21 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
|
||||
f, err := h.fs.Open(absPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open %s: %w", e.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
hash, bytesRead, err := hashFile(f, h.reportProgress)
|
||||
_ = f.Close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to hash %s: %w", e.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
h.hashedBytes += bytesRead
|
||||
h.hashedFiles++
|
||||
|
||||
// Add to builder with computed hash
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
// Add to builder with computed hash; a refused path is named in the error
|
||||
return addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
|
||||
}
|
||||
|
||||
// writeFreshenedManifest writes the manifest atomically (write to a
|
||||
@@ -325,7 +314,7 @@ func writeFreshenedManifest(
|
||||
|
||||
outFile, err := afs.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = builder.Build(ctx, outFile)
|
||||
@@ -334,7 +323,7 @@ func writeFreshenedManifest(
|
||||
if err != nil {
|
||||
_ = afs.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to write manifest: %w", err)
|
||||
return fmt.Errorf("build manifest: %w", err)
|
||||
}
|
||||
|
||||
// Rename temp to final
|
||||
@@ -342,7 +331,7 @@ func writeFreshenedManifest(
|
||||
if err != nil {
|
||||
_ = afs.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to rename manifest: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -408,7 +397,7 @@ func (mfa *CLIApp) freshenScan(
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to scan filesystem: %w", err)
|
||||
return nil, 0, fmt.Errorf("scan filesystem: %w", err)
|
||||
}
|
||||
|
||||
// Remaining entries in existingByPath are removed files
|
||||
@@ -477,7 +466,7 @@ func (mfa *CLIApp) loadExistingEntries(
|
||||
Fs: mfa.Fs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
||||
return nil, fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
|
||||
existingFiles := manifest.Files()
|
||||
@@ -497,13 +486,12 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
) error {
|
||||
log.Debug("freshenManifestOperation()")
|
||||
|
||||
basePath := cmd.String("base")
|
||||
showProgress := cmd.Bool("progress")
|
||||
|
||||
// Find manifest file
|
||||
manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
//nolint:contextcheck // mfer loads a manifest without a context
|
||||
@@ -512,9 +500,9 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
return err
|
||||
}
|
||||
|
||||
absBase, err := filepath.Abs(basePath)
|
||||
absBase, err := filepath.Abs(resolveBasePath(cmd, manifestPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: invalid base path: %w", err)
|
||||
return fmt.Errorf("invalid base path: %w", err)
|
||||
}
|
||||
|
||||
// Phase 1: Scan filesystem
|
||||
@@ -606,9 +594,7 @@ func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
|
||||
break
|
||||
}
|
||||
|
||||
// Returned unwrapped: the caller renders this as
|
||||
// "failed to hash <path>: <err>" and adding a second layer here
|
||||
// would change that message.
|
||||
// Returned unwrapped: a read error already names the file.
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -300,6 +301,41 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
manifestPaths(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
|
||||
// runs freshen from the directory above sub, on the manifest in sub.
|
||||
// Without --base, the manifest then lists the files in sub, whether freshen
|
||||
// is given the manifest or sub. --base names the directory to list instead,
|
||||
// the current one included.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestFreshenResolvesEntriesAgainstManifestDirectory(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
want []string // the paths the manifest lists afterwards
|
||||
}{
|
||||
{[]string{testSubdir}, []string{"b.txt", "c.txt"}},
|
||||
{[]string{testSubdirManifest}, []string{"b.txt", "c.txt"}},
|
||||
{
|
||||
[]string{testFlagBase, ".", testSubdirManifest},
|
||||
[]string{testFileTxt, "sub/b.txt", "sub/c.txt"},
|
||||
},
|
||||
} {
|
||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
root := setupManifestInSubdir(t)
|
||||
writeTestFile(t, fs, filepath.Join(root, testSubdir, "c.txt"), "added")
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdFreshen, "-q"}, tc.args,
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, tc.want, manifestPaths(t, fs,
|
||||
filepath.Join(root, testSubdirManifest)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
||||
// proto3 field presence and may legitimately be absent.
|
||||
|
||||
+82
-34
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
@@ -26,6 +27,8 @@ var (
|
||||
// rendered message stays exactly as mfer has always printed it.
|
||||
errOutputExists = errors.New(
|
||||
"already exists (use --force to overwrite)")
|
||||
// errEmptyOutput indicates --output given with an empty value.
|
||||
errEmptyOutput = errors.New("--output must not be empty")
|
||||
)
|
||||
|
||||
// reportEnumProgress renders enumeration progress until the channel
|
||||
@@ -74,7 +77,7 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
|
||||
ap, err := filepath.Abs(inputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate: invalid path %q: %w", inputPath, err)
|
||||
return nil, fmt.Errorf("invalid path %q: %w", inputPath, err)
|
||||
}
|
||||
// Validate path exists before adding to list
|
||||
if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
|
||||
@@ -88,9 +91,40 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||
func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions {
|
||||
output := cmd.String("output")
|
||||
// outputPath returns the file gen writes the manifest to: the one --output
|
||||
// names, or else index.mf in the directory the only argument names, or
|
||||
// beside the file it names, or else in the current directory. An --output
|
||||
// given with an empty value is refused.
|
||||
func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
|
||||
if cmd.IsSet("output") {
|
||||
output := cmd.String("output")
|
||||
if output == "" {
|
||||
return "", errEmptyOutput
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
if cmd.Args().Len() != 1 {
|
||||
return defaultManifestName, nil
|
||||
}
|
||||
|
||||
arg := cmd.Args().First()
|
||||
|
||||
// A path that does not exist is refused when it is enumerated.
|
||||
info, err := mfa.Fs.Stat(arg)
|
||||
if err == nil && !info.IsDir() {
|
||||
return filepath.Join(filepath.Dir(arg), defaultManifestName), nil
|
||||
}
|
||||
|
||||
return filepath.Join(arg, defaultManifestName), nil
|
||||
}
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags and
|
||||
// the path the manifest is written to.
|
||||
func (mfa *CLIApp) buildScannerOptions(
|
||||
cmd *cli.Command, output string,
|
||||
) *mfer.ScannerOptions {
|
||||
opts := &mfer.ScannerOptions{
|
||||
IncludeDotfiles: cmd.Bool("include-dotfiles"),
|
||||
FollowSymLinks: cmd.Bool("follow-symlinks"),
|
||||
@@ -129,8 +163,7 @@ func (mfa *CLIApp) enumerateInputs(
|
||||
// Default to current directory
|
||||
err := s.EnumeratePath(".", enumProgress)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"generate: failed to enumerate current directory: %w", err)
|
||||
return fmt.Errorf("enumerate current directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -144,7 +177,7 @@ func (mfa *CLIApp) enumerateInputs(
|
||||
|
||||
err = s.EnumeratePaths(enumProgress, paths...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate: failed to enumerate paths: %w", err)
|
||||
return fmt.Errorf("enumerate files: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -203,23 +236,54 @@ func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runScanPhase reads the enumerated files and writes the manifest to out,
|
||||
// with optional progress reporting.
|
||||
func (mfa *CLIApp) runScanPhase(
|
||||
ctx context.Context, cmd *cli.Command, s *mfer.Scanner, out io.Writer,
|
||||
) error {
|
||||
var (
|
||||
scanProgress chan mfer.ScanStatus
|
||||
scanWg sync.WaitGroup
|
||||
)
|
||||
|
||||
if cmd.Bool("progress") {
|
||||
scanProgress = make(chan mfer.ScanStatus, 1)
|
||||
|
||||
scanWg.Add(1)
|
||||
|
||||
go reportScanProgress(scanProgress, &scanWg)
|
||||
}
|
||||
|
||||
err := s.ToManifest(ctx, out, scanProgress)
|
||||
|
||||
scanWg.Wait()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate manifest: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) generateManifestOperation(
|
||||
ctx context.Context, cmd *cli.Command,
|
||||
) error {
|
||||
log.Debug("generateManifestOperation()")
|
||||
|
||||
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd))
|
||||
|
||||
// Phase 1: Enumeration - collect paths and stat files
|
||||
err := mfa.runEnumeratePhase(cmd, s)
|
||||
outputPath, err := mfa.outputPath(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
showProgress := cmd.Bool("progress")
|
||||
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd, outputPath))
|
||||
|
||||
// Phase 1: Enumeration - collect paths and stat files
|
||||
err = mfa.runEnumeratePhase(cmd, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if output file exists
|
||||
outputPath := cmd.String("output")
|
||||
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") {
|
||||
return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
|
||||
}
|
||||
@@ -229,7 +293,7 @@ func (mfa *CLIApp) generateManifestOperation(
|
||||
|
||||
outFile, err := mfa.Fs.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Set up signal handler to clean up temp file on Ctrl-C
|
||||
@@ -250,37 +314,21 @@ func (mfa *CLIApp) generateManifestOperation(
|
||||
}()
|
||||
|
||||
// Phase 2: Scan - read file contents and generate manifest
|
||||
var (
|
||||
scanProgress chan mfer.ScanStatus
|
||||
scanWg sync.WaitGroup
|
||||
)
|
||||
|
||||
if showProgress {
|
||||
scanProgress = make(chan mfer.ScanStatus, 1)
|
||||
|
||||
scanWg.Add(1)
|
||||
|
||||
go reportScanProgress(scanProgress, &scanWg)
|
||||
}
|
||||
|
||||
err = s.ToManifest(ctx, outFile, scanProgress)
|
||||
|
||||
scanWg.Wait()
|
||||
|
||||
err = mfa.runScanPhase(ctx, cmd, s, outFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate manifest: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Close file before rename to ensure all data is flushed
|
||||
err = outFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
err = mfa.Fs.Rename(tmpPath, outputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
success = true
|
||||
|
||||
@@ -19,12 +19,12 @@ func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command)
|
||||
|
||||
pathOrURL, err := mfa.resolveManifestArg(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
rc, err := mfa.openManifestReader(ctx, pathOrURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = rc.Close() }()
|
||||
@@ -32,7 +32,7 @@ func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command)
|
||||
//nolint:contextcheck // mfer loads a manifest without a context
|
||||
manifest, err := mfer.NewManifestFromReader(rc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list: failed to parse manifest: %w", err)
|
||||
return fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
|
||||
files := manifest.Files()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -15,17 +16,13 @@ import (
|
||||
// manifestFetchTimeout bounds HTTP requests made to fetch a manifest.
|
||||
const manifestFetchTimeout = 30 * time.Second
|
||||
|
||||
// errHTTPStatus indicates an HTTP response with a non-OK status code.
|
||||
//
|
||||
// Its text is the literal "HTTP" prefix of the rendered "HTTP <code>"
|
||||
// message that mfer has always printed, so that wrapping it does not
|
||||
// change any user-visible output. Match it with errors.Is; do not read
|
||||
// its message.
|
||||
var errHTTPStatus = errors.New("HTTP")
|
||||
// errHTTPStatus indicates an HTTP response with a non-OK status code. It is
|
||||
// followed by the code, as in "unexpected HTTP status 404".
|
||||
var errHTTPStatus = errors.New("unexpected HTTP status")
|
||||
|
||||
// errManifestTooLarge indicates a manifest download that passed
|
||||
// CLIApp.maxManifestSize.
|
||||
var errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
|
||||
var errManifestTooLarge = errors.New("file exceeds maximum allowed size")
|
||||
|
||||
// isHTTPURL returns true if the string starts with http:// or https://.
|
||||
func isHTTPURL(s string) bool {
|
||||
@@ -40,20 +37,22 @@ func (mfa *CLIApp) openManifestReader(
|
||||
if isHTTPURL(pathOrURL) {
|
||||
client := &http.Client{Timeout: manifestFetchTimeout}
|
||||
|
||||
// The *url.Error that NewRequestWithContext and Do return names
|
||||
// the URL.
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pathOrURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
|
||||
return nil, fmt.Errorf("download manifest: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
|
||||
return nil, fmt.Errorf("download manifest: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_ = resp.Body.Close()
|
||||
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w %d",
|
||||
return nil, fmt.Errorf("download manifest %s: %w %d",
|
||||
pathOrURL, errHTTPStatus, resp.StatusCode)
|
||||
}
|
||||
|
||||
@@ -88,3 +87,17 @@ func (mfa *CLIApp) resolveManifestArg(cmd *cli.Command) (string, error) {
|
||||
|
||||
return findManifest(mfa.Fs, ".")
|
||||
}
|
||||
|
||||
// resolveBasePath returns the directory a manifest's paths are resolved
|
||||
// against: the one --base names, or else the directory holding the manifest,
|
||||
// or the current directory for a manifest URL.
|
||||
func resolveBasePath(cmd *cli.Command, manifestPath string) string {
|
||||
switch {
|
||||
case cmd.IsSet(flagBase):
|
||||
return cmd.String(flagBase)
|
||||
case isHTTPURL(manifestPath):
|
||||
return "."
|
||||
default:
|
||||
return filepath.Dir(manifestPath)
|
||||
}
|
||||
}
|
||||
|
||||
+11
-8
@@ -24,6 +24,7 @@ const (
|
||||
cmdList = "list"
|
||||
cmdVersion = "version"
|
||||
|
||||
flagBase = "base"
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagDest = "dest"
|
||||
@@ -210,9 +211,10 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "output",
|
||||
Value: defaultManifestName,
|
||||
Aliases: []string{"o"},
|
||||
Usage: "Specify output filename",
|
||||
Usage: "File to write the manifest to (default: index.mf in " +
|
||||
"the directory given, or beside the file given; with no " +
|
||||
"path or several, index.mf in the current directory)",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "force",
|
||||
@@ -259,10 +261,11 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
},
|
||||
Flags: append(commonFlags(),
|
||||
&cli.StringFlag{
|
||||
Name: "base",
|
||||
Name: flagBase,
|
||||
Aliases: []string{"b"},
|
||||
Value: ".",
|
||||
Usage: "Base directory for resolving relative paths from manifest",
|
||||
Usage: "Base directory for resolving relative paths from manifest " +
|
||||
"(by default the directory holding the manifest, or the " +
|
||||
"current directory for a manifest URL)",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: flagProgress,
|
||||
@@ -292,10 +295,10 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
},
|
||||
Flags: append(commonFlags(),
|
||||
&cli.StringFlag{
|
||||
Name: "base",
|
||||
Name: flagBase,
|
||||
Aliases: []string{"b"},
|
||||
Value: ".",
|
||||
Usage: "Base directory for resolving relative paths",
|
||||
Usage: "Base directory for resolving relative paths " +
|
||||
"(by default the directory holding the manifest)",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "follow-symlinks",
|
||||
|
||||
+4
-7
@@ -246,7 +246,7 @@ func (b *Builder) AddFileWithHash(
|
||||
) error {
|
||||
err := ValidatePath(string(path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("add file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if size < 0 {
|
||||
@@ -329,22 +329,19 @@ func (b *Builder) Build(ctx context.Context, w io.Writer) error {
|
||||
// Generate outer wrapper
|
||||
err := m.generateOuter(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: generate outer: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Generate final output
|
||||
err = m.generate(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: generate: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Write to output
|
||||
_, err = w.Write(m.output.Bytes())
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: write output: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// addEntry adds entry to the manifest unless an entry with its path is
|
||||
|
||||
+13
-16
@@ -24,13 +24,12 @@ var (
|
||||
errCompressedHashWrong = errors.New("compressed data hash mismatch")
|
||||
errSignatureNoPubKey = errors.New("signature present but no public key")
|
||||
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
|
||||
errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
|
||||
errManifestTooLarge = errors.New("file exceeds maximum allowed size")
|
||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||
errInvalidFileFormat = errors.New("invalid file format")
|
||||
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
||||
errDecodedTooLarge = errors.New(
|
||||
"manifest would take too much memory to decode")
|
||||
errSignerNotSigningKey = errors.New(
|
||||
errInvalidManifestPath = errors.New("invalid file entry")
|
||||
errDecodedTooLarge = errors.New("too much memory needed")
|
||||
errSignerNotSigningKey = errors.New(
|
||||
"signer is not the fingerprint of the key that made the signature")
|
||||
)
|
||||
|
||||
@@ -58,7 +57,7 @@ func (m *manifest) validateOuterHeader() error {
|
||||
// Validate outer UUID before any decompression
|
||||
err := validateUUID(m.pbOuter.GetUuid())
|
||||
if err != nil {
|
||||
return fmt.Errorf("outer UUID invalid: %w", err)
|
||||
return fmt.Errorf("outer message: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -73,7 +72,7 @@ func (m *manifest) verifyOuterIntegrity() error {
|
||||
|
||||
_, err := h.Write(m.pbOuter.GetInnerMessage())
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: hash write: %w", err)
|
||||
return fmt.Errorf("hash inner message: %w", err)
|
||||
}
|
||||
|
||||
sha256Hash := h.Sum(nil)
|
||||
@@ -91,9 +90,7 @@ func (m *manifest) verifyOuterIntegrity() error {
|
||||
|
||||
sigString, err := m.signatureString()
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to generate signature string for verification: %w", err,
|
||||
)
|
||||
return fmt.Errorf("build signature string: %w", err)
|
||||
}
|
||||
|
||||
// Loading a manifest takes no context; gpgTimeout still bounds gpg.
|
||||
@@ -104,7 +101,7 @@ func (m *manifest) verifyOuterIntegrity() error {
|
||||
m.pbOuter.GetSigningPubKey(),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("signature verification failed: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.EqualFold(string(m.pbOuter.GetSigner()), signingKey) {
|
||||
@@ -135,7 +132,7 @@ func (m *manifest) decompressInner() ([]byte, error) {
|
||||
zstd.WithDecodeBuffersBelow(0),
|
||||
zstd.WithDecoderMaxWindow(zstdWindowSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
|
||||
return nil, fmt.Errorf("create decompressor: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
@@ -150,7 +147,7 @@ func (m *manifest) decompressInner() ([]byte, error) {
|
||||
|
||||
dat, err := io.ReadAll(limitedReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deserialize: decompress: %w", err)
|
||||
return nil, fmt.Errorf("decompress inner message: %w", err)
|
||||
}
|
||||
|
||||
if int64(len(dat)) >= MaxDecompressedSize {
|
||||
@@ -266,7 +263,7 @@ func (m *manifest) deserializeInner() error {
|
||||
|
||||
err = checkDecodedSize(dat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||
return fmt.Errorf("unmarshal inner message: %w", err)
|
||||
}
|
||||
|
||||
// Deserialize inner message
|
||||
@@ -275,7 +272,7 @@ func (m *manifest) deserializeInner() error {
|
||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbInner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||
return fmt.Errorf("unmarshal inner message: %w", err)
|
||||
}
|
||||
|
||||
if m.pbInner.GetVersion() != MFFile_VERSION_ONE {
|
||||
@@ -285,7 +282,7 @@ func (m *manifest) deserializeInner() error {
|
||||
// Validate inner UUID
|
||||
err = validateUUID(m.pbInner.GetUuid())
|
||||
if err != nil {
|
||||
return fmt.Errorf("inner UUID invalid: %w", err)
|
||||
return fmt.Errorf("inner message: %w", err)
|
||||
}
|
||||
|
||||
// Verify UUIDs match
|
||||
|
||||
@@ -49,6 +49,6 @@ func TestReadAtMost(t *testing.T) {
|
||||
_, err = readAtMost(input, maxSize)
|
||||
require.ErrorIs(t, err, errManifestTooLarge)
|
||||
require.EqualError(t, err,
|
||||
"manifest exceeds maximum allowed size of 65536 bytes")
|
||||
"file exceeds maximum allowed size of 65536 bytes")
|
||||
assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
|
||||
}
|
||||
|
||||
+11
-7
@@ -74,14 +74,18 @@ func TestValidatePathMessagesVerbatim(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSerializeInternalErrorMessagesVerbatim pins the two distinct
|
||||
// "internal error" messages, which differ between generate and
|
||||
// generateOuter and have always done so.
|
||||
func TestSerializeInternalErrorMessagesVerbatim(t *testing.T) {
|
||||
// TestSerializeInnerNotSetMessagesVerbatim pins the messages generate and
|
||||
// generateOuter return when the inner message is missing.
|
||||
func TestSerializeInnerNotSetMessagesVerbatim(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := &manifest{}
|
||||
require.EqualError(t, m.generate(context.Background()),
|
||||
"internal error: pbInner not set")
|
||||
require.EqualError(t, m.generateOuter(context.Background()), "internal error")
|
||||
|
||||
err := m.generate(context.Background())
|
||||
require.ErrorIs(t, err, errInnerNotSet)
|
||||
require.EqualError(t, err, "inner message not set")
|
||||
|
||||
err = m.generateOuter(context.Background())
|
||||
require.ErrorIs(t, err, errInternal)
|
||||
require.EqualError(t, err, "inner message not set")
|
||||
}
|
||||
|
||||
+37
-27
@@ -53,7 +53,7 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
errGPGKeyNotFound = errors.New("gpg key not found")
|
||||
errGPGKeyNotFound = errors.New("GPG key not found")
|
||||
errFingerprintNotFound = errors.New("fingerprint not found for key")
|
||||
errSigningKeyCount = errors.New(
|
||||
"embedded public key block must hold exactly one key")
|
||||
@@ -88,8 +88,9 @@ func gpgArgs(opts []string, positional ...string) []string {
|
||||
}
|
||||
|
||||
// runGPG runs the gpg binary in batch mode with the given arguments and
|
||||
// optional stdin, returning captured stdout and stderr. gpg is killed when
|
||||
// ctx ends or gpgTimeout passes, whichever comes first.
|
||||
// optional stdin, returning captured stdout and stderr. If gpg fails, the
|
||||
// error ends with what gpg wrote to stderr. gpg is killed when ctx ends or
|
||||
// gpgTimeout passes, whichever comes first.
|
||||
func runGPG(
|
||||
ctx context.Context, stdin io.Reader, args ...string,
|
||||
) (*bytes.Buffer, *bytes.Buffer, error) {
|
||||
@@ -125,13 +126,28 @@ func runGPG(
|
||||
// "signal: killed"; return the reason instead.
|
||||
err = ctx.Err()
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = fmt.Errorf("gpg timed out: %w", err)
|
||||
err = fmt.Errorf("timed out: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
err = withStderr(err, &stderr)
|
||||
}
|
||||
|
||||
return &stdout, &stderr, err
|
||||
}
|
||||
|
||||
// withStderr returns err followed by what gpg wrote to stderr, or err alone
|
||||
// when gpg wrote nothing.
|
||||
func withStderr(err error, stderr *bytes.Buffer) error {
|
||||
messages := strings.TrimSpace(stderr.String())
|
||||
if messages == "" {
|
||||
return err
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: %s", err, messages)
|
||||
}
|
||||
|
||||
// parseFingerprint extracts the first fingerprint from gpg --with-colons
|
||||
// output, or returns ok=false if none is present.
|
||||
func parseFingerprint(colonOutput string) (string, bool) {
|
||||
@@ -174,7 +190,7 @@ func gpgSign(
|
||||
) ([]byte, string, error) {
|
||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-sign-*")
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to create temp dir: %w", err)
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
@@ -191,19 +207,19 @@ func gpgSign(
|
||||
"--local-user", string(keyID),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("gpg sign failed: %w: %s", err, stderr.String())
|
||||
return nil, "", fmt.Errorf("gpg sign: %w", err)
|
||||
}
|
||||
|
||||
// The last argument of SIG_CREATED is the fingerprint of the key that
|
||||
// made the signature.
|
||||
created, ok := parseStatusLine(stdout.String(), "SIG_CREATED")
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("%w: %s", errSigningKeyNotReported, stderr.String())
|
||||
return nil, "", withStderr(errSigningKeyNotReported, stderr)
|
||||
}
|
||||
|
||||
sig, err := os.ReadFile(sigFile) //nolint:gosec // G304: inside tmpDir, made above
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to read signature: %w", err)
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
return sig, created[len(created)-1], nil
|
||||
@@ -212,11 +228,11 @@ func gpgSign(
|
||||
// gpgExportPublicKey exports the public key for the specified key ID.
|
||||
// Returns the armored public key.
|
||||
func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
||||
stdout, stderr, err := runGPG(ctx, nil,
|
||||
stdout, _, err := runGPG(ctx, nil,
|
||||
gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gpg export failed: %w: %s", err, stderr.String())
|
||||
return nil, fmt.Errorf("gpg export: %w", err)
|
||||
}
|
||||
|
||||
if stdout.Len() == 0 {
|
||||
@@ -228,13 +244,11 @@ func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
||||
|
||||
// gpgGetKeyFingerprint gets the full fingerprint for a key ID.
|
||||
func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
||||
stdout, stderr, err := runGPG(ctx, nil,
|
||||
stdout, _, err := runGPG(ctx, nil,
|
||||
gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"gpg fingerprint lookup failed: %w: %s", err, stderr.String(),
|
||||
)
|
||||
return nil, fmt.Errorf("gpg fingerprint lookup: %w", err)
|
||||
}
|
||||
|
||||
fpr, ok := parseFingerprint(stdout.String())
|
||||
@@ -250,14 +264,12 @@ func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
||||
func gpgImportOneKey(ctx context.Context, gpgHome, pubKeyFile string) error {
|
||||
// --status-fd 1 sends gpg's status lines to stdout, which importing
|
||||
// otherwise leaves empty; its messages go to stderr.
|
||||
importStdout, importStderr, err := runGPG(ctx, nil,
|
||||
importStdout, _, err := runGPG(ctx, nil,
|
||||
gpgArgs([]string{gpgOptHomedir, gpgHome, gpgOptStatusFD, "1", "--import"},
|
||||
pubKeyFile)...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to import public key: %w: %s", err, importStderr.String(),
|
||||
)
|
||||
return fmt.Errorf("gpg import: %w", err)
|
||||
}
|
||||
|
||||
// The first argument of IMPORT_RES counts the primary keys gpg read
|
||||
@@ -284,7 +296,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
// Create temporary directory for GPG operations
|
||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-verify-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp dir: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
@@ -292,7 +304,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
// Set restrictive permissions
|
||||
err = os.Chmod(tmpDir, privateDirPerms)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to set temp dir permissions: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write public key to temp file
|
||||
@@ -300,7 +312,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
|
||||
err = os.WriteFile(pubKeyFile, pubKey, privateFilePerms)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to write public key: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write signature to temp file
|
||||
@@ -308,7 +320,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
|
||||
err = os.WriteFile(sigFile, signature, privateFilePerms)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to write signature: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write data to temp file
|
||||
@@ -316,7 +328,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
|
||||
err = os.WriteFile(dataFile, data, privateFilePerms)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to write data: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = gpgImportOneKey(ctx, tmpDir, pubKeyFile)
|
||||
@@ -326,14 +338,12 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
|
||||
|
||||
// --status-fd 1 sends gpg's status lines to stdout, which verifying a
|
||||
// detached signature otherwise leaves empty; its messages go to stderr.
|
||||
verifyStdout, verifyStderr, err := runGPG(ctx, nil,
|
||||
verifyStdout, _, err := runGPG(ctx, nil,
|
||||
gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptStatusFD, "1", gpgOptVerify},
|
||||
sigFile, dataFile)...,
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"signature verification failed: %w: %s", err, verifyStderr.String(),
|
||||
)
|
||||
return "", fmt.Errorf("gpg verify: %w", err)
|
||||
}
|
||||
|
||||
// gpg writes a VALIDSIG line for each good signature. Its first
|
||||
|
||||
+28
-2
@@ -603,7 +603,8 @@ func fakeGPGPath(t *testing.T, script string) string {
|
||||
|
||||
// TestGPGTimeoutKillsGPG puts a fake gpg that never finishes first on
|
||||
// PATH and checks that a run past its deadline is killed and reported as
|
||||
// a timeout of the named operation, instead of hanging.
|
||||
// a timeout of the named operation, instead of hanging. The fake gpg writes
|
||||
// nothing to stderr, so the message ends with the timeout.
|
||||
func TestGPGTimeoutKillsGPG(t *testing.T) {
|
||||
t.Setenv("PATH", fakeGPGPath(t, "#!/bin/sh\nexec sleep 10\n"))
|
||||
|
||||
@@ -612,7 +613,32 @@ func TestGPGTimeoutKillsGPG(t *testing.T) {
|
||||
|
||||
_, _, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Contains(t, err.Error(), "gpg sign failed: gpg timed out")
|
||||
assert.EqualError(t, err, "gpg sign: timed out: context deadline exceeded")
|
||||
}
|
||||
|
||||
// TestGPGSignKeyNotReportedKeepsStderr puts a fake gpg first on PATH that
|
||||
// exits cleanly without reporting the key that signed, and checks that what
|
||||
// it wrote to stderr is in the message.
|
||||
func TestGPGSignKeyNotReportedKeepsStderr(t *testing.T) {
|
||||
t.Setenv("PATH", fakeGPGPath(t,
|
||||
"#!/bin/sh\necho 'gpg: note from the fake gpg' >&2\n"))
|
||||
|
||||
_, _, err := gpgSign(context.Background(), []byte("data"), GPGKeyID("any"))
|
||||
require.ErrorIs(t, err, errSigningKeyNotReported)
|
||||
assert.EqualError(t, err,
|
||||
"gpg did not report the key that made the signature: "+
|
||||
"gpg: note from the fake gpg")
|
||||
}
|
||||
|
||||
// TestGPGFailureKeepsStderr puts a fake gpg first on PATH that writes to
|
||||
// stderr and exits non-zero, and checks that what it wrote ends the message.
|
||||
func TestGPGFailureKeepsStderr(t *testing.T) {
|
||||
t.Setenv("PATH", fakeGPGPath(t,
|
||||
"#!/bin/sh\necho 'gpg: signing failed: No secret key' >&2\nexit 2\n"))
|
||||
|
||||
_, _, err := gpgSign(context.Background(), []byte("data"), GPGKeyID("any"))
|
||||
assert.EqualError(t, err,
|
||||
"gpg sign: exit status 2: gpg: signing failed: No secret key")
|
||||
}
|
||||
|
||||
// TestGPGCancelWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
errOuterNotSet = errors.New("pbOuter not set")
|
||||
errOuterNotSet = errors.New("outer message not set")
|
||||
errUUIDNotSet = errors.New("UUID not set")
|
||||
errSHA256NotSet = errors.New("SHA256 hash not set")
|
||||
)
|
||||
@@ -65,7 +65,7 @@ func (m *manifest) signatureString() (string, error) {
|
||||
|
||||
mh, err := multihash.Encode(m.pbOuter.GetSha256(), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to encode multihash: %w", err)
|
||||
return "", fmt.Errorf("encode multihash: %w", err)
|
||||
}
|
||||
|
||||
uuidStr := hex.EncodeToString(m.pbOuter.GetUuid())
|
||||
|
||||
+15
-3
@@ -160,7 +160,9 @@ func (s *Scanner) EnumeratePath(
|
||||
return s.enumerateFS(afs, abs, progress)
|
||||
}
|
||||
|
||||
// EnumeratePaths walks multiple directory paths and adds all files to the scanner.
|
||||
// EnumeratePaths adds to the scanner the files under each directory path,
|
||||
// listed by their paths under it, and each file path, listed by its name
|
||||
// as EnumerateFile lists it.
|
||||
// If progress is non-nil, status updates are sent as files are discovered.
|
||||
// The progress channel is closed when the method returns.
|
||||
func (s *Scanner) EnumeratePaths(
|
||||
@@ -177,9 +179,19 @@ func (s *Scanner) EnumeratePaths(
|
||||
return err
|
||||
}
|
||||
|
||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
||||
info, err := s.fs.Stat(abs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
||||
err = s.enumerateFS(afs, abs, progress)
|
||||
} else {
|
||||
err = s.enumerateFileWithInfo(
|
||||
filepath.Base(abs), filepath.Dir(abs), info, progress)
|
||||
}
|
||||
|
||||
err = s.enumerateFS(afs, abs, progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -134,6 +134,28 @@ func TestScannerEnumeratePaths(t *testing.T) {
|
||||
assert.Equal(t, FileCount(2), s.FileCount())
|
||||
}
|
||||
|
||||
// TestScannerEnumeratePathsFile gives EnumeratePaths a directory and a
|
||||
// file: the file is listed by its name, as EnumerateFile lists it.
|
||||
func TestScannerEnumeratePathsFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll("/dir/sub", 0o755))
|
||||
require.NoError(t, fs.MkdirAll("/other", 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, "/dir/sub/one.txt", []byte("1"), 0o644))
|
||||
require.NoError(t, afero.WriteFile(fs, "/other/two.txt", []byte("2"), 0o644))
|
||||
|
||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
||||
require.NoError(t, s.EnumeratePaths(nil, "/dir", "/other/two.txt"))
|
||||
|
||||
paths := make([]RelFilePath, 0, s.FileCount())
|
||||
for _, f := range s.Files() {
|
||||
paths = append(paths, f.Path)
|
||||
}
|
||||
|
||||
assert.Equal(t, []RelFilePath{"sub/one.txt", "two.txt"}, paths)
|
||||
}
|
||||
|
||||
func TestScannerExcludeDotfiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+12
-14
@@ -20,11 +20,9 @@ const MAGIC string = "ZNAVSRFG"
|
||||
var (
|
||||
// errInnerNotSet is returned by generate when the inner manifest is
|
||||
// missing.
|
||||
errInnerNotSet = errors.New("internal error: pbInner not set")
|
||||
errInnerNotSet = errors.New("inner message not set")
|
||||
// errInternal is returned by generateOuter for the same condition.
|
||||
// The two messages differ, and both are load-bearing for callers that
|
||||
// match on text, so they are kept distinct.
|
||||
errInternal = errors.New("internal error")
|
||||
errInternal = errors.New("inner message not set")
|
||||
)
|
||||
|
||||
// nanosecondsInt32 converts t's nanosecond component to int32.
|
||||
@@ -65,14 +63,14 @@ func (m *manifest) generate(ctx context.Context) error {
|
||||
|
||||
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: marshal outer: %w", err)
|
||||
return fmt.Errorf("marshal outer message: %w", err)
|
||||
}
|
||||
|
||||
m.output = bytes.NewBufferString(MAGIC)
|
||||
|
||||
_, err = m.output.Write(dat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: write output: %w", err)
|
||||
return fmt.Errorf("write outer message: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -95,7 +93,7 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
||||
|
||||
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: marshal inner: %w", err)
|
||||
return fmt.Errorf("marshal inner message: %w", err)
|
||||
}
|
||||
|
||||
// Compress the inner data
|
||||
@@ -103,12 +101,12 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
||||
|
||||
zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: create compressor: %w", err)
|
||||
return fmt.Errorf("create compressor: %w", err)
|
||||
}
|
||||
|
||||
_, err = zw.Write(innerData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: compress: %w", err)
|
||||
return fmt.Errorf("compress inner message: %w", err)
|
||||
}
|
||||
|
||||
_ = zw.Close()
|
||||
@@ -120,7 +118,7 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
||||
|
||||
_, err = h.Write(compressedData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize: hash write: %w", err)
|
||||
return fmt.Errorf("hash inner message: %w", err)
|
||||
}
|
||||
|
||||
sha256Hash := h.Sum(nil)
|
||||
@@ -149,12 +147,12 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
||||
func (m *manifest) signOuter(ctx context.Context) error {
|
||||
sigString, err := m.signatureString()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate signature string: %w", err)
|
||||
return fmt.Errorf("build signature string: %w", err)
|
||||
}
|
||||
|
||||
sig, signingKey, err := gpgSign(ctx, []byte(sigString), m.signingOptions.KeyID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to sign manifest: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
m.pbOuter.Signature = sig
|
||||
@@ -163,14 +161,14 @@ func (m *manifest) signOuter(ctx context.Context) error {
|
||||
// fingerprint first.
|
||||
fingerprint, err := gpgGetKeyFingerprint(ctx, GPGKeyID(signingKey))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get key fingerprint: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
m.pbOuter.Signer = fingerprint
|
||||
|
||||
pubKey, err := gpgExportPublicKey(ctx, GPGKeyID(fingerprint))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to export public key: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
m.pbOuter.SigningPubKey = pubKey
|
||||
|
||||
Reference in New Issue
Block a user