Resolve check and freshen paths against the manifest's directory (closes #177)

Without --base, check and freshen now look for a manifest's files in the
directory that holds it, the file named or the one found in a directory
argument, instead of the current directory. So `mfer check /media/drive`
checks a drive against its own index.mf from anywhere. check of a manifest
given by URL still uses the current directory, and --base still overrides,
even when it names the current directory. The --base help text of both
commands and the README's Tool Examples state the default; the README gains
a freshen entry for this, which also names the hidden files and symlinks
freshen leaves out by default and the flags that include them.

Model: opus-5-5

README.md

@@ -9,8 +9,9 @@ downloading, streaming, and mirroring. It was first published in 2022. The
 manifest files' data is serialized with Google's
 [protobuf serialization format](https://developers.google.com/protocol-buffers).
 The structure of these files can be found
-[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
-which is included in the [project repository](https://git.eeqj.de/sneak/mfer).
+[in the format specification](docs/FORMAT.md), which refers to the protobuf
+schema `mfer/mf.proto` for exact field numbers and types. Both are included in
+the [project repository](https://git.eeqj.de/sneak/mfer).
 
 The current version is pre-1.0 and while the repo was published in 2022, there
 has not yet been any versioned release. [SemVer](https://semver.org) will be
@@ -272,9 +273,20 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
     - verifies checksums of all files in manifest, displaying error and exiting
       nonzero if any files are missing or corrupted, or have permission bits
       other than the mode the manifest records, unless that is `0000`
+    - looks for those files under the base directory: the one `--base` names, or
+      else the directory holding the manifest, or the current directory for a
+      manifest given by URL. So `mfer check /media/drive` checks a drive against
+      the `index.mf` at its root, from any directory
     - warns about each file under the base directory that the manifest does not
       list, hidden files included; with `--no-extra-files` each one is a failure
       instead
+- `mfer freshen` / `mfer freshen .`
+    - rewrites `index.mf` to list the files now under the directory holding it,
+      or under the one `--base` names, hashing only the files that are new or
+      changed
+    - leaves out hidden files unless given `--include-dotfiles`, and symlinks
+      unless given `--follow-symlinks`, which lists each symlink to a file under
+      its own name with the contents of the file it points to
 - `mfer fetch https://example.com/stuff/`
     - fetches `/stuff/index.mf` and downloads all files listed in manifest into
       the current directory, or the one given with `--dest`, and assures

internal/cli/check.go

@@ -299,6 +299,10 @@ func (mfa *CLIApp) checkManifestOperation(
 		return err
 	}
 
+	// Done before a URL is swapped for the temp file it is downloaded to,
+	// whose directory is not the base.
+	basePath := resolveBasePath(cmd, manifestPath)
+
 	// URL manifests need to be downloaded to a temp file for the checker
 	if isHTTPURL(manifestPath) {
 		tmpPath, tmpErr := mfa.fetchManifestToTemp(ctx, manifestPath)
@@ -311,7 +315,6 @@ func (mfa *CLIApp) checkManifestOperation(
 		manifestPath = tmpPath
 	}
 
-	basePath := cmd.String("base")
 	showProgress := cmd.Bool("progress")
 
 	log.Infof("checking manifest %s with base %s", manifestPath, basePath)

internal/cli/entry_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"slices"
@@ -979,6 +980,90 @@ func TestCheckNeverReportsManifest(t *testing.T) {
 	}
 }
 
+// The directory setupManifestInSubdir makes and the manifest it writes there,
+// relative to the working directory it sets.
+const (
+	testSubdir         = "sub"
+	testSubdirManifest = testSubdir + "/" + defaultManifestName
+)
+
+// setupManifestInSubdir makes a temp dir holding file.txt and sub/b.txt,
+// where sub/index.mf is the manifest gen writes for sub, and makes it the
+// working directory, so a test calling it cannot run in parallel. It returns
+// the temp dir.
+func setupManifestInSubdir(t *testing.T) string {
+	t.Helper()
+
+	root := t.TempDir()
+	sub := filepath.Join(root, testSubdir)
+
+	fs := afero.NewOsFs()
+	require.NoError(t, fs.MkdirAll(sub, 0o750))
+	writeTestFile(t, fs, filepath.Join(root, testFileTxt), "not in the manifest")
+	writeTestFile(t, fs, filepath.Join(sub, "b.txt"), "in the manifest")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", "-o", filepath.Join(sub, defaultManifestName), sub,
+	}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	t.Chdir(root)
+
+	return root
+}
+
+// TestCheckResolvesEntriesAgainstManifestDirectory runs check from the
+// directory above sub, on the manifest in sub. Without --base, the
+// manifest's entries are looked for in sub, whether check is given the
+// manifest or sub, and the files above sub are not reported. --base names
+// the directory to look in instead, the current one included.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestCheckResolvesEntriesAgainstManifestDirectory(t *testing.T) {
+	root := setupManifestInSubdir(t)
+
+	for _, tc := range []struct {
+		args     []string
+		exitCode int
+		failure  string // a line check must print, if any
+	}{
+		{[]string{testSubdir}, 0, ""},
+		{[]string{testSubdirManifest}, 0, ""},
+		{[]string{filepath.Join(root, testSubdir)}, 0, ""},
+		{[]string{testFlagBase, testSubdir, testSubdirManifest}, 0, ""},
+		{[]string{testFlagBase, ".", testSubdirManifest}, 1, "MISSING: b.txt"},
+	} {
+		t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
+			opts := testOpts(slices.Concat(
+				[]string{testApp, cmdCheck, testFlagNoExtra}, tc.args,
+			), afero.NewOsFs())
+			assert.Equal(t, tc.exitCode, runCLI(opts), "stderr: %s", testStderr(t, opts))
+			assert.Contains(t, testStderr(t, opts), tc.failure)
+		})
+	}
+}
+
+// TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory runs check on
+// a manifest given by URL, from a directory holding the file it lists: the
+// file is looked for there.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory(t *testing.T) {
+	files := map[string][]byte{testFileTxt: []byte("hello")}
+
+	server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
+	defer server.Close()
+
+	cwd := chdirTemp(t)
+	require.NoError(t,
+		os.WriteFile(filepath.Join(cwd, testFileTxt), files[testFileTxt], 0o600))
+
+	opts := testOpts([]string{
+		testApp, cmdCheck, testFlagNoExtra, server.URL + "/" + defaultManifestName,
+	}, afero.NewOsFs())
+	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+}
+
 // unlistableDirFs is a filesystem on which one directory cannot be listed.
 type unlistableDirFs struct {
 	afero.Fs

internal/cli/errmsg_test.go

@@ -405,6 +405,48 @@ func TestFetchHashMismatchMessage(t *testing.T) {
 	assert.EqualError(t, err, "download "+testFileTxt+": hash mismatch")
 }
 
+// TestFreshenBackslashPathMessage runs freshen on a tree that has gained a
+// file whose name holds a backslash, which a manifest path may not contain.
+func TestFreshenBackslashPathMessage(t *testing.T) {
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	root, manifestPath := setupFreshenDir(t, fs,
+		map[string]string{testFileTxt: "content"})
+	writeTestFile(t, fs, filepath.Join(root, `a\b.txt`), "new")
+
+	mfa := &CLIApp{Fs: fs}
+	cmd := mfa.freshenCommand()
+	cmd.Action = mfa.freshenManifestOperation
+
+	// freshenManifestOperation logs to the process-global logger.
+	err := runLocked(func() error {
+		return cmd.Run(context.Background(),
+			[]string{cmdFreshen, testFlagBase, root, manifestPath})
+	})
+	assert.EqualError(t, err,
+		`path "a\\b.txt" contains backslash; use forward slashes only`)
+}
+
+// TestFreshenReadErrorMessage has freshen hash a directory as though it
+// were a file, so reading it fails.
+func TestFreshenReadErrorMessage(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	require.NoError(t, os.Mkdir(filepath.Join(root, "sub"), 0o750))
+
+	hasher := &freshenHasher{
+		fs:      afero.NewOsFs(),
+		absBase: root,
+		builder: mfer.NewBuilder(),
+	}
+
+	err := hasher.processEntry(&freshenEntry{path: "sub", needsHash: true})
+	assert.EqualError(t, err,
+		"read "+filepath.Join(root, "sub")+": is a directory")
+}
+
 func TestURLRequiredMessage(t *testing.T) {
 	t.Parallel()
 

internal/cli/freshen.go

@@ -295,19 +295,14 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
 	_ = f.Close()
 
 	if err != nil {
-		return fmt.Errorf("hash %s: %w", e.path, err)
+		return err
 	}
 
 	h.hashedBytes += bytesRead
 	h.hashedFiles++
 
-	// Add to builder with computed hash
-	err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
-	if err != nil {
-		return fmt.Errorf("add %s: %w", e.path, err)
-	}
-
-	return nil
+	// Add to builder with computed hash; a refused path is named in the error
+	return addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
 }
 
 // writeFreshenedManifest writes the manifest atomically (write to a
@@ -491,7 +486,6 @@ func (mfa *CLIApp) freshenManifestOperation(
 ) error {
 	log.Debug("freshenManifestOperation()")
 
-	basePath := cmd.String("base")
 	showProgress := cmd.Bool("progress")
 
 	// Find manifest file
@@ -506,7 +500,7 @@ func (mfa *CLIApp) freshenManifestOperation(
 		return err
 	}
 
-	absBase, err := filepath.Abs(basePath)
+	absBase, err := filepath.Abs(resolveBasePath(cmd, manifestPath))
 	if err != nil {
 		return fmt.Errorf("invalid base path: %w", err)
 	}
@@ -600,7 +594,7 @@ func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
 			break
 		}
 
-		// Returned unwrapped: the caller adds the path.
+		// Returned unwrapped: a read error already names the file.
 		if err != nil {
 			return nil, total, err
 		}

internal/cli/freshen_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -300,6 +301,41 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
+// TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
+// runs freshen from the directory above sub, on the manifest in sub.
+// Without --base, the manifest then lists the files in sub, whether freshen
+// is given the manifest or sub. --base names the directory to list instead,
+// the current one included.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestFreshenResolvesEntriesAgainstManifestDirectory(t *testing.T) {
+	for _, tc := range []struct {
+		args []string
+		want []string // the paths the manifest lists afterwards
+	}{
+		{[]string{testSubdir}, []string{"b.txt", "c.txt"}},
+		{[]string{testSubdirManifest}, []string{"b.txt", "c.txt"}},
+		{
+			[]string{testFlagBase, ".", testSubdirManifest},
+			[]string{testFileTxt, "sub/b.txt", "sub/c.txt"},
+		},
+	} {
+		t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
+			fs := afero.NewOsFs()
+			root := setupManifestInSubdir(t)
+			writeTestFile(t, fs, filepath.Join(root, testSubdir, "c.txt"), "added")
+
+			opts := testOpts(slices.Concat(
+				[]string{testApp, cmdFreshen, "-q"}, tc.args,
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, tc.want, manifestPaths(t, fs,
+				filepath.Join(root, testSubdirManifest)))
+		})
+	}
+}
+
 // TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
 // with respect to MFFilePath.Mtime, which is a message pointer with
 // proto3 field presence and may legitimately be absent.

internal/cli/manifest_loader.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -86,3 +87,17 @@ func (mfa *CLIApp) resolveManifestArg(cmd *cli.Command) (string, error) {
 
 	return findManifest(mfa.Fs, ".")
 }
+
+// resolveBasePath returns the directory a manifest's paths are resolved
+// against: the one --base names, or else the directory holding the manifest,
+// or the current directory for a manifest URL.
+func resolveBasePath(cmd *cli.Command, manifestPath string) string {
+	switch {
+	case cmd.IsSet(flagBase):
+		return cmd.String(flagBase)
+	case isHTTPURL(manifestPath):
+		return "."
+	default:
+		return filepath.Dir(manifestPath)
+	}
+}

internal/cli/mfer.go

@@ -24,6 +24,7 @@ const (
 	cmdList     = "list"
 	cmdVersion  = "version"
 
+	flagBase               = "base"
 	flagProgress           = "progress"
 	flagTimeout            = "timeout"
 	flagDest               = "dest"
@@ -259,10 +260,11 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
 		},
 		Flags: append(commonFlags(),
 			&cli.StringFlag{
-				Name:    "base",
+				Name:    flagBase,
 				Aliases: []string{"b"},
-				Value:   ".",
-				Usage:   "Base directory for resolving relative paths from manifest",
+				Usage: "Base directory for resolving relative paths from manifest " +
+					"(by default the directory holding the manifest, or the " +
+					"current directory for a manifest URL)",
 			},
 			&cli.BoolFlag{
 				Name:    flagProgress,
@@ -292,10 +294,10 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
 		},
 		Flags: append(commonFlags(),
 			&cli.StringFlag{
-				Name:    "base",
+				Name:    flagBase,
 				Aliases: []string{"b"},
-				Value:   ".",
-				Usage:   "Base directory for resolving relative paths",
+				Usage: "Base directory for resolving relative paths " +
+					"(by default the directory holding the manifest)",
 			},
 			&cli.BoolFlag{
 				Name:    "follow-symlinks",

mfer/gpg.go

@@ -88,12 +88,12 @@ func gpgArgs(opts []string, positional ...string) []string {
 }
 
 // runGPG runs the gpg binary in batch mode with the given arguments and
-// optional stdin, returning captured stdout. If gpg fails, the error ends
-// with what gpg wrote to stderr. gpg is killed when ctx ends or gpgTimeout
-// passes, whichever comes first.
+// optional stdin, returning captured stdout and stderr. If gpg fails, the
+// error ends with what gpg wrote to stderr. gpg is killed when ctx ends or
+// gpgTimeout passes, whichever comes first.
 func runGPG(
 	ctx context.Context, stdin io.Reader, args ...string,
-) (*bytes.Buffer, error) {
+) (*bytes.Buffer, *bytes.Buffer, error) {
 	// exec.CommandContext kills only gpg itself. A gpg-agent that gpg
 	// starts runs detached and holds none of gpg's output, but another
 	// process gpg leaves behind (a wrapper script that runs the real gpg
@@ -130,12 +130,22 @@ func runGPG(
 		}
 	}
 
-	messages := strings.TrimSpace(stderr.String())
-	if err != nil && messages != "" {
-		err = fmt.Errorf("%w: %s", err, messages)
+	if err != nil {
+		err = withStderr(err, &stderr)
 	}
 
-	return &stdout, err
+	return &stdout, &stderr, err
+}
+
+// withStderr returns err followed by what gpg wrote to stderr, or err alone
+// when gpg wrote nothing.
+func withStderr(err error, stderr *bytes.Buffer) error {
+	messages := strings.TrimSpace(stderr.String())
+	if messages == "" {
+		return err
+	}
+
+	return fmt.Errorf("%w: %s", err, messages)
 }
 
 // parseFingerprint extracts the first fingerprint from gpg --with-colons
@@ -189,7 +199,7 @@ func gpgSign(
 
 	// The signature goes to sigFile, so --status-fd 1 can send gpg's status
 	// lines to stdout; its messages go to stderr.
-	stdout, err := runGPG(ctx, bytes.NewReader(data),
+	stdout, stderr, err := runGPG(ctx, bytes.NewReader(data),
 		"--detach-sign",
 		gpgOptArmor,
 		"--output", sigFile,
@@ -204,7 +214,7 @@ func gpgSign(
 	// made the signature.
 	created, ok := parseStatusLine(stdout.String(), "SIG_CREATED")
 	if !ok {
-		return nil, "", errSigningKeyNotReported
+		return nil, "", withStderr(errSigningKeyNotReported, stderr)
 	}
 
 	sig, err := os.ReadFile(sigFile) //nolint:gosec // G304: inside tmpDir, made above
@@ -218,7 +228,7 @@ func gpgSign(
 // gpgExportPublicKey exports the public key for the specified key ID.
 // Returns the armored public key.
 func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
-	stdout, err := runGPG(ctx, nil,
+	stdout, _, err := runGPG(ctx, nil,
 		gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
 	)
 	if err != nil {
@@ -234,7 +244,7 @@ func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
 
 // gpgGetKeyFingerprint gets the full fingerprint for a key ID.
 func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
-	stdout, err := runGPG(ctx, nil,
+	stdout, _, err := runGPG(ctx, nil,
 		gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
 	)
 	if err != nil {
@@ -254,7 +264,7 @@ func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
 func gpgImportOneKey(ctx context.Context, gpgHome, pubKeyFile string) error {
 	// --status-fd 1 sends gpg's status lines to stdout, which importing
 	// otherwise leaves empty; its messages go to stderr.
-	importStdout, err := runGPG(ctx, nil,
+	importStdout, _, err := runGPG(ctx, nil,
 		gpgArgs([]string{gpgOptHomedir, gpgHome, gpgOptStatusFD, "1", "--import"},
 			pubKeyFile)...,
 	)
@@ -328,7 +338,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) (string, err
 
 	// --status-fd 1 sends gpg's status lines to stdout, which verifying a
 	// detached signature otherwise leaves empty; its messages go to stderr.
-	verifyStdout, err := runGPG(ctx, nil,
+	verifyStdout, _, err := runGPG(ctx, nil,
 		gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptStatusFD, "1", gpgOptVerify},
 			sigFile, dataFile)...,
 	)

mfer/gpg_test.go

@@ -451,7 +451,7 @@ func TestManifestRefusesSecondEmbeddedKeyWithoutUserID(t *testing.T) {
 	t.Setenv("GNUPGHOME", otherHome)
 
 	// Keeping only the user IDs that match "nobody" exports none.
-	otherPubKey, err := runGPG(context.Background(), nil,
+	otherPubKey, _, err := runGPG(context.Background(), nil,
 		gpgArgs([]string{
 			"--export", gpgOptArmor, "--export-filter", "keep-uid=uid = nobody",
 		}, string(otherKey))...)
@@ -616,6 +616,31 @@ func TestGPGTimeoutKillsGPG(t *testing.T) {
 	assert.EqualError(t, err, "gpg sign: timed out: context deadline exceeded")
 }
 
+// TestGPGSignKeyNotReportedKeepsStderr puts a fake gpg first on PATH that
+// exits cleanly without reporting the key that signed, and checks that what
+// it wrote to stderr is in the message.
+func TestGPGSignKeyNotReportedKeepsStderr(t *testing.T) {
+	t.Setenv("PATH", fakeGPGPath(t,
+		"#!/bin/sh\necho 'gpg: note from the fake gpg' >&2\n"))
+
+	_, _, err := gpgSign(context.Background(), []byte("data"), GPGKeyID("any"))
+	require.ErrorIs(t, err, errSigningKeyNotReported)
+	assert.EqualError(t, err,
+		"gpg did not report the key that made the signature: "+
+			"gpg: note from the fake gpg")
+}
+
+// TestGPGFailureKeepsStderr puts a fake gpg first on PATH that writes to
+// stderr and exits non-zero, and checks that what it wrote ends the message.
+func TestGPGFailureKeepsStderr(t *testing.T) {
+	t.Setenv("PATH", fakeGPGPath(t,
+		"#!/bin/sh\necho 'gpg: signing failed: No secret key' >&2\nexit 2\n"))
+
+	_, _, err := gpgSign(context.Background(), []byte("data"), GPGKeyID("any"))
+	assert.EqualError(t, err,
+		"gpg sign: exit status 2: gpg: signing failed: No secret key")
+}
+
 // TestGPGCancelWhenChildHoldsOutput uses a fake gpg that runs sleep as a
 // child instead of exec-ing it, the way a wrapper script around the real
 // gpg might. Killing the fake gpg leaves sleep holding its stdout and