Return the context's error when it ends during a fetch request (closes #221)

retryingClient.get checked its context right after each request and
returned that request's error, so a deadline that passed while a 503 was
on its way came back as the 503. TestGetStopsWaitingWhenCanceled failed
on that now and then on a loaded host. When the context ends before a
retry, during the request or during the wait, get now returns an error
wrapping both the context's error and the last try's error.

A new test covers the context ending during the request, and
TestGetStopsWaitingWhenCanceled now covers it ending during the wait.
Both answer through a transport rather than a server, so neither depends
on how fast the 503 arrives.

Model: opus-5-5

README.md

@@ -13,11 +13,14 @@ The structure of these files can be found
 schema `mfer/mf.proto` for exact field numbers and types. Both are included in
 the [project repository](https://git.eeqj.de/sneak/mfer).
 
-Releases are git tags named by [SemVer](https://semver.org), starting with
-`v1.0.0`. No file in the tree records a version number. `mfer version` prints
-the revision the build stamped into the binary, which for `make build` and the
-Docker image is the output of `git describe` on the checkout they build. A
-binary built without that stamp, as
+The current version is pre-1.0 and while the repo was published in 2022, there
+has not yet been any versioned release. [SemVer](https://semver.org) will be
+used for releases.
+
+A version is a git tag, and no file in the tree records a version number.
+`mfer version` prints the revision the build stamped into the binary, which for
+`make build` and the Docker image is the output of `git describe` on the
+checkout they build. A binary built without that stamp, as
 `go install sneak.berlin/go/mfer/cmd/mfer@<tag>` builds it, prints the module
 version Go records in it instead. A program that imports the `mfer` package
 finds the version it was built against in its own build information
@@ -287,9 +290,6 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       only on the paths it is given and the files under them, so with the same
       `--seed` and an unchanged tree it writes the same bytes wherever the
       manifest goes. The file it writes to is never listed
-    - never lists `index.mf` at the top of a directory it is given, or a file it
-      is given by that name, wherever it writes the manifest, since `fetch`
-      saves the manifest there; `index.mf` in a subdirectory is listed
 - `mfer check` / `mfer check .`
     - verifies checksums of all files in manifest, displaying error and exiting
       nonzero if any files are missing or corrupted, or have permission bits
@@ -304,15 +304,11 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       `--no-extra-files` each one is a failure instead. A named pipe, socket or
       device, a symlink to a directory and a broken symlink are never reported,
       since gen cannot list them
-    - never reports `index.mf` at the top of the base directory as a file the
-      manifest does not list, even when given another manifest or a URL, since
-      gen and fetch write the manifest there
 - `mfer freshen` / `mfer freshen .`
     - rewrites `index.mf` to list the files now under the directory holding it,
       or under the one `--base` names, hashing only the files that are new or
       changed
-    - lists regular files only, and never `index.mf` at the top of that
-      directory, as gen does
+    - lists regular files only, as gen does
     - leaves out hidden files unless given `--include-dotfiles`, and symlinks
       unless given `--follow-symlinks`, which lists each symlink to a regular
       file under its own name with the contents of the file it points to

docs/FORMAT.md

@@ -80,28 +80,29 @@ After decompressing `innerMessage`, the result is a serialized `MFFile`
 
 Each file entry contains:
 
-| Field      | Number | Type                      | Description                                                  |
-| ---------- | ------ | ------------------------- | ------------------------------------------------------------ |
-| `path`     | 1      | string                    | Relative file path (see Path Rules)                          |
-| `size`     | 2      | int64                     | File size in bytes                                           |
-| `hashes`   | 3      | repeated `MFFileChecksum` | Must include exactly one SHA-256 multihash (see Hash Format) |
-| `mimeType` | 301    | string (optional)         | MIME type                                                    |
-| `mtime`    | 302    | Timestamp (optional)      | Modification time                                            |
-| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                          |
-| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                              |
+| Field      | Number | Type                      | Description                                                   |
+| ---------- | ------ | ------------------------- | ------------------------------------------------------------- |
+| `path`     | 1      | string                    | Relative file path (see Path Rules)                           |
+| `size`     | 2      | int64                     | File size in bytes                                            |
+| `hashes`   | 3      | repeated `MFFileChecksum` | At least one hash required; readers reject an entry with none |
+| `mimeType` | 301    | string (optional)         | MIME type                                                     |
+| `mtime`    | 302    | Timestamp (optional)      | Modification time                                             |
+| `ctime`    | 303    | Timestamp (optional)      | Change time (inode metadata change)                           |
+| `mode`     | 304    | uint32                    | Permission bits (see File Mode)                               |
 
 ## File Mode
 
 `mode` holds a file's Unix permission bits, the nine `rwx` bits, so it is never
-above `0777` (octal); the setuid, setgid and sticky bits are never recorded. A
-reader must reject a manifest with an entry whose mode is above `0777`. Writers
-record `0000` unless whoever creates the manifest asks for permissions. `0000`,
-the proto3 default, means no mode was recorded: readers never check or apply it.
+above `0777` (octal); the setuid, setgid and sticky bits are never recorded.
+Writers record `0000` unless whoever creates the manifest asks for permissions.
+`0000`, the proto3 default, means no mode was recorded: readers never check or
+apply it.
 
 The reference implementation records modes when `gen` or `freshen` is given
 `--include-permissions`. `check` fails a file whose permission bits differ from
 a recorded mode other than `0000`. `fetch` sets a recorded mode other than
-`0000` on each file it writes.
+`0000` on each file it writes, and refuses a manifest that records a mode above
+`0777` before it requests any file.
 
 ## Path Rules
 
@@ -132,16 +133,8 @@ Each checksum is a single `bytes multiHash` field containing a
 self-describing: the encoded bytes include a varint algorithm identifier
 followed by a varint digest length followed by the digest itself.
 
-A writer must give every file entry exactly one SHA-256 multihash of the file's
-content: the algorithm code `0x12`, the digest length `0x20` and the 32 bytes of
-the digest. It may list multihashes of other algorithms beside it. A reader must
-refuse a manifest with an entry that has no SHA-256 multihash or more than one,
-whatever other hashes the entry lists. A verifier then needs only SHA-256,
-checking each file against its entry's SHA-256 multihash; it may ignore the
-other hashes.
-
-The reference implementation writes one hash for each entry, the SHA-256
-multihash, and its `check` and `fetch` compute only SHA-256.
+The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
+Implementations must be able to verify SHA-256 multihashes at minimum.
 
 ## Signature Scheme
 
@@ -219,11 +212,10 @@ compresses the same inner message to the same bytes:
 - Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
   and `mode`. The hash is the SHA-256 multihash of the file's content, except
   that `freshen` reads only the files that are new or whose size, `mtime` or
-  `mode` changed, and for every other file carries the SHA-256 multihash of its
-  existing entry over unchanged, leaving out any other hash the entry lists.
-  `mtime` is the file's modification time as the file system gives it, to the
-  nanosecond, and `mode` is `0000` unless `gen` or `freshen` is given
-  `--include-permissions`.
+  `mode` changed, and for every other file carries the first hash of its
+  existing entry over unchanged. `mtime` is the file's modification time as the
+  file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
+  `freshen` is given `--include-permissions`.
 - `createdAt` is left out unless `gen` or `freshen` is given
   `--include-timestamps`. Then it holds the time the run started building the
   manifest, to the nanosecond, so it differs on every run.

internal/cli/check.go

@@ -2,7 +2,6 @@
 package cli
 
 import (
-	"bytes"
 	"context"
 	"encoding/hex"
 	"errors"
@@ -95,36 +94,47 @@ func findManifest(fs afero.Fs, dir string) (string, error) {
 	return path, nil
 }
 
-// downloadManifest downloads the manifest at url into memory. It refuses a
-// manifest larger than mfa.maxManifestSize.
-func (mfa *CLIApp) downloadManifest(ctx context.Context, url string) ([]byte, error) {
-	rc, err := mfa.openManifestReader(ctx, url)
-	if err != nil {
-		return nil, err
+// fetchManifestToTemp downloads a manifest URL to a temporary file and
+// returns the temp file path. The caller is responsible for removing it.
+func (mfa *CLIApp) fetchManifestToTemp(
+	ctx context.Context, url string,
+) (string, error) {
+	rc, fetchErr := mfa.openManifestReader(ctx, url)
+	if fetchErr != nil {
+		return "", fetchErr
 	}
 
-	defer func() { _ = rc.Close() }()
+	tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf")
+	if tmpErr != nil {
+		_ = rc.Close()
 
-	// Reading stops one byte past mfa.maxManifestSize, which is enough to
+		return "", tmpErr
+	}
+
+	tmpPath := tmpFile.Name()
+
+	// Copying stops one byte past mfa.maxManifestSize, which is enough to
 	// tell that the manifest is too large.
-	manifestData, err := io.ReadAll(io.LimitReader(rc, mfa.maxManifestSize+1))
-	if err != nil {
-		return nil, fmt.Errorf("download manifest: %w", err)
+	written, cpErr := io.Copy(tmpFile, io.LimitReader(rc, mfa.maxManifestSize+1))
+	_ = rc.Close()
+	_ = tmpFile.Close()
+
+	if cpErr == nil && written > mfa.maxManifestSize {
+		cpErr = fmt.Errorf("%w of %d bytes", errManifestTooLarge, mfa.maxManifestSize)
 	}
 
-	if int64(len(manifestData)) > mfa.maxManifestSize {
-		return nil, fmt.Errorf("download manifest: %w of %d bytes",
-			errManifestTooLarge, mfa.maxManifestSize)
+	if cpErr != nil {
+		_ = mfa.Fs.Remove(tmpPath)
+
+		return "", fmt.Errorf("download manifest: %w", cpErr)
 	}
 
-	return manifestData, nil
+	return tmpPath, nil
 }
 
 // verifyRequiredSigner enforces the --require-signature fingerprint
-// against signer, the fingerprint of the key that made the manifest's
-// signature, which is empty when the manifest is not signed. Loading the
-// manifest checked that signer is that key's fingerprint.
-func verifyRequiredSigner(signer, requiredSigner string) error {
+// against the key that made the manifest's signature.
+func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
 	// Validate fingerprint format: must be exactly 40 hex characters
 	if len(requiredSigner) != fingerprintHexLen {
 		return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
@@ -135,11 +145,15 @@ func verifyRequiredSigner(signer, requiredSigner string) error {
 		return fmt.Errorf("invalid fingerprint: %w", err)
 	}
 
-	if signer == "" {
+	if !chk.IsSigned() {
 		return fmt.Errorf("%w, but signature from %s is required",
 			errManifestNotSigned, requiredSigner)
 	}
 
+	// Loading the manifest checked that the signer is the fingerprint of
+	// the key that made the signature.
+	signer := string(chk.Signer())
+
 	// Compare fingerprints - must be exact match (case-insensitive)
 	if !strings.EqualFold(signer, requiredSigner) {
 		return fmt.Errorf("embedded signing key fingerprint %s %w %s",
@@ -195,10 +209,7 @@ func countCheckFailures(
 
 // findExtraFiles reports files present on disk but absent from the
 // manifest, and anything the search cannot read: each is a failure, logged
-// at error level, under --no-extra-files, otherwise a warning. index.mf at
-// the top of the base directory is never reported as absent from the
-// manifest, even when check is given another manifest or a URL, since gen
-// and fetch write the manifest there.
+// at error level, under --no-extra-files, otherwise a warning.
 func findExtraFiles(
 	ctx context.Context, cmd *cli.Command, chk *mfer.Checker, failures *int64,
 ) error {
@@ -207,10 +218,6 @@ func findExtraFiles(
 
 	go func() {
 		for result := range extraResults {
-			if result.Status == mfer.StatusExtra && result.Path == defaultManifestName {
-				continue
-			}
-
 			if cmd.Bool("no-extra-files") {
 				*failures++
 
@@ -292,28 +299,32 @@ func (mfa *CLIApp) checkManifestOperation(
 		return err
 	}
 
+	// Done before a URL is swapped for the temp file it is downloaded to,
+	// whose directory is not the base.
 	basePath := resolveBasePath(cmd, manifestPath)
 
-	opts := &mfer.CheckerOptions{BasePath: basePath, Fs: mfa.Fs}
-
-	// A manifest given by URL is downloaded into memory and read from
-	// there.
+	// URL manifests need to be downloaded to a temp file for the checker
 	if isHTTPURL(manifestPath) {
-		manifestData, downloadErr := mfa.downloadManifest(ctx, manifestPath)
-		if downloadErr != nil {
-			return downloadErr
+		tmpPath, tmpErr := mfa.fetchManifestToTemp(ctx, manifestPath)
+		if tmpErr != nil {
+			return tmpErr
 		}
 
-		opts.ManifestReader = bytes.NewReader(manifestData)
-	} else {
-		opts.ManifestPath = manifestPath
+		defer func() { _ = mfa.Fs.Remove(tmpPath) }()
+
+		manifestPath = tmpPath
 	}
 
 	showProgress := cmd.Bool("progress")
 
 	log.Infof("checking manifest %s with base %s", manifestPath, basePath)
 
-	chk, err := mfer.NewChecker(opts)
+	// Create checker
+	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
+		ManifestPath: manifestPath,
+		BasePath:     basePath,
+		Fs:           mfa.Fs,
+	})
 	if err != nil {
 		return fmt.Errorf("load manifest: %w", err)
 	}
@@ -321,7 +332,7 @@ func (mfa *CLIApp) checkManifestOperation(
 	// Check signature requirement. An empty fingerprint is checked too,
 	// and refused as one of the wrong length.
 	if cmd.IsSet(flagRequireSignature) {
-		err = verifyRequiredSigner(string(chk.Signer()), cmd.String(flagRequireSignature))
+		err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
 		if err != nil {
 			return err
 		}

internal/cli/entry_test.go

@@ -27,20 +27,19 @@ import (
 )
 
 const (
-	testApp                = "mfer"
-	testGitrev             = "abc123"
-	testDir                = "/testdir"
-	testFile1              = "/testdir/file1.txt"
-	testMF                 = "/testdir/test.mf"
-	testOutput             = "/output.mf"
-	testManifest           = "/manifest.mf"
-	testFlagBase           = "--base"
-	testFlagNoExtra        = "--no-extra-files"
-	testFlagFollowSymlinks = "--follow-symlinks"
-	testFlagVersion        = "--version"
-	testFlagVerbose        = "--verbose"
-	testFlagHelp           = "--help"
-	testCmdHelp            = "help"
+	testApp         = "mfer"
+	testGitrev      = "abc123"
+	testDir         = "/testdir"
+	testFile1       = "/testdir/file1.txt"
+	testMF          = "/testdir/test.mf"
+	testOutput      = "/output.mf"
+	testManifest    = "/manifest.mf"
+	testFlagBase    = "--base"
+	testFlagNoExtra = "--no-extra-files"
+	testFlagVersion = "--version"
+	testFlagVerbose = "--verbose"
+	testFlagHelp    = "--help"
+	testCmdHelp     = "help"
 )
 
 var errSimulatedWrite = errors.New("simulated write failure")
@@ -1003,7 +1002,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
 		exitCode int
 		extras   []string // what check reports as not in the manifest
 	}{
-		"gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil},
+		"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
 		"plain gen":             {nil, 1, []string{"filelink"}},
 	} {
 		t.Run(name, func(t *testing.T) {
@@ -1168,122 +1167,6 @@ func TestCheckURLManifestResolvesEntriesAgainstCurrentDirectory(t *testing.T) {
 	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 }
 
-// TestCheckURLManifestWithoutTempDirectory checks the tree under --base
-// against a manifest given by URL while TMPDIR names a directory that does
-// not exist. check keeps the manifest it downloads in memory, so it passes.
-// Setting TMPDIR keeps it from running in parallel with other tests.
-func TestCheckURLManifestWithoutTempDirectory(t *testing.T) {
-	files := map[string][]byte{testFileTxt: []byte("hello")}
-
-	server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
-	defer server.Close()
-
-	base := t.TempDir()
-	require.NoError(t,
-		os.WriteFile(filepath.Join(base, testFileTxt), files[testFileTxt], 0o600))
-
-	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
-
-	opts := testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, base,
-		server.URL + "/" + defaultManifestName,
-	}, afero.NewOsFs())
-	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-}
-
-// TestCheckNeverReportsIndexMFAtTopOfBase checks a tree holding index.mf at
-// its top, where gen and fetch write the manifest, against a manifest that
-// does not list it, given by URL and by a path outside the tree: check never
-// reports it, and passes with and without --no-extra-files. index.mf in a
-// subdirectory, and the temp file fetch saves index.mf through, are still
-// reported.
-func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	files := map[string][]byte{testFileTxt: []byte("hello")}
-	manifest := manifestOf(t, files)
-
-	server := httptest.NewServer(fetchTestHandler(manifest, files))
-	t.Cleanup(server.Close)
-
-	manifestOutsideTree := filepath.Join(t.TempDir(), "other.mf")
-	require.NoError(t, os.WriteFile(manifestOutsideTree, manifest, 0o600))
-
-	for name, manifestArg := range map[string]string{
-		"manifest by URL":  server.URL + "/" + defaultManifestName,
-		"manifest by path": manifestOutsideTree,
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-			require.NoError(t, os.WriteFile(
-				filepath.Join(base, testFileTxt), files[testFileTxt], 0o600))
-			require.NoError(t, os.WriteFile(
-				filepath.Join(base, defaultManifestName), manifest, 0o600))
-
-			check := func(flags ...string) (int, string) {
-				opts := testOpts(slices.Concat(
-					[]string{testApp, cmdCheck}, flags,
-					[]string{testFlagBase, base, manifestArg},
-				), afero.NewOsFs())
-
-				return runCLI(opts), testStderr(t, opts)
-			}
-
-			exitCode, stderr := check()
-			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			exitCode, stderr = check(testFlagNoExtra)
-			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			reported := []string{
-				testSubdirManifest,
-				tempPathFor(defaultManifestName),
-			}
-
-			require.NoError(t, os.Mkdir(filepath.Join(base, testSubdir), 0o750))
-
-			for _, path := range reported {
-				require.NoError(t,
-					os.WriteFile(filepath.Join(base, path), manifest, 0o600))
-			}
-
-			exitCode, stderr = check(testFlagNoExtra)
-			assert.Equal(t, 1, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			for _, path := range reported {
-				assert.Contains(t, stderr, "EXTRA: "+path)
-			}
-		})
-	}
-}
-
-// TestCheckChecksListedIndexMF checks a tree against a manifest that lists
-// index.mf at the top of the tree, where that file has changed since: check
-// reports it like any other changed file.
-func TestCheckChecksListedIndexMF(t *testing.T) {
-	t.Parallel()
-
-	manifestPath := filepath.Join(t.TempDir(), "other.mf")
-	require.NoError(t, os.WriteFile(manifestPath, manifestOf(t, map[string][]byte{
-		defaultManifestName: []byte("listed"),
-	}), 0o600))
-
-	base := t.TempDir()
-	require.NoError(t, os.WriteFile(
-		filepath.Join(base, defaultManifestName), []byte("edited"), 0o600))
-
-	opts := testOpts([]string{
-		testApp, cmdCheck, testFlagBase, base, manifestPath,
-	}, afero.NewOsFs())
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "HASH_MISMATCH: "+defaultManifestName)
-}
-
 // unlistableDirFs is a filesystem on which one directory cannot be listed.
 type unlistableDirFs struct {
 	afero.Fs
@@ -1332,60 +1215,6 @@ func TestCheckWithUnlistableDirectory(t *testing.T) {
 	assert.Contains(t, testStderr(t, opts), "unlisted.txt")
 }
 
-// TestCheckReportsUnreadableIndexMFAtTopOfBase has a directory named
-// index.mf at the top of the base that cannot be listed. check leaves out
-// only the report that index.mf is not in the manifest, so it reports the
-// error and fails under --no-extra-files.
-func TestCheckReportsUnreadableIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	unreadable := filepath.Join(testDir, defaultManifestName)
-
-	mem := afero.NewMemMapFs()
-	require.NoError(t, mem.MkdirAll(unreadable, 0o755))
-	writeTestFile(t, mem, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, mem)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	fs := unlistableDirFs{Fs: mem, dir: unreadable}
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "ERROR: "+defaultManifestName)
-}
-
-// TestCheckReportsUpperCaseIndexMFAtTopOfBase checks a tree holding
-// INDEX.MF at its top against a manifest that does not list it. Only the
-// name index.mf, in exactly that letter case, is left out, so check reports
-// INDEX.MF. The tree is in memory, where the two names are two files
-// whatever the letter case rules of the host's filesystem.
-func TestCheckReportsUpperCaseIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	upperCase := strings.ToUpper(defaultManifestName)
-
-	fs := afero.NewMemMapFs()
-	writeTestFile(t, fs, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	writeTestFile(t, fs, filepath.Join(testDir, upperCase), "not listed")
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "EXTRA: "+upperCase)
-}
-
 func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
 	t.Parallel()
 
@@ -1568,141 +1397,6 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 	assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
 }
 
-// TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside
-// the paths it is given: index.mf at the top of each directory given, and
-// a file given by that name, are not listed, since fetch saves the manifest
-// as index.mf at the top of the tree, while sub/index.mf is listed.
-func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
-	t.Parallel()
-
-	// Paths are relative to a temp dir holding tree, which holds file.txt,
-	// index.mf and sub/index.mf, and second, which holds second.txt and
-	// index.mf.
-	for name, tc := range map[string]struct {
-		paths  []string
-		listed []string
-	}{
-		"directory": {
-			[]string{"tree"},
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"two directories": {
-			[]string{"tree", "second"},
-			[]string{testFileTxt, testSubdirManifest, "second.txt"},
-		},
-		"files": {
-			[]string{"tree/" + testFileTxt, "tree/" + defaultManifestName},
-			[]string{testFileTxt},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-			second := filepath.Join(root, "second")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			require.NoError(t, fs.MkdirAll(second, 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-			writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			writeTestFile(t, fs, filepath.Join(second, "second.txt"), "second")
-			writeTestFile(t, fs, filepath.Join(second, defaultManifestName), "an ordinary file")
-
-			paths := make([]string, 0, len(tc.paths))
-			for _, path := range tc.paths {
-				paths = append(paths, filepath.Join(root, path))
-			}
-
-			output := filepath.Join(root, "other.mf")
-			opts := testOpts(slices.Concat(
-				[]string{testApp, cmdGenerate, "-q", "-o", output}, paths,
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
-		})
-	}
-}
-
-// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of
-// the tree a symlink to another file in it, and runs gen with --output
-// outside the tree: the file it points to is listed under its own name,
-// and the symlink, which gen lists only with --follow-symlinks, is left out
-// like any index.mf at the top of the tree.
-func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	// Paths are relative to tree, which holds file.txt, sub/index.mf and
-	// index.mf, a symlink to target.
-	for name, tc := range map[string]struct {
-		target string
-		flags  []string
-		path   string // the path gen is given
-		listed []string
-	}{
-		"directory given": {
-			testSubdirManifest, nil, ".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"directory given, --follow-symlinks": {
-			testSubdirManifest,
-			[]string{testFlagFollowSymlinks},
-			".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"file given": {
-			testFileTxt, nil, testFileTxt,
-			[]string{testFileTxt},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t,
-				os.Symlink(tc.target, filepath.Join(tree, defaultManifestName)))
-
-			output := filepath.Join(root, "other.mf")
-			opts := testOpts(slices.Concat(
-				[]string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags,
-				[]string{filepath.Join(tree, tc.path)},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
-		})
-	}
-}
-
-// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside
-// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file
-// given, is listed, since a file given is left out only when its own name
-// is index.mf.
-//
-//nolint:paralleltest // changes the process-global working directory
-func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) {
-	setupManifestInSubdir(t)
-
-	fs := afero.NewOsFs()
-	output := filepath.Join(t.TempDir(), "other.mf")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".",
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest)
-}
-
 // writeTestTree writes file.txt and sub/nested.txt under dir.
 func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
 	t.Helper()

internal/cli/errmsg_test.go

@@ -50,6 +50,38 @@ func runLocked(fn func() error) error {
 	return fn()
 }
 
+// unsignedChecker builds a Checker over a freshly scanned, unsigned manifest.
+func unsignedChecker(t *testing.T) *mfer.Checker {
+	t.Helper()
+
+	// The scan and the manifest load log to the process-global logger, so
+	// serialize with the CLI runs.
+	runMu.Lock()
+	defer runMu.Unlock()
+
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/d", 0o755))
+	require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
+
+	s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: fs})
+	require.NoError(t, s.EnumeratePath("/d", nil))
+
+	var buf bytes.Buffer
+
+	require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
+	require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
+
+	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
+		ManifestPath: "/d/index.mf",
+		BasePath:     "/d",
+		Fs:           fs,
+	})
+	require.NoError(t, err)
+	require.False(t, chk.IsSigned())
+
+	return chk
+}
+
 func TestNoManifestFoundMessage(t *testing.T) {
 	t.Parallel()
 
@@ -65,7 +97,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 	t.Run("invalid fingerprint length", func(t *testing.T) {
 		t.Parallel()
 
-		err := verifyRequiredSigner("", "12345678")
+		err := verifyRequiredSigner(unsignedChecker(t), "12345678")
 		require.ErrorIs(t, err, errInvalidFingerprint)
 		assert.EqualError(t, err,
 			"invalid fingerprint: must be exactly 40 hex characters, got 8")
@@ -74,7 +106,7 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 	t.Run("manifest not signed", func(t *testing.T) {
 		t.Parallel()
 
-		err := verifyRequiredSigner("", msgFpA)
+		err := verifyRequiredSigner(unsignedChecker(t), msgFpA)
 		require.ErrorIs(t, err, errManifestNotSigned)
 		assert.EqualError(t, err,
 			"manifest is not signed, but signature from "+msgFpA+" is required")
@@ -89,13 +121,13 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
 func TestSignerMismatchMessage(t *testing.T) {
 	t.Parallel()
 
-	signer := string(signedChecker(t,
-		signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")})).Signer())
+	chk := signedChecker(t,
+		signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
 
-	err := verifyRequiredSigner(signer, msgFpB)
+	err := verifyRequiredSigner(chk, msgFpB)
 	require.ErrorIs(t, err, errSignerMismatch)
 	assert.EqualError(t, err,
-		"embedded signing key fingerprint "+signer+
+		"embedded signing key fingerprint "+string(chk.Signer())+
 			" does not match required "+msgFpB)
 }
 

internal/cli/fetch.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/multiformats/go-multihash"
+	"github.com/spf13/afero"
 	"github.com/urfave/cli/v3"
 	"sneak.berlin/go/mfer/internal/log"
 	"sneak.berlin/go/mfer/mfer"
@@ -95,6 +96,9 @@ var (
 	// writes another file.
 	errNameClash = errors.New(
 		"manifest lists a file where fetch writes another file")
+	// errModeOutOfRange indicates a manifest that lists a mode with more
+	// than the permission bits, such as setuid.
+	errModeOutOfRange = errors.New("manifest lists a mode outside 0777")
 )
 
 // DownloadProgress reports the progress of a single file download.
@@ -425,9 +429,9 @@ func (mfa *CLIApp) fetchManifestOperation(
 
 // fetchManifest downloads the manifest at manifestURL and parses it,
 // enforcing --require-signature if it is given and refusing a manifest
-// that lists a file where fetch writes another. It returns the manifest as
-// downloaded, to be saved once the files are in place, and the files it
-// lists.
+// that lists a file where fetch writes another or a mode outside 0777. It
+// returns the manifest as downloaded, to be saved once the files are in
+// place, and the files it lists.
 func (mfa *CLIApp) fetchManifest(
 	ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
 ) ([]byte, []*mfer.MFFilePath, error) {
@@ -471,7 +475,7 @@ func (mfa *CLIApp) fetchManifest(
 	}
 
 	if requiredSigner != "" {
-		err = verifyRequiredSigner(string(manifest.Signer()), requiredSigner)
+		err = verifyFetchedSigner(manifestData, requiredSigner)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -484,6 +488,16 @@ func (mfa *CLIApp) fetchManifest(
 		return nil, nil, err
 	}
 
+	// fetch sets each recorded mode on the file it writes, so a mode above
+	// 0777, which could carry setuid, setgid or sticky bits, is refused
+	// before any file is requested.
+	for _, f := range files {
+		if f.GetMode() > uint32(os.ModePerm) {
+			return nil, nil, fmt.Errorf("%w: %s (%#o)",
+				errModeOutOfRange, f.GetPath(), f.GetMode())
+		}
+	}
+
 	log.Infof("manifest contains %d files", len(files))
 
 	return manifestData, files, nil
@@ -539,6 +553,31 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
 	return nil
 }
 
+// verifyFetchedSigner enforces --require-signature on the fetched manifest
+// exactly as check does. verifyRequiredSigner takes a Checker, which loads
+// its manifest from a file, so the manifest is handed to it as a file in
+// memory.
+func verifyFetchedSigner(manifestData []byte, requiredSigner string) error {
+	memFs := afero.NewMemMapFs()
+	manifestPath := "/" + defaultManifestName
+
+	err := afero.WriteFile(memFs, manifestPath, manifestData, filePerms)
+	if err != nil {
+		return err
+	}
+
+	chk, err := mfer.NewChecker(&mfer.CheckerOptions{
+		ManifestPath: manifestPath,
+		BasePath:     "/",
+		Fs:           memFs,
+	})
+	if err != nil {
+		return fmt.Errorf("load manifest: %w", err)
+	}
+
+	return verifyRequiredSigner(chk, requiredSigner)
+}
+
 // saveManifest writes the fetched manifest into dest under the default
 // manifest name, the way fetch writes every file: to a new temp file that
 // is then renamed into place.

internal/cli/fetch_test.go

@@ -19,12 +19,15 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+	"uuid"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/multiformats/go-multihash"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	urfcli "github.com/urfave/cli/v3"
+	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/mfer"
 )
 
@@ -486,17 +489,18 @@ func TestFetchStopsReadingFilePastListedSize(t *testing.T) {
 
 // TestManifestDownloadStopsAtLimit serves a manifest that never ends to
 // fetch and to check, with the most they download of a manifest lowered
-// to 64 KiB. Each must stop reading at that limit and fail with an error
-// naming it.
+// to 64 KiB. Each must stop reading at that limit, fail with an error
+// naming it and leave no temp file.
 func TestManifestDownloadStopsAtLimit(t *testing.T) {
-	t.Parallel()
-
 	server := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			_, _ = io.Copy(w, zeros{})
 		}))
 	defer server.Close()
 
+	tmpDir := t.TempDir()
+	t.Setenv("TMPDIR", tmpDir)
+
 	mfa := &CLIApp{Fs: afero.NewOsFs(), maxManifestSize: 64 << 10}
 
 	fetch := mfa.fetchCommand()
@@ -516,6 +520,10 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
 			"download manifest: file exceeds maximum allowed size of 65536 bytes",
 			cmd.Name)
 	}
+
+	leftover, err := os.ReadDir(tmpDir)
+	require.NoError(t, err)
+	assert.Empty(t, leftover)
 }
 
 //nolint:paralleltest // changes the process-global working directory
@@ -1532,9 +1540,27 @@ func TestFetchSetsRecordedMode(t *testing.T) {
 		[]string{"/" + defaultManifestName, "/" + testFileTxt}, requested)
 }
 
+// TestFetchRefusesModeOutsidePermissionBits fetches manifests that record
+// a mode with the setuid bit, once as Unix writes it (04755) and once as
+// Go keeps it in os.FileMode. fetch must refuse both before it creates the
+// destination or requests any file.
+func TestFetchRefusesModeOutsidePermissionBits(t *testing.T) {
+	t.Parallel()
+
+	files := map[string][]byte{testFileTxt: []byte("a file")}
+
+	assertFetchRefused(t, manifestWithMode(t, testFileTxt, files[testFileTxt], 0o4755),
+		files, "manifest lists a mode outside 0777: file.txt (04755)")
+
+	assertFetchRefused(t,
+		manifestWithMode(t, testFileTxt, files[testFileTxt],
+			uint32(os.ModeSetuid|0o755)),
+		files, "manifest lists a mode outside 0777: file.txt (040000755)")
+}
+
 // TestDownloadFileSetsOnlyPermissionBits downloads a file whose entry
 // records mode 0755 together with setuid, setgid or sticky, as Go keeps
-// them in os.FileMode. Loading a manifest refuses such an entry; called
+// them in os.FileMode. fetchManifest would refuse such an entry; called
 // directly, downloadFile must still set only the permission bits.
 func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
 	t.Parallel()
@@ -1571,6 +1597,51 @@ func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
 	}
 }
 
+// manifestWithMode returns a manifest listing one file, path, with content
+// and the given recorded mode. It is assembled by hand, since the builder
+// never records a mode outside 0777.
+func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []byte {
+	t.Helper()
+
+	digest := sha256.Sum256(content)
+	hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
+	require.NoError(t, err)
+
+	id := uuid.NewV4()
+
+	inner, err := proto.Marshal(&mfer.MFFile{
+		Version: mfer.MFFile_VERSION_ONE,
+		Files: []*mfer.MFFilePath{{
+			Path:   path,
+			Size:   int64(len(content)),
+			Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
+			Mode:   mode,
+		}},
+		Uuid: id[:],
+	})
+	require.NoError(t, err)
+
+	encoder, err := zstd.NewWriter(nil)
+	require.NoError(t, err)
+
+	compressed := encoder.EncodeAll(inner, nil)
+	require.NoError(t, encoder.Close())
+
+	sum := sha256.Sum256(compressed)
+
+	outer, err := proto.Marshal(&mfer.MFFileOuter{
+		Version:         mfer.MFFileOuter_VERSION_ONE,
+		CompressionType: mfer.MFFileOuter_COMPRESSION_ZSTD,
+		Size:            int64(len(inner)),
+		Sha256:          sum[:],
+		Uuid:            id[:],
+		InnerMessage:    compressed,
+	})
+	require.NoError(t, err)
+
+	return append([]byte(mfer.MAGIC), outer...)
+}
+
 // builtManifest returns a manifest of files, built directly rather than
 // scanned, since a scan lists no hidden files.
 func builtManifest(t *testing.T, files map[string][]byte) []byte {

internal/cli/freshen.go

@@ -33,9 +33,9 @@ var (
 	// modification time where one is required to carry it forward unchanged.
 	errEntryMissingMtime = errors.New("manifest entry has no mtime")
 
-	// errEntryMissingHash indicates a manifest entry that carries no
-	// SHA-256 hash to carry forward.
-	errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
+	// errEntryMissingHash indicates a manifest entry that carries no hash
+	// to carry forward.
+	errEntryMissingHash = errors.New("manifest entry has no hash")
 )
 
 // errSignKeyNeeded indicates a signed manifest given to freshen with no
@@ -192,12 +192,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
 		return nil
 	}
 
-	// Leave out a file listed as index.mf, at the top of the base
-	// directory, since fetch saves the manifest there, as gen does.
-	if relPath == defaultManifestName {
-		return nil
-	}
-
 	// Handle symlinks
 	if info.Mode()&fs.ModeSymlink != 0 {
 		realInfo, keep := s.resolveSymlink(path)
@@ -705,26 +699,26 @@ func addFileToBuilder(
 // Entries reach this path only when recordEntry classified them as
 // unchanged, which requires a recorded mtime, so an absent mtime here is
 // an error rather than something to paper over with the Unix epoch.
-// Only the entry's SHA-256 hash is carried over, wherever the entry lists
-// it, since check and fetch compare with no other. Loading refuses an
-// entry with no SHA-256 hash; one that got here anyway is an error too,
-// not a file to leave out of the manifest.
+// Loading refuses an entry with no hash; one that got here anyway is an
+// error too, not a file to leave out of the manifest.
 func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
 	mtime, ok := entryMtime(entry)
 	if !ok {
 		return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
 	}
 
-	hash := mfer.SHA256Multihash(entry)
-	if hash == nil {
+	if len(entry.GetHashes()) == 0 {
 		return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
 	}
 
 	err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
 		mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
-		fs.FileMode(entry.GetMode()), hash)
+		fs.FileMode(entry.GetMode()), entry.GetHashes()[0].GetMultiHash())
 	if err != nil {
-		return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
+		return fmt.Errorf(
+			"manifest entry %s: %w (regenerate the manifest with mfer generate)",
+			entry.GetPath(), err,
+		)
 	}
 
 	return nil

internal/cli/freshen_test.go

@@ -2,10 +2,7 @@
 package cli
 
 import (
-	"bytes"
-	"context"
 	"crypto/sha256"
-	"crypto/sha512"
 	"maps"
 	"os"
 	"path/filepath"
@@ -413,77 +410,6 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
-// TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside
-// the directory --base names, after index.mf and sub/index.mf are added
-// there: index.mf at the top is not listed, since fetch saves the manifest
-// there, while sub/index.mf is listed.
-func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
-	t.Parallel()
-
-	root := t.TempDir()
-	tree := filepath.Join(root, "tree")
-	manifestPath := filepath.Join(root, "other.mf")
-
-	fs := afero.NewOsFs()
-	require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-	writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-
-	opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, tree}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
-	writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-
-	runFreshen(t, fs, tree, manifestPath)
-
-	assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
-		manifestPaths(t, fs, manifestPath))
-}
-
-// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept
-// outside the directory --base names, after sub/index.mf is added there
-// and index.mf at the top is made a symlink to it: sub/index.mf is listed,
-// and the symlink, which freshen lists only with --follow-symlinks, is left
-// out like any index.mf at the top.
-func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	for name, flags := range map[string][]string{
-		"plain freshen":             nil,
-		"freshen --follow-symlinks": {testFlagFollowSymlinks},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-			manifestPath := filepath.Join(root, "other.mf")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-
-			opts := testOpts([]string{
-				testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
-			}, fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t, os.Symlink(testSubdirManifest,
-				filepath.Join(tree, defaultManifestName)))
-
-			opts = testOpts(slices.Concat(
-				[]string{testApp, cmdFreshen, "-q"}, flags,
-				[]string{testFlagBase, tree, manifestPath},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
-				manifestPaths(t, fs, manifestPath))
-		})
-	}
-}
-
 // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
 // runs freshen from the directory above sub, on the manifest in sub.
 // Without --base, the manifest then lists the files in sub, whether freshen
@@ -658,76 +584,48 @@ func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
 	assert.Contains(t, err.Error(), "file1.txt")
 }
 
-// TestFreshenAddExistingRejectsMissingHash pins that an entry with no
-// SHA-256 hash, whether it lists no hash or only a SHA-1 one, is an error
-// rather than a file left out of the rebuilt manifest.
+// TestFreshenAddExistingRejectsMissingHash pins that an entry with no hash
+// is an error rather than a file left out of the rebuilt manifest.
 func TestFreshenAddExistingRejectsMissingHash(t *testing.T) {
 	t.Parallel()
 
+	b := mfer.NewBuilder()
+	entry := &mfer.MFFilePath{
+		Path:  "data.bin",
+		Size:  8,
+		Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
+	}
+
+	err := addExistingToBuilder(b, entry)
+	require.ErrorIs(t, err, errEntryMissingHash)
+	assert.Contains(t, err.Error(), "data.bin")
+	assert.Zero(t, b.FileCount())
+}
+
+// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
+// entry whose hash the builder refuses is reported as a problem with the
+// manifest, with the command that fixes it.
+func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
+	t.Parallel()
+
 	sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
 	require.NoError(t, err)
 
-	tests := []struct {
-		name   string
-		hashes []*mfer.MFFileChecksum
-	}{
-		{"no hash", nil},
-		{"SHA-1 hash only", []*mfer.MFFileChecksum{{MultiHash: sha1Hash}}},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			b := mfer.NewBuilder()
-			entry := &mfer.MFFilePath{
-				Path:   "data.bin",
-				Size:   8,
-				Mtime:  &mfer.Timestamp{Seconds: 1_700_000_000},
-				Hashes: tt.hashes,
-			}
-
-			err := addExistingToBuilder(b, entry)
-			require.ErrorIs(t, err, errEntryMissingHash)
-			assert.Contains(t, err.Error(), "data.bin")
-			assert.Zero(t, b.FileCount())
-		})
-	}
-}
-
-// TestFreshenAddExistingKeepsSHA256Hash pins that an unchanged entry that
-// lists another hash before its SHA-256 hash is carried over with the
-// SHA-256 hash, the one check and fetch compare with, and nothing else.
-func TestFreshenAddExistingKeepsSHA256Hash(t *testing.T) {
-	t.Parallel()
-
-	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
-	require.NoError(t, err)
-
-	sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
-	require.NoError(t, err)
-
 	b := mfer.NewBuilder()
 	entry := &mfer.MFFilePath{
-		Path:  "kept.txt",
-		Size:  6,
+		Path:  "old.txt",
+		Size:  8,
 		Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
 		Hashes: []*mfer.MFFileChecksum{
-			{MultiHash: sha512Hash},
-			{MultiHash: sha256Hash},
+			{MultiHash: sha1Hash},
 		},
 	}
 
-	require.NoError(t, addExistingToBuilder(b, entry))
-
-	var buf bytes.Buffer
-	require.NoError(t, b.Build(context.Background(), &buf))
-
-	m, err := mfer.NewManifestFromReader(&buf)
-	require.NoError(t, err)
-	require.Len(t, m.Files(), 1)
-	require.Len(t, m.Files()[0].GetHashes(), 1)
-	assert.Equal(t, sha256Hash, m.Files()[0].GetHashes()[0].GetMultiHash())
+	err = addExistingToBuilder(b, entry)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "manifest entry old.txt")
+	assert.Contains(t, err.Error(), "mfer generate")
+	assert.Zero(t, b.FileCount())
 }
 
 // TestEntryMtime pins the presence semantics the callers depend on.

internal/cli/gen.go

@@ -137,9 +137,6 @@ func (mfa *CLIApp) buildScannerOptions(
 		// Neither a manifest being replaced nor a temp file left by an
 		// interrupted run belongs in the new manifest.
 		ExcludePaths: []string{output, manifestTempPath(output)},
-		// Nor does a file listed as index.mf, at the top of the tree, since
-		// fetch saves the manifest there and refuses a manifest that lists it.
-		ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName},
 	}
 
 	// Set seed for deterministic UUID if provided

internal/cli/pipe_test.go

@@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) {
 	require.NoError(t, os.Symlink(target, filepath.Join(root, testLink)))
 
 	opts := testOpts([]string{
-		testApp, cmdFreshen, "-v", testFlagFollowSymlinks,
+		testApp, cmdFreshen, "-v", "--follow-symlinks",
 		testFlagBase, root, manifestPath,
 	}, fs)
 	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))

mfer/builder.go

@@ -37,7 +37,8 @@ var (
 	errPathDot          = errors.New("contains '.' segment")
 	errSizeMismatch     = errors.New("size mismatch")
 	errNegativeSize     = errors.New("size cannot be negative")
-	errHashNotSHA256    = errors.New("hash is not a SHA-256 multihash")
+	errHashNotMultihash = errors.New("hash is not a valid multihash")
+	errHashTooShort     = errors.New("hash digest is too short")
 	errDuplicatePath    = errors.New("duplicate path")
 )
 
@@ -112,28 +113,6 @@ func (m ModTime) Timestamp() *Timestamp {
 // Multihash represents a multihash-encoded file hash (typically SHA2-256).
 type Multihash []byte
 
-// SHA256Multihash returns the first of entry's hashes that is a SHA-256
-// multihash, or nil if none is. Loading a manifest refuses an entry that
-// does not list exactly one, so for a loaded entry it is the only one.
-func SHA256Multihash(entry *MFFilePath) Multihash {
-	for _, h := range entry.GetHashes() {
-		if isSHA256Multihash(h.GetMultiHash()) {
-			return h.GetMultiHash()
-		}
-	}
-
-	return nil
-}
-
-// isSHA256Multihash reports whether hash is a SHA-256 multihash: the
-// multihash code for SHA-256, then a 32-byte digest.
-func isSHA256Multihash(hash []byte) bool {
-	decoded, err := multihash.Decode(hash)
-
-	return err == nil && decoded.Code == multihash.SHA2_256 &&
-		len(decoded.Digest) == sha256.Size
-}
-
 // FileHashProgress reports progress during file hashing.
 type FileHashProgress struct {
 	BytesRead FileSize // Total bytes read so far for the current file
@@ -262,7 +241,8 @@ func (b *Builder) FileCount() int {
 // This is useful when the hash is already known (e.g., from an existing manifest).
 // Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
 // Returns an error if path is invalid or already added, size is negative,
-// or hash is not a SHA-256 multihash, the hash every entry must carry.
+// or hash is not a multihash with a digest of at least 32 bytes, as long
+// as SHA-256's.
 func (b *Builder) AddFileWithHash(
 	path RelFilePath,
 	size FileSize,
@@ -279,8 +259,19 @@ func (b *Builder) AddFileWithHash(
 		return errNegativeSize
 	}
 
-	if !isSHA256Multihash(hash) {
-		return errHashNotSHA256
+	decoded, err := multihash.Decode(hash)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errHashNotMultihash, err)
+	}
+
+	// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
+	// hash is at least as long as a SHA-256 multihash, so a manifest of
+	// shorter ones could fail to load.
+	if len(decoded.Digest) < sha256.Size {
+		return fmt.Errorf(
+			"%w: %d bytes, at least %d needed",
+			errHashTooShort, len(decoded.Digest), sha256.Size,
+		)
 	}
 
 	entry := &MFFilePath{

mfer/builder_test.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"crypto/sha512"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -95,25 +94,23 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
 	sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
 	require.NoError(t, err)
 
-	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
-	require.NoError(t, err)
-
 	truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
 	require.NoError(t, err)
 
 	tests := []struct {
 		name string
 		hash Multihash
+		want error
 	}{
-		{"nil hash", nil},
-		{"empty hash", []byte{}},
-		{"one-byte hash", []byte{0x12}},
+		{"nil hash", nil, errHashNotMultihash},
+		{"empty hash", []byte{}, errHashNotMultihash},
+		{"one-byte hash", []byte{0x12}, errHashNotMultihash},
 		// A SHA-256 code and 32-byte length, then only two bytes of digest.
-		{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}},
-		{"SHA-1 multihash", sha1Hash},
-		{"SHA-512 multihash", sha512Hash},
-		// A SHA-256 code with a 31-byte digest.
-		{"31-byte digest", truncatedHash},
+		{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
+		// A valid multihash, but its 20-byte SHA-1 digest is too short.
+		{"SHA-1 multihash", sha1Hash, errHashTooShort},
+		// A valid multihash whose 31-byte digest is one byte short.
+		{"31-byte digest", truncatedHash, errHashTooShort},
 	}
 
 	for _, tt := range tests {
@@ -122,7 +119,7 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
 
 			b := NewBuilder()
 			err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), 0, tt.hash)
-			require.ErrorIs(t, err, errHashNotSHA256)
+			require.ErrorIs(t, err, tt.want)
 			assert.Equal(t, 0, b.FileCount())
 		})
 	}

mfer/checker.go

@@ -80,8 +80,7 @@ type Checker struct {
 	// manifestPaths is a set of paths in the manifest for quick lookup
 	manifestPaths map[RelFilePath]struct{}
 	// manifestInfo is the manifest file, which FindExtraFiles leaves out,
-	// matched with os.SameFile. It is nil for a manifest read from
-	// CheckerOptions.ManifestReader.
+	// matched with os.SameFile.
 	manifestInfo os.FileInfo
 	// signature info from the manifest
 	signature     []byte
@@ -91,13 +90,8 @@ type Checker struct {
 
 // CheckerOptions configures a Checker.
 type CheckerOptions struct {
-	// ManifestPath is the manifest file to check against. It is required
-	// unless ManifestReader is given.
+	// ManifestPath is the manifest file to check against (required).
 	ManifestPath string
-	// ManifestReader, if not nil, is read for the manifest to check against
-	// in place of ManifestPath, such as a manifest downloaded into memory.
-	// FindExtraFiles then has no manifest file to leave out.
-	ManifestReader io.Reader
 	// BasePath is the directory relative to which manifest paths are
 	// resolved (required).
 	BasePath string
@@ -106,10 +100,9 @@ type CheckerOptions struct {
 }
 
 // NewChecker creates a new Checker with the given options. It returns an
-// error if opts is nil, the base path is empty, or neither a manifest path
-// nor a manifest reader is given.
+// error if opts is nil or either path is empty.
 func NewChecker(opts *CheckerOptions) (*Checker, error) {
-	if opts == nil || (opts.ManifestPath == "" && opts.ManifestReader == nil) {
+	if opts == nil || opts.ManifestPath == "" {
 		return nil, errManifestPathEmpty
 	}
 
@@ -122,30 +115,12 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
 		fs = afero.NewOsFs()
 	}
 
-	var (
-		m            *manifest
-		manifestInfo os.FileInfo
-		err          error
-	)
-
-	if opts.ManifestReader != nil {
-		m, err = NewManifestFromReader(opts.ManifestReader)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		m, err = NewManifestFromFile(&ManifestFromFileOptions{
-			Path: opts.ManifestPath,
-			Fs:   fs,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		manifestInfo, err = fs.Stat(opts.ManifestPath)
-		if err != nil {
-			return nil, err
-		}
+	m, err := NewManifestFromFile(&ManifestFromFileOptions{
+		Path: opts.ManifestPath,
+		Fs:   fs,
+	})
+	if err != nil {
+		return nil, err
 	}
 
 	abs, err := filepath.Abs(opts.BasePath)
@@ -160,6 +135,11 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
 		manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
 	}
 
+	manifestInfo, err := fs.Stat(opts.ManifestPath)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Checker{
 		basePath:      AbsFilePath(abs),
 		files:         files,

mfer/checker_test.go

@@ -189,38 +189,6 @@ func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
 	assert.Equal(t, StatusOK, (<-results).Status)
 }
 
-// TestNewCheckerReadsManifestFromReader gives NewChecker the manifest as a
-// reader and no manifest path, on a filesystem that holds only the files it
-// lists: it checks those files and finds nothing unlisted.
-func TestNewCheckerReadsManifestFromReader(t *testing.T) {
-	t.Parallel()
-
-	files := map[string][]byte{testFile1: []byte("hello")}
-
-	manifestFs := afero.NewMemMapFs()
-	createTestManifest(t, manifestFs, testManifestPath, files)
-	manifestData, err := afero.ReadFile(manifestFs, testManifestPath)
-	require.NoError(t, err)
-
-	fs := afero.NewMemMapFs()
-	createFilesOnDisk(t, fs, files)
-
-	chk, err := NewChecker(&CheckerOptions{
-		ManifestReader: bytes.NewReader(manifestData),
-		BasePath:       testDataDir,
-		Fs:             fs,
-	})
-	require.NoError(t, err)
-
-	results := make(chan Result, 1)
-	require.NoError(t, chk.Check(context.Background(), results, nil))
-	assert.Equal(t, Result{Path: testFile1, Status: StatusOK}, <-results)
-
-	extras := make(chan Result, len(files))
-	require.NoError(t, chk.FindExtraFiles(context.Background(), extras))
-	assert.Empty(t, extras)
-}
-
 func TestCheckerFileCountAndTotalBytes(t *testing.T) {
 	t.Parallel()
 

mfer/deserialize.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
 	"github.com/klauspost/compress/zstd"
@@ -28,9 +27,7 @@ var (
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("invalid file entry")
-	errEntryMissingHash     = errors.New("file entry has no SHA-256 hash")
-	errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
-	errModeOutOfRange       = errors.New("file entry has a mode above 0777")
+	errEntryMissingHash     = errors.New("file entry has no hash")
 	errDecodedTooLarge      = errors.New("too much memory needed")
 	errSignerNotSigningKey  = errors.New(
 		"signer is not the fingerprint of the key that made the signature")
@@ -239,30 +236,6 @@ func forEachBytesField(
 	return nil
 }
 
-// validateEntryHashes refuses, naming its path, an entry that does not list
-// exactly one SHA-256 multihash; hashes of other algorithms beside it are
-// allowed. check and fetch compute only SHA-256, so with none they would
-// have nothing to check the file against, and with two check would pass a
-// file that matches either.
-func validateEntryHashes(entry *MFFilePath) error {
-	sha256Hashes := 0
-
-	for _, h := range entry.GetHashes() {
-		if isSHA256Multihash(h.GetMultiHash()) {
-			sha256Hashes++
-		}
-	}
-
-	switch {
-	case sha256Hashes == 0:
-		return fmt.Errorf("%w: %q", errEntryMissingHash, entry.GetPath())
-	case sha256Hashes > 1:
-		return fmt.Errorf("%w: %q", errEntryExtraSHA256Hash, entry.GetPath())
-	default:
-		return nil
-	}
-}
-
 func (m *manifest) deserializeInner() error {
 	err := m.validateOuterHeader()
 	if err != nil {
@@ -321,10 +294,8 @@ func (m *manifest) deserializeInner() error {
 	// untrusted .mf. Reject loudly on the first offender rather than
 	// dropping entries, which would let a hostile manifest hide files from a
 	// check. A path listed twice is refused too: check would check the one
-	// file against both entries. So is an entry that does not list exactly
-	// one SHA-256 hash, and one that records a mode above 0777, such as one
-	// with setuid: a mode holds the permission bits alone, which are all
-	// check compares and fetch sets.
+	// file against both entries. So is an entry with no hash: there is
+	// nothing to check the file against.
 	seen := make(map[string]bool, len(m.pbInner.GetFiles()))
 
 	for _, f := range m.pbInner.GetFiles() {
@@ -333,13 +304,8 @@ func (m *manifest) deserializeInner() error {
 			return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
 		}
 
-		err = validateEntryHashes(f)
-		if err != nil {
-			return err
-		}
-
-		if f.GetMode() > uint32(os.ModePerm) {
-			return fmt.Errorf("%w: %q (%#o)", errModeOutOfRange, f.GetPath(), f.GetMode())
+		if len(f.GetHashes()) == 0 {
+			return fmt.Errorf("%w: %q", errEntryMissingHash, f.GetPath())
 		}
 
 		if seen[f.GetPath()] {

mfer/deserialize_path_test.go

@@ -5,11 +5,8 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"crypto/sha512"
 	"fmt"
-	"os"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 	"uuid"
@@ -170,154 +167,40 @@ func TestDeserializeRefusesPathListedTwice(t *testing.T) {
 	}
 }
 
-// A file entry must list exactly one SHA-256 hash, and may list hashes of
-// other algorithms beside it. An entry with none, even one that lists a
-// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
-// each refusal names the path. An entry listing a SHA-512 hash and then a
-// SHA-256 one loads with both.
-func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
+// A manifest holding a file entry with no hash is refused as it is loaded,
+// naming the path: there is nothing to check the file against.
+func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
 	t.Parallel()
 
-	sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	id := uuid.NewV4()
+	inner, err := proto.Marshal(&MFFile{
+		Version: MFFile_VERSION_ONE,
+		Files:   []*MFFilePath{{Path: "x", Size: 1}},
+		Uuid:    id[:],
+	})
 	require.NoError(t, err)
 
-	otherSHA256Hash, err := multihash.Encode(
-		bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
-	require.NoError(t, err)
-
-	sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
-	require.NoError(t, err)
-
-	tests := []struct {
-		name   string
-		hashes [][]byte
-		want   error // nil for an entry that loads
-	}{
-		{"no hash", nil, errEntryMissingHash},
-		{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
-		{
-			"two SHA-256 hashes",
-			[][]byte{sha256Hash, otherSHA256Hash},
-			errEntryExtraSHA256Hash,
-		},
-		{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			entry := &MFFilePath{Path: "x", Size: 1}
-			for _, h := range tt.hashes {
-				entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
-			}
-
-			id := uuid.NewV4()
-			inner, err := proto.Marshal(&MFFile{
-				Version: MFFile_VERSION_ONE,
-				Files:   []*MFFilePath{entry},
-				Uuid:    id[:],
-			})
-			require.NoError(t, err)
-
-			m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
-			if tt.want == nil {
-				require.NoError(t, err)
-				require.Len(t, m.Files(), 1)
-				assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
-
-				return
-			}
-
-			require.ErrorIs(t, err, tt.want)
-			assert.Contains(t, err.Error(), `"x"`)
-		})
-	}
+	_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+	require.ErrorIs(t, err, errEntryMissingHash)
+	require.EqualError(t, err, `file entry has no hash: "x"`)
 }
 
-// A file entry that records a mode with more than the permission bits is
-// refused as it is loaded, naming the path and the mode: once with setuid as
-// Unix writes it (04755) and once as Go keeps it in os.FileMode. An entry
-// recording 0777 loads.
-func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) {
-	t.Parallel()
-
-	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
-	require.NoError(t, err)
-
-	tests := []struct {
-		name string
-		mode uint32
-		want string // "" for an entry that loads
-	}{
-		{
-			"setuid as Unix writes it",
-			0o4755,
-			`file entry has a mode above 0777: "x" (04755)`,
-		},
-		{
-			"setuid as Go keeps it",
-			uint32(os.ModeSetuid | 0o755),
-			`file entry has a mode above 0777: "x" (040000755)`,
-		},
-		{"permission bits only", 0o777, ""},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			id := uuid.NewV4()
-			inner, err := proto.Marshal(&MFFile{
-				Version: MFFile_VERSION_ONE,
-				Files: []*MFFilePath{{
-					Path:   "x",
-					Size:   1,
-					Hashes: []*MFFileChecksum{{MultiHash: hash}},
-					Mode:   tt.mode,
-				}},
-				Uuid: id[:],
-			})
-			require.NoError(t, err)
-
-			m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
-			if tt.want == "" {
-				require.NoError(t, err)
-				require.Len(t, m.Files(), 1)
-				assert.Equal(t, tt.mode, m.Files()[0].GetMode())
-
-				return
-			}
-
-			require.ErrorIs(t, err, errModeOutOfRange)
-			require.EqualError(t, err, tt.want)
-		})
-	}
-}
-
-// Entries of a path, a SHA-256 hash, an empty MIME type and empty
-// modification and change times are counted at 432 bytes each (176 + 112 +
-// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
-// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
-// the five uncounted, even the MIME type, brings it under 8. A two-character
-// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
-// each base-36 digit, and each entry's path is its number in base 36, padded
-// with zeros to that length, since a manifest lists a path only once.
+// Entries of a path, an empty hash, an empty MIME type and empty modification
+// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
+// and take 16 bytes plus the path to encode. A 37-character path makes that
+// 53 bytes, about 8.2 times: refused, and leaving any one of the five
+// uncounted, even the MIME type, brings it under 8. A 39-character path makes
+// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
+// padded with zeros to that length, since a manifest lists a path only once.
 func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 	t.Parallel()
 
-	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
-	require.NoError(t, err)
-
-	checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
-	require.NoError(t, err)
-
 	tests := []struct {
 		pathLen int
 		refused bool
 	}{
-		{1, true},
-		{2, false},
+		{37, true},
+		{39, false},
 	}
 
 	for _, tt := range tests {
@@ -330,14 +213,11 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
 			inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
 			inner = protowire.AppendBytes(inner, id[:])
 
-			for i := range 36 {
-				path := strconv.FormatInt(int64(i), 36)
-				path = strings.Repeat("0", tt.pathLen-len(path)) + path
-
+			for i := range 1000 {
 				entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
-				entry = protowire.AppendString(entry, path)
+				entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
 				entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
-				entry = protowire.AppendBytes(entry, checksum)
+				entry = protowire.AppendBytes(entry, nil)
 				entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
 				entry = protowire.AppendBytes(entry, nil)
 				entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
@@ -367,16 +247,10 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
 	unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
 	unknown = protowire.AppendBytes(unknown, []byte("not known"))
 
-	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
-	require.NoError(t, err)
-
-	checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
-	require.NoError(t, err)
-
 	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
 	entry = protowire.AppendString(entry, "a")
 	entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
-	entry = protowire.AppendBytes(entry, checksum)
+	entry = protowire.AppendBytes(entry, nil)
 	entry = append(entry, unknown...)
 
 	id := uuid.NewV4()

mfer/scanner.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -65,9 +64,6 @@ type ScannerOptions struct {
 	Seed string
 	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
 	ExcludePaths []string
-	// ExcludeRelPaths lists paths to leave out of the listing, matched
-	// against the path a file would be listed by, with no file looked up.
-	ExcludeRelPaths []RelFilePath
 }
 
 // FileEntry represents a file that has been enumerated.
@@ -443,10 +439,9 @@ func (s *Scanner) enumerateFileWithInfo(
 	}
 
 	// Clean the path - remove leading slash if present
-	cleanPath := strings.TrimPrefix(filePath, "/")
-
-	if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) {
-		return nil
+	cleanPath := filePath
+	if len(cleanPath) > 0 && cleanPath[0] == '/' {
+		cleanPath = cleanPath[1:]
 	}
 
 	// Compute absolute path for file reading