Never list index.mf at the top of the tree in gen and freshen (closes #233)

fetch saves the manifest as index.mf at the top of the tree and refuses a
manifest that lists a file there. gen given --output, and freshen with its
manifest outside the base directory, listed such a file, so they wrote
manifests fetch would not take.

Both now leave out each file the manifest would list as index.mf: one at
the top of the tree, or a file given to gen by that name. The rule goes by
that path alone and looks up no file, so a symlink named index.mf never
hides the file it points to. index.mf in a subdirectory is still listed.
The scanner in package mfer gets an option to leave out files by the path
they would be listed by.

Model: opus-5-5

README.md

@@ -13,14 +13,11 @@ The structure of these files can be found
 schema `mfer/mf.proto` for exact field numbers and types. Both are included in
 the [project repository](https://git.eeqj.de/sneak/mfer).
 
-The current version is pre-1.0 and while the repo was published in 2022, there
+Releases are git tags named by [SemVer](https://semver.org), starting with
-has not yet been any versioned release. [SemVer](https://semver.org) will be
+`v1.0.0`. No file in the tree records a version number. `mfer version` prints
-used for releases.
+the revision the build stamped into the binary, which for `make build` and the
-
+Docker image is the output of `git describe` on the checkout they build. A
-A version is a git tag, and no file in the tree records a version number.
+binary built without that stamp, as
-`mfer version` prints the revision the build stamped into the binary, which for
-`make build` and the Docker image is the output of `git describe` on the
-checkout they build. A binary built without that stamp, as
 `go install sneak.berlin/go/mfer/cmd/mfer@<tag>` builds it, prints the module
 version Go records in it instead. A program that imports the `mfer` package
 finds the version it was built against in its own build information
@@ -290,6 +287,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       only on the paths it is given and the files under them, so with the same
       `--seed` and an unchanged tree it writes the same bytes wherever the
       manifest goes. The file it writes to is never listed
+    - never lists `index.mf` at the top of a directory it is given, or a file it
+      is given by that name, wherever it writes the manifest, since `fetch`
+      saves the manifest there; `index.mf` in a subdirectory is listed
 - `mfer check` / `mfer check .`
     - verifies checksums of all files in manifest, displaying error and exiting
       nonzero if any files are missing or corrupted, or have permission bits
@@ -304,11 +304,15 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       `--no-extra-files` each one is a failure instead. A named pipe, socket or
       device, a symlink to a directory and a broken symlink are never reported,
       since gen cannot list them
+    - never reports `index.mf` at the top of the base directory as a file the
+      manifest does not list, even when given another manifest or a URL, since
+      gen and fetch write the manifest there
 - `mfer freshen` / `mfer freshen .`
     - rewrites `index.mf` to list the files now under the directory holding it,
       or under the one `--base` names, hashing only the files that are new or
       changed
-    - lists regular files only, as gen does
+    - lists regular files only, and never `index.mf` at the top of that
+      directory, as gen does
     - leaves out hidden files unless given `--include-dotfiles`, and symlinks
       unless given `--follow-symlinks`, which lists each symlink to a regular
       file under its own name with the contents of the file it points to

docs/FORMAT.md

@@ -93,16 +93,15 @@ Each file entry contains:
 ## File Mode
 
 `mode` holds a file's Unix permission bits, the nine `rwx` bits, so it is never
-above `0777` (octal); the setuid, setgid and sticky bits are never recorded.
+above `0777` (octal); the setuid, setgid and sticky bits are never recorded. A
-Writers record `0000` unless whoever creates the manifest asks for permissions.
+reader must reject a manifest with an entry whose mode is above `0777`. Writers
-`0000`, the proto3 default, means no mode was recorded: readers never check or
+record `0000` unless whoever creates the manifest asks for permissions. `0000`,
-apply it.
+the proto3 default, means no mode was recorded: readers never check or apply it.
 
 The reference implementation records modes when `gen` or `freshen` is given
 `--include-permissions`. `check` fails a file whose permission bits differ from
 a recorded mode other than `0000`. `fetch` sets a recorded mode other than
-`0000` on each file it writes, and refuses a manifest that records a mode above
+`0000` on each file it writes.
-`0777` before it requests any file.
 
 ## Path Rules
 

internal/cli/check.go

@@ -195,7 +195,10 @@ func countCheckFailures(
 
 // findExtraFiles reports files present on disk but absent from the
 // manifest, and anything the search cannot read: each is a failure, logged
-// at error level, under --no-extra-files, otherwise a warning.
+// at error level, under --no-extra-files, otherwise a warning. index.mf at
+// the top of the base directory is never reported as absent from the
+// manifest, even when check is given another manifest or a URL, since gen
+// and fetch write the manifest there.
 func findExtraFiles(
 	ctx context.Context, cmd *cli.Command, chk *mfer.Checker, failures *int64,
 ) error {
@@ -204,6 +207,10 @@ func findExtraFiles(
 
 	go func() {
 		for result := range extraResults {
+			if result.Status == mfer.StatusExtra && result.Path == defaultManifestName {
+				continue
+			}
+
 			if cmd.Bool("no-extra-files") {
 				*failures++
 

internal/cli/entry_test.go

@@ -27,19 +27,20 @@ import (
 )
 
 const (
-	testApp         = "mfer"
+	testApp                = "mfer"
-	testGitrev      = "abc123"
+	testGitrev             = "abc123"
-	testDir         = "/testdir"
+	testDir                = "/testdir"
-	testFile1       = "/testdir/file1.txt"
+	testFile1              = "/testdir/file1.txt"
-	testMF          = "/testdir/test.mf"
+	testMF                 = "/testdir/test.mf"
-	testOutput      = "/output.mf"
+	testOutput             = "/output.mf"
-	testManifest    = "/manifest.mf"
+	testManifest           = "/manifest.mf"
-	testFlagBase    = "--base"
+	testFlagBase           = "--base"
-	testFlagNoExtra = "--no-extra-files"
+	testFlagNoExtra        = "--no-extra-files"
-	testFlagVersion = "--version"
+	testFlagFollowSymlinks = "--follow-symlinks"
-	testFlagVerbose = "--verbose"
+	testFlagVersion        = "--version"
-	testFlagHelp    = "--help"
+	testFlagVerbose        = "--verbose"
-	testCmdHelp     = "help"
+	testFlagHelp           = "--help"
+	testCmdHelp            = "help"
 )
 
 var errSimulatedWrite = errors.New("simulated write failure")
@@ -1002,7 +1003,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
 		exitCode int
 		extras   []string // what check reports as not in the manifest
 	}{
-		"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
+		"gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil},
 		"plain gen":             {nil, 1, []string{"filelink"}},
 	} {
 		t.Run(name, func(t *testing.T) {
@@ -1190,6 +1191,99 @@ func TestCheckURLManifestWithoutTempDirectory(t *testing.T) {
 	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 }
 
+// TestCheckNeverReportsIndexMFAtTopOfBase checks a tree holding index.mf at
+// its top, where gen and fetch write the manifest, against a manifest that
+// does not list it, given by URL and by a path outside the tree: check never
+// reports it, and passes with and without --no-extra-files. index.mf in a
+// subdirectory, and the temp file fetch saves index.mf through, are still
+// reported.
+func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
+	t.Parallel()
+
+	files := map[string][]byte{testFileTxt: []byte("hello")}
+	manifest := manifestOf(t, files)
+
+	server := httptest.NewServer(fetchTestHandler(manifest, files))
+	t.Cleanup(server.Close)
+
+	manifestOutsideTree := filepath.Join(t.TempDir(), "other.mf")
+	require.NoError(t, os.WriteFile(manifestOutsideTree, manifest, 0o600))
+
+	for name, manifestArg := range map[string]string{
+		"manifest by URL":  server.URL + "/" + defaultManifestName,
+		"manifest by path": manifestOutsideTree,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+			require.NoError(t, os.WriteFile(
+				filepath.Join(base, testFileTxt), files[testFileTxt], 0o600))
+			require.NoError(t, os.WriteFile(
+				filepath.Join(base, defaultManifestName), manifest, 0o600))
+
+			check := func(flags ...string) (int, string) {
+				opts := testOpts(slices.Concat(
+					[]string{testApp, cmdCheck}, flags,
+					[]string{testFlagBase, base, manifestArg},
+				), afero.NewOsFs())
+
+				return runCLI(opts), testStderr(t, opts)
+			}
+
+			exitCode, stderr := check()
+			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
+			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
+
+			exitCode, stderr = check(testFlagNoExtra)
+			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
+			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
+
+			reported := []string{
+				testSubdirManifest,
+				tempPathFor(defaultManifestName),
+			}
+
+			require.NoError(t, os.Mkdir(filepath.Join(base, testSubdir), 0o750))
+
+			for _, path := range reported {
+				require.NoError(t,
+					os.WriteFile(filepath.Join(base, path), manifest, 0o600))
+			}
+
+			exitCode, stderr = check(testFlagNoExtra)
+			assert.Equal(t, 1, exitCode, "stderr: %s", stderr)
+			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
+
+			for _, path := range reported {
+				assert.Contains(t, stderr, "EXTRA: "+path)
+			}
+		})
+	}
+}
+
+// TestCheckChecksListedIndexMF checks a tree against a manifest that lists
+// index.mf at the top of the tree, where that file has changed since: check
+// reports it like any other changed file.
+func TestCheckChecksListedIndexMF(t *testing.T) {
+	t.Parallel()
+
+	manifestPath := filepath.Join(t.TempDir(), "other.mf")
+	require.NoError(t, os.WriteFile(manifestPath, manifestOf(t, map[string][]byte{
+		defaultManifestName: []byte("listed"),
+	}), 0o600))
+
+	base := t.TempDir()
+	require.NoError(t, os.WriteFile(
+		filepath.Join(base, defaultManifestName), []byte("edited"), 0o600))
+
+	opts := testOpts([]string{
+		testApp, cmdCheck, testFlagBase, base, manifestPath,
+	}, afero.NewOsFs())
+	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
+	assert.Contains(t, testStderr(t, opts), "HASH_MISMATCH: "+defaultManifestName)
+}
+
 // unlistableDirFs is a filesystem on which one directory cannot be listed.
 type unlistableDirFs struct {
 	afero.Fs
@@ -1238,6 +1332,60 @@ func TestCheckWithUnlistableDirectory(t *testing.T) {
 	assert.Contains(t, testStderr(t, opts), "unlisted.txt")
 }
 
+// TestCheckReportsUnreadableIndexMFAtTopOfBase has a directory named
+// index.mf at the top of the base that cannot be listed. check leaves out
+// only the report that index.mf is not in the manifest, so it reports the
+// error and fails under --no-extra-files.
+func TestCheckReportsUnreadableIndexMFAtTopOfBase(t *testing.T) {
+	t.Parallel()
+
+	unreadable := filepath.Join(testDir, defaultManifestName)
+
+	mem := afero.NewMemMapFs()
+	require.NoError(t, mem.MkdirAll(unreadable, 0o755))
+	writeTestFile(t, mem, testFile1, "hello")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
+	}, mem)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	fs := unlistableDirFs{Fs: mem, dir: unreadable}
+
+	opts = testOpts([]string{
+		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
+	}, fs)
+	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
+	assert.Contains(t, testStderr(t, opts), "ERROR: "+defaultManifestName)
+}
+
+// TestCheckReportsUpperCaseIndexMFAtTopOfBase checks a tree holding
+// INDEX.MF at its top against a manifest that does not list it. Only the
+// name index.mf, in exactly that letter case, is left out, so check reports
+// INDEX.MF. The tree is in memory, where the two names are two files
+// whatever the letter case rules of the host's filesystem.
+func TestCheckReportsUpperCaseIndexMFAtTopOfBase(t *testing.T) {
+	t.Parallel()
+
+	upperCase := strings.ToUpper(defaultManifestName)
+
+	fs := afero.NewMemMapFs()
+	writeTestFile(t, fs, testFile1, "hello")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
+	}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	writeTestFile(t, fs, filepath.Join(testDir, upperCase), "not listed")
+
+	opts = testOpts([]string{
+		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
+	}, fs)
+	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
+	assert.Contains(t, testStderr(t, opts), "EXTRA: "+upperCase)
+}
+
 func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
 	t.Parallel()
 
@@ -1420,6 +1568,141 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 	assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
 }
 
+// TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside
+// the paths it is given: index.mf at the top of each directory given, and
+// a file given by that name, are not listed, since fetch saves the manifest
+// as index.mf at the top of the tree, while sub/index.mf is listed.
+func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
+	t.Parallel()
+
+	// Paths are relative to a temp dir holding tree, which holds file.txt,
+	// index.mf and sub/index.mf, and second, which holds second.txt and
+	// index.mf.
+	for name, tc := range map[string]struct {
+		paths  []string
+		listed []string
+	}{
+		"directory": {
+			[]string{"tree"},
+			[]string{testFileTxt, testSubdirManifest},
+		},
+		"two directories": {
+			[]string{"tree", "second"},
+			[]string{testFileTxt, testSubdirManifest, "second.txt"},
+		},
+		"files": {
+			[]string{"tree/" + testFileTxt, "tree/" + defaultManifestName},
+			[]string{testFileTxt},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			root := t.TempDir()
+			tree := filepath.Join(root, "tree")
+			second := filepath.Join(root, "second")
+
+			fs := afero.NewOsFs()
+			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+			require.NoError(t, fs.MkdirAll(second, 0o750))
+			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+			writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
+			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+			writeTestFile(t, fs, filepath.Join(second, "second.txt"), "second")
+			writeTestFile(t, fs, filepath.Join(second, defaultManifestName), "an ordinary file")
+
+			paths := make([]string, 0, len(tc.paths))
+			for _, path := range tc.paths {
+				paths = append(paths, filepath.Join(root, path))
+			}
+
+			output := filepath.Join(root, "other.mf")
+			opts := testOpts(slices.Concat(
+				[]string{testApp, cmdGenerate, "-q", "-o", output}, paths,
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
+		})
+	}
+}
+
+// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of
+// the tree a symlink to another file in it, and runs gen with --output
+// outside the tree: the file it points to is listed under its own name,
+// and the symlink, which gen lists only with --follow-symlinks, is left out
+// like any index.mf at the top of the tree.
+func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) {
+	t.Parallel()
+
+	// Paths are relative to tree, which holds file.txt, sub/index.mf and
+	// index.mf, a symlink to target.
+	for name, tc := range map[string]struct {
+		target string
+		flags  []string
+		path   string // the path gen is given
+		listed []string
+	}{
+		"directory given": {
+			testSubdirManifest, nil, ".",
+			[]string{testFileTxt, testSubdirManifest},
+		},
+		"directory given, --follow-symlinks": {
+			testSubdirManifest,
+			[]string{testFlagFollowSymlinks},
+			".",
+			[]string{testFileTxt, testSubdirManifest},
+		},
+		"file given": {
+			testFileTxt, nil, testFileTxt,
+			[]string{testFileTxt},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			root := t.TempDir()
+			tree := filepath.Join(root, "tree")
+
+			fs := afero.NewOsFs()
+			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+			require.NoError(t,
+				os.Symlink(tc.target, filepath.Join(tree, defaultManifestName)))
+
+			output := filepath.Join(root, "other.mf")
+			opts := testOpts(slices.Concat(
+				[]string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags,
+				[]string{filepath.Join(tree, tc.path)},
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
+		})
+	}
+}
+
+// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside
+// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file
+// given, is listed, since a file given is left out only when its own name
+// is index.mf.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) {
+	setupManifestInSubdir(t)
+
+	fs := afero.NewOsFs()
+	output := filepath.Join(t.TempDir(), "other.mf")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".",
+	}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest)
+}
+
 // writeTestTree writes file.txt and sub/nested.txt under dir.
 func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
 	t.Helper()

internal/cli/fetch.go

@@ -95,9 +95,6 @@ var (
 	// writes another file.
 	errNameClash = errors.New(
 		"manifest lists a file where fetch writes another file")
-	// errModeOutOfRange indicates a manifest that lists a mode with more
-	// than the permission bits, such as setuid.
-	errModeOutOfRange = errors.New("manifest lists a mode outside 0777")
 )
 
 // DownloadProgress reports the progress of a single file download.
@@ -428,9 +425,9 @@ func (mfa *CLIApp) fetchManifestOperation(
 
 // fetchManifest downloads the manifest at manifestURL and parses it,
 // enforcing --require-signature if it is given and refusing a manifest
-// that lists a file where fetch writes another or a mode outside 0777. It
+// that lists a file where fetch writes another. It returns the manifest as
-// returns the manifest as downloaded, to be saved once the files are in
+// downloaded, to be saved once the files are in place, and the files it
-// place, and the files it lists.
+// lists.
 func (mfa *CLIApp) fetchManifest(
 	ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
 ) ([]byte, []*mfer.MFFilePath, error) {
@@ -487,16 +484,6 @@ func (mfa *CLIApp) fetchManifest(
 		return nil, nil, err
 	}
 
-	// fetch sets each recorded mode on the file it writes, so a mode above
-	// 0777, which could carry setuid, setgid or sticky bits, is refused
-	// before any file is requested.
-	for _, f := range files {
-		if f.GetMode() > uint32(os.ModePerm) {
-			return nil, nil, fmt.Errorf("%w: %s (%#o)",
-				errModeOutOfRange, f.GetPath(), f.GetMode())
-		}
-	}
-
 	log.Infof("manifest contains %d files", len(files))
 
 	return manifestData, files, nil

internal/cli/fetch_test.go

@@ -19,15 +19,12 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
-	"uuid"
 
-	"github.com/klauspost/compress/zstd"
 	"github.com/multiformats/go-multihash"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	urfcli "github.com/urfave/cli/v3"
-	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/mfer"
 )
 
@@ -1535,27 +1532,9 @@ func TestFetchSetsRecordedMode(t *testing.T) {
 		[]string{"/" + defaultManifestName, "/" + testFileTxt}, requested)
 }
 
-// TestFetchRefusesModeOutsidePermissionBits fetches manifests that record
-// a mode with the setuid bit, once as Unix writes it (04755) and once as
-// Go keeps it in os.FileMode. fetch must refuse both before it creates the
-// destination or requests any file.
-func TestFetchRefusesModeOutsidePermissionBits(t *testing.T) {
-	t.Parallel()
-
-	files := map[string][]byte{testFileTxt: []byte("a file")}
-
-	assertFetchRefused(t, manifestWithMode(t, testFileTxt, files[testFileTxt], 0o4755),
-		files, "manifest lists a mode outside 0777: file.txt (04755)")
-
-	assertFetchRefused(t,
-		manifestWithMode(t, testFileTxt, files[testFileTxt],
-			uint32(os.ModeSetuid|0o755)),
-		files, "manifest lists a mode outside 0777: file.txt (040000755)")
-}
-
 // TestDownloadFileSetsOnlyPermissionBits downloads a file whose entry
 // records mode 0755 together with setuid, setgid or sticky, as Go keeps
-// them in os.FileMode. fetchManifest would refuse such an entry; called
+// them in os.FileMode. Loading a manifest refuses such an entry; called
 // directly, downloadFile must still set only the permission bits.
 func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
 	t.Parallel()
@@ -1592,51 +1571,6 @@ func TestDownloadFileSetsOnlyPermissionBits(t *testing.T) {
 	}
 }
 
-// manifestWithMode returns a manifest listing one file, path, with content
-// and the given recorded mode. It is assembled by hand, since the builder
-// never records a mode outside 0777.
-func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []byte {
-	t.Helper()
-
-	digest := sha256.Sum256(content)
-	hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
-	require.NoError(t, err)
-
-	id := uuid.NewV4()
-
-	inner, err := proto.Marshal(&mfer.MFFile{
-		Version: mfer.MFFile_VERSION_ONE,
-		Files: []*mfer.MFFilePath{{
-			Path:   path,
-			Size:   int64(len(content)),
-			Hashes: []*mfer.MFFileChecksum{{MultiHash: hash}},
-			Mode:   mode,
-		}},
-		Uuid: id[:],
-	})
-	require.NoError(t, err)
-
-	encoder, err := zstd.NewWriter(nil)
-	require.NoError(t, err)
-
-	compressed := encoder.EncodeAll(inner, nil)
-	require.NoError(t, encoder.Close())
-
-	sum := sha256.Sum256(compressed)
-
-	outer, err := proto.Marshal(&mfer.MFFileOuter{
-		Version:         mfer.MFFileOuter_VERSION_ONE,
-		CompressionType: mfer.MFFileOuter_COMPRESSION_ZSTD,
-		Size:            int64(len(inner)),
-		Sha256:          sum[:],
-		Uuid:            id[:],
-		InnerMessage:    compressed,
-	})
-	require.NoError(t, err)
-
-	return append([]byte(mfer.MAGIC), outer...)
-}
-
 // builtManifest returns a manifest of files, built directly rather than
 // scanned, since a scan lists no hidden files.
 func builtManifest(t *testing.T, files map[string][]byte) []byte {

internal/cli/freshen.go

@@ -192,6 +192,12 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
 		return nil
 	}
 
+	// Leave out a file listed as index.mf, at the top of the base
+	// directory, since fetch saves the manifest there, as gen does.
+	if relPath == defaultManifestName {
+		return nil
+	}
+
 	// Handle symlinks
 	if info.Mode()&fs.ModeSymlink != 0 {
 		realInfo, keep := s.resolveSymlink(path)

internal/cli/freshen_test.go

@@ -413,6 +413,77 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
+// TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside
+// the directory --base names, after index.mf and sub/index.mf are added
+// there: index.mf at the top is not listed, since fetch saves the manifest
+// there, while sub/index.mf is listed.
+func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	tree := filepath.Join(root, "tree")
+	manifestPath := filepath.Join(root, "other.mf")
+
+	fs := afero.NewOsFs()
+	require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+	writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+
+	opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, tree}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
+	writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+
+	runFreshen(t, fs, tree, manifestPath)
+
+	assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
+		manifestPaths(t, fs, manifestPath))
+}
+
+// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept
+// outside the directory --base names, after sub/index.mf is added there
+// and index.mf at the top is made a symlink to it: sub/index.mf is listed,
+// and the symlink, which freshen lists only with --follow-symlinks, is left
+// out like any index.mf at the top.
+func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) {
+	t.Parallel()
+
+	for name, flags := range map[string][]string{
+		"plain freshen":             nil,
+		"freshen --follow-symlinks": {testFlagFollowSymlinks},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			root := t.TempDir()
+			tree := filepath.Join(root, "tree")
+			manifestPath := filepath.Join(root, "other.mf")
+
+			fs := afero.NewOsFs()
+			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+
+			opts := testOpts([]string{
+				testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
+			}, fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+			require.NoError(t, os.Symlink(testSubdirManifest,
+				filepath.Join(tree, defaultManifestName)))
+
+			opts = testOpts(slices.Concat(
+				[]string{testApp, cmdFreshen, "-q"}, flags,
+				[]string{testFlagBase, tree, manifestPath},
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
+				manifestPaths(t, fs, manifestPath))
+		})
+	}
+}
+
 // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
 // runs freshen from the directory above sub, on the manifest in sub.
 // Without --base, the manifest then lists the files in sub, whether freshen

internal/cli/gen.go

@@ -137,6 +137,9 @@ func (mfa *CLIApp) buildScannerOptions(
 		// Neither a manifest being replaced nor a temp file left by an
 		// interrupted run belongs in the new manifest.
 		ExcludePaths: []string{output, manifestTempPath(output)},
+		// Nor does a file listed as index.mf, at the top of the tree, since
+		// fetch saves the manifest there and refuses a manifest that lists it.
+		ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName},
 	}
 
 	// Set seed for deterministic UUID if provided

internal/cli/pipe_test.go

@@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) {
 	require.NoError(t, os.Symlink(target, filepath.Join(root, testLink)))
 
 	opts := testOpts([]string{
-		testApp, cmdFreshen, "-v", "--follow-symlinks",
+		testApp, cmdFreshen, "-v", testFlagFollowSymlinks,
 		testFlagBase, root, manifestPath,
 	}, fs)
 	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))

mfer/deserialize.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"strings"
 
 	"github.com/klauspost/compress/zstd"
@@ -29,6 +30,7 @@ var (
 	errInvalidManifestPath  = errors.New("invalid file entry")
 	errEntryMissingHash     = errors.New("file entry has no SHA-256 hash")
 	errEntryExtraSHA256Hash = errors.New("file entry has more than one SHA-256 hash")
+	errModeOutOfRange       = errors.New("file entry has a mode above 0777")
 	errDecodedTooLarge      = errors.New("too much memory needed")
 	errSignerNotSigningKey  = errors.New(
 		"signer is not the fingerprint of the key that made the signature")
@@ -320,7 +322,9 @@ func (m *manifest) deserializeInner() error {
 	// dropping entries, which would let a hostile manifest hide files from a
 	// check. A path listed twice is refused too: check would check the one
 	// file against both entries. So is an entry that does not list exactly
-	// one SHA-256 hash.
+	// one SHA-256 hash, and one that records a mode above 0777, such as one
+	// with setuid: a mode holds the permission bits alone, which are all
+	// check compares and fetch sets.
 	seen := make(map[string]bool, len(m.pbInner.GetFiles()))
 
 	for _, f := range m.pbInner.GetFiles() {
@@ -334,6 +338,10 @@ func (m *manifest) deserializeInner() error {
 			return err
 		}
 
+		if f.GetMode() > uint32(os.ModePerm) {
+			return fmt.Errorf("%w: %q (%#o)", errModeOutOfRange, f.GetPath(), f.GetMode())
+		}
+
 		if seen[f.GetPath()] {
 			return fmt.Errorf("%w %q", errDuplicatePath, f.GetPath())
 		}

mfer/deserialize_path_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"crypto/sha512"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -234,6 +235,66 @@ func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
 	}
 }
 
+// A file entry that records a mode with more than the permission bits is
+// refused as it is loaded, naming the path and the mode: once with setuid as
+// Unix writes it (04755) and once as Go keeps it in os.FileMode. An entry
+// recording 0777 loads.
+func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) {
+	t.Parallel()
+
+	hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string
+		mode uint32
+		want string // "" for an entry that loads
+	}{
+		{
+			"setuid as Unix writes it",
+			0o4755,
+			`file entry has a mode above 0777: "x" (04755)`,
+		},
+		{
+			"setuid as Go keeps it",
+			uint32(os.ModeSetuid | 0o755),
+			`file entry has a mode above 0777: "x" (040000755)`,
+		},
+		{"permission bits only", 0o777, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			id := uuid.NewV4()
+			inner, err := proto.Marshal(&MFFile{
+				Version: MFFile_VERSION_ONE,
+				Files: []*MFFilePath{{
+					Path:   "x",
+					Size:   1,
+					Hashes: []*MFFileChecksum{{MultiHash: hash}},
+					Mode:   tt.mode,
+				}},
+				Uuid: id[:],
+			})
+			require.NoError(t, err)
+
+			m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+			if tt.want == "" {
+				require.NoError(t, err)
+				require.Len(t, m.Files(), 1)
+				assert.Equal(t, tt.mode, m.Files()[0].GetMode())
+
+				return
+			}
+
+			require.ErrorIs(t, err, errModeOutOfRange)
+			require.EqualError(t, err, tt.want)
+		})
+	}
+}
+
 // Entries of a path, a SHA-256 hash, an empty MIME type and empty
 // modification and change times are counted at 432 bytes each (176 + 112 +
 // 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character

mfer/scanner.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -64,6 +65,9 @@ type ScannerOptions struct {
 	Seed string
 	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
 	ExcludePaths []string
+	// ExcludeRelPaths lists paths to leave out of the listing, matched
+	// against the path a file would be listed by, with no file looked up.
+	ExcludeRelPaths []RelFilePath
 }
 
 // FileEntry represents a file that has been enumerated.
@@ -439,9 +443,10 @@ func (s *Scanner) enumerateFileWithInfo(
 	}
 
 	// Clean the path - remove leading slash if present
-	cleanPath := filePath
+	cleanPath := strings.TrimPrefix(filePath, "/")
-	if len(cleanPath) > 0 && cleanPath[0] == '/' {
+
-		cleanPath = cleanPath[1:]
+	if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) {
+		return nil
 	}
 
 	// Compute absolute path for file reading