Never list index.mf at the top of the tree in gen and freshen (closes #233)

fetch saves the manifest as index.mf at the top of the tree and refuses a
manifest that lists a file there. gen given --output, and freshen with its
manifest outside the base directory, listed such a file, so they wrote
manifests fetch would not take.

gen now leaves out index.mf in each directory it is given, or beside each
file it is given, the same way it leaves out the file it writes. freshen
leaves out index.mf at the top of the base directory. index.mf in a
subdirectory is still listed. The README says so for both commands.

Model: opus-5-5

README.md

@@ -304,9 +304,6 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       `--no-extra-files` each one is a failure instead. A named pipe, socket or
       device, a symlink to a directory and a broken symlink are never reported,
       since gen cannot list them
-    - never reports `index.mf` at the top of the base directory as a file the
-      manifest does not list, even when given another manifest or a URL, since
-      gen and fetch write the manifest there
 - `mfer freshen` / `mfer freshen .`
     - rewrites `index.mf` to list the files now under the directory holding it,
       or under the one `--base` names, hashing only the files that are new or

internal/cli/check.go

@@ -195,10 +195,7 @@ func countCheckFailures(
 
 // findExtraFiles reports files present on disk but absent from the
 // manifest, and anything the search cannot read: each is a failure, logged
-// at error level, under --no-extra-files, otherwise a warning. index.mf at
+// at error level, under --no-extra-files, otherwise a warning.
-// the top of the base directory is never reported as absent from the
-// manifest, even when check is given another manifest or a URL, since gen
-// and fetch write the manifest there.
 func findExtraFiles(
 	ctx context.Context, cmd *cli.Command, chk *mfer.Checker, failures *int64,
 ) error {
@@ -207,10 +204,6 @@ func findExtraFiles(
 
 	go func() {
 		for result := range extraResults {
-			if result.Status == mfer.StatusExtra && result.Path == defaultManifestName {
-				continue
-			}
-
 			if cmd.Bool("no-extra-files") {
 				*failures++
 

internal/cli/entry_test.go

@@ -36,7 +36,6 @@ const (
 	testManifest    = "/manifest.mf"
 	testFlagBase    = "--base"
 	testFlagNoExtra = "--no-extra-files"
-	testFlagFollowSymlinks = "--follow-symlinks"
 	testFlagVersion = "--version"
 	testFlagVerbose = "--verbose"
 	testFlagHelp    = "--help"
@@ -1003,7 +1002,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
 		exitCode int
 		extras   []string // what check reports as not in the manifest
 	}{
-		"gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil},
+		"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
 		"plain gen":             {nil, 1, []string{"filelink"}},
 	} {
 		t.Run(name, func(t *testing.T) {
@@ -1191,99 +1190,6 @@ func TestCheckURLManifestWithoutTempDirectory(t *testing.T) {
 	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 }
 
-// TestCheckNeverReportsIndexMFAtTopOfBase checks a tree holding index.mf at
-// its top, where gen and fetch write the manifest, against a manifest that
-// does not list it, given by URL and by a path outside the tree: check never
-// reports it, and passes with and without --no-extra-files. index.mf in a
-// subdirectory, and the temp file fetch saves index.mf through, are still
-// reported.
-func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	files := map[string][]byte{testFileTxt: []byte("hello")}
-	manifest := manifestOf(t, files)
-
-	server := httptest.NewServer(fetchTestHandler(manifest, files))
-	t.Cleanup(server.Close)
-
-	manifestOutsideTree := filepath.Join(t.TempDir(), "other.mf")
-	require.NoError(t, os.WriteFile(manifestOutsideTree, manifest, 0o600))
-
-	for name, manifestArg := range map[string]string{
-		"manifest by URL":  server.URL + "/" + defaultManifestName,
-		"manifest by path": manifestOutsideTree,
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-			require.NoError(t, os.WriteFile(
-				filepath.Join(base, testFileTxt), files[testFileTxt], 0o600))
-			require.NoError(t, os.WriteFile(
-				filepath.Join(base, defaultManifestName), manifest, 0o600))
-
-			check := func(flags ...string) (int, string) {
-				opts := testOpts(slices.Concat(
-					[]string{testApp, cmdCheck}, flags,
-					[]string{testFlagBase, base, manifestArg},
-				), afero.NewOsFs())
-
-				return runCLI(opts), testStderr(t, opts)
-			}
-
-			exitCode, stderr := check()
-			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			exitCode, stderr = check(testFlagNoExtra)
-			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			reported := []string{
-				testSubdirManifest,
-				tempPathFor(defaultManifestName),
-			}
-
-			require.NoError(t, os.Mkdir(filepath.Join(base, testSubdir), 0o750))
-
-			for _, path := range reported {
-				require.NoError(t,
-					os.WriteFile(filepath.Join(base, path), manifest, 0o600))
-			}
-
-			exitCode, stderr = check(testFlagNoExtra)
-			assert.Equal(t, 1, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			for _, path := range reported {
-				assert.Contains(t, stderr, "EXTRA: "+path)
-			}
-		})
-	}
-}
-
-// TestCheckChecksListedIndexMF checks a tree against a manifest that lists
-// index.mf at the top of the tree, where that file has changed since: check
-// reports it like any other changed file.
-func TestCheckChecksListedIndexMF(t *testing.T) {
-	t.Parallel()
-
-	manifestPath := filepath.Join(t.TempDir(), "other.mf")
-	require.NoError(t, os.WriteFile(manifestPath, manifestOf(t, map[string][]byte{
-		defaultManifestName: []byte("listed"),
-	}), 0o600))
-
-	base := t.TempDir()
-	require.NoError(t, os.WriteFile(
-		filepath.Join(base, defaultManifestName), []byte("edited"), 0o600))
-
-	opts := testOpts([]string{
-		testApp, cmdCheck, testFlagBase, base, manifestPath,
-	}, afero.NewOsFs())
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "HASH_MISMATCH: "+defaultManifestName)
-}
-
 // unlistableDirFs is a filesystem on which one directory cannot be listed.
 type unlistableDirFs struct {
 	afero.Fs
@@ -1332,60 +1238,6 @@ func TestCheckWithUnlistableDirectory(t *testing.T) {
 	assert.Contains(t, testStderr(t, opts), "unlisted.txt")
 }
 
-// TestCheckReportsUnreadableIndexMFAtTopOfBase has a directory named
-// index.mf at the top of the base that cannot be listed. check leaves out
-// only the report that index.mf is not in the manifest, so it reports the
-// error and fails under --no-extra-files.
-func TestCheckReportsUnreadableIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	unreadable := filepath.Join(testDir, defaultManifestName)
-
-	mem := afero.NewMemMapFs()
-	require.NoError(t, mem.MkdirAll(unreadable, 0o755))
-	writeTestFile(t, mem, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, mem)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	fs := unlistableDirFs{Fs: mem, dir: unreadable}
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "ERROR: "+defaultManifestName)
-}
-
-// TestCheckReportsUpperCaseIndexMFAtTopOfBase checks a tree holding
-// INDEX.MF at its top against a manifest that does not list it. Only the
-// name index.mf, in exactly that letter case, is left out, so check reports
-// INDEX.MF. The tree is in memory, where the two names are two files
-// whatever the letter case rules of the host's filesystem.
-func TestCheckReportsUpperCaseIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	upperCase := strings.ToUpper(defaultManifestName)
-
-	fs := afero.NewMemMapFs()
-	writeTestFile(t, fs, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	writeTestFile(t, fs, filepath.Join(testDir, upperCase), "not listed")
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "EXTRA: "+upperCase)
-}
-
 func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
 	t.Parallel()
 
@@ -1571,7 +1423,8 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 // TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside
 // the paths it is given: index.mf at the top of each directory given, and
 // a file given by that name, are not listed, since fetch saves the manifest
-// as index.mf at the top of the tree, while sub/index.mf is listed.
+// as index.mf at the top of the tree, while sub/index.mf is listed. The
+// files are recognized by file identity, which needs the real filesystem.
 func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
 	t.Parallel()
 
@@ -1627,82 +1480,6 @@ func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
 	}
 }
 
-// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of
-// the tree a symlink to another file in it, and runs gen with --output
-// outside the tree: the file it points to is listed under its own name,
-// and the symlink, which gen lists only with --follow-symlinks, is left out
-// like any index.mf at the top of the tree.
-func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	// Paths are relative to tree, which holds file.txt, sub/index.mf and
-	// index.mf, a symlink to target.
-	for name, tc := range map[string]struct {
-		target string
-		flags  []string
-		path   string // the path gen is given
-		listed []string
-	}{
-		"directory given": {
-			testSubdirManifest, nil, ".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"directory given, --follow-symlinks": {
-			testSubdirManifest,
-			[]string{testFlagFollowSymlinks},
-			".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"file given": {
-			testFileTxt, nil, testFileTxt,
-			[]string{testFileTxt},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t,
-				os.Symlink(tc.target, filepath.Join(tree, defaultManifestName)))
-
-			output := filepath.Join(root, "other.mf")
-			opts := testOpts(slices.Concat(
-				[]string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags,
-				[]string{filepath.Join(tree, tc.path)},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
-		})
-	}
-}
-
-// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside
-// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file
-// given, is listed, since a file given is left out only when its own name
-// is index.mf.
-//
-//nolint:paralleltest // changes the process-global working directory
-func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) {
-	setupManifestInSubdir(t)
-
-	fs := afero.NewOsFs()
-	output := filepath.Join(t.TempDir(), "other.mf")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".",
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest)
-}
-
 // writeTestTree writes file.txt and sub/nested.txt under dir.
 func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
 	t.Helper()

internal/cli/freshen.go

@@ -192,12 +192,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
 		return nil
 	}
 
-	// Leave out a file listed as index.mf, at the top of the base
-	// directory, since fetch saves the manifest there, as gen does.
-	if relPath == defaultManifestName {
-		return nil
-	}
-
 	// Handle symlinks
 	if info.Mode()&fs.ModeSymlink != 0 {
 		realInfo, keep := s.resolveSymlink(path)
@@ -417,12 +411,17 @@ func (mfa *CLIApp) freshenScan(
 	startScan := time.Now()
 	showProgress := cmd.Bool("progress")
 
-	// Leave out the manifest and a temp file left by an interrupted run,
+	// Leave out the manifest, a temp file left by an interrupted run, and
-	// as gen does. A path that cannot be stat'd, normally because no file
+	// index.mf at the top of the base directory, where fetch saves the
-	// is there, needs no leaving out.
+	// manifest, as gen does. A path that cannot be stat'd, normally because
+	// no file is there, needs no leaving out.
 	var excluded []fs.FileInfo
 
-	for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
+	for _, p := range []string{
+		manifestPath,
+		manifestTempPath(manifestPath),
+		filepath.Join(absBase, defaultManifestName),
+	} {
 		info, err := mfa.Fs.Stat(p)
 		if err == nil {
 			excluded = append(excluded, info)

internal/cli/freshen_test.go

@@ -416,7 +416,8 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 // TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside
 // the directory --base names, after index.mf and sub/index.mf are added
 // there: index.mf at the top is not listed, since fetch saves the manifest
-// there, while sub/index.mf is listed.
+// there, while sub/index.mf is listed. The file is recognized by file
+// identity, which needs the real filesystem.
 func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
 	t.Parallel()
 
@@ -440,50 +441,6 @@ func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
-// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept
-// outside the directory --base names, after sub/index.mf is added there
-// and index.mf at the top is made a symlink to it: sub/index.mf is listed,
-// and the symlink, which freshen lists only with --follow-symlinks, is left
-// out like any index.mf at the top.
-func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	for name, flags := range map[string][]string{
-		"plain freshen":             nil,
-		"freshen --follow-symlinks": {testFlagFollowSymlinks},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-			manifestPath := filepath.Join(root, "other.mf")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-
-			opts := testOpts([]string{
-				testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
-			}, fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t, os.Symlink(testSubdirManifest,
-				filepath.Join(tree, defaultManifestName)))
-
-			opts = testOpts(slices.Concat(
-				[]string{testApp, cmdFreshen, "-q"}, flags,
-				[]string{testFlagBase, tree, manifestPath},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
-				manifestPaths(t, fs, manifestPath))
-		})
-	}
-}
-
 // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
 // runs freshen from the directory above sub, on the manifest in sub.
 // Without --base, the manifest then lists the files in sub, whether freshen

internal/cli/gen.go

@@ -111,15 +111,19 @@ func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
 		return defaultManifestName, nil
 	}
 
-	arg := cmd.Args().First()
+	return mfa.topIndexMFPath(cmd.Args().First()), nil
+}
 
+// topIndexMFPath returns index.mf in the directory path names, or beside
+// the file it names, which is the top of the tree gen lists for that path.
+func (mfa *CLIApp) topIndexMFPath(path string) string {
 	// A path that does not exist is refused when it is enumerated.
-	info, err := mfa.Fs.Stat(arg)
+	info, err := mfa.Fs.Stat(path)
 	if err == nil && !info.IsDir() {
-		return filepath.Join(filepath.Dir(arg), defaultManifestName), nil
+		return filepath.Join(filepath.Dir(path), defaultManifestName)
 	}
 
-	return filepath.Join(arg, defaultManifestName), nil
+	return filepath.Join(path, defaultManifestName)
 }
 
 // buildScannerOptions constructs scanner options from the CLI flags and
@@ -137,9 +141,18 @@ func (mfa *CLIApp) buildScannerOptions(
 		// Neither a manifest being replaced nor a temp file left by an
 		// interrupted run belongs in the new manifest.
 		ExcludePaths: []string{output, manifestTempPath(output)},
-		// Nor does a file listed as index.mf, at the top of the tree, since
+	}
-		// fetch saves the manifest there and refuses a manifest that lists it.
+
-		ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName},
+	// Nor does index.mf at the top of the tree, since fetch saves the
+	// manifest there and refuses a manifest that lists it. With no path
+	// given, gen lists the current directory.
+	paths := cmd.Args().Slice()
+	if len(paths) == 0 {
+		paths = []string{"."}
+	}
+
+	for _, path := range paths {
+		opts.ExcludePaths = append(opts.ExcludePaths, mfa.topIndexMFPath(path))
 	}
 
 	// Set seed for deterministic UUID if provided

internal/cli/pipe_test.go

@@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) {
 	require.NoError(t, os.Symlink(target, filepath.Join(root, testLink)))
 
 	opts := testOpts([]string{
-		testApp, cmdFreshen, "-v", testFlagFollowSymlinks,
+		testApp, cmdFreshen, "-v", "--follow-symlinks",
 		testFlagBase, root, manifestPath,
 	}, fs)
 	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))

mfer/scanner.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -65,9 +64,6 @@ type ScannerOptions struct {
 	Seed string
 	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
 	ExcludePaths []string
-	// ExcludeRelPaths lists paths to leave out of the listing, matched
-	// against the path a file would be listed by, with no file looked up.
-	ExcludeRelPaths []RelFilePath
 }
 
 // FileEntry represents a file that has been enumerated.
@@ -443,10 +439,9 @@ func (s *Scanner) enumerateFileWithInfo(
 	}
 
 	// Clean the path - remove leading slash if present
-	cleanPath := strings.TrimPrefix(filePath, "/")
+	cleanPath := filePath
-
+	if len(cleanPath) > 0 && cleanPath[0] == '/' {
-	if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) {
+		cleanPath = cleanPath[1:]
-		return nil
 	}
 
 	// Compute absolute path for file reading