Never list index.mf at the top of the tree in gen and freshen (closes #233)

fetch saves the manifest as index.mf at the top of the tree and refuses a
manifest that lists a file there. gen given --output, and freshen with its
manifest outside the base directory, listed such a file, so they wrote
manifests fetch would not take.

Both now leave out each file the manifest would list as index.mf: one at
the top of the tree, or a file given to gen by that name. The rule goes by
that path alone and looks up no file, so a symlink named index.mf never
hides the file it points to. index.mf in a subdirectory is still listed.
The scanner in package mfer gets an option to leave out files by the path
they would be listed by.

Model: opus-5-5

internal/cli/entry_test.go

@@ -36,6 +36,7 @@ const (
 	testManifest           = "/manifest.mf"
 	testFlagBase           = "--base"
 	testFlagNoExtra        = "--no-extra-files"
+	testFlagFollowSymlinks = "--follow-symlinks"
 	testFlagVersion        = "--version"
 	testFlagVerbose        = "--verbose"
 	testFlagHelp           = "--help"
@@ -1002,7 +1003,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
 		exitCode int
 		extras   []string // what check reports as not in the manifest
 	}{
-		"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
+		"gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil},
 		"plain gen":             {nil, 1, []string{"filelink"}},
 	} {
 		t.Run(name, func(t *testing.T) {
@@ -1423,8 +1424,7 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 // TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside
 // the paths it is given: index.mf at the top of each directory given, and
 // a file given by that name, are not listed, since fetch saves the manifest
-// as index.mf at the top of the tree, while sub/index.mf is listed. The
+// as index.mf at the top of the tree, while sub/index.mf is listed.
-// files are recognized by file identity, which needs the real filesystem.
 func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
 	t.Parallel()
 
@@ -1480,6 +1480,82 @@ func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
 	}
 }
 
+// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of
+// the tree a symlink to another file in it, and runs gen with --output
+// outside the tree: the file it points to is listed under its own name,
+// and the symlink, which gen lists only with --follow-symlinks, is left out
+// like any index.mf at the top of the tree.
+func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) {
+	t.Parallel()
+
+	// Paths are relative to tree, which holds file.txt, sub/index.mf and
+	// index.mf, a symlink to target.
+	for name, tc := range map[string]struct {
+		target string
+		flags  []string
+		path   string // the path gen is given
+		listed []string
+	}{
+		"directory given": {
+			testSubdirManifest, nil, ".",
+			[]string{testFileTxt, testSubdirManifest},
+		},
+		"directory given, --follow-symlinks": {
+			testSubdirManifest,
+			[]string{testFlagFollowSymlinks},
+			".",
+			[]string{testFileTxt, testSubdirManifest},
+		},
+		"file given": {
+			testFileTxt, nil, testFileTxt,
+			[]string{testFileTxt},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			root := t.TempDir()
+			tree := filepath.Join(root, "tree")
+
+			fs := afero.NewOsFs()
+			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+			require.NoError(t,
+				os.Symlink(tc.target, filepath.Join(tree, defaultManifestName)))
+
+			output := filepath.Join(root, "other.mf")
+			opts := testOpts(slices.Concat(
+				[]string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags,
+				[]string{filepath.Join(tree, tc.path)},
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
+		})
+	}
+}
+
+// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside
+// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file
+// given, is listed, since a file given is left out only when its own name
+// is index.mf.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) {
+	setupManifestInSubdir(t)
+
+	fs := afero.NewOsFs()
+	output := filepath.Join(t.TempDir(), "other.mf")
+
+	opts := testOpts([]string{
+		testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".",
+	}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+	assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest)
+}
+
 // writeTestTree writes file.txt and sub/nested.txt under dir.
 func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
 	t.Helper()

internal/cli/freshen.go

@@ -192,6 +192,12 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
 		return nil
 	}
 
+	// Leave out a file listed as index.mf, at the top of the base
+	// directory, since fetch saves the manifest there, as gen does.
+	if relPath == defaultManifestName {
+		return nil
+	}
+
 	// Handle symlinks
 	if info.Mode()&fs.ModeSymlink != 0 {
 		realInfo, keep := s.resolveSymlink(path)
@@ -411,17 +417,12 @@ func (mfa *CLIApp) freshenScan(
 	startScan := time.Now()
 	showProgress := cmd.Bool("progress")
 
-	// Leave out the manifest, a temp file left by an interrupted run, and
+	// Leave out the manifest and a temp file left by an interrupted run,
-	// index.mf at the top of the base directory, where fetch saves the
+	// as gen does. A path that cannot be stat'd, normally because no file
-	// manifest, as gen does. A path that cannot be stat'd, normally because
+	// is there, needs no leaving out.
-	// no file is there, needs no leaving out.
 	var excluded []fs.FileInfo
 
-	for _, p := range []string{
+	for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
-		manifestPath,
-		manifestTempPath(manifestPath),
-		filepath.Join(absBase, defaultManifestName),
-	} {
 		info, err := mfa.Fs.Stat(p)
 		if err == nil {
 			excluded = append(excluded, info)

internal/cli/freshen_test.go

@@ -416,8 +416,7 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 // TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside
 // the directory --base names, after index.mf and sub/index.mf are added
 // there: index.mf at the top is not listed, since fetch saves the manifest
-// there, while sub/index.mf is listed. The file is recognized by file
+// there, while sub/index.mf is listed.
-// identity, which needs the real filesystem.
 func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
 	t.Parallel()
 
@@ -441,6 +440,50 @@ func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
+// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept
+// outside the directory --base names, after sub/index.mf is added there
+// and index.mf at the top is made a symlink to it: sub/index.mf is listed,
+// and the symlink, which freshen lists only with --follow-symlinks, is left
+// out like any index.mf at the top.
+func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) {
+	t.Parallel()
+
+	for name, flags := range map[string][]string{
+		"plain freshen":             nil,
+		"freshen --follow-symlinks": {testFlagFollowSymlinks},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			root := t.TempDir()
+			tree := filepath.Join(root, "tree")
+			manifestPath := filepath.Join(root, "other.mf")
+
+			fs := afero.NewOsFs()
+			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
+			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
+
+			opts := testOpts([]string{
+				testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
+			}, fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
+			require.NoError(t, os.Symlink(testSubdirManifest,
+				filepath.Join(tree, defaultManifestName)))
+
+			opts = testOpts(slices.Concat(
+				[]string{testApp, cmdFreshen, "-q"}, flags,
+				[]string{testFlagBase, tree, manifestPath},
+			), fs)
+			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+
+			assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
+				manifestPaths(t, fs, manifestPath))
+		})
+	}
+}
+
 // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
 // runs freshen from the directory above sub, on the manifest in sub.
 // Without --base, the manifest then lists the files in sub, whether freshen

internal/cli/gen.go

@@ -111,19 +111,15 @@ func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
 		return defaultManifestName, nil
 	}
 
-	return mfa.topIndexMFPath(cmd.Args().First()), nil
+	arg := cmd.Args().First()
-}
 
-// topIndexMFPath returns index.mf in the directory path names, or beside
-// the file it names, which is the top of the tree gen lists for that path.
-func (mfa *CLIApp) topIndexMFPath(path string) string {
 	// A path that does not exist is refused when it is enumerated.
-	info, err := mfa.Fs.Stat(path)
+	info, err := mfa.Fs.Stat(arg)
 	if err == nil && !info.IsDir() {
-		return filepath.Join(filepath.Dir(path), defaultManifestName)
+		return filepath.Join(filepath.Dir(arg), defaultManifestName), nil
 	}
 
-	return filepath.Join(path, defaultManifestName)
+	return filepath.Join(arg, defaultManifestName), nil
 }
 
 // buildScannerOptions constructs scanner options from the CLI flags and
@@ -141,18 +137,9 @@ func (mfa *CLIApp) buildScannerOptions(
 		// Neither a manifest being replaced nor a temp file left by an
 		// interrupted run belongs in the new manifest.
 		ExcludePaths: []string{output, manifestTempPath(output)},
-	}
+		// Nor does a file listed as index.mf, at the top of the tree, since
-
+		// fetch saves the manifest there and refuses a manifest that lists it.
-	// Nor does index.mf at the top of the tree, since fetch saves the
+		ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName},
-	// manifest there and refuses a manifest that lists it. With no path
-	// given, gen lists the current directory.
-	paths := cmd.Args().Slice()
-	if len(paths) == 0 {
-		paths = []string{"."}
-	}
-
-	for _, path := range paths {
-		opts.ExcludePaths = append(opts.ExcludePaths, mfa.topIndexMFPath(path))
 	}
 
 	// Set seed for deterministic UUID if provided

internal/cli/pipe_test.go

@@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) {
 	require.NoError(t, os.Symlink(target, filepath.Join(root, testLink)))
 
 	opts := testOpts([]string{
-		testApp, cmdFreshen, "-v", "--follow-symlinks",
+		testApp, cmdFreshen, "-v", testFlagFollowSymlinks,
 		testFlagBase, root, manifestPath,
 	}, fs)
 	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))

mfer/scanner.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -64,6 +65,9 @@ type ScannerOptions struct {
 	Seed string
 	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
 	ExcludePaths []string
+	// ExcludeRelPaths lists paths to leave out of the listing, matched
+	// against the path a file would be listed by, with no file looked up.
+	ExcludeRelPaths []RelFilePath
 }
 
 // FileEntry represents a file that has been enumerated.
@@ -439,9 +443,10 @@ func (s *Scanner) enumerateFileWithInfo(
 	}
 
 	// Clean the path - remove leading slash if present
-	cleanPath := filePath
+	cleanPath := strings.TrimPrefix(filePath, "/")
-	if len(cleanPath) > 0 && cleanPath[0] == '/' {
+
-		cleanPath = cleanPath[1:]
+	if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) {
+		return nil
 	}
 
 	// Compute absolute path for file reading