Never report index.mf at the top of the base as unlisted in check (closes #229)

gen and fetch write the manifest as index.mf at the top of the tree.
check left out only the manifest file it was given, so given a URL, or a
manifest stored elsewhere, it reported that index.mf as not in the
manifest, and --no-extra-files failed a tree fetch had just verified.
check now drops that one report for index.mf at the top of the base
directory. A manifest that lists index.mf still has it checked, and
index.mf in a subdirectory or fetch's leftover .index.mf.tmp are still
reported. The change is in package cli, which owns the default manifest
name; the Checker in package mfer is unchanged.

Model: opus-5-5

README.md

@@ -13,11 +13,14 @@ The structure of these files can be found
 schema `mfer/mf.proto` for exact field numbers and types. Both are included in
 the [project repository](https://git.eeqj.de/sneak/mfer).
 
-Releases are git tags named by [SemVer](https://semver.org), starting with
+The current version is pre-1.0 and while the repo was published in 2022, there
-`v1.0.0`. No file in the tree records a version number. `mfer version` prints
+has not yet been any versioned release. [SemVer](https://semver.org) will be
-the revision the build stamped into the binary, which for `make build` and the
+used for releases.
-Docker image is the output of `git describe` on the checkout they build. A
+
-binary built without that stamp, as
+A version is a git tag, and no file in the tree records a version number.
+`mfer version` prints the revision the build stamped into the binary, which for
+`make build` and the Docker image is the output of `git describe` on the
+checkout they build. A binary built without that stamp, as
 `go install sneak.berlin/go/mfer/cmd/mfer@<tag>` builds it, prints the module
 version Go records in it instead. A program that imports the `mfer` package
 finds the version it was built against in its own build information
@@ -287,9 +290,6 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
       only on the paths it is given and the files under them, so with the same
       `--seed` and an unchanged tree it writes the same bytes wherever the
       manifest goes. The file it writes to is never listed
-    - never lists `index.mf` at the top of a directory it is given, or a file it
-      is given by that name, wherever it writes the manifest, since `fetch`
-      saves the manifest there; `index.mf` in a subdirectory is listed
 - `mfer check` / `mfer check .`
     - verifies checksums of all files in manifest, displaying error and exiting
       nonzero if any files are missing or corrupted, or have permission bits
@@ -311,8 +311,7 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
     - rewrites `index.mf` to list the files now under the directory holding it,
       or under the one `--base` names, hashing only the files that are new or
       changed
-    - lists regular files only, and never `index.mf` at the top of that
+    - lists regular files only, as gen does
-      directory, as gen does
     - leaves out hidden files unless given `--include-dotfiles`, and symlinks
       unless given `--follow-symlinks`, which lists each symlink to a regular
       file under its own name with the contents of the file it points to

internal/cli/check.go

@@ -197,8 +197,8 @@ func countCheckFailures(
 // manifest, and anything the search cannot read: each is a failure, logged
 // at error level, under --no-extra-files, otherwise a warning. index.mf at
 // the top of the base directory is never reported as absent from the
-// manifest, even when check is given another manifest or a URL, since gen
+// manifest, since gen and fetch write the manifest there, even when check
-// and fetch write the manifest there.
+// is given another manifest or a URL.
 func findExtraFiles(
 	ctx context.Context, cmd *cli.Command, chk *mfer.Checker, failures *int64,
 ) error {

internal/cli/entry_test.go

@@ -27,20 +27,19 @@ import (
 )
 
 const (
-	testApp                = "mfer"
+	testApp         = "mfer"
-	testGitrev             = "abc123"
+	testGitrev      = "abc123"
-	testDir                = "/testdir"
+	testDir         = "/testdir"
-	testFile1              = "/testdir/file1.txt"
+	testFile1       = "/testdir/file1.txt"
-	testMF                 = "/testdir/test.mf"
+	testMF          = "/testdir/test.mf"
-	testOutput             = "/output.mf"
+	testOutput      = "/output.mf"
-	testManifest           = "/manifest.mf"
+	testManifest    = "/manifest.mf"
-	testFlagBase           = "--base"
+	testFlagBase    = "--base"
-	testFlagNoExtra        = "--no-extra-files"
+	testFlagNoExtra = "--no-extra-files"
-	testFlagFollowSymlinks = "--follow-symlinks"
+	testFlagVersion = "--version"
-	testFlagVersion        = "--version"
+	testFlagVerbose = "--verbose"
-	testFlagVerbose        = "--verbose"
+	testFlagHelp    = "--help"
-	testFlagHelp           = "--help"
+	testCmdHelp     = "help"
-	testCmdHelp            = "help"
 )
 
 var errSimulatedWrite = errors.New("simulated write failure")
@@ -1003,7 +1002,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
 		exitCode int
 		extras   []string // what check reports as not in the manifest
 	}{
-		"gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil},
+		"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
 		"plain gen":             {nil, 1, []string{"filelink"}},
 	} {
 		t.Run(name, func(t *testing.T) {
@@ -1191,13 +1190,12 @@ func TestCheckURLManifestWithoutTempDirectory(t *testing.T) {
 	assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 }
 
-// TestCheckNeverReportsIndexMFAtTopOfBase checks a tree holding index.mf at
+// TestCheckNeverReportsIndexMfAtTopOfBase checks a tree holding index.mf at
 // its top, where gen and fetch write the manifest, against a manifest that
-// does not list it, given by URL and by a path outside the tree: check never
+// does not list it, given by URL and by a path outside the tree: the check
-// reports it, and passes with and without --no-extra-files. index.mf in a
+// passes under --no-extra-files. index.mf in a subdirectory, and the temp
-// subdirectory, and the temp file fetch saves index.mf through, are still
+// file fetch saves index.mf through, are still reported.
-// reported.
+func TestCheckNeverReportsIndexMfAtTopOfBase(t *testing.T) {
-func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
 	t.Parallel()
 
 	files := map[string][]byte{testFileTxt: []byte("hello")}
@@ -1222,25 +1220,19 @@ func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
 			require.NoError(t, os.WriteFile(
 				filepath.Join(base, defaultManifestName), manifest, 0o600))
 
-			check := func(flags ...string) (int, string) {
+			check := func() (int, string) {
-				opts := testOpts(slices.Concat(
+				opts := testOpts([]string{
-					[]string{testApp, cmdCheck}, flags,
+					testApp, cmdCheck, testFlagNoExtra, testFlagBase, base, manifestArg,
-					[]string{testFlagBase, base, manifestArg},
+				}, afero.NewOsFs())
-				), afero.NewOsFs())
 
 				return runCLI(opts), testStderr(t, opts)
 			}
 
 			exitCode, stderr := check()
 			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
-
-			exitCode, stderr = check(testFlagNoExtra)
-			assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
 
 			reported := []string{
-				testSubdirManifest,
+				testSubdir + "/" + defaultManifestName,
 				tempPathFor(defaultManifestName),
 			}
 
@@ -1251,9 +1243,8 @@ func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
 					os.WriteFile(filepath.Join(base, path), manifest, 0o600))
 			}
 
-			exitCode, stderr = check(testFlagNoExtra)
+			exitCode, stderr = check()
 			assert.Equal(t, 1, exitCode, "stderr: %s", stderr)
-			assert.NotContains(t, stderr, "EXTRA: "+defaultManifestName)
 
 			for _, path := range reported {
 				assert.Contains(t, stderr, "EXTRA: "+path)
@@ -1262,10 +1253,10 @@ func TestCheckNeverReportsIndexMFAtTopOfBase(t *testing.T) {
 	}
 }
 
-// TestCheckChecksListedIndexMF checks a tree against a manifest that lists
+// TestCheckChecksListedIndexMf checks a tree against a manifest that lists
 // index.mf at the top of the tree, where that file has changed since: check
 // reports it like any other changed file.
-func TestCheckChecksListedIndexMF(t *testing.T) {
+func TestCheckChecksListedIndexMf(t *testing.T) {
 	t.Parallel()
 
 	manifestPath := filepath.Join(t.TempDir(), "other.mf")
@@ -1332,60 +1323,6 @@ func TestCheckWithUnlistableDirectory(t *testing.T) {
 	assert.Contains(t, testStderr(t, opts), "unlisted.txt")
 }
 
-// TestCheckReportsUnreadableIndexMFAtTopOfBase has a directory named
-// index.mf at the top of the base that cannot be listed. check leaves out
-// only the report that index.mf is not in the manifest, so it reports the
-// error and fails under --no-extra-files.
-func TestCheckReportsUnreadableIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	unreadable := filepath.Join(testDir, defaultManifestName)
-
-	mem := afero.NewMemMapFs()
-	require.NoError(t, mem.MkdirAll(unreadable, 0o755))
-	writeTestFile(t, mem, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, mem)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	fs := unlistableDirFs{Fs: mem, dir: unreadable}
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "ERROR: "+defaultManifestName)
-}
-
-// TestCheckReportsUpperCaseIndexMFAtTopOfBase checks a tree holding
-// INDEX.MF at its top against a manifest that does not list it. Only the
-// name index.mf, in exactly that letter case, is left out, so check reports
-// INDEX.MF. The tree is in memory, where the two names are two files
-// whatever the letter case rules of the host's filesystem.
-func TestCheckReportsUpperCaseIndexMFAtTopOfBase(t *testing.T) {
-	t.Parallel()
-
-	upperCase := strings.ToUpper(defaultManifestName)
-
-	fs := afero.NewMemMapFs()
-	writeTestFile(t, fs, testFile1, "hello")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	writeTestFile(t, fs, filepath.Join(testDir, upperCase), "not listed")
-
-	opts = testOpts([]string{
-		testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
-	}, fs)
-	assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
-	assert.Contains(t, testStderr(t, opts), "EXTRA: "+upperCase)
-}
-
 func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
 	t.Parallel()
 
@@ -1568,141 +1505,6 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 	assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
 }
 
-// TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside
-// the paths it is given: index.mf at the top of each directory given, and
-// a file given by that name, are not listed, since fetch saves the manifest
-// as index.mf at the top of the tree, while sub/index.mf is listed.
-func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) {
-	t.Parallel()
-
-	// Paths are relative to a temp dir holding tree, which holds file.txt,
-	// index.mf and sub/index.mf, and second, which holds second.txt and
-	// index.mf.
-	for name, tc := range map[string]struct {
-		paths  []string
-		listed []string
-	}{
-		"directory": {
-			[]string{"tree"},
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"two directories": {
-			[]string{"tree", "second"},
-			[]string{testFileTxt, testSubdirManifest, "second.txt"},
-		},
-		"files": {
-			[]string{"tree/" + testFileTxt, "tree/" + defaultManifestName},
-			[]string{testFileTxt},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-			second := filepath.Join(root, "second")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			require.NoError(t, fs.MkdirAll(second, 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-			writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			writeTestFile(t, fs, filepath.Join(second, "second.txt"), "second")
-			writeTestFile(t, fs, filepath.Join(second, defaultManifestName), "an ordinary file")
-
-			paths := make([]string, 0, len(tc.paths))
-			for _, path := range tc.paths {
-				paths = append(paths, filepath.Join(root, path))
-			}
-
-			output := filepath.Join(root, "other.mf")
-			opts := testOpts(slices.Concat(
-				[]string{testApp, cmdGenerate, "-q", "-o", output}, paths,
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
-		})
-	}
-}
-
-// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of
-// the tree a symlink to another file in it, and runs gen with --output
-// outside the tree: the file it points to is listed under its own name,
-// and the symlink, which gen lists only with --follow-symlinks, is left out
-// like any index.mf at the top of the tree.
-func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	// Paths are relative to tree, which holds file.txt, sub/index.mf and
-	// index.mf, a symlink to target.
-	for name, tc := range map[string]struct {
-		target string
-		flags  []string
-		path   string // the path gen is given
-		listed []string
-	}{
-		"directory given": {
-			testSubdirManifest, nil, ".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"directory given, --follow-symlinks": {
-			testSubdirManifest,
-			[]string{testFlagFollowSymlinks},
-			".",
-			[]string{testFileTxt, testSubdirManifest},
-		},
-		"file given": {
-			testFileTxt, nil, testFileTxt,
-			[]string{testFileTxt},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t,
-				os.Symlink(tc.target, filepath.Join(tree, defaultManifestName)))
-
-			output := filepath.Join(root, "other.mf")
-			opts := testOpts(slices.Concat(
-				[]string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags,
-				[]string{filepath.Join(tree, tc.path)},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
-		})
-	}
-}
-
-// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside
-// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file
-// given, is listed, since a file given is left out only when its own name
-// is index.mf.
-//
-//nolint:paralleltest // changes the process-global working directory
-func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) {
-	setupManifestInSubdir(t)
-
-	fs := afero.NewOsFs()
-	output := filepath.Join(t.TempDir(), "other.mf")
-
-	opts := testOpts([]string{
-		testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".",
-	}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest)
-}
-
 // writeTestTree writes file.txt and sub/nested.txt under dir.
 func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
 	t.Helper()

internal/cli/freshen.go

@@ -192,12 +192,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
 		return nil
 	}
 
-	// Leave out a file listed as index.mf, at the top of the base
-	// directory, since fetch saves the manifest there, as gen does.
-	if relPath == defaultManifestName {
-		return nil
-	}
-
 	// Handle symlinks
 	if info.Mode()&fs.ModeSymlink != 0 {
 		realInfo, keep := s.resolveSymlink(path)

internal/cli/freshen_test.go

@@ -413,77 +413,6 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
 		manifestPaths(t, fs, manifestPath))
 }
 
-// TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside
-// the directory --base names, after index.mf and sub/index.mf are added
-// there: index.mf at the top is not listed, since fetch saves the manifest
-// there, while sub/index.mf is listed.
-func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) {
-	t.Parallel()
-
-	root := t.TempDir()
-	tree := filepath.Join(root, "tree")
-	manifestPath := filepath.Join(root, "other.mf")
-
-	fs := afero.NewOsFs()
-	require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-	writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-
-	opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, tree}, fs)
-	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-	writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file")
-	writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-
-	runFreshen(t, fs, tree, manifestPath)
-
-	assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
-		manifestPaths(t, fs, manifestPath))
-}
-
-// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept
-// outside the directory --base names, after sub/index.mf is added there
-// and index.mf at the top is made a symlink to it: sub/index.mf is listed,
-// and the symlink, which freshen lists only with --follow-symlinks, is left
-// out like any index.mf at the top.
-func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) {
-	t.Parallel()
-
-	for name, flags := range map[string][]string{
-		"plain freshen":             nil,
-		"freshen --follow-symlinks": {testFlagFollowSymlinks},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			root := t.TempDir()
-			tree := filepath.Join(root, "tree")
-			manifestPath := filepath.Join(root, "other.mf")
-
-			fs := afero.NewOsFs()
-			require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750))
-			writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
-
-			opts := testOpts([]string{
-				testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
-			}, fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file")
-			require.NoError(t, os.Symlink(testSubdirManifest,
-				filepath.Join(tree, defaultManifestName)))
-
-			opts = testOpts(slices.Concat(
-				[]string{testApp, cmdFreshen, "-q"}, flags,
-				[]string{testFlagBase, tree, manifestPath},
-			), fs)
-			require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
-
-			assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest},
-				manifestPaths(t, fs, manifestPath))
-		})
-	}
-}
-
 // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then
 // runs freshen from the directory above sub, on the manifest in sub.
 // Without --base, the manifest then lists the files in sub, whether freshen

internal/cli/gen.go

@@ -137,9 +137,6 @@ func (mfa *CLIApp) buildScannerOptions(
 		// Neither a manifest being replaced nor a temp file left by an
 		// interrupted run belongs in the new manifest.
 		ExcludePaths: []string{output, manifestTempPath(output)},
-		// Nor does a file listed as index.mf, at the top of the tree, since
-		// fetch saves the manifest there and refuses a manifest that lists it.
-		ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName},
 	}
 
 	// Set seed for deterministic UUID if provided

internal/cli/pipe_test.go

@@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) {
 	require.NoError(t, os.Symlink(target, filepath.Join(root, testLink)))
 
 	opts := testOpts([]string{
-		testApp, cmdFreshen, "-v", testFlagFollowSymlinks,
+		testApp, cmdFreshen, "-v", "--follow-symlinks",
 		testFlagBase, root, manifestPath,
 	}, fs)
 	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))

mfer/scanner.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -65,9 +64,6 @@ type ScannerOptions struct {
 	Seed string
 	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
 	ExcludePaths []string
-	// ExcludeRelPaths lists paths to leave out of the listing, matched
-	// against the path a file would be listed by, with no file looked up.
-	ExcludeRelPaths []RelFilePath
 }
 
 // FileEntry represents a file that has been enumerated.
@@ -443,10 +439,9 @@ func (s *Scanner) enumerateFileWithInfo(
 	}
 
 	// Clean the path - remove leading slash if present
-	cleanPath := strings.TrimPrefix(filePath, "/")
+	cleanPath := filePath
-
+	if len(cleanPath) > 0 && cleanPath[0] == '/' {
-	if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) {
+		cleanPath = cleanPath[1:]
-		return nil
 	}
 
 	// Compute absolute path for file reading