From 033d8be5beb147c05a9e68cddecaad8bf8857abb Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 00:58:56 +0200 Subject: [PATCH] Never list index.mf at the top of the tree in gen and freshen (closes #233) fetch saves the manifest as index.mf at the top of the tree and refuses a manifest that lists a file there. gen given --output, and freshen with its manifest outside the base directory, listed such a file, so they wrote manifests fetch would not take. Both now leave out each file the manifest would list as index.mf: one at the top of the tree, or a file given to gen by that name. The rule goes by that path alone and looks up no file, so a symlink named index.mf never hides the file it points to. index.mf in a subdirectory is still listed. The scanner in package mfer gets an option to leave out files by the path they would be listed by. Model: opus-5-5 --- README.md | 6 +- internal/cli/entry_test.go | 164 ++++++++++++++++++++++++++++++++--- internal/cli/freshen.go | 6 ++ internal/cli/freshen_test.go | 71 +++++++++++++++ internal/cli/gen.go | 3 + internal/cli/pipe_test.go | 2 +- mfer/scanner.go | 11 ++- 7 files changed, 244 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c24a2ab..172c0a9 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues). only on the paths it is given and the files under them, so with the same `--seed` and an unchanged tree it writes the same bytes wherever the manifest goes. The file it writes to is never listed + - never lists `index.mf` at the top of a directory it is given, or a file it + is given by that name, wherever it writes the manifest, since `fetch` + saves the manifest there; `index.mf` in a subdirectory is listed - `mfer check` / `mfer check .` - verifies checksums of all files in manifest, displaying error and exiting nonzero if any files are missing or corrupted, or have permission bits @@ -308,7 +311,8 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues). - rewrites `index.mf` to list the files now under the directory holding it, or under the one `--base` names, hashing only the files that are new or changed - - lists regular files only, as gen does + - lists regular files only, and never `index.mf` at the top of that + directory, as gen does - leaves out hidden files unless given `--include-dotfiles`, and symlinks unless given `--follow-symlinks`, which lists each symlink to a regular file under its own name with the contents of the file it points to diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index f025383..6249bbe 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -27,19 +27,20 @@ import ( ) const ( - testApp = "mfer" - testGitrev = "abc123" - testDir = "/testdir" - testFile1 = "/testdir/file1.txt" - testMF = "/testdir/test.mf" - testOutput = "/output.mf" - testManifest = "/manifest.mf" - testFlagBase = "--base" - testFlagNoExtra = "--no-extra-files" - testFlagVersion = "--version" - testFlagVerbose = "--verbose" - testFlagHelp = "--help" - testCmdHelp = "help" + testApp = "mfer" + testGitrev = "abc123" + testDir = "/testdir" + testFile1 = "/testdir/file1.txt" + testMF = "/testdir/test.mf" + testOutput = "/output.mf" + testManifest = "/manifest.mf" + testFlagBase = "--base" + testFlagNoExtra = "--no-extra-files" + testFlagFollowSymlinks = "--follow-symlinks" + testFlagVersion = "--version" + testFlagVerbose = "--verbose" + testFlagHelp = "--help" + testCmdHelp = "help" ) var errSimulatedWrite = errors.New("simulated write failure") @@ -1002,7 +1003,7 @@ func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) { exitCode int extras []string // what check reports as not in the manifest }{ - "gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil}, + "gen --follow-symlinks": {[]string{testFlagFollowSymlinks}, 0, nil}, "plain gen": {nil, 1, []string{"filelink"}}, } { t.Run(name, func(t *testing.T) { @@ -1567,6 +1568,141 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) { assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output)) } +// TestGenerateLeavesTopIndexMFOutOfListing runs gen with --output outside +// the paths it is given: index.mf at the top of each directory given, and +// a file given by that name, are not listed, since fetch saves the manifest +// as index.mf at the top of the tree, while sub/index.mf is listed. +func TestGenerateLeavesTopIndexMFOutOfListing(t *testing.T) { + t.Parallel() + + // Paths are relative to a temp dir holding tree, which holds file.txt, + // index.mf and sub/index.mf, and second, which holds second.txt and + // index.mf. + for name, tc := range map[string]struct { + paths []string + listed []string + }{ + "directory": { + []string{"tree"}, + []string{testFileTxt, testSubdirManifest}, + }, + "two directories": { + []string{"tree", "second"}, + []string{testFileTxt, testSubdirManifest, "second.txt"}, + }, + "files": { + []string{"tree/" + testFileTxt, "tree/" + defaultManifestName}, + []string{testFileTxt}, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + tree := filepath.Join(root, "tree") + second := filepath.Join(root, "second") + + fs := afero.NewOsFs() + require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750)) + require.NoError(t, fs.MkdirAll(second, 0o750)) + writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello") + writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file") + writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file") + writeTestFile(t, fs, filepath.Join(second, "second.txt"), "second") + writeTestFile(t, fs, filepath.Join(second, defaultManifestName), "an ordinary file") + + paths := make([]string, 0, len(tc.paths)) + for _, path := range tc.paths { + paths = append(paths, filepath.Join(root, path)) + } + + output := filepath.Join(root, "other.mf") + opts := testOpts(slices.Concat( + []string{testApp, cmdGenerate, "-q", "-o", output}, paths, + ), fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output)) + }) + } +} + +// TestGenerateListsTargetOfTopIndexMFSymlink makes index.mf at the top of +// the tree a symlink to another file in it, and runs gen with --output +// outside the tree: the file it points to is listed under its own name, +// and the symlink, which gen lists only with --follow-symlinks, is left out +// like any index.mf at the top of the tree. +func TestGenerateListsTargetOfTopIndexMFSymlink(t *testing.T) { + t.Parallel() + + // Paths are relative to tree, which holds file.txt, sub/index.mf and + // index.mf, a symlink to target. + for name, tc := range map[string]struct { + target string + flags []string + path string // the path gen is given + listed []string + }{ + "directory given": { + testSubdirManifest, nil, ".", + []string{testFileTxt, testSubdirManifest}, + }, + "directory given, --follow-symlinks": { + testSubdirManifest, + []string{testFlagFollowSymlinks}, + ".", + []string{testFileTxt, testSubdirManifest}, + }, + "file given": { + testFileTxt, nil, testFileTxt, + []string{testFileTxt}, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + tree := filepath.Join(root, "tree") + + fs := afero.NewOsFs() + require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750)) + writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello") + writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file") + require.NoError(t, + os.Symlink(tc.target, filepath.Join(tree, defaultManifestName))) + + output := filepath.Join(root, "other.mf") + opts := testOpts(slices.Concat( + []string{testApp, cmdGenerate, "-q", "-o", output}, tc.flags, + []string{filepath.Join(tree, tc.path)}, + ), fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output)) + }) + } +} + +// TestGenerateListsIndexMFBesideFileGiven runs gen with --output outside +// the tree, given sub/b.txt and the tree: sub/index.mf, beside the file +// given, is listed, since a file given is left out only when its own name +// is index.mf. +// +//nolint:paralleltest // changes the process-global working directory +func TestGenerateListsIndexMFBesideFileGiven(t *testing.T) { + setupManifestInSubdir(t) + + fs := afero.NewOsFs() + output := filepath.Join(t.TempDir(), "other.mf") + + opts := testOpts([]string{ + testApp, cmdGenerate, "-q", "-o", output, filepath.Join(testSubdir, "b.txt"), ".", + }, fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + assert.Contains(t, manifestPaths(t, fs, output), testSubdirManifest) +} + // writeTestTree writes file.txt and sub/nested.txt under dir. func writeTestTree(t *testing.T, fs afero.Fs, dir string) { t.Helper() diff --git a/internal/cli/freshen.go b/internal/cli/freshen.go index 6e01b53..59982c1 100644 --- a/internal/cli/freshen.go +++ b/internal/cli/freshen.go @@ -192,6 +192,12 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro return nil } + // Leave out a file listed as index.mf, at the top of the base + // directory, since fetch saves the manifest there, as gen does. + if relPath == defaultManifestName { + return nil + } + // Handle symlinks if info.Mode()&fs.ModeSymlink != 0 { realInfo, keep := s.resolveSymlink(path) diff --git a/internal/cli/freshen_test.go b/internal/cli/freshen_test.go index e31d480..fdbb662 100644 --- a/internal/cli/freshen_test.go +++ b/internal/cli/freshen_test.go @@ -413,6 +413,77 @@ func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) { manifestPaths(t, fs, manifestPath)) } +// TestFreshenLeavesTopIndexMFOutOfListing freshens a manifest kept outside +// the directory --base names, after index.mf and sub/index.mf are added +// there: index.mf at the top is not listed, since fetch saves the manifest +// there, while sub/index.mf is listed. +func TestFreshenLeavesTopIndexMFOutOfListing(t *testing.T) { + t.Parallel() + + root := t.TempDir() + tree := filepath.Join(root, "tree") + manifestPath := filepath.Join(root, "other.mf") + + fs := afero.NewOsFs() + require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750)) + writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello") + + opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, tree}, fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + writeTestFile(t, fs, filepath.Join(tree, defaultManifestName), "an ordinary file") + writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file") + + runFreshen(t, fs, tree, manifestPath) + + assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest}, + manifestPaths(t, fs, manifestPath)) +} + +// TestFreshenListsTargetOfTopIndexMFSymlink freshens a manifest kept +// outside the directory --base names, after sub/index.mf is added there +// and index.mf at the top is made a symlink to it: sub/index.mf is listed, +// and the symlink, which freshen lists only with --follow-symlinks, is left +// out like any index.mf at the top. +func TestFreshenListsTargetOfTopIndexMFSymlink(t *testing.T) { + t.Parallel() + + for name, flags := range map[string][]string{ + "plain freshen": nil, + "freshen --follow-symlinks": {testFlagFollowSymlinks}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + tree := filepath.Join(root, "tree") + manifestPath := filepath.Join(root, "other.mf") + + fs := afero.NewOsFs() + require.NoError(t, fs.MkdirAll(filepath.Join(tree, testSubdir), 0o750)) + writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello") + + opts := testOpts([]string{ + testApp, cmdGenerate, "-q", "-o", manifestPath, tree, + }, fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + writeTestFile(t, fs, filepath.Join(tree, testSubdirManifest), "an ordinary file") + require.NoError(t, os.Symlink(testSubdirManifest, + filepath.Join(tree, defaultManifestName))) + + opts = testOpts(slices.Concat( + []string{testApp, cmdFreshen, "-q"}, flags, + []string{testFlagBase, tree, manifestPath}, + ), fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + assert.ElementsMatch(t, []string{testFileTxt, testSubdirManifest}, + manifestPaths(t, fs, manifestPath)) + }) + } +} + // TestFreshenResolvesEntriesAgainstManifestDirectory adds sub/c.txt, then // runs freshen from the directory above sub, on the manifest in sub. // Without --base, the manifest then lists the files in sub, whether freshen diff --git a/internal/cli/gen.go b/internal/cli/gen.go index 0dda37a..0c34e06 100644 --- a/internal/cli/gen.go +++ b/internal/cli/gen.go @@ -137,6 +137,9 @@ func (mfa *CLIApp) buildScannerOptions( // Neither a manifest being replaced nor a temp file left by an // interrupted run belongs in the new manifest. ExcludePaths: []string{output, manifestTempPath(output)}, + // Nor does a file listed as index.mf, at the top of the tree, since + // fetch saves the manifest there and refuses a manifest that lists it. + ExcludeRelPaths: []mfer.RelFilePath{defaultManifestName}, } // Set seed for deterministic UUID if provided diff --git a/internal/cli/pipe_test.go b/internal/cli/pipe_test.go index 47953ab..58111f9 100644 --- a/internal/cli/pipe_test.go +++ b/internal/cli/pipe_test.go @@ -81,7 +81,7 @@ func TestFreshenSkipsPipes(t *testing.T) { require.NoError(t, os.Symlink(target, filepath.Join(root, testLink))) opts := testOpts([]string{ - testApp, cmdFreshen, "-v", "--follow-symlinks", + testApp, cmdFreshen, "-v", testFlagFollowSymlinks, testFlagBase, root, manifestPath, }, fs) require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) diff --git a/mfer/scanner.go b/mfer/scanner.go index 358ffdb..3c3db4e 100644 --- a/mfer/scanner.go +++ b/mfer/scanner.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "sync" "time" @@ -64,6 +65,9 @@ type ScannerOptions struct { Seed string // ExcludePaths lists files to leave out of the listing, matched with os.SameFile. ExcludePaths []string + // ExcludeRelPaths lists paths to leave out of the listing, matched + // against the path a file would be listed by, with no file looked up. + ExcludeRelPaths []RelFilePath } // FileEntry represents a file that has been enumerated. @@ -439,9 +443,10 @@ func (s *Scanner) enumerateFileWithInfo( } // Clean the path - remove leading slash if present - cleanPath := filePath - if len(cleanPath) > 0 && cleanPath[0] == '/' { - cleanPath = cleanPath[1:] + cleanPath := strings.TrimPrefix(filePath, "/") + + if slices.Contains(s.options.ExcludeRelPaths, RelFilePath(cleanPath)) { + return nil } // Compute absolute path for file reading