Compare commits

..
1 Commits
Author SHA1 Message Date
sneak 73b97ac2e4 Default the manifest to index.mf and keep it out of its own listing (closes #100)
check / check (push) Failing after 2s
mfer gen now writes index.mf instead of .index.mf, the name fetch
requests and the README calls the standard filename. Given a
directory, check, freshen, list and export look only for index.mf;
.index.mf is no longer recognized. Because index.mf is not hidden, the
scanner now leaves the output file out of the listing by its path: gen
--force over an existing manifest no longer lists the old one, while a
file named index.mf in a subdirectory is still listed.

Model: opus-5-5
2026-10-04 08:52:52 +00:00
5 changed files with 47 additions and 128 deletions
+25 -62
View File
@@ -8,7 +8,6 @@ import (
"io" "io"
"math/rand" "math/rand"
"os" "os"
"path/filepath"
"sync" "sync"
"testing" "testing"
@@ -639,11 +638,31 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
assert.Equal(t, "existing", string(content), "original file should be preserved") assert.Equal(t, "existing", string(content), "original file should be preserved")
} }
// manifestPaths returns the file paths listed by the manifest at path. // TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string { // scanned tree with --force: the old file is not listed, while a file named
t.Helper() // index.mf in a subdirectory still is.
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
t.Parallel()
manifest, err := mfer.NewManifestFromFile(fs, path) for name, output := range map[string]string{
"default name": "/testdir/index.mf",
"other name in a subdirectory": "/testdir/sub/listing.mf",
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir/sub", 0o755))
writeTestFile(t, fs, "/testdir/readme.txt", "hello")
writeTestFile(t, fs, "/testdir/sub/index.mf", "an ordinary file")
writeTestFile(t, fs, output, "previous manifest")
opts := testOpts([]string{
testApp, cmdGenerate, "-q", "--force", "-o", output, testDir,
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
manifest, err := mfer.NewManifestFromFile(fs, output)
require.NoError(t, err) require.NoError(t, err)
paths := make([]string, 0, len(manifest.Files())) paths := make([]string, 0, len(manifest.Files()))
@@ -651,67 +670,11 @@ func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
paths = append(paths, f.GetPath()) paths = append(paths, f.GetPath())
} }
return paths assert.ElementsMatch(t, []string{"readme.txt", "sub/index.mf"}, paths)
}
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
// scanned tree with --force: the old file is not listed, even when the tree
// is named through a symlink, while a file named index.mf in a subdirectory
// still is. The output file is recognized by file identity, which needs
// the real filesystem.
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
t.Parallel()
// Paths are relative to a temp dir holding data/tree and link, a
// symlink to data.
for name, tc := range map[string]struct{ input, output string }{
"default name": {"data/tree", "data/tree/index.mf"},
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
root := t.TempDir()
tree := filepath.Join(root, "data", "tree")
output := filepath.Join(root, tc.output)
fs := afero.NewOsFs()
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
require.NoError(t, os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
writeTestFile(t, fs, filepath.Join(tree, "readme.txt"), "hello")
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
writeTestFile(t, fs, output, "previous manifest")
opts := testOpts([]string{
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{"readme.txt", "sub/index.mf"},
manifestPaths(t, fs, output))
}) })
} }
} }
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
// other arguments, so it scans the current directory and writes the
// relative path index.mf: the index.mf already there is not listed.
//
//nolint:paralleltest // changes the process-global working directory
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
chdirTemp(t)
fs := afero.NewOsFs()
writeTestFile(t, fs, "readme.txt", "hello")
writeTestFile(t, fs, "index.mf", "previous manifest")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.Equal(t, []string{"readme.txt"}, manifestPaths(t, fs, "index.mf"))
}
func TestGenerateAtomicWriteUsesTemp(t *testing.T) { func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
t.Parallel() t.Parallel()
+8 -15
View File
@@ -7,7 +7,6 @@ import (
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"path/filepath" "path/filepath"
"time" "time"
@@ -57,7 +56,7 @@ type freshenEntry struct {
type freshenScanner struct { type freshenScanner struct {
fs afero.Fs fs afero.Fs
absBase string absBase string
manifestInfo fs.FileInfo // the manifest being freshened manifestBase string
includeDotfiles bool includeDotfiles bool
followSymlinks bool followSymlinks bool
showProgress bool showProgress bool
@@ -157,6 +156,11 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
"freshen: failed to compute relative path for %s: %w", path, err) "freshen: failed to compute relative path for %s: %w", path, err)
} }
// Skip the manifest file itself
if relPath == s.manifestBase {
return nil
}
// Handle dotfiles // Handle dotfiles
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) { if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
if info.IsDir() { if info.IsDir() {
@@ -181,12 +185,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
info = realInfo info = realInfo
} }
// Skip the manifest file itself, however its path is spelled; gen
// leaves out its output file the same way.
if os.SameFile(info, s.manifestInfo) {
return nil
}
s.scanCount++ s.scanCount++
// Check against existing manifest // Check against existing manifest
@@ -366,22 +364,17 @@ func (mfa *CLIApp) freshenScan(
startScan := time.Now() startScan := time.Now()
showProgress := ctx.Bool("progress") showProgress := ctx.Bool("progress")
manifestInfo, err := mfa.Fs.Stat(manifestPath)
if err != nil {
return nil, 0, fmt.Errorf("freshen: %w", err)
}
scanner := &freshenScanner{ scanner := &freshenScanner{
fs: mfa.Fs, fs: mfa.Fs,
absBase: absBase, absBase: absBase,
manifestInfo: manifestInfo, manifestBase: filepath.Base(manifestPath),
includeDotfiles: ctx.Bool("include-dotfiles"), includeDotfiles: ctx.Bool("include-dotfiles"),
followSymlinks: ctx.Bool("follow-symlinks"), followSymlinks: ctx.Bool("follow-symlinks"),
showProgress: showProgress, showProgress: showProgress,
existingByPath: existingByPath, existingByPath: existingByPath,
} }
err = afero.Walk(mfa.Fs, absBase, scanner.walk) err := afero.Walk(mfa.Fs, absBase, scanner.walk)
if showProgress { if showProgress {
log.ProgressDone() log.ProgressDone()
+4 -33
View File
@@ -5,7 +5,6 @@ import (
"bytes" "bytes"
"context" "context"
"os" "os"
"path/filepath"
"testing" "testing"
"time" "time"
@@ -30,7 +29,7 @@ func (s stubFileInfo) IsDir() bool { return false }
func (s stubFileInfo) Sys() any { return nil } func (s stubFileInfo) Sys() any { return nil }
// setupFreshenDir populates /testdir with two files, scans it, and // setupFreshenDir populates /testdir with two files, scans it, and
// writes the resulting manifest to /testdir/index.mf. // writes the resulting manifest to /testdir/.index.mf.
func setupFreshenDir(t *testing.T, fs afero.Fs) { func setupFreshenDir(t *testing.T, fs afero.Fs) {
t.Helper() t.Helper()
@@ -49,7 +48,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
// Write manifest to filesystem // Write manifest to filesystem
require.NoError(t, require.NoError(t,
afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644)) afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
} }
func TestFreshenUnchanged(t *testing.T) { func TestFreshenUnchanged(t *testing.T) {
@@ -59,7 +58,7 @@ func TestFreshenUnchanged(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Parse manifest to verify // Parse manifest to verify
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/index.mf") manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
} }
@@ -71,7 +70,7 @@ func TestFreshenWithChanges(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Verify initial manifest has 2 files // Verify initial manifest has 2 files
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/index.mf") manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
@@ -96,34 +95,6 @@ func TestFreshenWithChanges(t *testing.T) {
assert.Equal(t, "modified content2", string(content)) assert.Equal(t, "modified content2", string(content))
} }
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
// subdirectory of the tree it lists: the manifest is not listed, while an
// ordinary file of the same name at the top of the tree is. The manifest
// is recognized by file identity, which needs the real filesystem.
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
t.Parallel()
root := t.TempDir()
manifestPath := filepath.Join(root, "sub", "listing.mf")
fs := afero.NewOsFs()
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
writeTestFile(t, fs, filepath.Join(root, "readme.txt"), "hello")
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
// A new file gives freshen something to write.
writeTestFile(t, fs, filepath.Join(root, "new.txt"), "new")
opts = testOpts([]string{testApp, "freshen", "-q", testFlagBase, root, manifestPath}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{"readme.txt", "listing.mf", "new.txt"},
manifestPaths(t, fs, manifestPath))
}
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry // TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
// with respect to MFFilePath.Mtime, which is a message pointer with // with respect to MFFilePath.Mtime, which is a message pointer with
// proto3 field presence and may legitimately be absent. // proto3 field presence and may legitimately be absent.
+5 -5
View File
@@ -360,7 +360,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
manifestFiles := map[string][]byte{ manifestFiles := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, "/data/index.mf", manifestFiles) createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
}) })
@@ -371,7 +371,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
require.NoError(t, fs.MkdirAll("/data", 0o755)) require.NoError(t, fs.MkdirAll("/data", 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
chk, err := NewChecker("/data/index.mf", "/data", fs) chk, err := NewChecker("/data/.index.mf", "/data", fs)
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -383,7 +383,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
extras = append(extras, r) extras = append(extras, r)
} }
// Should only report extra.txt, not .hidden, .config/settings, or index.mf // Should only report extra.txt, not .hidden, .config/settings, or .index.mf
for _, e := range extras { for _, e := range extras {
t.Logf("extra: %s", e.Path) t.Logf("extra: %s", e.Path)
} }
@@ -528,7 +528,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, "/data/index.mf", files) createTestManifest(t, fs, "/data/.index.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
// Add dotfiles and manifest file on disk // Add dotfiles and manifest file on disk
@@ -537,7 +537,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
require.NoError(t, require.NoError(t,
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644)) afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker("/data/index.mf", "/data", fs) chk, err := NewChecker("/data/.index.mf", "/data", fs)
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
+11 -19
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"io" "io"
"io/fs" "io/fs"
"os"
"path" "path"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -59,12 +58,8 @@ type ScannerOptions struct {
// Seed, if set, derives a deterministic UUID from this seed. // Seed, if set, derives a deterministic UUID from this seed.
Seed string Seed string
// OutputPath, if set, is the file the manifest will be written to. // OutputPath, if set, is the file the manifest will be written to.
// If that file exists when the scanner is created, enumeration leaves // Enumeration skips that one file, so a manifest written inside the
// it out however its path is spelled (through a symlink, a hard link, // scanned tree, or one it replaces there, is never listed.
// or in different letter case), so a manifest being replaced is never
// listed.
// Files are matched with os.SameFile, which only recognizes files on
// the operating system's filesystem.
OutputPath string OutputPath string
} }
@@ -84,7 +79,7 @@ type Scanner struct {
totalBytes FileSize // cached sum of all file sizes totalBytes FileSize // cached sum of all file sizes
options *ScannerOptions options *ScannerOptions
fs afero.Fs fs afero.Fs
outputInfo fs.FileInfo // the existing output file; nil when there is none outputPath string // options.OutputPath made absolute; "" when unset
} }
// NewScanner creates a new Scanner with default options. // NewScanner creates a new Scanner with default options.
@@ -110,12 +105,9 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
} }
if opts.OutputPath != "" { if opts.OutputPath != "" {
// If the output file cannot be stat'd, normally because it does // Abs fails only when there is no working directory, and then a
// not exist yet, there is nothing to leave out. // relative output path could not be written either.
info, err := s.fs.Stat(opts.OutputPath) s.outputPath, _ = filepath.Abs(opts.OutputPath)
if err == nil {
s.outputInfo = info
}
} }
return s return s
@@ -429,6 +421,11 @@ func (s *Scanner) enumerateFileWithInfo(
// Compute absolute path for file reading // Compute absolute path for file reading
absPath := filepath.Join(basePath, cleanPath) absPath := filepath.Join(basePath, cleanPath)
// The manifest being written is not one of the files it lists.
if absPath == s.outputPath {
return nil
}
// Handle symlinks // Handle symlinks
if info.Mode()&fs.ModeSymlink != 0 { if info.Mode()&fs.ModeSymlink != 0 {
if !s.options.FollowSymLinks { if !s.options.FollowSymLinks {
@@ -456,11 +453,6 @@ func (s *Scanner) enumerateFileWithInfo(
info = realInfo info = realInfo
} }
// The manifest being written is not one of the files it lists.
if s.outputInfo != nil && os.SameFile(info, s.outputInfo) {
return nil
}
entry := &FileEntry{ entry := &FileEntry{
Path: RelFilePath(cleanPath), Path: RelFilePath(cleanPath),
AbsPath: AbsFilePath(absPath), AbsPath: AbsFilePath(absPath),