Write gen DIR's manifest to DIR/index.mf (closes #178) #180

Merged
clawbot merged 1 commits from issue-178-gen-output into next 2026-10-08 03:08:41 +02:00
6 changed files with 299 additions and 32 deletions
+9
View File
@@ -269,6 +269,15 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
- recurses under current directory and writes out an `index.mf` - recurses under current directory and writes out an `index.mf`
- records every file's mode as `0000` unless given `--include-permissions`, - records every file's mode as `0000` unless given `--include-permissions`,
which records each file's permission bits (`0777` at most) which records each file's permission bits (`0777` at most)
- `mfer gen /media/drive`
- writes `/media/drive/index.mf`, listing each file by its path under
`/media/drive`, so `mfer check /media/drive` verifies it. Given a file,
gen writes `index.mf` beside it and lists the file by its name; given
several paths, it writes `index.mf` in the current directory
- `--output` names another file to write instead. What gen lists depends
only on the paths it is given and the files under them, so with the same
`--seed` and an unchanged tree it writes the same bytes wherever the
manifest goes. The file it writes to is never listed
- `mfer check` / `mfer check .` - `mfer check` / `mfer check .`
- verifies checksums of all files in manifest, displaying error and exiting - verifies checksums of all files in manifest, displaying error and exiting
nonzero if any files are missing or corrupted, or have permission bits nonzero if any files are missing or corrupted, or have permission bits
+174
View File
@@ -1289,6 +1289,180 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output)) assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
} }
// writeTestTree writes file.txt and sub/nested.txt under dir.
func writeTestTree(t *testing.T, fs afero.Fs, dir string) {
t.Helper()
require.NoError(t, fs.MkdirAll(filepath.Join(dir, testSubdir), 0o750))
writeTestFile(t, fs, filepath.Join(dir, testFileTxt), "hello")
writeTestFile(t, fs, filepath.Join(dir, testSubdir, "nested.txt"), "in sub")
}
// TestGenerateDefaultOutput runs gen without --output on one directory or
// one file: it writes index.mf in that directory, or beside that file,
// listing each file by its path under the directory index.mf is in, and
// check given that directory passes.
func TestGenerateDefaultOutput(t *testing.T) {
t.Parallel()
// Paths are relative to a temp dir holding file.txt and sub/nested.txt.
for name, tc := range map[string]struct {
input, output string
listed []string
}{
"directory": {
".", defaultManifestName, []string{testFileTxt, "sub/nested.txt"},
},
"subdirectory": {testSubdir, testSubdirManifest, []string{"nested.txt"}},
"file": {testFileTxt, defaultManifestName, []string{testFileTxt}},
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
root := t.TempDir()
fs := afero.NewOsFs()
writeTestTree(t, fs, root)
opts := testOpts([]string{
testApp, cmdGenerate, "-q", filepath.Join(root, tc.input),
}, fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
output := filepath.Join(root, tc.output)
assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output))
opts = testOpts([]string{
testApp, cmdCheck, "-q", filepath.Dir(output),
}, fs)
assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
})
}
}
// TestGenerateSeveralPathsDefaultOutput runs gen without --output on two
// directories: it writes index.mf in the current directory, listing both
// directories' files, and writes no index.mf in either directory.
//
//nolint:paralleltest // changes the process-global working directory
func TestGenerateSeveralPathsDefaultOutput(t *testing.T) {
root := t.TempDir()
fs := afero.NewOsFs()
dirs := []string{"first", "second"}
for _, dir := range dirs {
require.NoError(t, fs.MkdirAll(filepath.Join(root, dir), 0o750))
writeTestFile(t, fs, filepath.Join(root, dir, dir+".txt"), dir)
}
t.Chdir(root)
opts := testOpts(append([]string{testApp, cmdGenerate, "-q"}, dirs...), fs)
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
assert.ElementsMatch(t, []string{"first.txt", "second.txt"},
manifestPaths(t, fs, filepath.Join(root, defaultManifestName)))
for _, dir := range dirs {
exists, err := afero.Exists(fs, filepath.Join(root, dir, defaultManifestName))
require.NoError(t, err)
assert.False(t, exists, "index.mf written in %s", dir)
}
}
// TestGenerateBytesDoNotDependOnOutput runs gen --seed on one tree, each
// time writing to another file, over a file already there and beside an
// earlier run's temp file: neither is listed, and what is listed depends
// only on the tree, so every manifest has the same bytes.
func TestGenerateBytesDoNotDependOnOutput(t *testing.T) {
t.Parallel()
root := t.TempDir()
tree := filepath.Join(root, "tree")
defaultOutput := filepath.Join(tree, defaultManifestName)
fs := afero.NewOsFs()
writeTestTree(t, fs, tree)
var first []byte
for _, output := range []string{
defaultOutput,
filepath.Join(tree, "listing.mf"),
filepath.Join(tree, testSubdir, "listing.mf"),
filepath.Join(root, "outside.mf"),
} {
writeTestFile(t, fs, output, "previous manifest")
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
args := []string{testApp, cmdGenerate, "-q", "-f", "--seed", "mfer"}
if output != defaultOutput {
args = append(args, "-o", output)
}
args = append(args, tree)
opts := testOpts(args, fs)
require.Equal(t, 0, runCLI(opts),
"output %s, stderr: %s", output, testStderr(t, opts))
got, err := afero.ReadFile(fs, output)
require.NoError(t, err)
require.NoError(t, fs.Remove(output))
if first == nil {
first = got
}
assert.Equal(t, first, got, "output %s", output)
}
}
// TestGenerateRefusesExistingDefaultOutput runs gen without --output or
// --force on a directory already holding index.mf: gen fails, naming that
// file, and leaves it as it was.
func TestGenerateRefusesExistingDefaultOutput(t *testing.T) {
t.Parallel()
output := filepath.Join(testDir, defaultManifestName)
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll(testDir, 0o755))
writeTestFile(t, fs, testFile1, "hello")
writeTestFile(t, fs, output, "previous manifest")
opts := testOpts([]string{testApp, cmdGenerate, "-q", testDir}, fs)
assert.Equal(t, 1, runCLI(opts))
assert.Contains(t, testStderr(t, opts),
"output file "+output+" already exists (use --force to overwrite)")
content, err := afero.ReadFile(fs, output)
require.NoError(t, err)
assert.Equal(t, "previous manifest", string(content))
}
// TestGenerateRefusesEmptyOutput runs gen with --force and an --output
// given an empty value, as an unset shell variable gives it, on a
// directory already holding index.mf: gen fails and leaves that file as it
// was.
func TestGenerateRefusesEmptyOutput(t *testing.T) {
t.Parallel()
output := filepath.Join(testDir, defaultManifestName)
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll(testDir, 0o755))
writeTestFile(t, fs, testFile1, "hello")
writeTestFile(t, fs, output, "previous manifest")
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-f", "-o", "", testDir}, fs)
assert.Equal(t, 1, runCLI(opts))
assert.Contains(t, testStderr(t, opts), errEmptyOutput.Error())
content, err := afero.ReadFile(fs, output)
require.NoError(t, err)
assert.Equal(t, "previous manifest", string(content))
}
func TestGenerateAtomicWriteUsesTemp(t *testing.T) { func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
t.Parallel() t.Parallel()
+76 -27
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"os" "os"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
@@ -26,6 +27,8 @@ var (
// rendered message stays exactly as mfer has always printed it. // rendered message stays exactly as mfer has always printed it.
errOutputExists = errors.New( errOutputExists = errors.New(
"already exists (use --force to overwrite)") "already exists (use --force to overwrite)")
// errEmptyOutput indicates --output given with an empty value.
errEmptyOutput = errors.New("--output must not be empty")
) )
// reportEnumProgress renders enumeration progress until the channel // reportEnumProgress renders enumeration progress until the channel
@@ -88,9 +91,40 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
return paths, nil return paths, nil
} }
// buildScannerOptions constructs scanner options from the CLI flags. // outputPath returns the file gen writes the manifest to: the one --output
func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions { // names, or else index.mf in the directory the only argument names, or
output := cmd.String("output") // beside the file it names, or else in the current directory. An --output
// given with an empty value is refused.
func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
if cmd.IsSet("output") {
output := cmd.String("output")
if output == "" {
return "", errEmptyOutput
}
return output, nil
}
if cmd.Args().Len() != 1 {
return defaultManifestName, nil
}
arg := cmd.Args().First()
// A path that does not exist is refused when it is enumerated.
info, err := mfa.Fs.Stat(arg)
if err == nil && !info.IsDir() {
return filepath.Join(filepath.Dir(arg), defaultManifestName), nil
}
return filepath.Join(arg, defaultManifestName), nil
}
// buildScannerOptions constructs scanner options from the CLI flags and
// the path the manifest is written to.
func (mfa *CLIApp) buildScannerOptions(
cmd *cli.Command, output string,
) *mfer.ScannerOptions {
opts := &mfer.ScannerOptions{ opts := &mfer.ScannerOptions{
IncludeDotfiles: cmd.Bool("include-dotfiles"), IncludeDotfiles: cmd.Bool("include-dotfiles"),
FollowSymLinks: cmd.Bool("follow-symlinks"), FollowSymLinks: cmd.Bool("follow-symlinks"),
@@ -202,23 +236,54 @@ func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
return nil return nil
} }
// runScanPhase reads the enumerated files and writes the manifest to out,
// with optional progress reporting.
func (mfa *CLIApp) runScanPhase(
ctx context.Context, cmd *cli.Command, s *mfer.Scanner, out io.Writer,
) error {
var (
scanProgress chan mfer.ScanStatus
scanWg sync.WaitGroup
)
if cmd.Bool("progress") {
scanProgress = make(chan mfer.ScanStatus, 1)
scanWg.Add(1)
go reportScanProgress(scanProgress, &scanWg)
}
err := s.ToManifest(ctx, out, scanProgress)
scanWg.Wait()
if err != nil {
return fmt.Errorf("generate manifest: %w", err)
}
return nil
}
func (mfa *CLIApp) generateManifestOperation( func (mfa *CLIApp) generateManifestOperation(
ctx context.Context, cmd *cli.Command, ctx context.Context, cmd *cli.Command,
) error { ) error {
log.Debug("generateManifestOperation()") log.Debug("generateManifestOperation()")
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd)) outputPath, err := mfa.outputPath(cmd)
// Phase 1: Enumeration - collect paths and stat files
err := mfa.runEnumeratePhase(cmd, s)
if err != nil { if err != nil {
return err return err
} }
showProgress := cmd.Bool("progress") s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd, outputPath))
// Phase 1: Enumeration - collect paths and stat files
err = mfa.runEnumeratePhase(cmd, s)
if err != nil {
return err
}
// Check if output file exists // Check if output file exists
outputPath := cmd.String("output")
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") { if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") {
return fmt.Errorf("output file %s %w", outputPath, errOutputExists) return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
} }
@@ -249,25 +314,9 @@ func (mfa *CLIApp) generateManifestOperation(
}() }()
// Phase 2: Scan - read file contents and generate manifest // Phase 2: Scan - read file contents and generate manifest
var ( err = mfa.runScanPhase(ctx, cmd, s, outFile)
scanProgress chan mfer.ScanStatus
scanWg sync.WaitGroup
)
if showProgress {
scanProgress = make(chan mfer.ScanStatus, 1)
scanWg.Add(1)
go reportScanProgress(scanProgress, &scanWg)
}
err = s.ToManifest(ctx, outFile, scanProgress)
scanWg.Wait()
if err != nil { if err != nil {
return fmt.Errorf("generate manifest: %w", err) return err
} }
// Close file before rename to ensure all data is flushed // Close file before rename to ensure all data is flushed
+3 -2
View File
@@ -211,9 +211,10 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "output", Name: "output",
Value: defaultManifestName,
Aliases: []string{"o"}, Aliases: []string{"o"},
Usage: "Specify output filename", Usage: "File to write the manifest to (default: index.mf in " +
"the directory given, or beside the file given; with no " +
"path or several, index.mf in the current directory)",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "force", Name: "force",
+15 -3
View File
@@ -160,7 +160,9 @@ func (s *Scanner) EnumeratePath(
return s.enumerateFS(afs, abs, progress) return s.enumerateFS(afs, abs, progress)
} }
// EnumeratePaths walks multiple directory paths and adds all files to the scanner. // EnumeratePaths adds to the scanner the files under each directory path,
// listed by their paths under it, and each file path, listed by its name
// as EnumerateFile lists it.
// If progress is non-nil, status updates are sent as files are discovered. // If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns. // The progress channel is closed when the method returns.
func (s *Scanner) EnumeratePaths( func (s *Scanner) EnumeratePaths(
@@ -177,9 +179,19 @@ func (s *Scanner) EnumeratePaths(
return err return err
} }
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs)) info, err := s.fs.Stat(abs)
if err != nil {
return err
}
if info.IsDir() {
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
err = s.enumerateFS(afs, abs, progress)
} else {
err = s.enumerateFileWithInfo(
filepath.Base(abs), filepath.Dir(abs), info, progress)
}
err = s.enumerateFS(afs, abs, progress)
if err != nil { if err != nil {
return err return err
} }
+22
View File
@@ -134,6 +134,28 @@ func TestScannerEnumeratePaths(t *testing.T) {
assert.Equal(t, FileCount(2), s.FileCount()) assert.Equal(t, FileCount(2), s.FileCount())
} }
// TestScannerEnumeratePathsFile gives EnumeratePaths a directory and a
// file: the file is listed by its name, as EnumerateFile lists it.
func TestScannerEnumeratePathsFile(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/dir/sub", 0o755))
require.NoError(t, fs.MkdirAll("/other", 0o755))
require.NoError(t, afero.WriteFile(fs, "/dir/sub/one.txt", []byte("1"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/other/two.txt", []byte("2"), 0o644))
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
require.NoError(t, s.EnumeratePaths(nil, "/dir", "/other/two.txt"))
paths := make([]RelFilePath, 0, s.FileCount())
for _, f := range s.Files() {
paths = append(paths, f.Path)
}
assert.Equal(t, []RelFilePath{"sub/one.txt", "two.txt"}, paths)
}
func TestScannerExcludeDotfiles(t *testing.T) { func TestScannerExcludeDotfiles(t *testing.T) {
t.Parallel() t.Parallel()