diff --git a/README.md b/README.md index 12a4215..4daf28f 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,15 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues). - recurses under current directory and writes out an `index.mf` - records every file's mode as `0000` unless given `--include-permissions`, which records each file's permission bits (`0777` at most) +- `mfer gen /media/drive` + - writes `/media/drive/index.mf`, listing each file by its path under + `/media/drive`, so `mfer check /media/drive` verifies it. Given a file, + gen writes `index.mf` beside it and lists the file by its name; given + several paths, it writes `index.mf` in the current directory + - `--output` names another file to write instead. What gen lists depends + only on the paths it is given and the files under them, so with the same + `--seed` and an unchanged tree it writes the same bytes wherever the + manifest goes. The file it writes to is never listed - `mfer check` / `mfer check .` - verifies checksums of all files in manifest, displaying error and exiting nonzero if any files are missing or corrupted, or have permission bits diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index c54489f..b27ea7e 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -1289,6 +1289,180 @@ func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) { assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output)) } +// writeTestTree writes file.txt and sub/nested.txt under dir. +func writeTestTree(t *testing.T, fs afero.Fs, dir string) { + t.Helper() + + require.NoError(t, fs.MkdirAll(filepath.Join(dir, testSubdir), 0o750)) + writeTestFile(t, fs, filepath.Join(dir, testFileTxt), "hello") + writeTestFile(t, fs, filepath.Join(dir, testSubdir, "nested.txt"), "in sub") +} + +// TestGenerateDefaultOutput runs gen without --output on one directory or +// one file: it writes index.mf in that directory, or beside that file, +// listing each file by its path under the directory index.mf is in, and +// check given that directory passes. +func TestGenerateDefaultOutput(t *testing.T) { + t.Parallel() + + // Paths are relative to a temp dir holding file.txt and sub/nested.txt. + for name, tc := range map[string]struct { + input, output string + listed []string + }{ + "directory": { + ".", defaultManifestName, []string{testFileTxt, "sub/nested.txt"}, + }, + "subdirectory": {testSubdir, testSubdirManifest, []string{"nested.txt"}}, + "file": {testFileTxt, defaultManifestName, []string{testFileTxt}}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + fs := afero.NewOsFs() + writeTestTree(t, fs, root) + + opts := testOpts([]string{ + testApp, cmdGenerate, "-q", filepath.Join(root, tc.input), + }, fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + output := filepath.Join(root, tc.output) + assert.ElementsMatch(t, tc.listed, manifestPaths(t, fs, output)) + + opts = testOpts([]string{ + testApp, cmdCheck, "-q", filepath.Dir(output), + }, fs) + assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + }) + } +} + +// TestGenerateSeveralPathsDefaultOutput runs gen without --output on two +// directories: it writes index.mf in the current directory, listing both +// directories' files, and writes no index.mf in either directory. +// +//nolint:paralleltest // changes the process-global working directory +func TestGenerateSeveralPathsDefaultOutput(t *testing.T) { + root := t.TempDir() + fs := afero.NewOsFs() + dirs := []string{"first", "second"} + + for _, dir := range dirs { + require.NoError(t, fs.MkdirAll(filepath.Join(root, dir), 0o750)) + writeTestFile(t, fs, filepath.Join(root, dir, dir+".txt"), dir) + } + + t.Chdir(root) + + opts := testOpts(append([]string{testApp, cmdGenerate, "-q"}, dirs...), fs) + require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts)) + + assert.ElementsMatch(t, []string{"first.txt", "second.txt"}, + manifestPaths(t, fs, filepath.Join(root, defaultManifestName))) + + for _, dir := range dirs { + exists, err := afero.Exists(fs, filepath.Join(root, dir, defaultManifestName)) + require.NoError(t, err) + assert.False(t, exists, "index.mf written in %s", dir) + } +} + +// TestGenerateBytesDoNotDependOnOutput runs gen --seed on one tree, each +// time writing to another file, over a file already there and beside an +// earlier run's temp file: neither is listed, and what is listed depends +// only on the tree, so every manifest has the same bytes. +func TestGenerateBytesDoNotDependOnOutput(t *testing.T) { + t.Parallel() + + root := t.TempDir() + tree := filepath.Join(root, "tree") + defaultOutput := filepath.Join(tree, defaultManifestName) + + fs := afero.NewOsFs() + writeTestTree(t, fs, tree) + + var first []byte + + for _, output := range []string{ + defaultOutput, + filepath.Join(tree, "listing.mf"), + filepath.Join(tree, testSubdir, "listing.mf"), + filepath.Join(root, "outside.mf"), + } { + writeTestFile(t, fs, output, "previous manifest") + writeTestFile(t, fs, manifestTempPath(output), "part of a manifest") + + args := []string{testApp, cmdGenerate, "-q", "-f", "--seed", "mfer"} + if output != defaultOutput { + args = append(args, "-o", output) + } + + args = append(args, tree) + + opts := testOpts(args, fs) + require.Equal(t, 0, runCLI(opts), + "output %s, stderr: %s", output, testStderr(t, opts)) + + got, err := afero.ReadFile(fs, output) + require.NoError(t, err) + require.NoError(t, fs.Remove(output)) + + if first == nil { + first = got + } + + assert.Equal(t, first, got, "output %s", output) + } +} + +// TestGenerateRefusesExistingDefaultOutput runs gen without --output or +// --force on a directory already holding index.mf: gen fails, naming that +// file, and leaves it as it was. +func TestGenerateRefusesExistingDefaultOutput(t *testing.T) { + t.Parallel() + + output := filepath.Join(testDir, defaultManifestName) + + fs := afero.NewMemMapFs() + require.NoError(t, fs.MkdirAll(testDir, 0o755)) + writeTestFile(t, fs, testFile1, "hello") + writeTestFile(t, fs, output, "previous manifest") + + opts := testOpts([]string{testApp, cmdGenerate, "-q", testDir}, fs) + assert.Equal(t, 1, runCLI(opts)) + assert.Contains(t, testStderr(t, opts), + "output file "+output+" already exists (use --force to overwrite)") + + content, err := afero.ReadFile(fs, output) + require.NoError(t, err) + assert.Equal(t, "previous manifest", string(content)) +} + +// TestGenerateRefusesEmptyOutput runs gen with --force and an --output +// given an empty value, as an unset shell variable gives it, on a +// directory already holding index.mf: gen fails and leaves that file as it +// was. +func TestGenerateRefusesEmptyOutput(t *testing.T) { + t.Parallel() + + output := filepath.Join(testDir, defaultManifestName) + + fs := afero.NewMemMapFs() + require.NoError(t, fs.MkdirAll(testDir, 0o755)) + writeTestFile(t, fs, testFile1, "hello") + writeTestFile(t, fs, output, "previous manifest") + + opts := testOpts([]string{testApp, cmdGenerate, "-q", "-f", "-o", "", testDir}, fs) + assert.Equal(t, 1, runCLI(opts)) + assert.Contains(t, testStderr(t, opts), errEmptyOutput.Error()) + + content, err := afero.ReadFile(fs, output) + require.NoError(t, err) + assert.Equal(t, "previous manifest", string(content)) +} + func TestGenerateAtomicWriteUsesTemp(t *testing.T) { t.Parallel() diff --git a/internal/cli/gen.go b/internal/cli/gen.go index b32170f..4ea3e36 100644 --- a/internal/cli/gen.go +++ b/internal/cli/gen.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "os/signal" "path/filepath" @@ -26,6 +27,8 @@ var ( // rendered message stays exactly as mfer has always printed it. errOutputExists = errors.New( "already exists (use --force to overwrite)") + // errEmptyOutput indicates --output given with an empty value. + errEmptyOutput = errors.New("--output must not be empty") ) // reportEnumProgress renders enumeration progress until the channel @@ -88,9 +91,40 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) { return paths, nil } -// buildScannerOptions constructs scanner options from the CLI flags. -func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions { - output := cmd.String("output") +// outputPath returns the file gen writes the manifest to: the one --output +// names, or else index.mf in the directory the only argument names, or +// beside the file it names, or else in the current directory. An --output +// given with an empty value is refused. +func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) { + if cmd.IsSet("output") { + output := cmd.String("output") + if output == "" { + return "", errEmptyOutput + } + + return output, nil + } + + if cmd.Args().Len() != 1 { + return defaultManifestName, nil + } + + arg := cmd.Args().First() + + // A path that does not exist is refused when it is enumerated. + info, err := mfa.Fs.Stat(arg) + if err == nil && !info.IsDir() { + return filepath.Join(filepath.Dir(arg), defaultManifestName), nil + } + + return filepath.Join(arg, defaultManifestName), nil +} + +// buildScannerOptions constructs scanner options from the CLI flags and +// the path the manifest is written to. +func (mfa *CLIApp) buildScannerOptions( + cmd *cli.Command, output string, +) *mfer.ScannerOptions { opts := &mfer.ScannerOptions{ IncludeDotfiles: cmd.Bool("include-dotfiles"), FollowSymLinks: cmd.Bool("follow-symlinks"), @@ -202,23 +236,54 @@ func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error { return nil } +// runScanPhase reads the enumerated files and writes the manifest to out, +// with optional progress reporting. +func (mfa *CLIApp) runScanPhase( + ctx context.Context, cmd *cli.Command, s *mfer.Scanner, out io.Writer, +) error { + var ( + scanProgress chan mfer.ScanStatus + scanWg sync.WaitGroup + ) + + if cmd.Bool("progress") { + scanProgress = make(chan mfer.ScanStatus, 1) + + scanWg.Add(1) + + go reportScanProgress(scanProgress, &scanWg) + } + + err := s.ToManifest(ctx, out, scanProgress) + + scanWg.Wait() + + if err != nil { + return fmt.Errorf("generate manifest: %w", err) + } + + return nil +} + func (mfa *CLIApp) generateManifestOperation( ctx context.Context, cmd *cli.Command, ) error { log.Debug("generateManifestOperation()") - s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd)) - - // Phase 1: Enumeration - collect paths and stat files - err := mfa.runEnumeratePhase(cmd, s) + outputPath, err := mfa.outputPath(cmd) if err != nil { return err } - showProgress := cmd.Bool("progress") + s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd, outputPath)) + + // Phase 1: Enumeration - collect paths and stat files + err = mfa.runEnumeratePhase(cmd, s) + if err != nil { + return err + } // Check if output file exists - outputPath := cmd.String("output") if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") { return fmt.Errorf("output file %s %w", outputPath, errOutputExists) } @@ -249,25 +314,9 @@ func (mfa *CLIApp) generateManifestOperation( }() // Phase 2: Scan - read file contents and generate manifest - var ( - scanProgress chan mfer.ScanStatus - scanWg sync.WaitGroup - ) - - if showProgress { - scanProgress = make(chan mfer.ScanStatus, 1) - - scanWg.Add(1) - - go reportScanProgress(scanProgress, &scanWg) - } - - err = s.ToManifest(ctx, outFile, scanProgress) - - scanWg.Wait() - + err = mfa.runScanPhase(ctx, cmd, s, outFile) if err != nil { - return fmt.Errorf("generate manifest: %w", err) + return err } // Close file before rename to ensure all data is flushed diff --git a/internal/cli/mfer.go b/internal/cli/mfer.go index 1b8d83d..a19db67 100644 --- a/internal/cli/mfer.go +++ b/internal/cli/mfer.go @@ -211,9 +211,10 @@ func (mfa *CLIApp) generateCommand() *cli.Command { }, &cli.StringFlag{ Name: "output", - Value: defaultManifestName, Aliases: []string{"o"}, - Usage: "Specify output filename", + Usage: "File to write the manifest to (default: index.mf in " + + "the directory given, or beside the file given; with no " + + "path or several, index.mf in the current directory)", }, &cli.BoolFlag{ Name: "force", diff --git a/mfer/scanner.go b/mfer/scanner.go index 92b5538..058c227 100644 --- a/mfer/scanner.go +++ b/mfer/scanner.go @@ -160,7 +160,9 @@ func (s *Scanner) EnumeratePath( return s.enumerateFS(afs, abs, progress) } -// EnumeratePaths walks multiple directory paths and adds all files to the scanner. +// EnumeratePaths adds to the scanner the files under each directory path, +// listed by their paths under it, and each file path, listed by its name +// as EnumerateFile lists it. // If progress is non-nil, status updates are sent as files are discovered. // The progress channel is closed when the method returns. func (s *Scanner) EnumeratePaths( @@ -177,9 +179,19 @@ func (s *Scanner) EnumeratePaths( return err } - afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs)) + info, err := s.fs.Stat(abs) + if err != nil { + return err + } + + if info.IsDir() { + afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs)) + err = s.enumerateFS(afs, abs, progress) + } else { + err = s.enumerateFileWithInfo( + filepath.Base(abs), filepath.Dir(abs), info, progress) + } - err = s.enumerateFS(afs, abs, progress) if err != nil { return err } diff --git a/mfer/scanner_test.go b/mfer/scanner_test.go index 75b4b5b..1ceb6b7 100644 --- a/mfer/scanner_test.go +++ b/mfer/scanner_test.go @@ -134,6 +134,28 @@ func TestScannerEnumeratePaths(t *testing.T) { assert.Equal(t, FileCount(2), s.FileCount()) } +// TestScannerEnumeratePathsFile gives EnumeratePaths a directory and a +// file: the file is listed by its name, as EnumerateFile lists it. +func TestScannerEnumeratePathsFile(t *testing.T) { + t.Parallel() + + fs := afero.NewMemMapFs() + require.NoError(t, fs.MkdirAll("/dir/sub", 0o755)) + require.NoError(t, fs.MkdirAll("/other", 0o755)) + require.NoError(t, afero.WriteFile(fs, "/dir/sub/one.txt", []byte("1"), 0o644)) + require.NoError(t, afero.WriteFile(fs, "/other/two.txt", []byte("2"), 0o644)) + + s := NewScannerWithOptions(&ScannerOptions{Fs: fs}) + require.NoError(t, s.EnumeratePaths(nil, "/dir", "/other/two.txt")) + + paths := make([]RelFilePath, 0, s.FileCount()) + for _, f := range s.Files() { + paths = append(paths, f.Path) + } + + assert.Equal(t, []RelFilePath{"sub/one.txt", "two.txt"}, paths) +} + func TestScannerExcludeDotfiles(t *testing.T) { t.Parallel()