dwim: take the scan flags (closes #162) (#193)

internal/sfdupes/cancel_test.go

@@ -358,8 +358,9 @@ func TestRunScanInterrupted(t *testing.T) {
 	stderr := captureStderr(t)
 	dir := buildSmokeTree(t)
 
-	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
-		false, nil)
+	opts := scanOptions{workers: walkCancelWorkers}
+
+	err := runScan(t.Context(), []string{dir}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -382,8 +383,7 @@ func TestRunScanInterrupted(t *testing.T) {
 	go func() {
 		defer close(done)
 
-		err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
-			false, false, nil)
+		err = runScan(cancelledContext(t), []string{dir}, opts)
 	}()
 
 	awaitReturn(t, done, "runScan")
@@ -410,8 +410,7 @@ func TestRunScanInterrupted(t *testing.T) {
 			got, before)
 	}
 
-	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
-		false, nil)
+	err = runScan(t.Context(), []string{dir}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
 	dir := buildWalkCancelTree(t)
 
 	err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
-		walkCancelWorkers, false, false, nil)
+		scanOptions{workers: walkCancelWorkers})
 	if !errors.Is(err, errInterrupted) {
 		t.Fatalf("runScan interrupted mid-hash = %v, want %v",
 			err, errInterrupted)

internal/sfdupes/cli.go

@@ -88,10 +88,10 @@ value, separated by a tab:
   size of recorded files  their total size, in bytes, then rounded
 `
 
-	dwimHelp = `dwim runs what "sfdupes scan /" runs, then what "sfdupes trees" runs.
-Run it as root, so that the scan can read every file. The scan is stored
-in the database, as scan stores it, so "sfdupes trees" prints the report
-again later.
+	dwimHelp = `dwim runs what "sfdupes scan /" runs with the same flags, then what
+"sfdupes trees" runs. Run it as root, so that the scan can read every
+file. The scan is stored in the database, as scan stores it, so
+"sfdupes trees" prints the report again later.
 `
 )
 
@@ -100,12 +100,11 @@ again later.
 // cobra prints for it is the whole message.
 var errNoSubcommand = errors.New("no subcommand")
 
-// errWorkersBelowOne is the usage error for a scan --workers value
-// below 1.
+// errWorkersBelowOne is the usage error for a --workers value below 1.
 var errWorkersBelowOne = errors.New("--workers must be at least 1")
 
-// errEmptyExcludeValue is the usage error for an empty scan --exclude
-// value, as from --exclude "$DIR" with DIR unset. Taken as a path, it
+// errEmptyExcludeValue is the usage error for an empty --exclude value,
+// as from --exclude "$DIR" with DIR unset. Taken as a path, it
 // would name the working directory.
 var errEmptyExcludeValue = errors.New("--exclude must not be empty")
 
@@ -256,83 +255,116 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 		}),
 	}
 
-	dwimCmd := &cobra.Command{
-		Use:   cmdDwim,
-		Short: "Scan / with the default excludes, then print the duplicate-tree report",
-		Args:  cobra.NoArgs,
-		RunE: runE(func(ctx context.Context, _ []string) error {
-			return runDwim(ctx, "/", stdout)
-		}),
-	}
-
 	root.Long = root.Short + ".\n\n" + databaseHelp
 	reportCmd.Long = reportCmd.Short + ".\n\n" + reportHelp
 	treesCmd.Long = treesCmd.Short + ".\n\n" + treesHelp
 	infoCmd.Long = infoCmd.Short + ".\n\n" + infoHelp
-	dwimCmd.Long = dwimCmd.Short + ".\n\n" + dwimHelp
 
-	root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
+	root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd,
+		newDwimCommand(stdout))
 
 	return root
 }
 
-// newScanCommand builds the scan subcommand, with its --workers, -x,
-// --no-default-excludes and --exclude flags. Its help names the
+// scanOptions holds the values of the flags that scan and dwim share:
+// --workers, -x, --no-default-excludes and --exclude.
+type scanOptions struct {
+	workers           int
+	oneFS             bool
+	noDefaultExcludes bool
+	excludePaths      []string
+}
+
+// scanFlagsUsage names the flags that scan and dwim share, for their
+// usage lines.
+const scanFlagsUsage = "[--workers N] [-x] [--no-default-excludes] " +
+	"[--exclude PATH]..."
+
+// addScanFlags adds the flags that scan and dwim share to cmd, which
+// stores their values in opts.
+func addScanFlags(cmd *cobra.Command, opts *scanOptions) {
+	cmd.Flags().IntVar(&opts.workers, "workers", runtime.NumCPU(),
+		"concurrent workers for the walk and hash phases")
+	cmd.Flags().BoolVarP(&opts.oneFS, "one-file-system", "x", false,
+		"do not cross filesystem boundaries")
+	cmd.Flags().BoolVar(&opts.noDefaultExcludes, "no-default-excludes",
+		false, "enter the default excludes, the paths listed above")
+	// A string array, not a slice, so a comma in a path is not a
+	// separator.
+	cmd.Flags().StringArrayVar(&opts.excludePaths, "exclude", nil,
+		"never enter or record `PATH` or anything below it; repeatable")
+}
+
+// check rejects the flag values that are usage errors, before the
+// command runs. It is the PreRunE of scan and dwim.
+func (o *scanOptions) check(cmd *cobra.Command, _ []string) error {
+	err := checkScanWorkers(cmd, o.workers)
+	if err != nil {
+		return err
+	}
+
+	return rejectEmptyExcludeValue(cmd, o.excludePaths)
+}
+
+// newScanCommand builds the scan subcommand. Its help names the
 // database, and lists the default excludes of the system it runs on and
 // the filesystem types it never records anything on.
 func newScanCommand() *cobra.Command {
-	var (
-		scanWorkers           int
-		scanOneFS             bool
-		scanNoDefaultExcludes bool
-		scanExclude           []string
-	)
+	var opts scanOptions
 
 	short := "Walk trees and synchronize the scan database"
 
 	scanCmd := &cobra.Command{
-		Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] " +
-			"[--exclude PATH]... PATH...",
+		Use: cmdScan + " " + scanFlagsUsage + " PATH...",
 		// The Use line already names every flag, so cobra must not
 		// append its generic "[flags]" to it.
 		DisableFlagsInUseLine: true,
 		Short:                 short,
 		Long: short + ".\n\n" + databaseHelp + "\n" +
 			defaultExcludesHelp(runtime.GOOS),
-		Args: cobra.MinimumNArgs(1),
-		PreRunE: func(cmd *cobra.Command, _ []string) error {
-			err := checkScanWorkers(cmd, scanWorkers)
-			if err != nil {
-				return err
-			}
-
-			return rejectEmptyExcludeValue(cmd, scanExclude)
-		},
+		Args:    cobra.MinimumNArgs(1),
+		PreRunE: opts.check,
 		RunE: runE(func(ctx context.Context, args []string) error {
 			ctx, stop := interruptContext(ctx)
 			defer stop()
 
-			return runScan(ctx, args, scanWorkers, scanOneFS,
-				scanNoDefaultExcludes, scanExclude)
+			return runScan(ctx, args, opts)
 		}),
 	}
-	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
-		"concurrent workers for the walk and hash phases")
-	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
-		"do not cross filesystem boundaries")
-	scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
-		false, "enter the default excludes, the paths listed above")
-	// A string array, not a slice, so a comma in a path is not a
-	// separator.
-	scanCmd.Flags().StringArrayVar(&scanExclude, "exclude", nil,
-		"never enter or record `PATH` or anything below it; repeatable")
+	addScanFlags(scanCmd, &opts)
 
 	return scanCmd
 }
 
-// checkScanWorkers rejects a scan --workers value below 1. That is a
-// usage error reported in one line: cobra prints the returned message
-// without the usage text, and run exits 2.
+// newDwimCommand builds the dwim subcommand, which takes the scan flags
+// with the meaning they have for scan. Its help says what dwim runs,
+// then lists the default excludes and the filesystem types the scan
+// never records anything on.
+func newDwimCommand(stdout io.Writer) *cobra.Command {
+	var opts scanOptions
+
+	short := "Scan /, then print the duplicate-tree report"
+
+	dwimCmd := &cobra.Command{
+		Use:                   cmdDwim + " " + scanFlagsUsage,
+		DisableFlagsInUseLine: true,
+		Short:                 short,
+		Long: short + ".\n\n" + dwimHelp + "\n" +
+			defaultExcludesHelp(runtime.GOOS),
+		Args:    cobra.NoArgs,
+		PreRunE: opts.check,
+		RunE: runE(func(ctx context.Context, _ []string) error {
+			return runDwim(ctx, "/", opts, stdout)
+		}),
+	}
+	addScanFlags(dwimCmd, &opts)
+
+	return dwimCmd
+}
+
+// checkScanWorkers rejects a --workers value below 1. That is a usage
+// error reported in one line: cobra prints the returned message without
+// the usage text, and run exits 2.
 func checkScanWorkers(cmd *cobra.Command, workers int) error {
 	if workers >= 1 {
 		return nil
@@ -343,7 +375,7 @@ func checkScanWorkers(cmd *cobra.Command, workers int) error {
 	return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
 }
 
-// rejectEmptyExcludeValue rejects an empty scan --exclude value. That
+// rejectEmptyExcludeValue rejects an empty --exclude value. That
 // is a usage error reported in one line, as for --workers below 1.
 func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
 	if !slices.Contains(excludePaths, "") {

internal/sfdupes/cli_test.go

@@ -25,6 +25,9 @@ const usageMarker = "Usage:"
 // helpFlag asks for the help text.
 const helpFlag = "--help"
 
+// workersFlag is the scan and dwim flag that sets the worker count.
+const workersFlag = "--workers"
+
 // TestMain stops the tests at once, saying why, when their temporary
 // directory is not on a persistent filesystem: scan records nothing
 // there (README "Rules for the walk"), so most scan tests would fail
@@ -573,8 +576,8 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
 	}{
 		cmdScan: {
 			run: func() error {
-				return runScan(t.Context(), []string{missing}, 1, false, false,
-					nil)
+				return runScan(t.Context(), []string{missing},
+					scanOptions{workers: 1})
 			},
 			want: scanWant,
 		},
@@ -665,7 +668,7 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 
 			var stdout, stderr bytes.Buffer
 
-			args := []string{cmdScan, "--workers", workers, t.TempDir()}
+			args := []string{cmdScan, workersFlag, workers, t.TempDir()}
 
 			code := run(args, &stdout, &stderr)
 			if code != exitUsage {

internal/sfdupes/dwim.go

@@ -5,20 +5,20 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"runtime"
 )
 
 // runDwim implements the dwim subcommand: it runs what "sfdupes scan
-// root" runs with no flags, then what "sfdupes trees" runs, and ends
-// with a line on stderr saying how to print the report again. The dwim
-// command passes "/" as root. It catches SIGINT and SIGTERM as scan
-// does, but only during the scan. An error from the scan, an interrupt
-// included, ends it before the report. During the report, a signal
-// ends it at once, as it ends trees.
-func runDwim(ctx context.Context, root string, stdout io.Writer) error {
+// root" runs with the scan flags in opts, then what "sfdupes trees"
+// runs, and ends with a line on stderr saying how to print the report
+// again. The dwim command passes "/" as root. It catches SIGINT and
+// SIGTERM as scan does, but only during the scan. An error from the
+// scan, an interrupt included, ends it before the report. During the
+// report, a signal ends it at once, as it ends trees.
+func runDwim(
+	ctx context.Context, root string, opts scanOptions, stdout io.Writer,
+) error {
 	scanCtx, stop := interruptContext(ctx)
-	err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false,
-		nil)
+	err := runScan(scanCtx, []string{root}, opts)
 
 	stop()
 

internal/sfdupes/dwim_test.go

@@ -2,7 +2,10 @@ package sfdupes //nolint:testpackage // calls the unexported runDwim
 
 import (
 	"bytes"
+	"errors"
+	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 )
@@ -28,7 +31,7 @@ func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
 
 	stderr := captureStderr(t)
 
-	err := runDwim(t.Context(), root, &stdout)
+	err := runDwim(t.Context(), root, scanOptions{workers: 1}, &stdout)
 	if err != nil {
 		t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
 	}
@@ -67,7 +70,7 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
 
 	stderr := captureStderr(t)
 
-	err := runDwim(t.Context(), missing, &stdout)
+	err := runDwim(t.Context(), missing, scanOptions{workers: 1}, &stdout)
 	if err == nil {
 		t.Fatal("runDwim on a missing root: error = nil, want it fatal")
 	}
@@ -81,3 +84,127 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
 		t.Errorf("stderr = %q, want nothing from trees or dwim", got)
 	}
 }
+
+func TestRunDwimExcludeLeavesTreeOut(t *testing.T) {
+	// dwim passes its --exclude paths to the scan: an excluded copy is
+	// not recorded, so its pair is not reported.
+	t.Setenv(databaseEnv, testDBPath(t))
+
+	root := realTempDir(t)
+
+	kept := []string{
+		filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
+		filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
+	}
+	writeFile(t, root, "b1/f.bin", pattern(2, 5000))
+	excluded := filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000)))
+
+	var stdout bytes.Buffer
+
+	stderr := captureStderr(t)
+
+	opts := scanOptions{workers: 1, excludePaths: []string{excluded}}
+
+	err := runDwim(t.Context(), root, opts, &stdout)
+	if err != nil {
+		t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
+	}
+
+	lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("stdout = %q, want a header and one group",
+			stdout.String())
+	}
+
+	want := kept[0] + "\t" + kept[1] + "\t1\t300\t"
+	if !strings.HasPrefix(lines[1], want) {
+		t.Errorf("group = %q, want it to start %q", lines[1], want)
+	}
+
+	skipped := "scan: skipped " + excluded + " (--exclude)\n"
+	if got := stderr(); !strings.Contains(got, skipped) {
+		t.Errorf("stderr = %q, want it to contain %q", got, skipped)
+	}
+}
+
+func TestRunDwimTakesScanFlags(t *testing.T) {
+	// The dwim command parses and checks the scan flags as scan does.
+	// Each case stops before the walk of / begins, and before the
+	// database is created. "dwim --exclude /" excludes its own operand.
+	cases := map[string]struct {
+		args []string
+		code int
+		want string
+	}{
+		"--exclude /": {
+			[]string{cmdDwim, excludeFlag, "/"},
+			exitFatal,
+			"sfdupes: scan /: operand excluded by --exclude /\n",
+		},
+		"--exclude empty": {
+			[]string{cmdDwim, excludeFlag, ""},
+			exitUsage,
+			"Error: --exclude must not be empty\n",
+		},
+		"--workers 0": {
+			[]string{cmdDwim, workersFlag, "0", "-x", "--no-default-excludes"},
+			exitUsage,
+			"Error: --workers must be at least 1, got 0\n",
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			path := testDBPath(t)
+			t.Setenv(databaseEnv, path)
+
+			stdout := captureStdout(t)
+			stderr := captureStderr(t)
+
+			code := run(tc.args, os.Stdout, os.Stderr)
+			if code != tc.code {
+				t.Errorf("run(%q) = %d, want %d", tc.args, code, tc.code)
+			}
+
+			if !strings.HasSuffix(stderr(), tc.want) || stdout() != "" {
+				t.Errorf("stdout, stderr = %q, %q, want \"\" and stderr "+
+					"ending %q", stdout(), stderr(), tc.want)
+			}
+
+			_, err := os.Lstat(path)
+			if !errors.Is(err, os.ErrNotExist) {
+				t.Errorf("lstat %s: %v, want no database", path, err)
+			}
+		})
+	}
+}
+
+func TestRunDwimHelpListsScanFlags(t *testing.T) {
+	t.Parallel()
+
+	// README "Subcommands" shows this usage line for dwim.
+	const want = "\n  sfdupes dwim [--workers N] [-x] [--no-default-excludes] " +
+		"[--exclude PATH]...\n"
+
+	var stdout, stderr bytes.Buffer
+
+	run([]string{cmdDwim, "-h"}, &stdout, &stderr)
+
+	if !strings.Contains(stderr.String(), want) {
+		t.Errorf("dwim -h stderr = %q, want the usage line %q",
+			stderr.String(), want)
+	}
+
+	if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
+		t.Errorf("dwim -h stderr = %q, want %s PATH", stderr.String(),
+			excludeFlag)
+	}
+
+	// README "Settings": the help lists the default excludes and the
+	// filesystem types the walk never records anything on.
+	help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
+	if !strings.Contains(stderr.String(), help) {
+		t.Errorf("dwim -h stderr = %q, want it to hold %q", stderr.String(),
+			help)
+	}
+}

internal/sfdupes/exclude.go

@@ -254,7 +254,7 @@ func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
 	return devices, lines.Err()
 }
 
-// defaultExcludesHelp describes, for the scan command's help, the
+// defaultExcludesHelp describes, for the scan and dwim help, the
 // default excludes of the system goos names and, where the scan lists
 // the mounted filesystems, the types it never records anything on.
 func defaultExcludesHelp(goos string) string {

internal/sfdupes/scan.go

@@ -98,17 +98,16 @@ func scanPhase(name string) string {
 // runs, and the lock is released after it. When ctx is cancelled, as by
 // the SIGINT or SIGTERM that interruptContext catches, the scan keeps
 // what it has hashed (see syncScan), prints how many files its walk
-// reached, and returns errInterrupted. workers must be at least 1; the
-// scan command rejects anything less. Before the lock is taken, the
-// scan lists the filesystems that are not persistent, on which the walk
-// records nothing; unless noDefaultExcludes is set, the walk does not
-// enter the default excludes either. The walk neither enters nor
-// records the paths in excludePaths, the --exclude values; an operand
-// at or below one is a fatal error. Once the database is open, the scan
-// notes its files, which the walk passes over.
-func runScan(ctx context.Context, roots []string, workers int,
-	oneFS, noDefaultExcludes bool, excludePaths []string,
-) error {
+// reached, and returns errInterrupted. opts.workers must be at least 1;
+// the scan and dwim commands reject anything less. Before the lock is
+// taken, the scan lists the filesystems that are not persistent, on
+// which the walk records nothing; unless opts.noDefaultExcludes is set,
+// the walk does not enter the default excludes either. The walk neither
+// enters nor records the paths in opts.excludePaths, the --exclude
+// values; an operand at or below one is a fatal error. Once the
+// database is open, the scan notes its files, which the walk passes
+// over.
+func runScan(ctx context.Context, roots []string, opts scanOptions) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
 		return err
@@ -123,7 +122,7 @@ func runScan(ctx context.Context, roots []string, workers int,
 
 	var rules walkRules
 
-	rules.excludePaths, err = excludePathInodes(excludePaths)
+	rules.excludePaths, err = excludePathInodes(opts.excludePaths)
 	if err != nil {
 		return fmt.Errorf("--exclude: %w", err)
 	}
@@ -138,7 +137,7 @@ func runScan(ctx context.Context, roots []string, workers int,
 		return fmt.Errorf("list the mounted filesystems: %w", err)
 	}
 
-	if !noDefaultExcludes {
+	if !opts.noDefaultExcludes {
 		rules.defaultExcludes = defaultExcludes(runtime.GOOS)
 	}
 
@@ -167,7 +166,7 @@ func runScan(ctx context.Context, roots []string, workers int,
 		return err
 	}
 
-	st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
+	st, err := syncScan(ctx, db, roots, opts.workers, opts.oneFS, rules)
 	if errors.Is(err, context.Canceled) {
 		return interrupted(st.walked)
 	}

internal/sfdupes/scan_test.go

@@ -2196,7 +2196,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) {
 	var stderr bytes.Buffer
 
 	code := run([]string{
-		cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir,
+		cmdScan, workersFlag, strconv.Itoa(hashLeakWorkers), dir,
 	}, io.Discard, &stderr)
 	if code != exitFatal {
 		t.Fatalf("run(scan) = %d, want %d; stderr: %s",