Add scan --exclude PATH (closes #163)

The walk neither enters nor records an --exclude path or anything
below it, and prints "scan: skipped PATH (--exclude)" for each one it
reaches. The flag repeats. Each value is resolved to its real path, as
an operand is. The records below an excluded path are kept, as for a
default exclude. An --exclude path that does not exist, or an operand
at or below one, is a fatal error before the scan starts.

Model: opus-5-5

internal/sfdupes/cancel_test.go

@@ -359,7 +359,7 @@ func TestRunScanInterrupted(t *testing.T) {
 	dir := buildSmokeTree(t)
 
 	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
-		false)
+		false, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -383,7 +383,7 @@ func TestRunScanInterrupted(t *testing.T) {
 		defer close(done)
 
 		err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
-			false, false)
+			false, false, nil)
 	}()
 
 	awaitReturn(t, done, "runScan")
@@ -411,7 +411,7 @@ func TestRunScanInterrupted(t *testing.T) {
 	}
 
 	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
-		false)
+		false, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,7 +437,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
 	dir := buildWalkCancelTree(t)
 
 	err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
-		walkCancelWorkers, false, false)
+		walkCancelWorkers, false, false, nil)
 	if !errors.Is(err, errInterrupted) {
 		t.Fatalf("runScan interrupted mid-hash = %v, want %v",
 			err, errInterrupted)

internal/sfdupes/cli.go

@@ -201,21 +201,23 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 	return root
 }
 
-// newScanCommand builds the scan subcommand, with its --workers, -x and
-// --no-default-excludes flags. Its help lists the default excludes of
-// the system it runs on, and the filesystem types it never records
-// anything on.
+// newScanCommand builds the scan subcommand, with its --workers, -x,
+// --no-default-excludes and --exclude flags. Its help lists the default
+// excludes of the system it runs on, and the filesystem types it never
+// records anything on.
 func newScanCommand() *cobra.Command {
 	var (
 		scanWorkers           int
 		scanOneFS             bool
 		scanNoDefaultExcludes bool
+		scanExclude           []string
 	)
 
 	short := "Walk trees and synchronize the scan database"
 
 	scanCmd := &cobra.Command{
-		Use:   cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
+		Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] " +
+			"[--exclude PATH]... PATH...",
 		Short: short,
 		Long:  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
 		Args:  cobra.MinimumNArgs(1),
@@ -227,7 +229,7 @@ func newScanCommand() *cobra.Command {
 			defer stop()
 
 			return runScan(ctx, args, scanWorkers, scanOneFS,
-				scanNoDefaultExcludes)
+				scanNoDefaultExcludes, scanExclude)
 		}),
 	}
 	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
@@ -236,6 +238,10 @@ func newScanCommand() *cobra.Command {
 		"do not cross filesystem boundaries")
 	scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
 		false, "enter the default excludes, the paths listed above")
+	// A string array, not a slice, so a comma in a path is not a
+	// separator.
+	scanCmd.Flags().StringArrayVar(&scanExclude, "exclude", nil,
+		"never enter or record `PATH` or anything below it; repeatable")
 
 	return scanCmd
 }

internal/sfdupes/cli_test.go

@@ -570,7 +570,8 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
 	}{
 		cmdScan: {
 			run: func() error {
-				return runScan(t.Context(), []string{missing}, 1, false, false)
+				return runScan(t.Context(), []string{missing}, 1, false, false,
+					nil)
 			},
 			want: scanWant,
 		},
@@ -712,6 +713,16 @@ func TestRunHelp(t *testing.T) {
 				args, got)
 		}
 	}
+
+	// The scan help lists --exclude with its argument.
+	var stdout, stderr bytes.Buffer
+
+	run([]string{cmdScan, "-h"}, &stdout, &stderr)
+
+	if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
+		t.Errorf("scan -h stderr = %q, want %s PATH", stderr.String(),
+			excludeFlag)
+	}
 }
 
 func TestRunVersion(t *testing.T) {

internal/sfdupes/dwim.go

@@ -17,7 +17,8 @@ import (
 // ends it at once, as it ends trees.
 func runDwim(ctx context.Context, root string, stdout io.Writer) error {
 	scanCtx, stop := interruptContext(ctx)
-	err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false)
+	err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false,
+		nil)
 
 	stop()
 

internal/sfdupes/exclude.go

@@ -1,8 +1,9 @@
 package sfdupes
 
-// This file holds two rules for the walk (README "Rules for the walk"):
+// This file holds three rules for the walk (README "Rules for the
+// walk"): the paths given with --exclude, which the scan never enters;
 // the default excludes, the paths a scan's walk does not enter below an
-// operand unless scan is given --no-default-excludes, and the
+// operand unless scan is given --no-default-excludes; and the
 // filesystems that are not persistent, on which the scan never records
 // or reads anything.
 
@@ -43,14 +44,20 @@ const typesPerHelpLine = 6
 // readMountInfo reads.
 var errMountInfoLine = errors.New("unexpected line")
 
+// errExcludedOperand reports an operand at or below a path given with
+// --exclude.
+var errExcludedOperand = errors.New("operand excluded by --exclude")
+
 // walkRules holds the rules for the walk that a scan sets up as it
-// starts. excludes are the default excludes, or none with
-// --no-default-excludes. nonPersistent maps the device ID of every
-// mounted filesystem that is not persistent to its type; the scan
-// records and reads nothing on one of those, whatever the flags.
+// starts. excluded are the real paths given with --exclude, which the
+// walk neither enters nor records. excludes are the default excludes,
+// or none with --no-default-excludes. nonPersistent maps the device ID
+// of every mounted filesystem that is not persistent to its type; the
+// scan records and reads nothing on one of those, whatever the flags.
 // databaseFiles are the scan's own database files, which the walk
 // passes over (see databaseFiles in db.go).
 type walkRules struct {
+	excluded      []string
 	excludes      []string
 	nonPersistent map[uint64]string
 	databaseFiles []fileRec
@@ -97,6 +104,36 @@ func isDefaultExclude(path string, excludes []string) bool {
 	return false
 }
 
+// resolveExcluded returns the real paths of excluded, the --exclude
+// values. One that cannot be resolved is a fatal error, and so is one
+// of roots at or below one of them. roots are real paths already.
+func resolveExcluded(roots, excluded []string) ([]string, error) {
+	excluded, err := resolveRoots(excluded)
+	if err != nil {
+		return nil, fmt.Errorf("--exclude: %w", err)
+	}
+
+	for _, root := range roots {
+		exclude := deepestRoot(root, excluded)
+		if exclude != "" {
+			return nil, fmt.Errorf("scan %s: %w %s", root,
+				errExcludedOperand, exclude)
+		}
+	}
+
+	return excluded, nil
+}
+
+// excludedLine returns the line the scan prints for path when it is one
+// of rules.excluded, and "" when it is not.
+func (rules walkRules) excludedLine(path string) string {
+	if !slices.Contains(rules.excluded, path) {
+		return ""
+	}
+
+	return fmt.Sprintf("scan: skipped %s (--exclude)", path)
+}
+
 // nonPersistentFilesystemTypes returns the filesystem types that are
 // not persistent: filesystems that the kernel makes up or that are kept
 // in memory. devfs is the macOS filesystem of /dev; the others are

internal/sfdupes/exclude_test.go

@@ -14,6 +14,9 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+// excludeFlag is the scan flag that names a path the walk never enters.
+const excludeFlag = "--exclude"
+
 func TestDefaultExcludes(t *testing.T) {
 	t.Parallel()
 
@@ -356,6 +359,121 @@ func TestRunScanNoDefaultExcludes(t *testing.T) {
 	assertRecorded(t, path, f, true)
 }
 
+// TestRunScanExcludeNested scans a tree through the command with an
+// --exclude path two levels below the operand. The walk records nothing
+// at or below it and says so. The record an earlier scan left there is
+// kept although its file is gone.
+func TestRunScanExcludeNested(t *testing.T) {
+	path := testDBPath(t)
+	t.Setenv(databaseEnv, path)
+
+	dir := realTempDir(t)
+	excluded := filepath.Join(dir, "a", "b")
+	kept := writeFile(t, dir, "a/f", pattern(1, 10))
+	gone := writeFile(t, excluded, "gone", pattern(2, 10))
+
+	scanOK(t, dir)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unseen := writeFile(t, excluded, "c/new", pattern(3, 10))
+
+	stderr := scanOK(t, excludeFlag, excluded, dir)
+
+	line := "scan: skipped " + excluded + " (--exclude)\n"
+	if !strings.Contains(stderr, line) {
+		t.Errorf("stderr = %q, want %q", stderr, line)
+	}
+
+	assertRecorded(t, path, kept, true)
+	assertRecorded(t, path, gone, true)
+	assertRecorded(t, path, unseen, false)
+}
+
+// TestRunScanExcludeRepeated gives --exclude twice, once for a
+// directory and once for a file, the file through a symlink to it. The
+// walk records neither, and records the file beside them.
+func TestRunScanExcludeRepeated(t *testing.T) {
+	path := testDBPath(t)
+	t.Setenv(databaseEnv, path)
+
+	dir := realTempDir(t)
+	inDir := writeFile(t, dir, "skip/f", pattern(1, 10))
+	file := writeFile(t, dir, "skip-file", pattern(2, 10))
+	kept := writeFile(t, dir, "keep", pattern(3, 10))
+
+	link := filepath.Join(t.TempDir(), "link")
+	symlink(t, file, link)
+
+	scanOK(t, excludeFlag, filepath.Dir(inDir), excludeFlag, link, dir)
+
+	assertRecorded(t, path, inDir, false)
+	assertRecorded(t, path, file, false)
+	assertRecorded(t, path, kept, true)
+}
+
+// TestRunScanExcludedOperand checks that an operand at or below an
+// --exclude path is a fatal error that names both, and so is an
+// --exclude path that does not exist. Neither scan writes a database.
+func TestRunScanExcludedOperand(t *testing.T) {
+	dir := realTempDir(t)
+	sub := filepath.Join(dir, "sub")
+	writeFile(t, sub, "f", pattern(1, 10))
+
+	cases := map[string]struct {
+		args []string
+		want string
+	}{
+		"at it": {
+			[]string{excludeFlag, sub, sub},
+			"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
+		},
+		"below it": {
+			[]string{excludeFlag, dir, sub},
+			"scan " + sub + ": operand excluded by --exclude " + dir + "\n",
+		},
+		"after an operand it does not exclude": {
+			[]string{dir, excludeFlag, sub, sub},
+			"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
+		},
+		"exclude missing": {
+			[]string{excludeFlag, filepath.Join(dir, "nope"), dir},
+			"--exclude: resolve " + filepath.Join(dir, "nope") + ": ",
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			path := testDBPath(t)
+			t.Setenv(databaseEnv, path)
+
+			stdout := captureStdout(t)
+			stderr := captureStderr(t)
+
+			code := run(append([]string{cmdScan}, tc.args...), os.Stdout,
+				os.Stderr)
+			if code != exitFatal {
+				t.Errorf("run(scan %q) = %d, want %d", tc.args, code,
+					exitFatal)
+			}
+
+			assertFatalOutput(t, stderr(), stdout())
+
+			if !strings.Contains(stderr(), "sfdupes: "+tc.want) {
+				t.Errorf("stderr = %q, want %q", stderr(), tc.want)
+			}
+
+			_, err := os.Lstat(path)
+			if !errors.Is(err, os.ErrNotExist) {
+				t.Errorf("lstat %s: %v, want no database", path, err)
+			}
+		})
+	}
+}
+
 // TestRunScanSkipsNonPersistentOperand scans, through the command, a
 // directory on a filesystem that is not persistent, with and without
 // --no-default-excludes: either way it is skipped, with a line saying

internal/sfdupes/scan.go

@@ -102,10 +102,12 @@ func scanPhase(name string) string {
 // scan command rejects anything less. Before the lock is taken, the
 // scan lists the filesystems that are not persistent, on which the walk
 // records nothing; unless noDefaultExcludes is set, the walk does not
-// enter the default excludes either. Once the database is open, the
-// scan notes its files, which the walk passes over.
+// enter the default excludes either. The walk neither enters nor
+// records the paths in excluded, the --exclude values; an operand at or
+// below one is a fatal error. Once the database is open, the scan notes
+// its files, which the walk passes over.
 func runScan(ctx context.Context, roots []string, workers int,
-	oneFS, noDefaultExcludes bool,
+	oneFS, noDefaultExcludes bool, excluded []string,
 ) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -128,12 +130,17 @@ func runScan(ctx context.Context, roots []string, workers int,
 		return err
 	}
 
+	excluded, err = resolveExcluded(roots, excluded)
+	if err != nil {
+		return err
+	}
+
 	nonPersistent, err := nonPersistentFilesystems()
 	if err != nil {
 		return fmt.Errorf("list the mounted filesystems: %w", err)
 	}
 
-	rules := walkRules{nonPersistent: nonPersistent}
+	rules := walkRules{excluded: excluded, nonPersistent: nonPersistent}
 	if !noDefaultExcludes {
 		rules.excludes = defaultExcludes(runtime.GOOS)
 	}
@@ -209,12 +216,12 @@ func interrupted(walked int) error {
 	return errInterrupted
 }
 
-// resolveRoots converts each PATH operand to its real path: absolute,
-// with every symlink in it followed. Database records are keyed by
-// that path, so scan results depend neither on the working directory
-// nor on the symlinks an operand was given through. An operand that
-// cannot be resolved, because it or a symlink's target does not exist
-// or cannot be read, is a fatal error before any scanning.
+// resolveRoots converts each PATH operand, or each --exclude value, to
+// its real path: absolute, with every symlink in it followed. Database
+// records are keyed by that path, so scan results depend neither on the
+// working directory nor on the symlinks an operand was given through.
+// A path that cannot be resolved, because it or a symlink's target does
+// not exist or cannot be read, is a fatal error before any scanning.
 func resolveRoots(roots []string) ([]string, error) {
 	resolved := make([]string, 0, len(roots))
 
@@ -493,10 +500,11 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 // as verified. When its disk_size differs from the record's, it is
 // queued for the update phase, which writes the new disk_size. A record
 // at or below a path the walk left out, because -x stopped at it, it is
-// a default exclude, or it is not on a persistent filesystem, counts as
-// outside the roots: it is not deleted. That does not hold below a root
-// at or inside that path, since that root's own walk verifies the
-// records below it; nor is a line printed for such a root.
+// an --exclude path or a default exclude, or it is not on a persistent
+// filesystem, counts as outside the roots: it is not deleted. That does
+// not hold below a root at or inside that path, since that root's own
+// walk verifies the records below it; nor is a line printed for such a
+// root.
 func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
 	// The records the last scan left under the roots estimate how many
 	// files the walk will find. A first scan has none, and no estimate.
@@ -844,9 +852,9 @@ type dirJob struct {
 // file's statted record, a warning when fail is set, or a path the walk
 // left out: when otherFSDir is set, a directory -x did not enter
 // because it is on another filesystem than its operand, and when
-// leftOut is set, a default exclude or a directory or file on a
-// filesystem that is not persistent, with warn holding the line that
-// says so.
+// leftOut is set, an --exclude path, a default exclude or a directory or
+// file on a filesystem that is not persistent, with warn holding the
+// line that says so.
 type walkEvent struct {
 	rec        fileRec
 	warn       string
@@ -1128,8 +1136,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 // no longer exists, and one that has stopped being a regular file is
 // skipped. A file on a filesystem in rules that is not persistent, as
 // through a bind mount of one file, is not recorded: it gets an event
-// naming it instead. One of the scan's own database files in rules gets
-// no event at all.
+// naming it instead, and so does a file given with --exclude. One of
+// the scan's own database files in rules gets no event at all.
 func emitFile(ctx context.Context, p string, e fs.DirEntry,
 	rules walkRules, events chan<- walkEvent,
 ) {
@@ -1153,6 +1161,10 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 	}
 
 	line := rules.nonPersistentLine(p, info)
+	if line == "" {
+		line = rules.excludedLine(p)
+	}
+
 	if line != "" {
 		sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
 
@@ -1177,11 +1189,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 // subdirJob applies the descent rules to directory p: never enter
 // .zfs (ZFS snapshot pseudo-dirs would list every file once per
 // snapshot), never enter a directory on a filesystem in rules that is
-// not persistent, nor a default exclude in rules, and with -x never
-// enter a directory on a different filesystem than its operand. For the
-// last three it sends an event that names the directory, so the scan
-// keeps the records below it, except those below an operand at or
-// inside it.
+// not persistent, nor an --exclude path or a default exclude in rules,
+// and with -x never enter a directory on a different filesystem than
+// its operand. For all but .zfs it sends an event that names the
+// directory, so the scan keeps the records below it, except those below
+// an operand at or inside it.
 func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 	parent dirJob, oneFS bool, rules walkRules,
 	events chan<- walkEvent,
@@ -1206,6 +1218,10 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 	}
 
 	line := rules.nonPersistentLine(p, info)
+	if line == "" {
+		line = rules.excludedLine(p)
+	}
+
 	if line != "" {
 		sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})