dwim now takes --workers, -x, --no-default-excludes and --exclude with the meaning they have for scan. One scanOptions struct, one function that adds the flags and one PreRunE check serve both commands, and runScan takes the struct instead of four separate values. The dwim help lists the default excludes, as the scan help does. Model: opus-5-5
This commit is contained in:
1 parent
6caaff935a
commit
56301dac74
10 files changed
+272
-109
No files matched your search
@@ -255,15 +255,14 @@ In order:
|
|||||||
- `report` prints the file-level duplicate report from the database.
|
- `report` prints the file-level duplicate report from the database.
|
||||||
- `trees` prints the duplicate-tree report from the database.
|
- `trees` prints the duplicate-tree report from the database.
|
||||||
- `info` prints the database's details.
|
- `info` prints the database's details.
|
||||||
- `dwim` scans `/` with the default excludes, then prints the duplicate-tree
|
- `dwim` scans `/`, then prints the duplicate-tree report.
|
||||||
report.
|
|
||||||
|
|
||||||
```
|
```
|
||||||
sfdupes scan [--workers N] [-x] [--no-default-excludes] [--exclude PATH]... PATH...
|
sfdupes scan [--workers N] [-x] [--no-default-excludes] [--exclude PATH]... PATH...
|
||||||
sfdupes report > dupes.tsv
|
sfdupes report > dupes.tsv
|
||||||
sfdupes trees > dupetrees.tsv
|
sfdupes trees > dupetrees.tsv
|
||||||
sfdupes info
|
sfdupes info
|
||||||
sfdupes dwim
|
sfdupes dwim [--workers N] [-x] [--no-default-excludes] [--exclude PATH]...
|
||||||
sfdupes --version
|
sfdupes --version
|
||||||
sfdupes [command] --help
|
sfdupes [command] --help
|
||||||
```
|
```
|
||||||
@@ -293,20 +292,20 @@ The reference below is also the program's specification.
|
|||||||
database is `/var/lib/sfdupes/db.sqlite`. There is no command-line flag for
|
database is `/var/lib/sfdupes/db.sqlite`. There is no command-line flag for
|
||||||
it. The path names the file exactly, whatever characters it holds. A relative
|
it. The path names the file exactly, whatever characters it holds. A relative
|
||||||
path is relative to the working directory.
|
path is relative to the working directory.
|
||||||
- `--workers N` (`scan`) sets how many directories `scan` reads at once while it
|
- `--workers N` (`scan`, `dwim`) sets how many directories `scan` reads at once
|
||||||
lists the trees. It also sets how many files `scan` reads at once while it
|
while it lists the trees. It also sets how many files `scan` reads at once
|
||||||
hashes them. The default is the number of CPUs. `N` must be at least 1. A
|
while it hashes them. The default is the number of CPUs. `N` must be at
|
||||||
smaller value is a usage error, reported in one line on stderr with exit 2
|
least 1. A smaller value is a usage error, reported in one line on stderr with
|
||||||
before anything is scanned.
|
exit 2 before anything is scanned.
|
||||||
- `-x`, `--one-file-system` (`scan`) keeps the walk of each operand out of
|
- `-x`, `--one-file-system` (`scan`, `dwim`) keeps the walk of each operand out
|
||||||
directories on other filesystems, as the same flag does for `du` and `rsync`.
|
of directories on other filesystems, as the same flag does for `du` and
|
||||||
By default the walk also enters directories on other filesystems.
|
`rsync`. By default the walk also enters directories on other filesystems.
|
||||||
- `--no-default-excludes` (`scan`) lets the walk enter the default excludes.
|
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
||||||
Those are the system and temporary directories the walk otherwise leaves out,
|
excludes. Those are the system and temporary directories the walk otherwise
|
||||||
such as `/var/tmp` on Linux (see "Rules for the walk").
|
leaves out, such as `/var/tmp` on Linux (see "Rules for the walk").
|
||||||
- `--exclude PATH` (`scan`) keeps the walk out of `PATH` and everything below
|
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
||||||
it. `PATH` may be a directory or a file. The flag may be given more than once.
|
below it. `PATH` may be a directory or a file. The flag may be given more than
|
||||||
The walk knows `PATH` by its device ID and inode number, as it knows the
|
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
||||||
database files (see "Rules for the walk"). So the walk leaves `PATH` out
|
database files (see "Rules for the walk"). So the walk leaves `PATH` out
|
||||||
whatever path leads to it. That includes a hard link to a file given as
|
whatever path leads to it. That includes a hard link to a file given as
|
||||||
`PATH`. A symlink given as `PATH` excludes its target. An empty `PATH`, as in
|
`PATH`. A symlink given as `PATH` excludes its target. An empty `PATH`, as in
|
||||||
@@ -317,9 +316,9 @@ The reference below is also the program's specification.
|
|||||||
- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and
|
- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and
|
||||||
exits 0. It writes nothing to stderr.
|
exits 0. It writes nothing to stderr.
|
||||||
- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr
|
- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr
|
||||||
and exits 0. It writes nothing to stdout. `sfdupes scan --help` also lists the
|
and exits 0. It writes nothing to stdout. `sfdupes scan --help` and
|
||||||
default excludes of the system it runs on, and the filesystem types the walk
|
`sfdupes dwim --help` also list the default excludes of the system they run
|
||||||
never records anything on.
|
on, and the filesystem types the walk never records anything on.
|
||||||
|
|
||||||
### Duplicate detection
|
### Duplicate detection
|
||||||
|
|
||||||
@@ -857,14 +856,25 @@ trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimab
|
|||||||
### `dwim` mode
|
### `dwim` mode
|
||||||
|
|
||||||
`dwim`, short for "do what I mean", scans a new machine and reports its
|
`dwim`, short for "do what I mean", scans a new machine and reports its
|
||||||
duplicate trees in one command. It takes no positional arguments and no flags of
|
duplicate trees in one command. It takes no positional arguments. Like `scan`,
|
||||||
its own. Like `scan`, it should run as root, so that it can read every file.
|
it should run as root, so that it can read every file.
|
||||||
|
|
||||||
`dwim` runs what `sfdupes scan /` runs with no flags. The walk skips the default
|
`dwim` takes these flags of `scan`, with the same meaning (see "Settings"):
|
||||||
excludes and crosses into other filesystems. `--workers` keeps its default. The
|
|
||||||
|
- `--workers N`;
|
||||||
|
- `-x`, `--one-file-system`;
|
||||||
|
- `--no-default-excludes`;
|
||||||
|
- `--exclude PATH`, which may be given more than once.
|
||||||
|
|
||||||
|
`dwim` runs what `sfdupes scan /` runs with the same flags. With no flags, the
|
||||||
|
walk skips the default excludes and crosses into other filesystems. The
|
||||||
database, its creation and the scan lock are those of `scan`. Then `dwim` runs
|
database, its creation and the scan lock are those of `scan`. Then `dwim` runs
|
||||||
what `sfdupes trees` runs. The largest trees come first.
|
what `sfdupes trees` runs. The largest trees come first.
|
||||||
|
|
||||||
|
```
|
||||||
|
sfdupes dwim -x --exclude /srv/cache
|
||||||
|
```
|
||||||
|
|
||||||
The report goes to stdout, exactly as `trees` prints it. On stderr comes what
|
The report goes to stdout, exactly as `trees` prints it. On stderr comes what
|
||||||
`scan` prints, then what `trees` prints, then one last line:
|
`scan` prints, then what `trees` prints, then one last line:
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,10 @@
|
|||||||
Newest first. An entry ends with its date and its issue or branch, where those
|
Newest first. An entry ends with its date and its issue or branch, where those
|
||||||
are known.
|
are known.
|
||||||
|
|
||||||
|
- `dwim` takes the `scan` flags `--workers`, `-x`, `--no-default-excludes` and
|
||||||
|
`--exclude`, with the same meaning. Both commands register and check them with
|
||||||
|
the same code (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/162)
|
||||||
|
|
||||||
- New `scan` flag `--exclude PATH`, which may be repeated. The walk neither
|
- New `scan` flag `--exclude PATH`, which may be repeated. The walk neither
|
||||||
enters nor records an excluded path, and keeps the records below it. An
|
enters nor records an excluded path, and keeps the records below it. An
|
||||||
operand at or below one is a fatal error (2026-11-11,
|
operand at or below one is a fatal error (2026-11-11,
|
||||||
|
|||||||
@@ -358,8 +358,9 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
dir := buildSmokeTree(t)
|
dir := buildSmokeTree(t)
|
||||||
|
|
||||||
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
opts := scanOptions{workers: walkCancelWorkers}
|
||||||
false, nil)
|
|
||||||
|
err := runScan(t.Context(), []string{dir}, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -382,8 +383,7 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
|
||||||
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
|
err = runScan(cancelledContext(t), []string{dir}, opts)
|
||||||
false, false, nil)
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
awaitReturn(t, done, "runScan")
|
awaitReturn(t, done, "runScan")
|
||||||
@@ -410,8 +410,7 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
got, before)
|
got, before)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
err = runScan(t.Context(), []string{dir}, opts)
|
||||||
false, nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -437,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
|
|||||||
dir := buildWalkCancelTree(t)
|
dir := buildWalkCancelTree(t)
|
||||||
|
|
||||||
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
||||||
walkCancelWorkers, false, false, nil)
|
scanOptions{workers: walkCancelWorkers})
|
||||||
if !errors.Is(err, errInterrupted) {
|
if !errors.Is(err, errInterrupted) {
|
||||||
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
||||||
err, errInterrupted)
|
err, errInterrupted)
|
||||||
|
|||||||
+79
-48
@@ -37,12 +37,11 @@ const (
|
|||||||
// cobra prints for it is the whole message.
|
// cobra prints for it is the whole message.
|
||||||
var errNoSubcommand = errors.New("no subcommand")
|
var errNoSubcommand = errors.New("no subcommand")
|
||||||
|
|
||||||
// errWorkersBelowOne is the usage error for a scan --workers value
|
// errWorkersBelowOne is the usage error for a --workers value below 1.
|
||||||
// below 1.
|
|
||||||
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
||||||
|
|
||||||
// errEmptyExcludeValue is the usage error for an empty scan --exclude
|
// errEmptyExcludeValue is the usage error for an empty --exclude value,
|
||||||
// value, as from --exclude "$DIR" with DIR unset. Taken as a path, it
|
// as from --exclude "$DIR" with DIR unset. Taken as a path, it
|
||||||
// would name the working directory.
|
// would name the working directory.
|
||||||
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
|
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
|
||||||
|
|
||||||
@@ -193,76 +192,108 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
dwimCmd := &cobra.Command{
|
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd,
|
||||||
Use: cmdDwim,
|
newDwimCommand(stdout))
|
||||||
Short: "Scan / with the default excludes, then print the duplicate-tree report",
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
|
||||||
return runDwim(ctx, "/", stdout)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
|
|
||||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
|
||||||
|
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|
||||||
// newScanCommand builds the scan subcommand, with its --workers, -x,
|
// scanOptions holds the values of the flags that scan and dwim share:
|
||||||
// --no-default-excludes and --exclude flags. Its help lists the default
|
// --workers, -x, --no-default-excludes and --exclude.
|
||||||
|
type scanOptions struct {
|
||||||
|
workers int
|
||||||
|
oneFS bool
|
||||||
|
noDefaultExcludes bool
|
||||||
|
excludePaths []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// scanFlagsUsage names the flags that scan and dwim share, for their
|
||||||
|
// usage lines.
|
||||||
|
const scanFlagsUsage = "[--workers N] [-x] [--no-default-excludes] " +
|
||||||
|
"[--exclude PATH]..."
|
||||||
|
|
||||||
|
// addScanFlags adds the flags that scan and dwim share to cmd, which
|
||||||
|
// stores their values in opts.
|
||||||
|
func addScanFlags(cmd *cobra.Command, opts *scanOptions) {
|
||||||
|
cmd.Flags().IntVar(&opts.workers, "workers", runtime.NumCPU(),
|
||||||
|
"concurrent workers for the walk and hash phases")
|
||||||
|
cmd.Flags().BoolVarP(&opts.oneFS, "one-file-system", "x", false,
|
||||||
|
"do not cross filesystem boundaries")
|
||||||
|
cmd.Flags().BoolVar(&opts.noDefaultExcludes, "no-default-excludes",
|
||||||
|
false, "enter the default excludes, the paths listed above")
|
||||||
|
// A string array, not a slice, so a comma in a path is not a
|
||||||
|
// separator.
|
||||||
|
cmd.Flags().StringArrayVar(&opts.excludePaths, "exclude", nil,
|
||||||
|
"never enter or record `PATH` or anything below it; repeatable")
|
||||||
|
}
|
||||||
|
|
||||||
|
// check rejects the flag values that are usage errors, before the
|
||||||
|
// command runs. It is the PreRunE of scan and dwim.
|
||||||
|
func (o *scanOptions) check(cmd *cobra.Command, _ []string) error {
|
||||||
|
err := checkScanWorkers(cmd, o.workers)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return rejectEmptyExcludeValue(cmd, o.excludePaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newScanCommand builds the scan subcommand. Its help lists the default
|
||||||
// excludes of the system it runs on, and the filesystem types it never
|
// excludes of the system it runs on, and the filesystem types it never
|
||||||
// records anything on.
|
// records anything on.
|
||||||
func newScanCommand() *cobra.Command {
|
func newScanCommand() *cobra.Command {
|
||||||
var (
|
var opts scanOptions
|
||||||
scanWorkers int
|
|
||||||
scanOneFS bool
|
|
||||||
scanNoDefaultExcludes bool
|
|
||||||
scanExclude []string
|
|
||||||
)
|
|
||||||
|
|
||||||
short := "Walk trees and synchronize the scan database"
|
short := "Walk trees and synchronize the scan database"
|
||||||
|
|
||||||
scanCmd := &cobra.Command{
|
scanCmd := &cobra.Command{
|
||||||
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] " +
|
Use: cmdScan + " " + scanFlagsUsage + " PATH...",
|
||||||
"[--exclude PATH]... PATH...",
|
|
||||||
// The Use line already names every flag, so cobra must not
|
// The Use line already names every flag, so cobra must not
|
||||||
// append its generic "[flags]" to it.
|
// append its generic "[flags]" to it.
|
||||||
DisableFlagsInUseLine: true,
|
DisableFlagsInUseLine: true,
|
||||||
Short: short,
|
Short: short,
|
||||||
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
||||||
Args: cobra.MinimumNArgs(1),
|
Args: cobra.MinimumNArgs(1),
|
||||||
PreRunE: func(cmd *cobra.Command, _ []string) error {
|
PreRunE: opts.check,
|
||||||
err := checkScanWorkers(cmd, scanWorkers)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rejectEmptyExcludeValue(cmd, scanExclude)
|
|
||||||
},
|
|
||||||
RunE: runE(func(ctx context.Context, args []string) error {
|
RunE: runE(func(ctx context.Context, args []string) error {
|
||||||
ctx, stop := interruptContext(ctx)
|
ctx, stop := interruptContext(ctx)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
return runScan(ctx, args, scanWorkers, scanOneFS,
|
return runScan(ctx, args, opts)
|
||||||
scanNoDefaultExcludes, scanExclude)
|
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
addScanFlags(scanCmd, &opts)
|
||||||
"concurrent workers for the walk and hash phases")
|
|
||||||
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
|
||||||
"do not cross filesystem boundaries")
|
|
||||||
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
|
|
||||||
false, "enter the default excludes, the paths listed above")
|
|
||||||
// A string array, not a slice, so a comma in a path is not a
|
|
||||||
// separator.
|
|
||||||
scanCmd.Flags().StringArrayVar(&scanExclude, "exclude", nil,
|
|
||||||
"never enter or record `PATH` or anything below it; repeatable")
|
|
||||||
|
|
||||||
return scanCmd
|
return scanCmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkScanWorkers rejects a scan --workers value below 1. That is a
|
// newDwimCommand builds the dwim subcommand, which takes the scan flags
|
||||||
// usage error reported in one line: cobra prints the returned message
|
// with the meaning they have for scan. Its help lists what the scan
|
||||||
// without the usage text, and run exits 2.
|
// help lists.
|
||||||
|
func newDwimCommand(stdout io.Writer) *cobra.Command {
|
||||||
|
var opts scanOptions
|
||||||
|
|
||||||
|
short := "Scan /, then print the duplicate-tree report"
|
||||||
|
|
||||||
|
dwimCmd := &cobra.Command{
|
||||||
|
Use: cmdDwim + " " + scanFlagsUsage,
|
||||||
|
DisableFlagsInUseLine: true,
|
||||||
|
Short: short,
|
||||||
|
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
PreRunE: opts.check,
|
||||||
|
RunE: runE(func(ctx context.Context, _ []string) error {
|
||||||
|
return runDwim(ctx, "/", opts, stdout)
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
addScanFlags(dwimCmd, &opts)
|
||||||
|
|
||||||
|
return dwimCmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkScanWorkers rejects a --workers value below 1. That is a usage
|
||||||
|
// error reported in one line: cobra prints the returned message without
|
||||||
|
// the usage text, and run exits 2.
|
||||||
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
||||||
if workers >= 1 {
|
if workers >= 1 {
|
||||||
return nil
|
return nil
|
||||||
@@ -273,7 +304,7 @@ func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
|||||||
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
||||||
}
|
}
|
||||||
|
|
||||||
// rejectEmptyExcludeValue rejects an empty scan --exclude value. That
|
// rejectEmptyExcludeValue rejects an empty --exclude value. That
|
||||||
// is a usage error reported in one line, as for --workers below 1.
|
// is a usage error reported in one line, as for --workers below 1.
|
||||||
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
|
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
|
||||||
if !slices.Contains(excludePaths, "") {
|
if !slices.Contains(excludePaths, "") {
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ import (
|
|||||||
// error prints and a runtime failure must not.
|
// error prints and a runtime failure must not.
|
||||||
const usageMarker = "Usage:"
|
const usageMarker = "Usage:"
|
||||||
|
|
||||||
|
// workersFlag is the scan and dwim flag that sets the worker count.
|
||||||
|
const workersFlag = "--workers"
|
||||||
|
|
||||||
// TestMain stops the tests at once, saying why, when their temporary
|
// TestMain stops the tests at once, saying why, when their temporary
|
||||||
// directory is not on a persistent filesystem: scan records nothing
|
// directory is not on a persistent filesystem: scan records nothing
|
||||||
// there (README "Rules for the walk"), so most scan tests would fail
|
// there (README "Rules for the walk"), so most scan tests would fail
|
||||||
@@ -570,8 +573,8 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
cmdScan: {
|
cmdScan: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runScan(t.Context(), []string{missing}, 1, false, false,
|
return runScan(t.Context(), []string{missing},
|
||||||
nil)
|
scanOptions{workers: 1})
|
||||||
},
|
},
|
||||||
want: scanWant,
|
want: scanWant,
|
||||||
},
|
},
|
||||||
@@ -662,7 +665,7 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
args := []string{cmdScan, "--workers", workers, t.TempDir()}
|
args := []string{cmdScan, workersFlag, workers, t.TempDir()}
|
||||||
|
|
||||||
code := run(args, &stdout, &stderr)
|
code := run(args, &stdout, &stderr)
|
||||||
if code != exitUsage {
|
if code != exitUsage {
|
||||||
|
|||||||
+10
-10
@@ -5,20 +5,20 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
||||||
// root" runs with no flags, then what "sfdupes trees" runs, and ends
|
// root" runs with the scan flags in opts, then what "sfdupes trees"
|
||||||
// with a line on stderr saying how to print the report again. The dwim
|
// runs, and ends with a line on stderr saying how to print the report
|
||||||
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
|
// again. The dwim command passes "/" as root. It catches SIGINT and
|
||||||
// does, but only during the scan. An error from the scan, an interrupt
|
// SIGTERM as scan does, but only during the scan. An error from the
|
||||||
// included, ends it before the report. During the report, a signal
|
// scan, an interrupt included, ends it before the report. During the
|
||||||
// ends it at once, as it ends trees.
|
// report, a signal ends it at once, as it ends trees.
|
||||||
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
|
func runDwim(
|
||||||
|
ctx context.Context, root string, opts scanOptions, stdout io.Writer,
|
||||||
|
) error {
|
||||||
scanCtx, stop := interruptContext(ctx)
|
scanCtx, stop := interruptContext(ctx)
|
||||||
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false,
|
err := runScan(scanCtx, []string{root}, opts)
|
||||||
nil)
|
|
||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package sfdupes //nolint:testpackage // calls the unexported runDwim
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -28,7 +30,7 @@ func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
|||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
err := runDwim(t.Context(), root, &stdout)
|
err := runDwim(t.Context(), root, scanOptions{workers: 1}, &stdout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
}
|
}
|
||||||
@@ -67,7 +69,7 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
|||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
err := runDwim(t.Context(), missing, &stdout)
|
err := runDwim(t.Context(), missing, scanOptions{workers: 1}, &stdout)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
||||||
}
|
}
|
||||||
@@ -81,3 +83,119 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
|||||||
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunDwimExcludeLeavesTreeOut(t *testing.T) {
|
||||||
|
// dwim passes its --exclude paths to the scan: an excluded copy is
|
||||||
|
// not recorded, so its pair is not reported.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
root := realTempDir(t)
|
||||||
|
|
||||||
|
kept := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
||||||
|
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
||||||
|
}
|
||||||
|
writeFile(t, root, "b1/f.bin", pattern(2, 5000))
|
||||||
|
excluded := filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000)))
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
opts := scanOptions{workers: 1, excludePaths: []string{excluded}}
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), root, opts, &stdout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
||||||
|
if len(lines) != 2 {
|
||||||
|
t.Fatalf("stdout = %q, want a header and one group",
|
||||||
|
stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
want := kept[0] + "\t" + kept[1] + "\t1\t300\t"
|
||||||
|
if !strings.HasPrefix(lines[1], want) {
|
||||||
|
t.Errorf("group = %q, want it to start %q", lines[1], want)
|
||||||
|
}
|
||||||
|
|
||||||
|
skipped := "scan: skipped " + excluded + " (--exclude)\n"
|
||||||
|
if got := stderr(); !strings.Contains(got, skipped) {
|
||||||
|
t.Errorf("stderr = %q, want it to contain %q", got, skipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimTakesScanFlags(t *testing.T) {
|
||||||
|
// The dwim command parses and checks the scan flags as scan does.
|
||||||
|
// Each case stops before the walk of / begins, and before the
|
||||||
|
// database is created. "dwim --exclude /" excludes its own operand.
|
||||||
|
cases := map[string]struct {
|
||||||
|
args []string
|
||||||
|
code int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
"--exclude /": {
|
||||||
|
[]string{cmdDwim, excludeFlag, "/"},
|
||||||
|
exitFatal,
|
||||||
|
"sfdupes: scan /: operand excluded by --exclude /\n",
|
||||||
|
},
|
||||||
|
"--exclude empty": {
|
||||||
|
[]string{cmdDwim, excludeFlag, ""},
|
||||||
|
exitUsage,
|
||||||
|
"Error: --exclude must not be empty\n",
|
||||||
|
},
|
||||||
|
"--workers 0": {
|
||||||
|
[]string{cmdDwim, workersFlag, "0", "-x", "--no-default-excludes"},
|
||||||
|
exitUsage,
|
||||||
|
"Error: --workers must be at least 1, got 0\n",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
stdout := captureStdout(t)
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
code := run(tc.args, os.Stdout, os.Stderr)
|
||||||
|
if code != tc.code {
|
||||||
|
t.Errorf("run(%q) = %d, want %d", tc.args, code, tc.code)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasSuffix(stderr(), tc.want) || stdout() != "" {
|
||||||
|
t.Errorf("stdout, stderr = %q, %q, want \"\" and stderr "+
|
||||||
|
"ending %q", stdout(), stderr(), tc.want)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := os.Lstat(path)
|
||||||
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Errorf("lstat %s: %v, want no database", path, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimHelpListsScanFlags(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// README "Subcommands" shows this usage line for dwim.
|
||||||
|
const want = "\n sfdupes dwim [--workers N] [-x] [--no-default-excludes] " +
|
||||||
|
"[--exclude PATH]...\n"
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
run([]string{cmdDwim, "-h"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if !strings.Contains(stderr.String(), want) {
|
||||||
|
t.Errorf("dwim -h stderr = %q, want the usage line %q",
|
||||||
|
stderr.String(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
|
||||||
|
t.Errorf("dwim -h stderr = %q, want %s PATH", stderr.String(),
|
||||||
|
excludeFlag)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -254,7 +254,7 @@ func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
|
|||||||
return devices, lines.Err()
|
return devices, lines.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultExcludesHelp describes, for the scan command's help, the
|
// defaultExcludesHelp describes, for the scan and dwim help, the
|
||||||
// default excludes of the system goos names and, where the scan lists
|
// default excludes of the system goos names and, where the scan lists
|
||||||
// the mounted filesystems, the types it never records anything on.
|
// the mounted filesystems, the types it never records anything on.
|
||||||
func defaultExcludesHelp(goos string) string {
|
func defaultExcludesHelp(goos string) string {
|
||||||
|
|||||||
+11
-13
@@ -98,17 +98,15 @@ func scanPhase(name string) string {
|
|||||||
// runs, and the lock is released after it. When ctx is cancelled, as by
|
// runs, and the lock is released after it. When ctx is cancelled, as by
|
||||||
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
||||||
// what it has hashed (see syncScan), prints how many files its walk
|
// what it has hashed (see syncScan), prints how many files its walk
|
||||||
// reached, and returns errInterrupted. workers must be at least 1; the
|
// reached, and returns errInterrupted. opts.workers must be at least 1;
|
||||||
// scan command rejects anything less. Before the lock is taken, the
|
// the scan command rejects anything less. Before the lock is taken, the
|
||||||
// scan lists the filesystems that are not persistent, on which the walk
|
// scan lists the filesystems that are not persistent, on which the walk
|
||||||
// records nothing; unless noDefaultExcludes is set, the walk does not
|
// records nothing; unless opts.noDefaultExcludes is set, the walk does
|
||||||
// enter the default excludes either. The walk neither enters nor
|
// not enter the default excludes either. The walk neither enters nor
|
||||||
// records the paths in excludePaths, the --exclude values; an operand
|
// records the paths in opts.excludePaths, the --exclude values; an
|
||||||
// at or below one is a fatal error. Once the database is open, the scan
|
// operand at or below one is a fatal error. Once the database is open,
|
||||||
// notes its files, which the walk passes over.
|
// the scan notes its files, which the walk passes over.
|
||||||
func runScan(ctx context.Context, roots []string, workers int,
|
func runScan(ctx context.Context, roots []string, opts scanOptions) error {
|
||||||
oneFS, noDefaultExcludes bool, excludePaths []string,
|
|
||||||
) error {
|
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -123,7 +121,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
|
|
||||||
var rules walkRules
|
var rules walkRules
|
||||||
|
|
||||||
rules.excludePaths, err = excludePathInodes(excludePaths)
|
rules.excludePaths, err = excludePathInodes(opts.excludePaths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("--exclude: %w", err)
|
return fmt.Errorf("--exclude: %w", err)
|
||||||
}
|
}
|
||||||
@@ -138,7 +136,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
return fmt.Errorf("list the mounted filesystems: %w", err)
|
return fmt.Errorf("list the mounted filesystems: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !noDefaultExcludes {
|
if !opts.noDefaultExcludes {
|
||||||
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
|
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,7 +165,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
|
st, err := syncScan(ctx, db, roots, opts.workers, opts.oneFS, rules)
|
||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
return interrupted(st.walked)
|
return interrupted(st.walked)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2196,7 +2196,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) {
|
|||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{
|
code := run([]string{
|
||||||
cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir,
|
cmdScan, workersFlag, strconv.Itoa(hashLeakWorkers), dir,
|
||||||
}, io.Discard, &stderr)
|
}, io.Discard, &stderr)
|
||||||
if code != exitFatal {
|
if code != exitFatal {
|
||||||
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
||||||
|
|||||||
Reference in new issue
Block a user