Limit report and trees to given paths (closes #174) (#191)
check / check (push) Canceled after 0s

This commit was merged in pull request #191.
This commit is contained in:
clawbot committed 2026-11-11 09:37:14 +01:00
1 parent 58d27a6df3
commit d4f90f43ff
8 files changed
+412 -75

No files matched your search

+34 -10
View File
@@ -49,6 +49,13 @@ const escapesHelp = `Four characters in a path are escaped, so each row stays on
\r a carriage return
`
// operandsHelp is the paragraph of the report and trees help on their
// PATH operands (README "Operands of `report` and `trees`").
const operandsHelp = `With PATH operands, only the records at or below them are used. A
relative PATH is joined to the working directory. No PATH is looked up
on the filesystem, so a symlink in one is not followed.
`
// reportHelp, treesHelp, infoHelp and dwimHelp follow the one-line
// summary in the help of their subcommands.
const (
@@ -61,7 +68,7 @@ row per duplicate file. The columns are:
Rows are ordered by size, largest first, then by path.
` + escapesHelp
` + escapesHelp + "\n" + operandsHelp
treesHelp = `The report is tab-separated text on stdout: a header line, then one
row per duplicate directory tree. The columns are:
@@ -74,7 +81,7 @@ row per duplicate directory tree. The columns are:
Rows are ordered by size, largest first, then by path.
` + escapesHelp
` + escapesHelp + "\n" + operandsHelp
infoHelp = `The details go to stdout, one per line. Each line holds a name and its
value, separated by a tab:
@@ -108,6 +115,11 @@ var errWorkersBelowOne = errors.New("--workers must be at least 1")
// would name the working directory.
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
// errEmptyOperand is the usage error for an empty PATH operand of
// report or trees, as from "$DIR" with DIR unset. Taken as a path, it
// would name the working directory.
var errEmptyOperand = errors.New("PATH must not be empty")
// Version is the build version, injected at link time via -ldflags
// (see the Makefile); empty when the linker set none, as for a binary
// built by go install. buildVersion turns it into what --version prints.
@@ -229,20 +241,20 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
"print the version to stdout")
reportCmd := &cobra.Command{
Use: cmdReport,
Use: cmdReport + " [PATH...]",
Short: "Read the scan database and print the file-level duplicates report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runReport(ctx, stdout)
Args: rejectEmptyOperand,
RunE: runE(func(ctx context.Context, args []string) error {
return runReport(ctx, args, stdout)
}),
}
treesCmd := &cobra.Command{
Use: cmdTrees,
Use: cmdTrees + " [PATH...]",
Short: "Read the scan database and print the duplicate-tree report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runTrees(ctx, stdout)
Args: rejectEmptyOperand,
RunE: runE(func(ctx context.Context, args []string) error {
return runTrees(ctx, args, stdout)
}),
}
@@ -387,6 +399,18 @@ func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
return errEmptyExcludeValue
}
// rejectEmptyOperand rejects an empty PATH operand of report or trees.
// That is a usage error reported in one line, as for --workers below 1.
func rejectEmptyOperand(cmd *cobra.Command, args []string) error {
if !slices.Contains(args, "") {
return nil
}
cmd.SilenceUsage = true
return errEmptyOperand
}
// runE adapts a subcommand implementation, or the version print, to
// cobra's RunE. Cobra prints the error and the command's usage text for
// every error RunE returns, but a subcommand that ran and failed has no
+19 -12
View File
@@ -583,14 +583,14 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
},
cmdReport: {
run: func() error {
return runReport(cancelledContext(t), io.Discard)
return runReport(cancelledContext(t), nil, io.Discard)
},
want: "report: database " + defaultDatabasePath + " (default)\n" +
reportOpenLine + "\n" + reportOpenLine + "\n",
},
cmdTrees: {
run: func() error {
return runTrees(cancelledContext(t), io.Discard)
return runTrees(cancelledContext(t), nil, io.Discard)
},
want: "trees: database " + defaultDatabasePath + " (default)\n" +
treesOpenLine + "\n" + treesOpenLine + "\n",
@@ -626,8 +626,6 @@ func TestRunUsageErrors(t *testing.T) {
}{
"no subcommand": {[]string{}, usageMarker},
"scan without paths": {[]string{cmdScan}, usageMarker},
"report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
@@ -764,11 +762,16 @@ func TestRunHelpDescribesDatabaseAndOutput(t *testing.T) {
t.Parallel()
// README §Settings: the help names the database, and says what
// report, trees, info and dwim print.
// report, trees, info and dwim print. The help of report and trees
// also explains their PATH operands.
database := []string{
"scan and dwim store", databaseEnv, defaultDatabasePath, "needs root",
}
escapes := []string{`\\`, `\t`, `\n`, `\r`}
operands := []string{
"With PATH operands", "at or below them", "working directory",
"No PATH is looked up", "not followed",
}
// The subcommand whose help is printed; none for the root.
cases := []struct {
@@ -779,17 +782,17 @@ func TestRunHelpDescribesDatabaseAndOutput(t *testing.T) {
{[]string{cmdScan}, database},
{
[]string{cmdReport},
append([]string{
slices.Concat([]string{
"tab-separated", "\n first ", "\n dupe ", "\n size ",
"largest first",
}, escapes...),
}, escapes, operands),
},
{
[]string{cmdTrees},
append([]string{
slices.Concat([]string{
"tab-separated", "\n first ", "\n dupe ", "\n files ",
"\n size ", "\n disk_size ", "largest first",
}, escapes...),
}, escapes, operands),
},
{
[]string{cmdInfo},
@@ -1839,9 +1842,13 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
scanFixture(t)
cases := map[string]func(context.Context, io.Writer) error{
cmdReport: runReport,
cmdTrees: runTrees,
cmdInfo: runInfo,
cmdReport: func(ctx context.Context, w io.Writer) error {
return runReport(ctx, nil, w)
},
cmdTrees: func(ctx context.Context, w io.Writer) error {
return runTrees(ctx, nil, w)
},
cmdInfo: runInfo,
}
for name, fn := range cases {
+1 -1
View File
@@ -26,7 +26,7 @@ func runDwim(
return err
}
err = runTrees(ctx, stdout)
err = runTrees(ctx, nil, stdout)
if err != nil {
return err
}
+70 -14
View File
@@ -9,6 +9,7 @@ import (
"io"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
@@ -44,10 +45,12 @@ func reportPhase(name string) string {
}
// runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. Of the scanned filesystem it
// only checks, with pathExists, that each path in a duplicate group is
// still there. It names the database on stderr before it opens it. Any
// database problem, including a missing database, is fatal.
// duplicates report as TSV on stdout, from the records at or below the
// operands in args, or from every record when args is empty. Of the
// scanned filesystem it only checks, with pathExists, that each path in
// a duplicate group is still there; it never looks up an operand. It
// names the database on stderr before it opens it. Any database
// problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (read the number of
@@ -55,7 +58,12 @@ func reportPhase(name string) string {
// order, writing each size's duplicate groups to stdout once that size
// is read; see readReport). Each display ends before the next phase
// starts, and before an error is returned to be printed.
func runReport(ctx context.Context, stdout io.Writer) error {
func runReport(ctx context.Context, args []string, stdout io.Writer) error {
operands, err := cleanOperands(args)
if err != nil {
return err
}
dbPath, source, err := databasePath()
if err != nil {
return err
@@ -87,7 +95,8 @@ func runReport(ctx context.Context, stdout io.Writer) error {
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db/disk/out",
})
report, err := readReport(ctx, db, dbPath, stdout, pathExists, prog)
report, err := readReport(ctx, db, dbPath, operands, stdout, pathExists,
prog)
prog.finish()
@@ -108,12 +117,14 @@ func runReport(ctx context.Context, stdout io.Writer) error {
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. A path for which exists returns false is left out of its group.
// stdout may be the terminal prog is drawn on, so the lines go through
// prog's writer. It returns what it wrote, with the totals for the
// summary line.
// read. Only the records that underOperand finds under operands join a
// group, and a path for which exists returns false is left out of its
// group. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
stdout io.Writer, exists func(path string) bool, prog *progress,
operands []string, stdout io.Writer, exists func(path string) bool,
prog *progress,
) (*reportWriter, error) {
out := prog.writer(stdout)
report := &reportWriter{out: out, exists: exists}
@@ -128,6 +139,12 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
prog.increment()
report.records++
if !underOperand(r.path, operands) {
return nil
}
writeErr = report.add(r)
return writeErr
@@ -159,7 +176,7 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
// records of one size, so it holds only the duplicate groups of the
// size being read, and writes them, ordered by first path, once a
// record of a smaller size comes. It counts what the summary line
// reports.
// reports; readReport counts the records read.
type reportWriter struct {
out io.Writer
@@ -179,8 +196,6 @@ type reportWriter struct {
// its paths; any other ends them, and one of a smaller size first
// writes the groups of the size before.
func (w *reportWriter) add(r scanRec) error {
w.records++
if r.size == w.sig.size && r.head == w.sig.head &&
r.tail == w.sig.tail && r.content == w.sig.content {
w.paths = append(w.paths, r.path)
@@ -273,6 +288,47 @@ func pathExists(path string) bool {
return !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ENOTDIR)
}
// cleanOperands returns the operands of report or trees, each made
// absolute by absolutePath and then cleaned. The records hold real
// paths, with no "." or ".." element and no repeated or trailing
// slash, so an operand is cleaned to match them. No operand is looked
// up on the filesystem. None is empty: rejectEmptyOperand refuses an
// empty operand on the command line.
func cleanOperands(args []string) ([]string, error) {
operands := make([]string, 0, len(args))
for _, a := range args {
abs, err := absolutePath(a)
if err != nil {
return nil, fmt.Errorf("resolve path %s: %w", escapePath(a), err)
}
operands = append(operands, filepath.Clean(abs))
}
return operands, nil
}
// underOperand reports whether path is one of operands or lies below
// one of them. Each operand is clean and absolute, as cleanOperands
// returns it. With no operands, a report covers the whole database, so
// underOperand is true for every path.
func underOperand(path string, operands []string) bool {
if len(operands) == 0 {
return true
}
for _, o := range operands {
// Only the root directory's clean path ends in "/".
dir := strings.TrimSuffix(o, "/") + "/"
if path == o || strings.HasPrefix(path, dir) {
return true
}
}
return false
}
// escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
+139 -6
View File
@@ -102,7 +102,8 @@ func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
var out bytes.Buffer
_, err := readReport(t.Context(), db, "", &out, everyPathExists, nil)
_, err := readReport(t.Context(), db, "", nil, &out, everyPathExists,
nil)
if err != nil {
t.Fatal(err)
}
@@ -148,6 +149,136 @@ func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup {
return dupeGroups(t, db)
}
// TestRunReportOperands runs report with operands on three copies of
// one file, two of them under srv/photos, and a fourth copy in
// srv/photos2, all in a new directory. Only the records at or below the
// operands form the groups and the totals, however each operand is
// written.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestRunReportOperands(t *testing.T) {
h := testHash("h")
recs := make([]scanRec, 0, 4)
for _, p := range []string{
"/a/copy", "/srv/photos/x", "/srv/photos/y", "/srv/photos2/z",
} {
recs = append(recs,
scanRec{size: 7, head: h, tail: h, content: h, path: p})
}
dir, recs := recsOnDisk(t, recs)
a, x, y, z := recs[0].path, recs[1].path, recs[2].path, recs[3].path
t.Setenv(databaseEnv, seedDatabase(t, recs))
header := "first\tdupe\tsize\n"
row := func(first, dupe string) string {
return first + "\t" + dupe + "\t7\n"
}
photos := header + row(x, y)
cases := map[string]struct {
operands []string
want string
summary string
}{
"directory": {[]string{dir + "/srv/photos"}, photos,
"1 duplicate groups, 1 dupe files, 7 B reclaimable"},
"trailing slash": {[]string{dir + "/srv/photos/"}, photos, ""},
"dot and dot-dot": {[]string{dir + "/srv/./a/../photos"}, photos, ""},
"files": {[]string{a, z}, header + row(a, z), ""},
"overlapping": {[]string{dir + "/srv", dir + "/srv/photos"},
header + row(x, y) + row(x, z), ""},
"root": {[]string{"/"},
header + row(a, x) + row(a, y) + row(a, z), ""},
"no record": {[]string{dir + "/srv/photo"}, header,
"0 duplicate groups, 0 dupe files, 0 B reclaimable"},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
var stdout bytes.Buffer
stderr := captureStderr(t)
code := run(append([]string{cmdReport}, tc.operands...),
&stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(report %q) = %d, want %d; stderr: %s",
tc.operands, code, exitOK, stderr())
}
if got := stdout.String(); got != tc.want {
t.Errorf("stdout = %q, want %q", got, tc.want)
}
// The records read are all four, whatever the operands.
summary := "report: 4 records read, " + tc.summary + "\n"
if got := stderr(); tc.summary != "" &&
!strings.HasSuffix(got, summary) {
t.Errorf("stderr = %q, want it to end %q", got, summary)
}
})
}
}
//nolint:paralleltest // t.Chdir changes the process-wide working directory
func TestCleanOperandsMakesRelativePathsAbsolute(t *testing.T) {
dir := t.TempDir()
t.Chdir(dir)
wd, err := absolutePath(".")
if err != nil {
t.Fatal(err)
}
got, err := cleanOperands([]string{"photos/", "a/../b", "/c//d"})
if err != nil {
t.Fatal(err)
}
want := []string{
filepath.Join(wd, "photos"), filepath.Join(wd, "b"), "/c/d",
}
if !slices.Equal(got, want) {
t.Errorf("cleanOperands = %q, want %q", got, want)
}
}
// TestRunReportsRejectEmptyOperand checks that an empty PATH operand of
// report or trees, alone or after another operand, is a usage error
// reported in one line before the database is opened (README "Error
// handling and exit codes").
func TestRunReportsRejectEmptyOperand(t *testing.T) {
cases := map[string][]string{
"report": {cmdReport, ""},
"trees": {cmdTrees, ""},
"after an operand": {cmdTrees, t.TempDir(), ""},
}
for name, args := range cases {
t.Run(name, func(t *testing.T) {
// No database exists there, so opening one would be a fatal
// error, exit 1.
t.Setenv(databaseEnv, testDBPath(t))
var stdout, stderr bytes.Buffer
code := run(args, &stdout, &stderr)
if code != exitUsage {
t.Errorf("run(%q) = %d, want %d", args, code, exitUsage)
}
want := "Error: PATH must not be empty\n"
if stdout.String() != "" || stderr.String() != want {
t.Errorf("stdout, stderr = %q, %q, want \"\", %q",
stdout.String(), stderr.String(), want)
}
})
}
}
func TestRunReportEscapesPaths(t *testing.T) {
dir, recs := recsOnDisk(t, awkwardPairRecs())
t.Setenv(databaseEnv, seedDatabase(t, recs))
@@ -195,7 +326,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
// Counts the records read, and draws nothing.
prog := &progress{now: time.Now}
_, err = readReport(t.Context(), db, "", failingWriter{},
_, err = readReport(t.Context(), db, "", nil, failingWriter{},
everyPathExists, prog)
if !errors.Is(err, errWriteFailed) ||
!strings.HasPrefix(err.Error(), "write stdout: ") {
@@ -292,7 +423,8 @@ func TestReadReportOnTerminal(t *testing.T) {
waitingFor: countingRecords,
}, terminalWidth, time.Hour, time.Now)
_, err = readReport(t.Context(), db, "", os.Stderr, everyPathExists, p)
_, err = readReport(t.Context(), db, "", nil, os.Stderr,
everyPathExists, p)
p.finish()
@@ -514,7 +646,7 @@ func TestReadReportChecksOnlyPossibleDuplicates(t *testing.T) {
return true
}
_, err = readReport(t.Context(), db, "", io.Discard, exists, nil)
_, err = readReport(t.Context(), db, "", nil, io.Discard, exists, nil)
if err != nil {
t.Fatal(err)
}
@@ -548,7 +680,8 @@ func TestReadReportDropsGroupLeftWithOnePath(t *testing.T) {
exists := func(path string) bool { return path != "/a/gone" }
report, err := readReport(t.Context(), db, "", io.Discard, exists, nil)
report, err := readReport(t.Context(), db, "", nil, io.Discard, exists,
nil)
if err != nil {
t.Fatal(err)
}
@@ -581,7 +714,7 @@ func TestReadReportKeepsGroupLeftWithTwoPaths(t *testing.T) {
var out bytes.Buffer
report, err := readReport(t.Context(), db, "", &out, exists, nil)
report, err := readReport(t.Context(), db, "", nil, &out, exists, nil)
if err != nil {
t.Fatal(err)
}
+45 -14
View File
@@ -37,12 +37,14 @@ func treesPhase(name string) string {
// runTrees implements the trees subcommand: it reads every record from
// the database in path order, reconstructs the directory hierarchy from
// the record paths, computes a Merkle-style digest per directory, and
// prints maximal duplicate-tree groups as TSV on stdout. Of the scanned
// filesystem it only checks, with pathExists, that the root of each
// duplicate tree is still there, never the files below it. It names the
// database on stderr before it opens it. Any database problem,
// including a missing database, is fatal.
// the paths of the records at or below the operands in args (of every
// record when args is empty), computes a Merkle-style digest per
// directory, and prints maximal duplicate-tree groups as TSV on stdout.
// Of the scanned filesystem it only checks, with pathExists, that the
// root of each duplicate tree is still there, never the files below it;
// it never looks up an operand. It names the database on stderr before
// it opens it. Any database problem, including a missing database, is
// fatal.
//
// The work runs in four phases, each under a progress display of its
// own on stderr: open (open the database), read (read the number of
@@ -50,7 +52,12 @@ func treesPhase(name string) string {
// hierarchy), group (group the directories by digest) and write (write
// the groups to stdout). Each display ends before the next phase
// starts, and before an error is returned to be printed.
func runTrees(ctx context.Context, stdout io.Writer) error {
func runTrees(ctx context.Context, args []string, stdout io.Writer) error {
operands, err := cleanOperands(args)
if err != nil {
return err
}
dbPath, source, err := databasePath()
if err != nil {
return err
@@ -79,7 +86,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
label: treesPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db",
})
records, super, allDirs, err := readTree(ctx, db, prog)
records, super, allDirs, err := readTree(ctx, db, operands, prog)
prog.finish()
@@ -118,10 +125,17 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
}
// readTree gives prog the number of records in db as its total, then
// reads every record in path order into the directory hierarchy,
// counting each on prog. It returns the number of records read, the
// super-root and every directory.
func readTree(ctx context.Context, db *sql.DB,
// reads every record in path order, counting each on prog. It adds the
// records that underOperand finds under operands to the directory
// hierarchy. It returns the number of records read, the super-root and
// every directory at or below one of operands.
//
// Any other directory lies above an operand and can hold files that are
// not used, so its digest does not describe it. It is left out of the
// directories returned. A directory whose parent is left out gets the
// super-root as its parent, so suppressed never judges its group by
// that parent's digest.
func readTree(ctx context.Context, db *sql.DB, operands []string,
prog *progress,
) (int, *treeNode, []*treeNode, error) {
records := 0
@@ -131,7 +145,10 @@ func readTree(ctx context.Context, db *sql.DB,
records++
prog.increment()
tree.add(r)
if underOperand(r.path, operands) {
tree.add(r)
}
})
if err != nil {
return 0, nil, nil, err
@@ -139,7 +156,21 @@ func readTree(ctx context.Context, db *sql.DB,
super, allDirs := tree.finish()
return records, super, allDirs, nil
dirs := allDirs[:0]
for _, d := range allDirs {
if !underOperand(d.path, operands) {
continue
}
if !underOperand(d.parent.path, operands) {
d.parent = super
}
dirs = append(dirs, d)
}
return records, super, dirs, nil
}
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
+51
View File
@@ -214,6 +214,57 @@ func TestRunTreesEscapesPaths(t *testing.T) {
}
}
// TestRunTreesOperands runs trees with operands on records in a new
// directory. x/a and y/a are copies, and so are y/a and srv/b. x and y
// also hold files that differ. Given x/a and y/a, trees reads x and y
// as holding one copy each of a, yet neither is reported: only the
// directories at or below an operand are.
func TestRunTreesOperands(t *testing.T) {
f, g, k := testHash("f"), testHash("g"), testHash("k")
dir, recs := recsOnDisk(t, []scanRec{
{size: 10, head: f, tail: f, content: f, path: "/srv/b/f"},
{size: 10, head: f, tail: f, content: f, path: "/x/a/f"},
{size: 20, head: g, tail: g, content: g, path: "/x/g"},
{size: 10, head: f, tail: f, content: f, path: "/y/a/f"},
{size: 30, head: k, tail: k, content: k, path: "/y/k"},
})
b, xa, ya := dir+"/srv/b", dir+"/x/a", dir+"/y/a"
header := "first\tdupe\tfiles\tsize\tdisk_size\n"
row := func(first, dupe string) string {
return first + "\t" + dupe + "\t1\t10\t0\n"
}
cases := map[string]struct {
operands []string
want string
}{
"none": {nil, header + row(b, xa) + row(b, ya)},
"two copies": {[]string{xa, ya}, header + row(xa, ya)},
"parents": {[]string{dir + "/x", dir + "/y"}, header + row(xa, ya)},
"one copy": {[]string{b}, header},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Setenv(databaseEnv, seedDatabase(t, recs))
var stdout, stderr bytes.Buffer
code := run(append([]string{cmdTrees}, tc.operands...),
&stdout, &stderr)
if code != exitOK {
t.Fatalf("run(trees %q) = %d, want %d; stderr: %s",
tc.operands, code, exitOK, stderr.String())
}
if got := stdout.String(); got != tc.want {
t.Errorf("stdout = %q, want %q", got, tc.want)
}
})
}
}
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
// take different space on disk. The row's disk_size is the sum over the
// dupe copy's files, those in its subdirectory included.