Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5eb7953ae9 | ||
|
|
a27d651532 | ||
|
|
a1ec1e0ca4 | ||
|
|
6c529fcc6d |
No files matched your search
@@ -208,8 +208,8 @@ makes the comparison exact. Reading samples cuts a 150 GiB file to about 150 MiB
|
||||
of reads, at the price of a comparison that is not exact.
|
||||
|
||||
The hashes live in a database that is kept between runs. A rescan reads only new
|
||||
files, and files whose size changed or whose modification time (mtime) is newer
|
||||
than recorded. All analysis then works from the database alone.
|
||||
files, and files whose size or modification time (mtime) differs from their
|
||||
record. All analysis then works from the database alone.
|
||||
|
||||
The goal is whole duplicated trees, not only single files. A download extracted
|
||||
twice, or a project copied once too often, is such a tree. It can be removed as
|
||||
@@ -300,9 +300,11 @@ The reference below is also the program's specification.
|
||||
- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and
|
||||
exits 0. It writes nothing to stderr.
|
||||
- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr
|
||||
and exits 0. It writes nothing to stdout. `sfdupes scan --help` also lists the
|
||||
default excludes of the system it runs on, and the filesystem types the walk
|
||||
never records anything on.
|
||||
and exits 0. It writes nothing to stdout. The help of `sfdupes` and of `scan`
|
||||
names the database. The help of `report`, `trees`, `info` and `dwim` says what
|
||||
each prints. `sfdupes scan --help` also lists the default excludes of the
|
||||
system it runs on, and the filesystem types the walk never records anything
|
||||
on.
|
||||
|
||||
### Duplicate detection
|
||||
|
||||
@@ -410,17 +412,16 @@ operands:
|
||||
|
||||
- A file not yet in the database is hashed and inserted.
|
||||
- A file already in the database is not read again when its size and mtime match
|
||||
its record. The mtime matches when it is not newer than the recorded one. This
|
||||
is what makes a daily rescan cheap.
|
||||
its record. This is what makes a daily rescan cheap.
|
||||
- When such a file's size on disk differs from its recorded `disk_size`, `scan`
|
||||
updates `disk_size` alone. It still does not read the file, and counts it as
|
||||
unchanged. ZFS, for example, counts a new file's blocks up to seconds after
|
||||
the write. A block is the unit in which a filesystem allocates disk space. A
|
||||
file scanned within those seconds gets its real `disk_size` at the next scan.
|
||||
- `scan` hashes a file again when its size differs or its mtime is newer. The
|
||||
mtime is compared to the nanosecond, as finely as the filesystem records it. A
|
||||
same-size rewrite counts as a change whenever the filesystem gives it a later
|
||||
mtime, even within the same second.
|
||||
- `scan` hashes a file again when its size or its mtime differs from its record.
|
||||
An older mtime counts as much as a newer one. A file put back by `cp -p`,
|
||||
`rsync -a` or a restore from backup often has an older mtime. The mtime is
|
||||
compared to the nanosecond, as finely as the filesystem records it.
|
||||
- The scan deletes each record under a scanned operand whose file it could not
|
||||
verify. That covers deleted files, and paths the scan could not stat or read
|
||||
this run. To stat a file is to read its details, such as its size, without
|
||||
|
||||
@@ -29,6 +29,14 @@
|
||||
Newest first. An entry ends with its date and its issue or branch, where those
|
||||
are known.
|
||||
|
||||
- The help of `sfdupes` and of `scan` names the database, its default path and
|
||||
`SFDUPES_DATABASE`. The help of `report` and `trees` lists the columns, the
|
||||
order of the rows and the escapes. The help of `info` lists its lines. The
|
||||
help of `dwim` says that it scans `/`, should run as root and keeps the scan
|
||||
in the database (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/177)
|
||||
- The usage line of `scan` no longer ends in `[flags]`. It is the line
|
||||
`README.md` shows under "Subcommands" (2026-11-11,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/169)
|
||||
- `report` and `trees` leave out empty files. `report` writes no group of size
|
||||
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
|
||||
counts toward a tree's digest (2026-11-11,
|
||||
|
||||
+75
-7
@@ -31,6 +31,64 @@ const (
|
||||
cmdDwim = "dwim"
|
||||
)
|
||||
|
||||
// databaseHelp is the paragraph of the root and scan help that names
|
||||
// the database (README "Location").
|
||||
const databaseHelp = "scan and dwim store what they find in a database " +
|
||||
"file. report, trees\nand info read it. The file is $" + databaseEnv +
|
||||
" when that is set and\nnot empty, and otherwise " +
|
||||
defaultDatabasePath + ", where creating it\nneeds root.\n"
|
||||
|
||||
// escapesHelp is the paragraph of the report and trees help on how
|
||||
// paths are escaped (README "Report output format").
|
||||
const escapesHelp = `Four characters in a path are escaped, so that each row stays one
|
||||
line: a backslash as \\, a tab as \t, a newline as \n and a carriage
|
||||
return as \r.
|
||||
`
|
||||
|
||||
// reportHelp, treesHelp, infoHelp and dwimHelp follow the one-line
|
||||
// summary in the help of their subcommands.
|
||||
const reportHelp = `The report is tab-separated text on stdout: a header line, then one
|
||||
row per duplicate file. The columns are:
|
||||
|
||||
first the path that sorts first, byte by byte, in its group of copies
|
||||
dupe another copy of that file
|
||||
size the file's size in bytes
|
||||
|
||||
Rows are ordered by size, largest first, then by path.
|
||||
|
||||
` + escapesHelp
|
||||
|
||||
const treesHelp = `The report is tab-separated text on stdout: a header line, then one
|
||||
row per duplicate directory tree. The columns are:
|
||||
|
||||
first the path that sorts first, byte by byte, in its group of copies
|
||||
dupe another copy of that tree
|
||||
files the number of regular files in one copy
|
||||
size their total size in bytes
|
||||
disk_size the bytes the dupe copy takes on disk
|
||||
|
||||
Rows are ordered by size, largest first, then by path.
|
||||
|
||||
` + escapesHelp
|
||||
|
||||
const infoHelp = `The details go to stdout, one per line. Each line holds a name and its
|
||||
value, separated by a tab:
|
||||
|
||||
database the path, and where it came from
|
||||
database file size in bytes
|
||||
-wal file size in bytes, only while a -wal file exists
|
||||
modified when the database file last changed, in UTC
|
||||
schema version the version of the database's layout
|
||||
records the number of records
|
||||
size of recorded files the total size of the recorded files, in bytes
|
||||
`
|
||||
|
||||
const dwimHelp = `dwim runs what "sfdupes scan /" runs, then what "sfdupes trees" runs.
|
||||
Run it as root, so that the scan can read every file. The scan is stored
|
||||
in the database, as scan stores it, so "sfdupes trees" prints the report
|
||||
again later.
|
||||
`
|
||||
|
||||
// errNoSubcommand is returned by the root command when it is invoked
|
||||
// without a subcommand. That is a usage error, and the usage text
|
||||
// cobra prints for it is the whole message.
|
||||
@@ -196,15 +254,21 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
||||
}),
|
||||
}
|
||||
|
||||
root.Long = root.Short + ".\n\n" + databaseHelp
|
||||
reportCmd.Long = reportCmd.Short + ".\n\n" + reportHelp
|
||||
treesCmd.Long = treesCmd.Short + ".\n\n" + treesHelp
|
||||
infoCmd.Long = infoCmd.Short + ".\n\n" + infoHelp
|
||||
dwimCmd.Long = dwimCmd.Short + ".\n\n" + dwimHelp
|
||||
|
||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
// newScanCommand builds the scan subcommand, with its --workers, -x and
|
||||
// --no-default-excludes flags. Its help lists the default excludes of
|
||||
// the system it runs on, and the filesystem types it never records
|
||||
// anything on.
|
||||
// --no-default-excludes flags. Its help names the database, and lists
|
||||
// the default excludes of the system it runs on and the filesystem
|
||||
// types it never records anything on.
|
||||
func newScanCommand() *cobra.Command {
|
||||
var (
|
||||
scanWorkers int
|
||||
@@ -215,10 +279,14 @@ func newScanCommand() *cobra.Command {
|
||||
short := "Walk trees and synchronize the scan database"
|
||||
|
||||
scanCmd := &cobra.Command{
|
||||
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
|
||||
Short: short,
|
||||
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
|
||||
// The Use line already names every flag, so cobra must not
|
||||
// append its generic "[flags]" to it.
|
||||
DisableFlagsInUseLine: true,
|
||||
Short: short,
|
||||
Long: short + ".\n\n" + databaseHelp + "\n" +
|
||||
defaultExcludesHelp(runtime.GOOS),
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
PreRunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return checkScanWorkers(cmd, scanWorkers)
|
||||
},
|
||||
|
||||
@@ -22,6 +22,9 @@ import (
|
||||
// error prints and a runtime failure must not.
|
||||
const usageMarker = "Usage:"
|
||||
|
||||
// helpFlag asks for the help text.
|
||||
const helpFlag = "--help"
|
||||
|
||||
// TestMain stops the tests at once, saying why, when their temporary
|
||||
// directory is not on a persistent filesystem: scan records nothing
|
||||
// there (README "Rules for the walk"), so most scan tests would fail
|
||||
@@ -692,7 +695,7 @@ func TestRunHelp(t *testing.T) {
|
||||
|
||||
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
||||
// empty.
|
||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||
cases := [][]string{{helpFlag}, {"-h"}, {cmdScan, helpFlag}}
|
||||
|
||||
for _, args := range cases {
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -714,6 +717,98 @@ func TestRunHelp(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScanUsageLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README §Subcommands shows this usage line for scan, and the help
|
||||
// and the usage error both print it.
|
||||
const want = "\n sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...\n"
|
||||
|
||||
cases := map[string][]string{
|
||||
"help": {cmdScan, "-h"},
|
||||
"missing paths": {cmdScan},
|
||||
}
|
||||
|
||||
for name, args := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
run(args, &stdout, &stderr)
|
||||
|
||||
if !strings.Contains(stderr.String(), want) {
|
||||
t.Errorf("run(%v) stderr = %q, want the usage line %q",
|
||||
args, stderr.String(), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHelpDescribesDatabaseAndOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README §Settings: the help names the database, and says what
|
||||
// report, trees, info and dwim print.
|
||||
database := []string{
|
||||
"scan and dwim store", databaseEnv, defaultDatabasePath, "needs root",
|
||||
}
|
||||
escapes := []string{`\\`, `\t`, `\n`, `\r`}
|
||||
|
||||
// The subcommand whose help is printed; none for the root.
|
||||
cases := []struct {
|
||||
subcommand []string
|
||||
want []string
|
||||
}{
|
||||
{nil, database},
|
||||
{[]string{cmdScan}, database},
|
||||
{
|
||||
[]string{cmdReport},
|
||||
append([]string{
|
||||
"tab-separated", "\n first ", "\n dupe ", "\n size ",
|
||||
}, escapes...),
|
||||
},
|
||||
{
|
||||
[]string{cmdTrees},
|
||||
append([]string{
|
||||
"tab-separated", "\n first ", "\n dupe ", "\n files ",
|
||||
"\n size ", "\n disk_size ",
|
||||
}, escapes...),
|
||||
},
|
||||
{
|
||||
[]string{cmdInfo},
|
||||
[]string{
|
||||
"\n database ", "\n database file size ",
|
||||
"\n -wal file size ", "\n modified ",
|
||||
"\n schema version ", "\n records ",
|
||||
"\n size of recorded files ",
|
||||
},
|
||||
},
|
||||
{
|
||||
[]string{cmdDwim},
|
||||
[]string{"sfdupes scan /", "as root", "sfdupes trees"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
args := slices.Concat(tc.subcommand, []string{helpFlag})
|
||||
|
||||
code := run(args, &stdout, &stderr)
|
||||
if code != exitOK {
|
||||
t.Fatalf("run(%v) = %d, want %d", args, code, exitOK)
|
||||
}
|
||||
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(stderr.String(), want) {
|
||||
t.Errorf("run(%v) help lacks %q:\n%s",
|
||||
args, want, stderr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -545,7 +545,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||
prog.increment()
|
||||
|
||||
old, ok := s.existing[ev.rec.path]
|
||||
if !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime) {
|
||||
if !ok || old.size != ev.rec.size || !ev.rec.mtime.Equal(old.mtime) {
|
||||
s.toHash = append(s.toHash, ev.rec)
|
||||
|
||||
continue
|
||||
@@ -780,16 +780,6 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||
}
|
||||
|
||||
// mtimeAfter reports whether mtime a is later than mtime b.
|
||||
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
|
||||
func mtimeAfter(a, b time.Time) bool {
|
||||
if a.Unix() != b.Unix() {
|
||||
return a.Unix() > b.Unix()
|
||||
}
|
||||
|
||||
return a.Nanosecond() > b.Nanosecond()
|
||||
}
|
||||
|
||||
// underAnyRoot reports whether path is any of the roots or lies under
|
||||
// one of them.
|
||||
func underAnyRoot(path string, roots []string) bool {
|
||||
|
||||
@@ -1384,6 +1384,47 @@ func TestSyncScanMtimeBump(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncScanMtimeMovedBack replaces a file with other content of the
|
||||
// same size and an older mtime, as cp -p or a restore from backup does.
|
||||
// The next scan must hash the file again.
|
||||
func TestSyncScanMtimeMovedBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
a := writeFile(t, dir, "a.bin", pattern(1, 500))
|
||||
|
||||
recorded := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
|
||||
err := os.Chtimes(a, recorded, recorded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
syncTree(t, db, dir)
|
||||
|
||||
replaced := pattern(3, 500)
|
||||
writeFile(t, dir, "a.bin", replaced)
|
||||
|
||||
older := recorded.Add(-time.Hour)
|
||||
|
||||
err = os.Chtimes(a, older, older)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
st := syncTree(t, db, dir)
|
||||
if st != (scanStats{walked: 1, updated: 1}) {
|
||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
||||
}
|
||||
|
||||
assertWholeFileHashed(t, db, a, replaced)
|
||||
|
||||
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(older) {
|
||||
t.Fatalf("mtime = %v, want %v", r.mtime, older)
|
||||
}
|
||||
}
|
||||
|
||||
// assertWholeFileHashed fails unless the record for path holds the
|
||||
// whole-file hash of data as its head, tail, and content.
|
||||
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
||||
|
||||
Reference in new issue
Block a user