Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6732172ed8 |
No files matched your search
@@ -208,8 +208,8 @@ makes the comparison exact. Reading samples cuts a 150 GiB file to about 150 MiB
|
|||||||
of reads, at the price of a comparison that is not exact.
|
of reads, at the price of a comparison that is not exact.
|
||||||
|
|
||||||
The hashes live in a database that is kept between runs. A rescan reads only new
|
The hashes live in a database that is kept between runs. A rescan reads only new
|
||||||
files, and files whose size or modification time (mtime) differs from their
|
files, and files whose size changed or whose modification time (mtime) is newer
|
||||||
record. All analysis then works from the database alone.
|
than recorded. All analysis then works from the database alone.
|
||||||
|
|
||||||
The goal is whole duplicated trees, not only single files. A download extracted
|
The goal is whole duplicated trees, not only single files. A download extracted
|
||||||
twice, or a project copied once too often, is such a tree. It can be removed as
|
twice, or a project copied once too often, is such a tree. It can be removed as
|
||||||
@@ -410,16 +410,17 @@ operands:
|
|||||||
|
|
||||||
- A file not yet in the database is hashed and inserted.
|
- A file not yet in the database is hashed and inserted.
|
||||||
- A file already in the database is not read again when its size and mtime match
|
- A file already in the database is not read again when its size and mtime match
|
||||||
its record. This is what makes a daily rescan cheap.
|
its record. The mtime matches when it is not newer than the recorded one. This
|
||||||
|
is what makes a daily rescan cheap.
|
||||||
- When such a file's size on disk differs from its recorded `disk_size`, `scan`
|
- When such a file's size on disk differs from its recorded `disk_size`, `scan`
|
||||||
updates `disk_size` alone. It still does not read the file, and counts it as
|
updates `disk_size` alone. It still does not read the file, and counts it as
|
||||||
unchanged. ZFS, for example, counts a new file's blocks up to seconds after
|
unchanged. ZFS, for example, counts a new file's blocks up to seconds after
|
||||||
the write. A block is the unit in which a filesystem allocates disk space. A
|
the write. A block is the unit in which a filesystem allocates disk space. A
|
||||||
file scanned within those seconds gets its real `disk_size` at the next scan.
|
file scanned within those seconds gets its real `disk_size` at the next scan.
|
||||||
- `scan` hashes a file again when its size or its mtime differs from its record.
|
- `scan` hashes a file again when its size differs or its mtime is newer. The
|
||||||
An older mtime counts as much as a newer one. A file put back by `cp -p`,
|
mtime is compared to the nanosecond, as finely as the filesystem records it. A
|
||||||
`rsync -a` or a restore from backup often has an older mtime. The mtime is
|
same-size rewrite counts as a change whenever the filesystem gives it a later
|
||||||
compared to the nanosecond, as finely as the filesystem records it.
|
mtime, even within the same second.
|
||||||
- The scan deletes each record under a scanned operand whose file it could not
|
- The scan deletes each record under a scanned operand whose file it could not
|
||||||
verify. That covers deleted files, and paths the scan could not stat or read
|
verify. That covers deleted files, and paths the scan could not stat or read
|
||||||
this run. To stat a file is to read its details, such as its size, without
|
this run. To stat a file is to read its details, such as its size, without
|
||||||
@@ -801,7 +802,8 @@ The summary line on stderr gives:
|
|||||||
- the records read;
|
- the records read;
|
||||||
- the duplicate-tree groups;
|
- the duplicate-tree groups;
|
||||||
- the dupe trees, which is the number of rows;
|
- the dupe trees, which is the number of rows;
|
||||||
- the reclaimable bytes, the sum of `size` over all rows.
|
- the reclaimable bytes, what removing every `dupe` path would free. A `dupe`
|
||||||
|
path inside another `dupe` path adds nothing, so each file is counted once.
|
||||||
|
|
||||||
```
|
```
|
||||||
trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable
|
trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable
|
||||||
|
|||||||
@@ -29,6 +29,9 @@
|
|||||||
Newest first. An entry ends with its date and its issue or branch, where those
|
Newest first. An entry ends with its date and its issue or branch, where those
|
||||||
are known.
|
are known.
|
||||||
|
|
||||||
|
- The reclaimable total of `trees` counts each file once. A `dupe` tree inside
|
||||||
|
another `dupe` tree adds nothing, so the total is what removing every `dupe`
|
||||||
|
path frees (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/168)
|
||||||
- `report` and `trees` leave out empty files. `report` writes no group of size
|
- `report` and `trees` leave out empty files. `report` writes no group of size
|
||||||
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
|
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
|
||||||
counts toward a tree's digest (2026-11-11,
|
counts toward a tree's digest (2026-11-11,
|
||||||
|
|||||||
@@ -545,7 +545,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
old, ok := s.existing[ev.rec.path]
|
old, ok := s.existing[ev.rec.path]
|
||||||
if !ok || old.size != ev.rec.size || !ev.rec.mtime.Equal(old.mtime) {
|
if !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime) {
|
||||||
s.toHash = append(s.toHash, ev.rec)
|
s.toHash = append(s.toHash, ev.rec)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@@ -780,6 +780,16 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mtimeAfter reports whether mtime a is later than mtime b.
|
||||||
|
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
|
||||||
|
func mtimeAfter(a, b time.Time) bool {
|
||||||
|
if a.Unix() != b.Unix() {
|
||||||
|
return a.Unix() > b.Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
return a.Nanosecond() > b.Nanosecond()
|
||||||
|
}
|
||||||
|
|
||||||
// underAnyRoot reports whether path is any of the roots or lies under
|
// underAnyRoot reports whether path is any of the roots or lies under
|
||||||
// one of them.
|
// one of them.
|
||||||
func underAnyRoot(path string, roots []string) bool {
|
func underAnyRoot(path string, roots []string) bool {
|
||||||
|
|||||||
@@ -1384,47 +1384,6 @@ func TestSyncScanMtimeBump(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSyncScanMtimeMovedBack replaces a file with other content of the
|
|
||||||
// same size and an older mtime, as cp -p or a restore from backup does.
|
|
||||||
// The next scan must hash the file again.
|
|
||||||
func TestSyncScanMtimeMovedBack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
a := writeFile(t, dir, "a.bin", pattern(1, 500))
|
|
||||||
|
|
||||||
recorded := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
err := os.Chtimes(a, recorded, recorded)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
replaced := pattern(3, 500)
|
|
||||||
writeFile(t, dir, "a.bin", replaced)
|
|
||||||
|
|
||||||
older := recorded.Add(-time.Hour)
|
|
||||||
|
|
||||||
err = os.Chtimes(a, older, older)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertWholeFileHashed(t, db, a, replaced)
|
|
||||||
|
|
||||||
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(older) {
|
|
||||||
t.Fatalf("mtime = %v, want %v", r.mtime, older)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertWholeFileHashed fails unless the record for path holds the
|
// assertWholeFileHashed fails unless the record for path holds the
|
||||||
// whole-file hash of data as its head, tail, and content.
|
// whole-file hash of data as its head, tail, and content.
|
||||||
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
||||||
|
|||||||
+42
-12
@@ -100,7 +100,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
|||||||
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
|
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
|
||||||
stalledOn: "out",
|
stalledOn: "out",
|
||||||
})
|
})
|
||||||
dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
|
dupeTrees, err := writeTreeGroups(stdout, dupes, prog)
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
@@ -111,7 +111,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
|||||||
fmt.Fprintf(os.Stderr,
|
fmt.Fprintf(os.Stderr,
|
||||||
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
|
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
|
||||||
"%s reclaimable\n",
|
"%s reclaimable\n",
|
||||||
records, len(dupes), dupeTrees, humanBytes(reclaimable))
|
records, len(dupes), dupeTrees, humanBytes(reclaimableBytes(dupes)))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -145,22 +145,19 @@ func readTree(ctx context.Context, db *sql.DB,
|
|||||||
// a row for every member of each group but its first, counting each
|
// a row for every member of each group but its first, counting each
|
||||||
// group on prog once its rows are written. The disk_size column is the
|
// group on prog once its rows are written. The disk_size column is the
|
||||||
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
|
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
|
||||||
// go through prog's writer. It returns the number of rows (dupe trees)
|
// go through prog's writer. It returns the number of rows (dupe trees).
|
||||||
// and the bytes they could reclaim.
|
|
||||||
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
) (int, int64, error) {
|
) (int, error) {
|
||||||
out := prog.writer(stdout)
|
out := prog.writer(stdout)
|
||||||
|
|
||||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
|
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dupeTrees := 0
|
dupeTrees := 0
|
||||||
|
|
||||||
var reclaimable int64
|
|
||||||
|
|
||||||
for _, g := range dupes {
|
for _, g := range dupes {
|
||||||
first := g[0]
|
first := g[0]
|
||||||
for _, n := range g[1:] {
|
for _, n := range g[1:] {
|
||||||
@@ -168,11 +165,10 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
|||||||
escapePath(first.path), escapePath(n.path),
|
escapePath(first.path), escapePath(n.path),
|
||||||
first.fileCount, first.totalSize, n.diskSize)
|
first.fileCount, first.totalSize, n.diskSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dupeTrees++
|
dupeTrees++
|
||||||
reclaimable += first.totalSize
|
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
@@ -180,10 +176,44 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
|||||||
|
|
||||||
err = out.Flush()
|
err = out.Flush()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return dupeTrees, reclaimable, nil
|
return dupeTrees, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reclaimableBytes returns the bytes that removing every dupe tree in
|
||||||
|
// dupes would free. A dupe tree inside another dupe tree adds nothing,
|
||||||
|
// since the outer one already counts its files.
|
||||||
|
func reclaimableBytes(dupes [][]*treeNode) int64 {
|
||||||
|
isDupe := make(map[*treeNode]bool)
|
||||||
|
|
||||||
|
for _, g := range dupes {
|
||||||
|
for _, n := range g[1:] {
|
||||||
|
isDupe[n] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
for n := range isDupe {
|
||||||
|
if !insideDupe(n, isDupe) {
|
||||||
|
total += n.totalSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// insideDupe reports whether a directory above n is in isDupe.
|
||||||
|
func insideDupe(n *treeNode, isDupe map[*treeNode]bool) bool {
|
||||||
|
for p := n.parent; p != nil; p = p.parent {
|
||||||
|
if isDupe[p] {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// treeBuilder reconstructs the directory hierarchy from records added
|
// treeBuilder reconstructs the directory hierarchy from records added
|
||||||
|
|||||||
@@ -303,7 +303,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
|||||||
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
|
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
|
||||||
}, terminalWidth, time.Hour, time.Now)
|
}, terminalWidth, time.Hour, time.Now)
|
||||||
|
|
||||||
_, _, err := writeTreeGroups(os.Stderr, dupes, p)
|
_, err := writeTreeGroups(os.Stderr, dupes, p)
|
||||||
|
|
||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
@@ -487,3 +487,33 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
|||||||
t.Fatalf("groups = %v, want %v", got, want)
|
t.Fatalf("groups = %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestReclaimableBytesCountsNestedDupeOnce checks that a dupe tree
|
||||||
|
// inside another dupe tree adds nothing to the reclaimable bytes:
|
||||||
|
// removing /B and /C/x frees 1500 + 1000 bytes, and /B/x lies in /B.
|
||||||
|
func TestReclaimableBytesCountsNestedDupeOnce(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fh, ft, fc := testHash("fh"), testHash("ft"), testHash("fc")
|
||||||
|
gh, gt, gc := testHash("gh"), testHash("gt"), testHash("gc")
|
||||||
|
recs := []scanRec{
|
||||||
|
{size: 1000, head: fh, tail: ft, content: fc, path: "/A/x/f"},
|
||||||
|
{size: 500, head: gh, tail: gt, content: gc, path: "/A/y/g"},
|
||||||
|
{size: 1000, head: fh, tail: ft, content: fc, path: "/B/x/f"},
|
||||||
|
{size: 500, head: gh, tail: gt, content: gc, path: "/B/y/g"},
|
||||||
|
{size: 1000, head: fh, tail: ft, content: fc, path: "/C/x/f"},
|
||||||
|
}
|
||||||
|
|
||||||
|
super, dirs := treeOf(t, recs)
|
||||||
|
dupes := collectTreeGroups(dirs, super, nil)
|
||||||
|
|
||||||
|
wantGroups := [][]string{{"/A", "/B"}, {"/A/x", "/B/x", "/C/x"}}
|
||||||
|
if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
|
||||||
|
slices.Equal) {
|
||||||
|
t.Fatalf("groups = %v, want %v", got, wantGroups)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := reclaimableBytes(dupes); got != 2500 {
|
||||||
|
t.Errorf("reclaimableBytes = %d, want 2500", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user