sneak/sfdupes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stat in the walk, hash only shared sizes, flush batches mid-scan

Browse Source 
Restructure scan into three phases: walk+stat, hash, update.

The stat pass is folded into the walk workers: each regular file is
lstatted as its directory is read, while the metadata is hot. The
walk builds a scan-wide size census (walked files plus records
outside the scan roots), and unchanged already-hashed files resolve
during the walk without further work.

Only files whose size at least one other file shares are ever read:
a size-unique file cannot be a duplicate, so it is recorded without
hashes (head and tail empty). When a later scan makes its size
shared, the file is hashed then, even if otherwise unchanged. report
excludes unhashed records; trees gives them a never-matching
signature so a tree containing one never compares equal to another.

Hashed records are committed in batched transactions while the hash
phase runs, so an interrupted scan keeps everything hashed so far
and the next run resumes cheaply. The hash phase total is exact,
giving a meaningful ETA.

Memory drops accordingly: the existing-record index holds only path,
size, mtime, and a hashed flag (no hash values); the walk carries one
small record per candidate file; overlapping operands are pruned up
front instead of deduplicating every walked path in a scan-wide set.
Files no bigger than one chunk are hashed with a single read.
This commit is contained in:
Jeffrey Paul
2026-07-25 06:06:22 +07:00
parent 340bdbe39e
commit b62b4f297f
8 changed files with 719 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
scan_test.go
View File

@@ -110,14 +110,51 @@ func TestHashHeadTailErrors(t *testing.T) {
}
}
func TestWalkPass(t *testing.T) {
// collectWalk runs a walk over roots and returns the emitted records
// and the number of warning events.
func collectWalk(t *testing.T, roots []string, oneFS bool,
workers int,
) ([]fileRec, int) {
t.Helper()
var (
recs []fileRec
errs int
)
for ev := range startWalk(roots, oneFS, workers) {
if ev.fail {
errs++
continue
}
recs = append(recs, ev.rec)
}
return recs, errs
}
// walkedPaths returns the sorted paths of the walked records.
func walkedPaths(recs []fileRec) []string {
paths := make([]string, 0, len(recs))
for _, r := range recs {
paths = append(paths, r.path)
}
slices.Sort(paths)
return paths
}
func TestWalk(t *testing.T) {
t.Parallel()
dir := t.TempDir()
want := []string{
writeFile(t, dir, "a.txt", []byte("a")),
writeFile(t, dir, "sub/b.txt", []byte("b")),
writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
writeFile(t, dir, "sub/b.txt", []byte("bb")),
writeFile(t, dir, "sub/deeper/c.txt", []byte("ccc")),
}
slices.Sort(want)
@@ -131,19 +168,29 @@ func TestWalkPass(t *testing.T) {
t.Fatal(err)
}
paths, errs := walkPass([]string{dir}, false, 4)
recs, errs := collectWalk(t, []string{dir}, false, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if got := walkedPaths(recs); !slices.Equal(got, want) {
t.Fatalf("paths = %q, want %q", got, want)
}
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
// The walk stats each file as it is discovered: every record must
// carry the real size and a plausible mtime.
for _, r := range recs {
if r.size < 1 || r.size > 3 {
t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
}
if r.mtime <= 0 {
t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
}
}
}
func TestWalkPassDeepAndWide(t *testing.T) {
func TestWalkDeepAndWide(t *testing.T) {
t.Parallel()
// Exercise the dispatcher with more directories than workers and
@@ -161,19 +208,17 @@ func TestWalkPassDeepAndWide(t *testing.T) {
slices.Sort(want)
paths, errs := walkPass([]string{dir}, false, 8)
recs, errs := collectWalk(t, []string{dir}, false, 8)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("walked %d paths, want %d", len(paths), len(want))
if got := walkedPaths(recs); !slices.Equal(got, want) {
t.Fatalf("walked %d paths, want %d", len(got), len(want))
}
}
func TestWalkPassMultipleRoots(t *testing.T) {
func TestWalkMultipleRoots(t *testing.T) {
t.Parallel()
rootA := t.TempDir()
@@ -188,19 +233,17 @@ func TestWalkPassMultipleRoots(t *testing.T) {
// Operands are enumerated concurrently by the shared pool; order
// is unspecified.
paths, errs := walkPass([]string{rootA, rootB}, false, 4)
recs, errs := collectWalk(t, []string{rootA, rootB}, false, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
if got := walkedPaths(recs); !slices.Equal(got, want) {
t.Fatalf("paths = %q, want %q", got, want)
}
}
func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
func TestWalkFileAndSymlinkOperands(t *testing.T) {
t.Parallel()
dir := t.TempDir()
@@ -213,20 +256,20 @@ func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
t.Fatal(err)
}
// A regular-file operand is emitted as itself.
paths, errs := walkPass([]string{f}, false, 2)
if errs != 0 || !slices.Equal(paths, []string{f}) {
t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
// A regular-file operand is emitted as itself, statted.
recs, errs := collectWalk(t, []string{f}, false, 2)
if errs != 0 || len(recs) != 1 || recs[0].path != f || recs[0].size != 4 {
t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs)
}
// A symlink operand is not followed and yields nothing.
paths, errs = walkPass([]string{link}, false, 2)
if errs != 0 || len(paths) != 0 {
t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
recs, errs = collectWalk(t, []string{link}, false, 2)
if errs != 0 || len(recs) != 0 {
t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs)
}
}
func TestWalkPassOneFilesystemSameFS(t *testing.T) {
func TestWalkOneFilesystemSameFS(t *testing.T) {
t.Parallel()
// Everything in one filesystem: -x must not skip anything.
@@ -236,41 +279,13 @@ func TestWalkPassOneFilesystemSameFS(t *testing.T) {
writeFile(t, dir, "sub/deep/b", []byte("b")),
}
paths, errs := walkPass([]string{dir}, true, 4)
recs, errs := collectWalk(t, []string{dir}, true, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
}
}
func TestStatPass(t *testing.T) {
t.Parallel()
dir := t.TempDir()
a := writeFile(t, dir, "a", pattern(1, 10))
b := writeFile(t, dir, "b", pattern(2, 20))
missing := filepath.Join(dir, "vanished")
recs, errs := statPass([]string{a, b, missing}, 2)
if errs != 1 {
t.Fatalf("errs = %d, want 1 for the vanished file", errs)
}
slices.SortFunc(recs, func(x, y fileRec) int {
return strings.Compare(x.path, y.path)
})
if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
}
if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
t.Fatalf("recs = %+v, want positive mtimes", recs)
if got := walkedPaths(recs); !slices.Equal(got, want) {
t.Fatalf("paths = %q, want %q", got, want)
}
}
@@ -662,14 +677,88 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
}
}
func TestUniquePaths(t *testing.T) {
func TestScanSkipsUniqueSizes(t *testing.T) {
t.Parallel()
got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"})
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
want := []string{"/a", "/b", "/c"}
writeFile(t, dir, "b.bin", pattern(2, 600))
// Neither size is shared, so neither file is read: both records
// are written without hashes and no duplicates are reported.
st := syncTree(t, db, dir)
if st != (scanStats{added: 2}) {
t.Fatalf("stats = %+v, want 2 added", st)
}
recs := dbRecords(t, db)
for _, r := range recs {
if r.head != "" || r.tail != "" {
t.Errorf("%s: head = %q tail = %q, want unhashed",
r.path, r.head, r.tail)
}
}
if groups := collectDupeGroups(recs); len(groups) != 0 {
t.Fatalf("groups = %+v, want none from unhashed records", groups)
}
// A new same-size file makes 500 a shared size: the next scan
// hashes both the new file and the previously unhashed unchanged
// one, and they group as duplicates.
c := writeFile(t, dir, "c.bin", pattern(1, 500))
st = syncTree(t, db, dir)
if st != (scanStats{added: 1, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged",
st)
}
groups := collectDupeGroups(dbRecords(t, db))
if len(groups) != 1 {
t.Fatalf("groups = %+v, want the a/c pair", groups)
}
if want := []string{a, c}; !slices.Equal(groups[0].paths, want) {
t.Fatalf("group paths = %q, want %q", groups[0].paths, want)
}
}
func TestTreesUnhashedNeverEqual(t *testing.T) {
t.Parallel()
// Two trees identical except for unhashed same-name, same-size
// files (possible when the trees were scanned separately) must not
// compare equal: unhashed content is unknown.
shared := pattern(1, 100)
recs := []scanRec{
{path: "/x/t1/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
{path: "/x/t2/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
{path: "/x/t1/u", size: 50},
{path: "/x/t2/u", size: 50},
}
super, dirs := buildHierarchy(recs)
super.compute()
if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
t.Fatalf("tree groups = %d, want 0 (unhashed files differ)",
len(tg))
}
}
func TestPruneRoots(t *testing.T) {
t.Parallel()
// Duplicates and operands under other operands are dropped; /cc is
// not under /c (sibling with a shared prefix).
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
want := []string{"/a", "/c", "/cc"}
if !slices.Equal(got, want) {
t.Fatalf("uniquePaths = %q, want %q", got, want)
t.Fatalf("pruneRoots = %q, want %q", got, want)
}
}