Hash in inode order, read hard links once, never open empty files

Sort the hash queue by (device, inode) so reads proceed in inode
order, which minimizes seeking on spinning disks. Paths that are hard
links to the same inode form one run: the run is read once and every
path shares the result, so link farms (rsync --link-dest backups)
cost one read per inode instead of one per path. A run that fails to
read skips all of its paths.

Zero-length files have constant head/tail hashes; return them without
opening the file.

The hash progress total now counts actual reads (runs, not paths).
Hard-linked paths still appear in reports as duplicates — their
content is identical — though they share storage; noted in README.
This commit is contained in:
2026-07-25 14:36:55 +07:00
parent 67bde6226d
commit b14b735c88
3 changed files with 247 additions and 57 deletions

View File

@@ -100,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) {
t.Error("no error for a missing file")
}
// A zero-length file has constant hashes and is never opened: even
// a missing path succeeds.
head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0)
if err != nil || head != emptyHash || tail != emptyHash {
t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes",
head, tail, err)
}
// A file that shrank between the stat and hash passes: reading at
// the stat-reported size must fail rather than emit wrong hashes.
p := writeFile(t, dir, "shrunk", []byte("tiny"))
@@ -749,6 +757,96 @@ func TestTreesUnhashedNeverEqual(t *testing.T) {
}
}
func TestScanHardlinksReadOnce(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 300))
b := filepath.Join(dir, "b.bin")
err := os.Link(a, b)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{added: 2}) {
t.Fatalf("stats = %+v, want 2 added", st)
}
// Both paths share the single read's hashes and group together.
recs := dbRecords(t, db)
ra := recordByPath(t, recs, a)
rb := recordByPath(t, recs, b)
if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
}
if groups := collectDupeGroups(recs); len(groups) != 1 {
t.Fatalf("groups = %+v, want the hardlink pair", groups)
}
}
func TestScanHardlinkRunFailsTogether(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 300))
err := os.Link(a, filepath.Join(dir, "b.bin"))
if err != nil {
t.Fatal(err)
}
// Unreadable inode: the run's single read fails, so both paths are
// skipped — proof that hard links are read once, not per path.
err = os.Chmod(a, 0)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st.skipped != 2 || st.added != 0 {
t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
}
}
func TestHashRuns(t *testing.T) {
t.Parallel()
rec := func(path string, dev, ino uint64) fileRec {
return fileRec{path: path, dev: dev, ino: ino}
}
runs := hashRuns([]fileRec{
rec("/c", 1, 7),
rec("/a", 1, 7),
rec("/b", 1, 9),
// No inode identity: never merged, even with matching zeros.
rec("/z1", 0, 0),
rec("/z2", 0, 0),
})
got := make([][]string, 0, len(runs))
for _, run := range runs {
paths := make([]string, 0, len(run))
for _, r := range run {
paths = append(paths, r.path)
}
got = append(got, paths)
}
want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("runs = %v, want %v", got, want)
}
}
func TestPruneRoots(t *testing.T) {
t.Parallel()