From b14b735c8837a676a0caaed90d3fc4d315ea2125 Mon Sep 17 00:00:00 2001 From: sneak Date: Sat, 25 Jul 2026 14:36:55 +0700 Subject: [PATCH] Hash in inode order, read hard links once, never open empty files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sort the hash queue by (device, inode) so reads proceed in inode order, which minimizes seeking on spinning disks. Paths that are hard links to the same inode form one run: the run is read once and every path shares the result, so link farms (rsync --link-dest backups) cost one read per inode instead of one per path. A run that fails to read skips all of its paths. Zero-length files have constant head/tail hashes; return them without opening the file. The hash progress total now counts actual reads (runs, not paths). Hard-linked paths still appear in reports as duplicates — their content is identical — though they share storage; noted in README. --- README.md | 18 +++-- scan.go | 188 ++++++++++++++++++++++++++++++++++++--------------- scan_test.go | 98 +++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index d7bd657..a6af79a 100644 --- a/README.md +++ b/README.md @@ -234,10 +234,14 @@ during the hash phase: unchanged unhashed ones simply keep their records. Every file with a shared size is hashed by the worker pool: read the first `min(1024, size)` bytes and the last `min(1024, size)` bytes - (one read when `size <= 1024`, since the two windows coincide; - for `size == 0` hash the empty input) and compute the SHA-256 of - each. The phase total is exact, so progress and ETA are - meaningful. Completed records are committed in batched + (one read when `size <= 1024`, since the two windows coincide) + and compute the SHA-256 of each. Zero-length files have constant + hashes and are never opened. Files are hashed in **inode order** + (minimizing seeks on spinning disks), and paths that are hard + links to the same inode are **read once**, all sharing the one + result — a hard-link backup farm costs one read per inode, not + per path. The phase total counts actual reads, so progress and + ETA are meaningful. Completed records are committed in batched transactions **while hashing runs**, so a scan interrupted after hours keeps everything hashed so far and the next scan resumes cheaply, skipping records already written. @@ -354,6 +358,12 @@ Definitions: equal. Equal digests imply equal recursive file count and equal total byte size. +Known limitation (accepted): hard-linked paths are reported as +duplicates by `report` and count toward duplicate trees — their +content is genuinely identical — even though they share storage, so +removing one reclaims no space. Inode identity is used during the +scan to avoid redundant reads but is not persisted in the database. + Known limitation (accepted): only regular files that appear in the database define a tree. Empty directories are invisible, and a file skipped during the scan (e.g. permission error) in one copy but not the diff --git a/scan.go b/scan.go index 4e74278..ecc6b37 100644 --- a/scan.go +++ b/scan.go @@ -1,6 +1,7 @@ package main import ( + "cmp" "crypto/sha256" "database/sql" "encoding/hex" @@ -21,11 +22,15 @@ const chunk = 1024 // and hash worker pools. const workQueueDepth = 1024 -// fileRec carries one statted file between the scan phases. +// fileRec carries one statted file between the scan phases. dev and +// ino identify the underlying inode so hard-linked paths can share +// one read; both are zero when the platform exposes no inode. type fileRec struct { path string size int64 mtime int64 + dev uint64 + ino uint64 } // fileMeta is the in-memory index entry for one existing database @@ -312,73 +317,120 @@ func (s *scanState) resolve(path string) { s.st.added++ } -// hashPhase hashes every queued file with the worker pool, committing -// completed records to the database in batches as results arrive, so -// a long scan persists its progress as it goes (an interrupted scan -// resumes cheaply: the next run skips everything already recorded). -// The total is exact, so the bar shows a real ETA. Files that fail to -// hash are warned about and skipped; their stale records, if any, are +// hashRuns sorts the queued files into inode order and groups paths +// sharing an inode into runs. Inode-ordered reads minimize seeking on +// spinning disks, and each run of hard-linked paths is read once, +// with every path sharing the result. Files without an inode identity +// are never merged. +func hashRuns(toHash []fileRec) [][]fileRec { + slices.SortFunc(toHash, func(a, b fileRec) int { + if c := cmp.Compare(a.dev, b.dev); c != 0 { + return c + } + + if c := cmp.Compare(a.ino, b.ino); c != 0 { + return c + } + + return strings.Compare(a.path, b.path) + }) + + var runs [][]fileRec + + start := 0 + + for i := 1; i <= len(toHash); i++ { + if i == len(toHash) || !sameInode(toHash[i-1], toHash[i]) { + runs = append(runs, toHash[start:i]) + start = i + } + } + + return runs +} + +// sameInode reports whether two records name the same underlying +// inode. Records without an inode identity never match. +func sameInode(a, b fileRec) bool { + return (a.dev != 0 || a.ino != 0) && a.dev == b.dev && a.ino == b.ino +} + +// hashPhase hashes every queued file with the worker pool — one read +// per inode run, in inode order — committing completed records to the +// database in batches as results arrive, so a long scan persists its +// progress as it goes (an interrupted scan resumes cheaply: the next +// run skips everything already recorded). The total counts actual +// reads, so the bar shows a real ETA. A run that fails to hash is +// warned about and skipped; stale records for its paths, if any, are // deleted by the update phase. func (s *scanState) hashPhase(workers int) error { - jobs := make(chan fileRec, workQueueDepth) + runs := hashRuns(s.toHash) + s.toHash = nil + + jobs := make(chan []fileRec, workQueueDepth) results := make(chan hashResult, workQueueDepth) startHashWorkers(jobs, results, workers) - // The feeder ranges over its own reference: s.toHash is released - // below while the feeder may still be running. - toHash := s.toHash - s.toHash = nil - go func() { - for _, rec := range toHash { - jobs <- rec + for _, run := range runs { + jobs <- run } close(jobs) }() - prog := newProgress("hash", int64(len(toHash))) + prog := newProgress("hash", int64(len(runs))) defer prog.finish() - for range toHash { + for range runs { r := <-results prog.increment() if r.err != nil { - s.st.skipped++ + s.st.skipped += len(r.run) - prog.warnf("hash %s: %v", r.rec.path, r.err) + prog.warnf("hash %s: %v", r.run[0].path, r.err) continue } - s.resolve(r.rec.path) - - s.batch = append(s.batch, scanRec{ - size: r.rec.size, - mtime: r.rec.mtime, - head: r.head, - tail: r.tail, - path: r.rec.path, - }) - - if len(s.batch) < updateBatchSize { - continue - } - - err := applyBatch(s.db, s.batch, nil, nil) + err := s.recordRun(r) if err != nil { return err } - - s.batch = s.batch[:0] } return nil } +// recordRun folds one hash result into the running batch: every path +// in the run (one file, or several hard links to it) gets a record +// with the shared hashes. +func (s *scanState) recordRun(r hashResult) error { + for _, rec := range r.run { + s.resolve(rec.path) + + s.batch = append(s.batch, scanRec{ + size: rec.size, + mtime: rec.mtime, + head: r.head, + tail: r.tail, + path: rec.path, + }) + } + + if len(s.batch) < updateBatchSize { + return nil + } + + err := applyBatch(s.db, s.batch, nil, nil) + s.batch = s.batch[:0] + + return err +} + // updatePhase writes the scan's tail under one progress display: the // final partial batch of hashed records, a hash-less record for every // size-unique new or changed file, and deletions for every record the @@ -515,10 +567,14 @@ func seedRoot(root string, events chan<- walkEvent) []dirJob { return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}} case fi.Mode().IsRegular(): + dev, ino := inodeOfInfo(fi) + events <- walkEvent{rec: fileRec{ path: root, size: fi.Size(), mtime: fi.ModTime().Unix(), + dev: dev, + ino: ino, }} return nil @@ -649,10 +705,14 @@ func emitFile(p string, e fs.DirEntry, events chan<- walkEvent) { return } + dev, ino := inodeOfInfo(info) + events <- walkEvent{rec: fileRec{ path: p, size: info.Size(), mtime: info.ModTime().Unix(), + dev: dev, + ino: ino, }} } @@ -700,38 +760,61 @@ func deviceOfInfo(fi fs.FileInfo) (uint64, bool) { return statDev(st), true } -// hashResult carries one file's head/tail hashes (or the error that -// prevented hashing it) from the hash workers to the hash phase. +// inodeOfInfo extracts the (device, inode) pair identifying a file's +// underlying inode; (0, 0) when the platform exposes none (such files +// are never merged as hard links). +func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + + return statDev(st), st.Ino +} + +// hashResult carries one inode run's head/tail hashes (or the error +// that prevented hashing it) from the hash workers to the hash phase. type hashResult struct { - rec fileRec + run []fileRec head string tail string err error } -// startHashWorkers starts the hash worker pool: workers read jobs, -// write one result per record, and exit when jobs is closed. -func startHashWorkers(jobs <-chan fileRec, results chan<- hashResult, +// startHashWorkers starts the hash worker pool: workers read inode +// runs, hash each run's first path (all paths in a run are hard links +// to the same inode), write one result per run, and exit when jobs is +// closed. +func startHashWorkers(jobs <-chan []fileRec, results chan<- hashResult, workers int, ) { for range workers { go func() { - for rec := range jobs { - head, tail, err := hashHeadTail(rec.path, rec.size) + for run := range jobs { + head, tail, err := hashHeadTail(run[0].path, run[0].size) results <- hashResult{ - rec: rec, head: head, tail: tail, err: err, + run: run, head: head, tail: tail, err: err, } } }() } } +// emptyHash is the lowercase-hex SHA-256 of the empty input: the head +// and tail hash of every zero-length file. +const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + + "27ae41e4649b934ca495991b7852b855" + // hashHeadTail returns the lowercase-hex SHA-256 of the first // min(chunk, size) bytes and of the last min(chunk, size) bytes of the -// file at path. The two reads overlap when size < 2*chunk; for -// size == 0 both hashes are of the empty input. size is the value -// recorded when the file was statted. +// file at path. The two reads overlap when size < 2*chunk. size is the +// value recorded when the file was statted; a zero-length file's +// hashes are constant, so it is never even opened. func hashHeadTail(path string, size int64) (string, string, error) { + if size == 0 { + return emptyHash, emptyHash, nil + } + //nolint:gosec // hashing operator-supplied paths is the tool's purpose f, err := os.Open(path) if err != nil { @@ -743,11 +826,10 @@ func hashHeadTail(path string, size int64) (string, string, error) { n := min(int64(chunk), size) buf := make([]byte, n) - if n > 0 { - _, err = f.ReadAt(buf, 0) - if err != nil { - return "", "", err - } + + _, err = f.ReadAt(buf, 0) + if err != nil { + return "", "", err } h := sha256.Sum256(buf) diff --git a/scan_test.go b/scan_test.go index e658e4e..9e563cd 100644 --- a/scan_test.go +++ b/scan_test.go @@ -100,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) { t.Error("no error for a missing file") } + // A zero-length file has constant hashes and is never opened: even + // a missing path succeeds. + head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0) + if err != nil || head != emptyHash || tail != emptyHash { + t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes", + head, tail, err) + } + // A file that shrank between the stat and hash passes: reading at // the stat-reported size must fail rather than emit wrong hashes. p := writeFile(t, dir, "shrunk", []byte("tiny")) @@ -749,6 +757,96 @@ func TestTreesUnhashedNeverEqual(t *testing.T) { } } +func TestScanHardlinksReadOnce(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 300)) + b := filepath.Join(dir, "b.bin") + + err := os.Link(a, b) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{added: 2}) { + t.Fatalf("stats = %+v, want 2 added", st) + } + + // Both paths share the single read's hashes and group together. + recs := dbRecords(t, db) + + ra := recordByPath(t, recs, a) + rb := recordByPath(t, recs, b) + + if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail { + t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb) + } + + if groups := collectDupeGroups(recs); len(groups) != 1 { + t.Fatalf("groups = %+v, want the hardlink pair", groups) + } +} + +func TestScanHardlinkRunFailsTogether(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 300)) + + err := os.Link(a, filepath.Join(dir, "b.bin")) + if err != nil { + t.Fatal(err) + } + + // Unreadable inode: the run's single read fails, so both paths are + // skipped — proof that hard links are read once, not per path. + err = os.Chmod(a, 0) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st.skipped != 2 || st.added != 0 { + t.Fatalf("stats = %+v, want both hardlink paths skipped", st) + } +} + +func TestHashRuns(t *testing.T) { + t.Parallel() + + rec := func(path string, dev, ino uint64) fileRec { + return fileRec{path: path, dev: dev, ino: ino} + } + + runs := hashRuns([]fileRec{ + rec("/c", 1, 7), + rec("/a", 1, 7), + rec("/b", 1, 9), + // No inode identity: never merged, even with matching zeros. + rec("/z1", 0, 0), + rec("/z2", 0, 0), + }) + + got := make([][]string, 0, len(runs)) + for _, run := range runs { + paths := make([]string, 0, len(run)) + for _, r := range run { + paths = append(paths, r.path) + } + + got = append(got, paths) + } + + want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}} + if !slices.EqualFunc(got, want, slices.Equal) { + t.Fatalf("runs = %v, want %v", got, want) + } +} + func TestPruneRoots(t *testing.T) { t.Parallel()