diff --git a/README.md b/README.md index d7bd657..a6af79a 100644 --- a/README.md +++ b/README.md @@ -234,10 +234,14 @@ during the hash phase: unchanged unhashed ones simply keep their records. Every file with a shared size is hashed by the worker pool: read the first `min(1024, size)` bytes and the last `min(1024, size)` bytes - (one read when `size <= 1024`, since the two windows coincide; - for `size == 0` hash the empty input) and compute the SHA-256 of - each. The phase total is exact, so progress and ETA are - meaningful. Completed records are committed in batched + (one read when `size <= 1024`, since the two windows coincide) + and compute the SHA-256 of each. Zero-length files have constant + hashes and are never opened. Files are hashed in **inode order** + (minimizing seeks on spinning disks), and paths that are hard + links to the same inode are **read once**, all sharing the one + result — a hard-link backup farm costs one read per inode, not + per path. The phase total counts actual reads, so progress and + ETA are meaningful. Completed records are committed in batched transactions **while hashing runs**, so a scan interrupted after hours keeps everything hashed so far and the next scan resumes cheaply, skipping records already written. @@ -354,6 +358,12 @@ Definitions: equal. Equal digests imply equal recursive file count and equal total byte size. +Known limitation (accepted): hard-linked paths are reported as +duplicates by `report` and count toward duplicate trees — their +content is genuinely identical — even though they share storage, so +removing one reclaims no space. Inode identity is used during the +scan to avoid redundant reads but is not persisted in the database. + Known limitation (accepted): only regular files that appear in the database define a tree. Empty directories are invisible, and a file skipped during the scan (e.g. permission error) in one copy but not the diff --git a/scan.go b/scan.go index 4e74278..ecc6b37 100644 --- a/scan.go +++ b/scan.go @@ -1,6 +1,7 @@ package main import ( + "cmp" "crypto/sha256" "database/sql" "encoding/hex" @@ -21,11 +22,15 @@ const chunk = 1024 // and hash worker pools. const workQueueDepth = 1024 -// fileRec carries one statted file between the scan phases. +// fileRec carries one statted file between the scan phases. dev and +// ino identify the underlying inode so hard-linked paths can share +// one read; both are zero when the platform exposes no inode. type fileRec struct { path string size int64 mtime int64 + dev uint64 + ino uint64 } // fileMeta is the in-memory index entry for one existing database @@ -312,73 +317,120 @@ func (s *scanState) resolve(path string) { s.st.added++ } -// hashPhase hashes every queued file with the worker pool, committing -// completed records to the database in batches as results arrive, so -// a long scan persists its progress as it goes (an interrupted scan -// resumes cheaply: the next run skips everything already recorded). -// The total is exact, so the bar shows a real ETA. Files that fail to -// hash are warned about and skipped; their stale records, if any, are +// hashRuns sorts the queued files into inode order and groups paths +// sharing an inode into runs. Inode-ordered reads minimize seeking on +// spinning disks, and each run of hard-linked paths is read once, +// with every path sharing the result. Files without an inode identity +// are never merged. +func hashRuns(toHash []fileRec) [][]fileRec { + slices.SortFunc(toHash, func(a, b fileRec) int { + if c := cmp.Compare(a.dev, b.dev); c != 0 { + return c + } + + if c := cmp.Compare(a.ino, b.ino); c != 0 { + return c + } + + return strings.Compare(a.path, b.path) + }) + + var runs [][]fileRec + + start := 0 + + for i := 1; i <= len(toHash); i++ { + if i == len(toHash) || !sameInode(toHash[i-1], toHash[i]) { + runs = append(runs, toHash[start:i]) + start = i + } + } + + return runs +} + +// sameInode reports whether two records name the same underlying +// inode. Records without an inode identity never match. +func sameInode(a, b fileRec) bool { + return (a.dev != 0 || a.ino != 0) && a.dev == b.dev && a.ino == b.ino +} + +// hashPhase hashes every queued file with the worker pool — one read +// per inode run, in inode order — committing completed records to the +// database in batches as results arrive, so a long scan persists its +// progress as it goes (an interrupted scan resumes cheaply: the next +// run skips everything already recorded). The total counts actual +// reads, so the bar shows a real ETA. A run that fails to hash is +// warned about and skipped; stale records for its paths, if any, are // deleted by the update phase. func (s *scanState) hashPhase(workers int) error { - jobs := make(chan fileRec, workQueueDepth) + runs := hashRuns(s.toHash) + s.toHash = nil + + jobs := make(chan []fileRec, workQueueDepth) results := make(chan hashResult, workQueueDepth) startHashWorkers(jobs, results, workers) - // The feeder ranges over its own reference: s.toHash is released - // below while the feeder may still be running. - toHash := s.toHash - s.toHash = nil - go func() { - for _, rec := range toHash { - jobs <- rec + for _, run := range runs { + jobs <- run } close(jobs) }() - prog := newProgress("hash", int64(len(toHash))) + prog := newProgress("hash", int64(len(runs))) defer prog.finish() - for range toHash { + for range runs { r := <-results prog.increment() if r.err != nil { - s.st.skipped++ + s.st.skipped += len(r.run) - prog.warnf("hash %s: %v", r.rec.path, r.err) + prog.warnf("hash %s: %v", r.run[0].path, r.err) continue } - s.resolve(r.rec.path) - - s.batch = append(s.batch, scanRec{ - size: r.rec.size, - mtime: r.rec.mtime, - head: r.head, - tail: r.tail, - path: r.rec.path, - }) - - if len(s.batch) < updateBatchSize { - continue - } - - err := applyBatch(s.db, s.batch, nil, nil) + err := s.recordRun(r) if err != nil { return err } - - s.batch = s.batch[:0] } return nil } +// recordRun folds one hash result into the running batch: every path +// in the run (one file, or several hard links to it) gets a record +// with the shared hashes. +func (s *scanState) recordRun(r hashResult) error { + for _, rec := range r.run { + s.resolve(rec.path) + + s.batch = append(s.batch, scanRec{ + size: rec.size, + mtime: rec.mtime, + head: r.head, + tail: r.tail, + path: rec.path, + }) + } + + if len(s.batch) < updateBatchSize { + return nil + } + + err := applyBatch(s.db, s.batch, nil, nil) + s.batch = s.batch[:0] + + return err +} + // updatePhase writes the scan's tail under one progress display: the // final partial batch of hashed records, a hash-less record for every // size-unique new or changed file, and deletions for every record the @@ -515,10 +567,14 @@ func seedRoot(root string, events chan<- walkEvent) []dirJob { return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}} case fi.Mode().IsRegular(): + dev, ino := inodeOfInfo(fi) + events <- walkEvent{rec: fileRec{ path: root, size: fi.Size(), mtime: fi.ModTime().Unix(), + dev: dev, + ino: ino, }} return nil @@ -649,10 +705,14 @@ func emitFile(p string, e fs.DirEntry, events chan<- walkEvent) { return } + dev, ino := inodeOfInfo(info) + events <- walkEvent{rec: fileRec{ path: p, size: info.Size(), mtime: info.ModTime().Unix(), + dev: dev, + ino: ino, }} } @@ -700,38 +760,61 @@ func deviceOfInfo(fi fs.FileInfo) (uint64, bool) { return statDev(st), true } -// hashResult carries one file's head/tail hashes (or the error that -// prevented hashing it) from the hash workers to the hash phase. +// inodeOfInfo extracts the (device, inode) pair identifying a file's +// underlying inode; (0, 0) when the platform exposes none (such files +// are never merged as hard links). +func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + + return statDev(st), st.Ino +} + +// hashResult carries one inode run's head/tail hashes (or the error +// that prevented hashing it) from the hash workers to the hash phase. type hashResult struct { - rec fileRec + run []fileRec head string tail string err error } -// startHashWorkers starts the hash worker pool: workers read jobs, -// write one result per record, and exit when jobs is closed. -func startHashWorkers(jobs <-chan fileRec, results chan<- hashResult, +// startHashWorkers starts the hash worker pool: workers read inode +// runs, hash each run's first path (all paths in a run are hard links +// to the same inode), write one result per run, and exit when jobs is +// closed. +func startHashWorkers(jobs <-chan []fileRec, results chan<- hashResult, workers int, ) { for range workers { go func() { - for rec := range jobs { - head, tail, err := hashHeadTail(rec.path, rec.size) + for run := range jobs { + head, tail, err := hashHeadTail(run[0].path, run[0].size) results <- hashResult{ - rec: rec, head: head, tail: tail, err: err, + run: run, head: head, tail: tail, err: err, } } }() } } +// emptyHash is the lowercase-hex SHA-256 of the empty input: the head +// and tail hash of every zero-length file. +const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + + "27ae41e4649b934ca495991b7852b855" + // hashHeadTail returns the lowercase-hex SHA-256 of the first // min(chunk, size) bytes and of the last min(chunk, size) bytes of the -// file at path. The two reads overlap when size < 2*chunk; for -// size == 0 both hashes are of the empty input. size is the value -// recorded when the file was statted. +// file at path. The two reads overlap when size < 2*chunk. size is the +// value recorded when the file was statted; a zero-length file's +// hashes are constant, so it is never even opened. func hashHeadTail(path string, size int64) (string, string, error) { + if size == 0 { + return emptyHash, emptyHash, nil + } + //nolint:gosec // hashing operator-supplied paths is the tool's purpose f, err := os.Open(path) if err != nil { @@ -743,11 +826,10 @@ func hashHeadTail(path string, size int64) (string, string, error) { n := min(int64(chunk), size) buf := make([]byte, n) - if n > 0 { - _, err = f.ReadAt(buf, 0) - if err != nil { - return "", "", err - } + + _, err = f.ReadAt(buf, 0) + if err != nil { + return "", "", err } h := sha256.Sum256(buf) diff --git a/scan_test.go b/scan_test.go index e658e4e..9e563cd 100644 --- a/scan_test.go +++ b/scan_test.go @@ -100,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) { t.Error("no error for a missing file") } + // A zero-length file has constant hashes and is never opened: even + // a missing path succeeds. + head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0) + if err != nil || head != emptyHash || tail != emptyHash { + t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes", + head, tail, err) + } + // A file that shrank between the stat and hash passes: reading at // the stat-reported size must fail rather than emit wrong hashes. p := writeFile(t, dir, "shrunk", []byte("tiny")) @@ -749,6 +757,96 @@ func TestTreesUnhashedNeverEqual(t *testing.T) { } } +func TestScanHardlinksReadOnce(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 300)) + b := filepath.Join(dir, "b.bin") + + err := os.Link(a, b) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{added: 2}) { + t.Fatalf("stats = %+v, want 2 added", st) + } + + // Both paths share the single read's hashes and group together. + recs := dbRecords(t, db) + + ra := recordByPath(t, recs, a) + rb := recordByPath(t, recs, b) + + if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail { + t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb) + } + + if groups := collectDupeGroups(recs); len(groups) != 1 { + t.Fatalf("groups = %+v, want the hardlink pair", groups) + } +} + +func TestScanHardlinkRunFailsTogether(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 300)) + + err := os.Link(a, filepath.Join(dir, "b.bin")) + if err != nil { + t.Fatal(err) + } + + // Unreadable inode: the run's single read fails, so both paths are + // skipped — proof that hard links are read once, not per path. + err = os.Chmod(a, 0) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st.skipped != 2 || st.added != 0 { + t.Fatalf("stats = %+v, want both hardlink paths skipped", st) + } +} + +func TestHashRuns(t *testing.T) { + t.Parallel() + + rec := func(path string, dev, ino uint64) fileRec { + return fileRec{path: path, dev: dev, ino: ino} + } + + runs := hashRuns([]fileRec{ + rec("/c", 1, 7), + rec("/a", 1, 7), + rec("/b", 1, 9), + // No inode identity: never merged, even with matching zeros. + rec("/z1", 0, 0), + rec("/z2", 0, 0), + }) + + got := make([][]string, 0, len(runs)) + for _, run := range runs { + paths := make([]string, 0, len(run)) + for _, r := range run { + paths = append(paths, r.path) + } + + got = append(got, paths) + } + + want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}} + if !slices.EqualFunc(got, want, slices.Equal) { + t.Fatalf("runs = %v, want %v", got, want) + } +} + func TestPruneRoots(t *testing.T) { t.Parallel()