Merge branch 'inode-runs': inode-ordered hashing, hard links read once

README.md

@@ -234,10 +234,14 @@ during the hash phase:
     unchanged unhashed ones simply keep their records. Every file
     with a shared size is hashed by the worker pool: read the first
     `min(1024, size)` bytes and the last `min(1024, size)` bytes
-    (one read when `size <= 1024`, since the two windows coincide;
-    for `size == 0` hash the empty input) and compute the SHA-256 of
-    each. The phase total is exact, so progress and ETA are
-    meaningful. Completed records are committed in batched
+    (one read when `size <= 1024`, since the two windows coincide)
+    and compute the SHA-256 of each. Zero-length files have constant
+    hashes and are never opened. Files are hashed in **inode order**
+    (minimizing seeks on spinning disks), and paths that are hard
+    links to the same inode are **read once**, all sharing the one
+    result — a hard-link backup farm costs one read per inode, not
+    per path. The phase total counts actual reads, so progress and
+    ETA are meaningful. Completed records are committed in batched
     transactions **while hashing runs**, so a scan interrupted after
     hours keeps everything hashed so far and the next scan resumes
     cheaply, skipping records already written.
@@ -354,6 +358,12 @@ Definitions:
     equal. Equal digests imply equal recursive file count and equal
     total byte size.
 
+Known limitation (accepted): hard-linked paths are reported as
+duplicates by `report` and count toward duplicate trees — their
+content is genuinely identical — even though they share storage, so
+removing one reclaims no space. Inode identity is used during the
+scan to avoid redundant reads but is not persisted in the database.
+
 Known limitation (accepted): only regular files that appear in the
 database define a tree. Empty directories are invisible, and a file
 skipped during the scan (e.g. permission error) in one copy but not the

scan.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"cmp"
 	"crypto/sha256"
 	"database/sql"
 	"encoding/hex"
@@ -21,11 +22,15 @@ const chunk = 1024
 // and hash worker pools.
 const workQueueDepth = 1024
 
-// fileRec carries one statted file between the scan phases.
+// fileRec carries one statted file between the scan phases. dev and
+// ino identify the underlying inode so hard-linked paths can share
+// one read; both are zero when the platform exposes no inode.
 type fileRec struct {
 	path  string
 	size  int64
 	mtime int64
+	dev   uint64
+	ino   uint64
 }
 
 // fileMeta is the in-memory index entry for one existing database
@@ -312,73 +317,120 @@ func (s *scanState) resolve(path string) {
 	s.st.added++
 }
 
-// hashPhase hashes every queued file with the worker pool, committing
-// completed records to the database in batches as results arrive, so
-// a long scan persists its progress as it goes (an interrupted scan
-// resumes cheaply: the next run skips everything already recorded).
-// The total is exact, so the bar shows a real ETA. Files that fail to
-// hash are warned about and skipped; their stale records, if any, are
+// hashRuns sorts the queued files into inode order and groups paths
+// sharing an inode into runs. Inode-ordered reads minimize seeking on
+// spinning disks, and each run of hard-linked paths is read once,
+// with every path sharing the result. Files without an inode identity
+// are never merged.
+func hashRuns(toHash []fileRec) [][]fileRec {
+	slices.SortFunc(toHash, func(a, b fileRec) int {
+		if c := cmp.Compare(a.dev, b.dev); c != 0 {
+			return c
+		}
+
+		if c := cmp.Compare(a.ino, b.ino); c != 0 {
+			return c
+		}
+
+		return strings.Compare(a.path, b.path)
+	})
+
+	var runs [][]fileRec
+
+	start := 0
+
+	for i := 1; i <= len(toHash); i++ {
+		if i == len(toHash) || !sameInode(toHash[i-1], toHash[i]) {
+			runs = append(runs, toHash[start:i])
+			start = i
+		}
+	}
+
+	return runs
+}
+
+// sameInode reports whether two records name the same underlying
+// inode. Records without an inode identity never match.
+func sameInode(a, b fileRec) bool {
+	return (a.dev != 0 || a.ino != 0) && a.dev == b.dev && a.ino == b.ino
+}
+
+// hashPhase hashes every queued file with the worker pool — one read
+// per inode run, in inode order — committing completed records to the
+// database in batches as results arrive, so a long scan persists its
+// progress as it goes (an interrupted scan resumes cheaply: the next
+// run skips everything already recorded). The total counts actual
+// reads, so the bar shows a real ETA. A run that fails to hash is
+// warned about and skipped; stale records for its paths, if any, are
 // deleted by the update phase.
 func (s *scanState) hashPhase(workers int) error {
-	jobs := make(chan fileRec, workQueueDepth)
+	runs := hashRuns(s.toHash)
+	s.toHash = nil
+
+	jobs := make(chan []fileRec, workQueueDepth)
 	results := make(chan hashResult, workQueueDepth)
 
 	startHashWorkers(jobs, results, workers)
 
-	// The feeder ranges over its own reference: s.toHash is released
-	// below while the feeder may still be running.
-	toHash := s.toHash
-	s.toHash = nil
-
 	go func() {
-		for _, rec := range toHash {
-			jobs <- rec
+		for _, run := range runs {
+			jobs <- run
 		}
 
 		close(jobs)
 	}()
 
-	prog := newProgress("hash", int64(len(toHash)))
+	prog := newProgress("hash", int64(len(runs)))
 	defer prog.finish()
 
-	for range toHash {
+	for range runs {
 		r := <-results
 
 		prog.increment()
 
 		if r.err != nil {
-			s.st.skipped++
+			s.st.skipped += len(r.run)
 
-			prog.warnf("hash %s: %v", r.rec.path, r.err)
+			prog.warnf("hash %s: %v", r.run[0].path, r.err)
 
 			continue
 		}
 
-		s.resolve(r.rec.path)
-
-		s.batch = append(s.batch, scanRec{
-			size:  r.rec.size,
-			mtime: r.rec.mtime,
-			head:  r.head,
-			tail:  r.tail,
-			path:  r.rec.path,
-		})
-
-		if len(s.batch) < updateBatchSize {
-			continue
-		}
-
-		err := applyBatch(s.db, s.batch, nil, nil)
+		err := s.recordRun(r)
 		if err != nil {
 			return err
 		}
-
-		s.batch = s.batch[:0]
 	}
 
 	return nil
 }
 
+// recordRun folds one hash result into the running batch: every path
+// in the run (one file, or several hard links to it) gets a record
+// with the shared hashes.
+func (s *scanState) recordRun(r hashResult) error {
+	for _, rec := range r.run {
+		s.resolve(rec.path)
+
+		s.batch = append(s.batch, scanRec{
+			size:  rec.size,
+			mtime: rec.mtime,
+			head:  r.head,
+			tail:  r.tail,
+			path:  rec.path,
+		})
+	}
+
+	if len(s.batch) < updateBatchSize {
+		return nil
+	}
+
+	err := applyBatch(s.db, s.batch, nil, nil)
+	s.batch = s.batch[:0]
+
+	return err
+}
+
 // updatePhase writes the scan's tail under one progress display: the
 // final partial batch of hashed records, a hash-less record for every
 // size-unique new or changed file, and deletions for every record the
@@ -515,10 +567,14 @@ func seedRoot(root string, events chan<- walkEvent) []dirJob {
 
 		return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}}
 	case fi.Mode().IsRegular():
+		dev, ino := inodeOfInfo(fi)
+
 		events <- walkEvent{rec: fileRec{
 			path:  root,
 			size:  fi.Size(),
 			mtime: fi.ModTime().Unix(),
+			dev:   dev,
+			ino:   ino,
 		}}
 
 		return nil
@@ -649,10 +705,14 @@ func emitFile(p string, e fs.DirEntry, events chan<- walkEvent) {
 		return
 	}
 
+	dev, ino := inodeOfInfo(info)
+
 	events <- walkEvent{rec: fileRec{
 		path:  p,
 		size:  info.Size(),
 		mtime: info.ModTime().Unix(),
+		dev:   dev,
+		ino:   ino,
 	}}
 }
 
@@ -700,38 +760,61 @@ func deviceOfInfo(fi fs.FileInfo) (uint64, bool) {
 	return statDev(st), true
 }
 
-// hashResult carries one file's head/tail hashes (or the error that
-// prevented hashing it) from the hash workers to the hash phase.
+// inodeOfInfo extracts the (device, inode) pair identifying a file's
+// underlying inode; (0, 0) when the platform exposes none (such files
+// are never merged as hard links).
+func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, 0
+	}
+
+	return statDev(st), st.Ino
+}
+
+// hashResult carries one inode run's head/tail hashes (or the error
+// that prevented hashing it) from the hash workers to the hash phase.
 type hashResult struct {
-	rec  fileRec
+	run  []fileRec
 	head string
 	tail string
 	err  error
 }
 
-// startHashWorkers starts the hash worker pool: workers read jobs,
-// write one result per record, and exit when jobs is closed.
-func startHashWorkers(jobs <-chan fileRec, results chan<- hashResult,
+// startHashWorkers starts the hash worker pool: workers read inode
+// runs, hash each run's first path (all paths in a run are hard links
+// to the same inode), write one result per run, and exit when jobs is
+// closed.
+func startHashWorkers(jobs <-chan []fileRec, results chan<- hashResult,
 	workers int,
 ) {
 	for range workers {
 		go func() {
-			for rec := range jobs {
-				head, tail, err := hashHeadTail(rec.path, rec.size)
+			for run := range jobs {
+				head, tail, err := hashHeadTail(run[0].path, run[0].size)
 				results <- hashResult{
-					rec: rec, head: head, tail: tail, err: err,
+					run: run, head: head, tail: tail, err: err,
 				}
 			}
 		}()
 	}
 }
 
+// emptyHash is the lowercase-hex SHA-256 of the empty input: the head
+// and tail hash of every zero-length file.
+const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
+	"27ae41e4649b934ca495991b7852b855"
+
 // hashHeadTail returns the lowercase-hex SHA-256 of the first
 // min(chunk, size) bytes and of the last min(chunk, size) bytes of the
-// file at path. The two reads overlap when size < 2*chunk; for
-// size == 0 both hashes are of the empty input. size is the value
-// recorded when the file was statted.
+// file at path. The two reads overlap when size < 2*chunk. size is the
+// value recorded when the file was statted; a zero-length file's
+// hashes are constant, so it is never even opened.
 func hashHeadTail(path string, size int64) (string, string, error) {
+	if size == 0 {
+		return emptyHash, emptyHash, nil
+	}
+
 	//nolint:gosec // hashing operator-supplied paths is the tool's purpose
 	f, err := os.Open(path)
 	if err != nil {
@@ -743,12 +826,11 @@ func hashHeadTail(path string, size int64) (string, string, error) {
 	n := min(int64(chunk), size)
 
 	buf := make([]byte, n)
-	if n > 0 {
+
 	_, err = f.ReadAt(buf, 0)
 	if err != nil {
 		return "", "", err
 	}
-	}
 
 	h := sha256.Sum256(buf)
 

scan_test.go

@@ -100,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) {
 		t.Error("no error for a missing file")
 	}
 
+	// A zero-length file has constant hashes and is never opened: even
+	// a missing path succeeds.
+	head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0)
+	if err != nil || head != emptyHash || tail != emptyHash {
+		t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes",
+			head, tail, err)
+	}
+
 	// A file that shrank between the stat and hash passes: reading at
 	// the stat-reported size must fail rather than emit wrong hashes.
 	p := writeFile(t, dir, "shrunk", []byte("tiny"))
@@ -749,6 +757,96 @@ func TestTreesUnhashedNeverEqual(t *testing.T) {
 	}
 }
 
+func TestScanHardlinksReadOnce(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+	b := filepath.Join(dir, "b.bin")
+
+	err := os.Link(a, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	// Both paths share the single read's hashes and group together.
+	recs := dbRecords(t, db)
+
+	ra := recordByPath(t, recs, a)
+	rb := recordByPath(t, recs, b)
+
+	if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
+		t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the hardlink pair", groups)
+	}
+}
+
+func TestScanHardlinkRunFailsTogether(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+
+	err := os.Link(a, filepath.Join(dir, "b.bin"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unreadable inode: the run's single read fails, so both paths are
+	// skipped — proof that hard links are read once, not per path.
+	err = os.Chmod(a, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.skipped != 2 || st.added != 0 {
+		t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
+	}
+}
+
+func TestHashRuns(t *testing.T) {
+	t.Parallel()
+
+	rec := func(path string, dev, ino uint64) fileRec {
+		return fileRec{path: path, dev: dev, ino: ino}
+	}
+
+	runs := hashRuns([]fileRec{
+		rec("/c", 1, 7),
+		rec("/a", 1, 7),
+		rec("/b", 1, 9),
+		// No inode identity: never merged, even with matching zeros.
+		rec("/z1", 0, 0),
+		rec("/z2", 0, 0),
+	})
+
+	got := make([][]string, 0, len(runs))
+	for _, run := range runs {
+		paths := make([]string, 0, len(run))
+		for _, r := range run {
+			paths = append(paths, r.path)
+		}
+
+		got = append(got, paths)
+	}
+
+	want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}}
+	if !slices.EqualFunc(got, want, slices.Equal) {
+		t.Fatalf("runs = %v, want %v", got, want)
+	}
+}
+
 func TestPruneRoots(t *testing.T) {
 	t.Parallel()