Compute the content hash only when head and tail match (closes #61)

A file of 10 MiB or more now gets only its 64 KiB head and tail in the
hash phase, so its content is read only when it can be a duplicate. A
new content phase after the update phase finds every group of records,
anywhere in the database, that share size, head and tail and include
one without a content hash. It checks every member with lstat and, when
at least two pass, reads those without a content hash through the
existing worker pool; a stale file does not count as a match. report
and trees leave out records without a content hash. The README, help
text and TODO entry describe the gate; the schema stays at version 1.

Lint suppressed: gosec on the file open in hashContentOnly, as in
hashSignature, and on one chmod in a test.

Model: opus-5-5

scan_test.go

@@ -106,6 +106,8 @@ func TestHashSignatureBelowThreshold(t *testing.T) {
 // difference in the last window changes only tail, and a difference
 // between the windows changes neither end hash but does change the
 // whole-file content rung (the file is below wholeFileMax).
+// hashSignature leaves the content hash of a file this size to the
+// content phase, so that rung is checked through a scan.
 func TestHashSignatureEnds(t *testing.T) {
 	t.Parallel()
 
@@ -125,8 +127,11 @@ func TestHashSignatureEnds(t *testing.T) {
 	pokeAt(t, midDiff, size/2, []byte{1})
 
 	bHead, bTail, bContent := sig(t, base, size)
+	if bContent != "" {
+		t.Errorf("content = %q, want none from the hash phase", bContent)
+	}
 
-	h, tl, c := sig(t, headDiff, size)
+	h, tl, _ := sig(t, headDiff, size)
 	if h == bHead {
 		t.Error("a byte in the first window did not change head")
 	}
@@ -135,11 +140,7 @@ func TestHashSignatureEnds(t *testing.T) {
 		t.Error("a byte in the first window changed tail")
 	}
 
-	if c == bContent {
-		t.Error("a byte in the first window did not change content")
-	}
-
-	h, tl, c = sig(t, tailDiff, size)
+	h, tl, _ = sig(t, tailDiff, size)
 	if tl == bTail {
 		t.Error("a byte in the last window did not change tail")
 	}
@@ -148,16 +149,15 @@ func TestHashSignatureEnds(t *testing.T) {
 		t.Error("a byte in the last window changed head")
 	}
 
-	if c == bContent {
-		t.Error("a byte in the last window did not change content")
-	}
-
-	h, tl, c = sig(t, midDiff, size)
+	h, tl, _ = sig(t, midDiff, size)
 	if h != bHead || tl != bTail {
 		t.Error("a byte between the windows changed an end hash")
 	}
 
-	if c == bContent {
+	// base and midDiff match on size, head, and tail, so the scan reads
+	// both for their content hashes.
+	c := scanContents(t, dir, base, midDiff)
+	if c[midDiff] == c[base] {
 		t.Error("whole-file content rung ignored a byte between the windows")
 	}
 }
@@ -246,19 +246,38 @@ func pokeAt(t *testing.T, path string, off int64, data []byte) {
 	}
 }
 
-// contentHash returns just the content rung of a file's signature.
-func contentHash(t *testing.T, path string, size int64) string {
+// scanContents scans dir into a fresh database and returns the content
+// hash recorded for each file, by path, failing the test if one of want
+// has none. A file of headTailMin or more gets a content hash only when
+// it is scanned with a file of the same size, head, and tail.
+func scanContents(t *testing.T, dir string,
+	want ...string,
+) map[string]string {
 	t.Helper()
 
-	_, _, content := sig(t, path, size)
+	db := openTestDB(t)
+	syncTree(t, db, dir)
 
-	return content
+	contents := make(map[string]string)
+	for _, r := range dbRecords(t, db) {
+		contents[r.path] = r.content
+	}
+
+	for _, p := range want {
+		if contents[p] == "" {
+			t.Fatalf("%s: no content hash", p)
+		}
+	}
+
+	return contents
 }
 
 // TestContentRungBoundary checks the 50 MiB boundary between the two
 // content rungs: just below it the whole file is hashed and any byte
 // difference shows; at the boundary only the gigabyte-spaced samples are
-// hashed, so a difference outside a sample window is invisible.
+// hashed, so a difference outside a sample window is invisible. The
+// files of each pair match on size, head, and tail, so the scan reads
+// both for their content hashes.
 func TestContentRungBoundary(t *testing.T) {
 	t.Parallel()
 
@@ -275,10 +294,6 @@ func TestContentRungBoundary(t *testing.T) {
 
 	pokeAt(t, underPoked, off, []byte{1})
 
-	if contentHash(t, underBase, under) == contentHash(t, underPoked, under) {
-		t.Error("whole-file rung ignored a byte difference below wholeFileMax")
-	}
-
 	// At the boundary: only [0, sampleWindow) is sampled, so the poked
 	// byte at off is invisible and the two content hashes match.
 	at := int64(wholeFileMax)
@@ -287,7 +302,13 @@ func TestContentRungBoundary(t *testing.T) {
 
 	pokeAt(t, atPoked, off, []byte{1})
 
-	if contentHash(t, atBase, at) != contentHash(t, atPoked, at) {
+	c := scanContents(t, dir, underBase, underPoked, atBase, atPoked)
+
+	if c[underBase] == c[underPoked] {
+		t.Error("whole-file rung ignored a byte difference below wholeFileMax")
+	}
+
+	if c[atBase] != c[atPoked] {
 		t.Error("sampled rung saw a byte outside every sample window")
 	}
 }
@@ -295,7 +316,8 @@ func TestContentRungBoundary(t *testing.T) {
 // TestContentRungMultiGigabyte exercises the sampled rung across several
 // gigabytes using sparse files: a difference inside the third sample
 // window (at offset 2*sampleStride) changes the hash, while a difference
-// in the gap after it does not.
+// in the gap after it does not. The three files match on size, head,
+// and tail, so the scan reads each for its content hash.
 func TestContentRungMultiGigabyte(t *testing.T) {
 	t.Parallel()
 
@@ -314,17 +336,374 @@ func TestContentRungMultiGigabyte(t *testing.T) {
 	pokeAt(t, inSample, thirdSample, []byte{1})
 	pokeAt(t, inGap, gap, []byte{1})
 
-	baseHash := contentHash(t, base, size)
+	c := scanContents(t, dir, base, inSample, inGap)
 
-	if contentHash(t, inSample, size) == baseHash {
+	if c[inSample] == c[base] {
 		t.Error("sample at 2 GiB was not read: difference there was invisible")
 	}
 
-	if contentHash(t, inGap, size) != baseHash {
+	if c[inGap] != c[base] {
 		t.Error("a byte in an unsampled gap changed the content hash")
 	}
 }
 
+// sparseFileWithoutMatch writes name in dir as a sparse file of size
+// bytes, next to another file of that size whose first byte differs. A
+// scan then reads the file's head and tail, since its size is shared,
+// but finds no file matching them, so it gets no content hash.
+func sparseFileWithoutMatch(t *testing.T, dir, name string,
+	size int64,
+) string {
+	t.Helper()
+
+	p := sparseFile(t, dir, name, size)
+	other := sparseFile(t, dir, name+"-other-head", size)
+
+	pokeAt(t, other, 0, []byte{1})
+
+	return p
+}
+
+// TestScanContentGate checks that a file of headTailMin or more is read
+// for its content hash only when its size, head, and tail match another
+// file's: a same-size pair whose heads differ and one whose tails differ
+// get no content hash and are not reported, while an identical pair is
+// read and reported.
+func TestScanContentGate(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	// Three sizes, so that no pair meets another.
+	headA := sparseFile(t, dir, "head-a", headTailMin)
+	headB := sparseFile(t, dir, "head-b", headTailMin)
+	tailA := sparseFile(t, dir, "tail-a", headTailMin+1)
+	tailB := sparseFile(t, dir, "tail-b", headTailMin+1)
+	same := []string{
+		sparseFile(t, dir, "same-a", headTailMin+2),
+		sparseFile(t, dir, "same-b", headTailMin+2),
+	}
+
+	pokeAt(t, headB, 0, []byte{1})
+	pokeAt(t, tailB, headTailMin, []byte{1}) // its last byte
+
+	syncTree(t, db, dir)
+
+	recs := dbRecords(t, db)
+	for _, p := range []string{headA, headB, tailA, tailB} {
+		r := recordByPath(t, recs, p)
+		if r.head == "" || r.tail == "" || r.content != "" {
+			t.Errorf("%s: head = %q tail = %q content = %q, "+
+				"want head and tail only", p, r.head, r.tail, r.content)
+		}
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 || !slices.Equal(groups[0].paths, same) {
+		t.Fatalf("groups = %+v, want only the identical pair %q",
+			groups, same)
+	}
+}
+
+// TestScanContentAcrossOperands checks that a stored file gets its
+// content hash when a later scan of a separate operand brings its
+// match: tree A's file has a head and tail but no content hash until
+// tree B, holding an identical file, is scanned.
+func TestScanContentAcrossOperands(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+	a := sparseFileWithoutMatch(t, t.TempDir(), "a", headTailMin)
+
+	syncTree(t, db, filepath.Dir(a))
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.head == "" || r.content != "" {
+		t.Fatalf("after scanning A: %+v, want head and tail only", r)
+	}
+
+	b := sparseFile(t, t.TempDir(), "b", headTailMin)
+
+	syncTree(t, db, filepath.Dir(b))
+
+	recs := dbRecords(t, db)
+	if r := recordByPath(t, recs, a); r.content == "" {
+		t.Fatalf("after scanning B: %+v, want A's file content-hashed", r)
+	}
+
+	want := []string{a, b}
+	slices.Sort(want)
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 || !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("groups = %+v, want the pair %q", groups, want)
+	}
+}
+
+// TestScanContentWithinOperand checks that a rescan adding a match next
+// to an unchanged stored file gives the stored file its content hash,
+// though the hash phase leaves it alone as unchanged.
+func TestScanContentWithinOperand(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	stored := sparseFileWithoutMatch(t, dir, "d1", headTailMin)
+
+	syncTree(t, db, dir)
+
+	added := sparseFile(t, dir, "d2", headTailMin)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 1, unchanged: 2}) {
+		t.Fatalf("rescan stats = %+v, want 1 added 2 unchanged", st)
+	}
+
+	want := []string{stored, added}
+
+	groups := collectDupeGroups(dbRecords(t, db))
+	if len(groups) != 1 || !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("groups = %+v, want the pair %q", groups, want)
+	}
+}
+
+// TestScanContentStalePartners checks that a stored file outside the
+// operand that has vanished, or changed, since it was recorded is not
+// read, and that its match inside the operand is not read either: the
+// match has no other partner left, so neither gets a content hash and
+// no duplicate is reported.
+func TestScanContentStalePartners(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+	dirA := t.TempDir()
+	gone := sparseFileWithoutMatch(t, dirA, "gone", headTailMin)
+	changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin+1)
+
+	syncTree(t, db, dirA)
+
+	before := dbRecords(t, db)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	future := time.Now().Add(time.Hour)
+
+	err = os.Chtimes(changed, future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dirB := t.TempDir()
+	sparseFile(t, dirB, "gone-copy", headTailMin)
+	sparseFile(t, dirB, "changed-copy", headTailMin+1)
+
+	st := syncTree(t, db, dirB)
+	if st != (scanStats{added: 2}) {
+		t.Errorf("stats = %+v, want 2 added and nothing skipped", st)
+	}
+
+	recs := dbRecords(t, db)
+	for _, r := range recs {
+		if r.content != "" {
+			t.Errorf("%s: content = %q, want none: its only match is stale",
+				r.path, r.content)
+		}
+	}
+
+	for _, old := range before {
+		if r := recordByPath(t, recs, old.path); r != old {
+			t.Errorf("record = %+v, want it left as %+v", r, old)
+		}
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 0 {
+		t.Errorf("groups = %+v, want none", groups)
+	}
+}
+
+// TestScanContentHashedStalePartners checks that stored matches outside
+// the operand that already have a content hash are checked like any
+// other: once one has vanished and the other has changed, a copy of
+// them scanned in another tree has no match left, so it is not read and
+// is not reported as their duplicate.
+func TestScanContentHashedStalePartners(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+	dirA := t.TempDir()
+	stored := []string{
+		sparseFile(t, dirA, "changed", headTailMin),
+		sparseFile(t, dirA, "gone", headTailMin),
+	}
+
+	// The two stored files match, so this scan gives both a content
+	// hash.
+	syncTree(t, db, dirA)
+
+	err := os.Remove(stored[1])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	future := time.Now().Add(time.Hour)
+
+	err = os.Chtimes(stored[0], future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := sparseFile(t, t.TempDir(), "copy", headTailMin)
+
+	st := syncTree(t, db, filepath.Dir(b))
+	if st != (scanStats{added: 1}) {
+		t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
+	}
+
+	recs := dbRecords(t, db)
+	if r := recordByPath(t, recs, b); r.content != "" {
+		t.Errorf("copy: content = %q, want none: its only matches are stale",
+			r.content)
+	}
+
+	// The stored records lie outside the operand and are left as they
+	// are, so they still group with each other, but not with the copy.
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 || !slices.Equal(groups[0].paths, stored) {
+		t.Errorf("groups = %+v, want only the stored pair %q", groups, stored)
+	}
+}
+
+// TestScanContentReadFailure checks that a failed content read is
+// counted as skipped and leaves the record without a content hash, and
+// that a later scan tries the read again.
+func TestScanContentReadFailure(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+	a := sparseFileWithoutMatch(t, t.TempDir(), "a", headTailMin)
+
+	syncTree(t, db, filepath.Dir(a))
+
+	// lstat still works on the unreadable file, so it passes the check
+	// and fails only when it is read.
+	err := os.Chmod(a, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dirB := t.TempDir()
+	b := sparseFile(t, dirB, "b", headTailMin)
+
+	st := syncTree(t, db, dirB)
+	if st != (scanStats{added: 1, skipped: 1}) {
+		t.Fatalf("stats = %+v, want 1 added 1 skipped", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.content != "" {
+		t.Fatalf("unreadable file: %+v, want no content hash", r)
+	}
+
+	err = os.Chmod(a, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st = syncTree(t, db, dirB)
+	if st != (scanStats{unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
+	}
+
+	want := []string{a, b}
+	slices.Sort(want)
+
+	groups := collectDupeGroups(dbRecords(t, db))
+	if len(groups) != 1 || !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("groups = %+v, want the pair %q after the retry",
+			groups, want)
+	}
+}
+
+// TestScanContentCheckError checks that a stored file the content phase
+// cannot lstat, for a reason other than its being gone, is counted as
+// skipped and does not count as a match.
+func TestScanContentCheckError(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+	sub := filepath.Join(t.TempDir(), "sub")
+
+	err := os.Mkdir(sub, 0o700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sparseFileWithoutMatch(t, sub, "a", headTailMin)
+	syncTree(t, db, sub)
+
+	// Without search permission on its directory, the stored file's
+	// lstat fails with permission denied.
+	err = os.Chmod(sub, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		//nolint:gosec // removing the directory needs its search bit back
+		_ = os.Chmod(sub, 0o700)
+	})
+
+	b := sparseFile(t, t.TempDir(), "b", headTailMin)
+
+	st := syncTree(t, db, filepath.Dir(b))
+	if st != (scanStats{added: 1, skipped: 1}) {
+		t.Fatalf("stats = %+v, want 1 added 1 skipped", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), b); r.content != "" {
+		t.Errorf("b: content = %q, want none: its only match could not be "+
+			"checked", r.content)
+	}
+}
+
+// TestScanContentHardlinks checks that the content phase stores the
+// content hash of a hard-linked file on every one of its links.
+func TestScanContentHardlinks(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := sparseFile(t, dir, "a", headTailMin)
+	b := filepath.Join(dir, "b")
+
+	err := os.Link(a, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := sparseFile(t, dir, "copy", headTailMin)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 3}) {
+		t.Fatalf("stats = %+v, want 3 added", st)
+	}
+
+	recs := dbRecords(t, db)
+
+	want := recordByPath(t, recs, c).content
+	if want == "" {
+		t.Fatal("the copy has no content hash")
+	}
+
+	for _, p := range []string{a, b} {
+		if got := recordByPath(t, recs, p).content; got != want {
+			t.Errorf("%s: content = %q, want %q", p, got, want)
+		}
+	}
+}
+
 // collectWalk runs a walk over roots and returns the emitted records
 // and the number of warning events.
 func collectWalk(t *testing.T, roots []string, oneFS bool,
@@ -910,9 +1289,9 @@ func TestScanSkipsUniqueSizes(t *testing.T) {
 
 	recs := dbRecords(t, db)
 	for _, r := range recs {
-		if r.head != "" || r.tail != "" {
-			t.Errorf("%s: head = %q tail = %q, want unhashed",
-				r.path, r.head, r.tail)
+		if r.head != "" || r.tail != "" || r.content != "" {
+			t.Errorf("%s: head = %q tail = %q content = %q, want unhashed",
+				r.path, r.head, r.tail, r.content)
 		}
 	}
 
@@ -944,23 +1323,36 @@ func TestScanSkipsUniqueSizes(t *testing.T) {
 func TestTreesUnhashedNeverEqual(t *testing.T) {
 	t.Parallel()
 
-	// Two trees identical except for unhashed same-name, same-size
-	// files (possible when the trees were scanned separately) must not
-	// compare equal: unhashed content is unknown.
-	shared := pattern(1, 100)
-	recs := []scanRec{
-		{path: "/x/t1/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
-		{path: "/x/t2/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
-		{path: "/x/t1/u", size: 50},
-		{path: "/x/t2/u", size: 50},
+	// Two trees identical except for same-name, same-size files without
+	// a content hash must not compare equal: their content is unknown.
+	// That holds for unhashed files (possible when the trees were
+	// scanned separately) and for files of headTailMin or more that
+	// have only a head and tail.
+	sum := hexSum(pattern(1, 100))
+	shared := []scanRec{
+		{path: "/x/t1/f1", size: 100, head: sum, tail: sum, content: sum},
+		{path: "/x/t2/f1", size: 100, head: sum, tail: sum, content: sum},
 	}
 
-	super, dirs := buildHierarchy(recs)
-	super.compute()
+	cases := map[string][]scanRec{
+		"unhashed": {
+			{path: "/x/t1/u", size: 50},
+			{path: "/x/t2/u", size: 50},
+		},
+		"head and tail only": {
+			{path: "/x/t1/u", size: headTailMin, head: "h", tail: "t"},
+			{path: "/x/t2/u", size: headTailMin, head: "h", tail: "t"},
+		},
+	}
 
-	if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
-		t.Fatalf("tree groups = %d, want 0 (unhashed files differ)",
-			len(tg))
+	for name, unknown := range cases {
+		super, dirs := buildHierarchy(append(slices.Clone(shared), unknown...))
+		super.compute()
+
+		if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
+			t.Errorf("%s: tree groups = %d, want 0 (the files may differ)",
+				name, len(tg))
+		}
 	}
 }