Add 64 KiB head/tail and content-hash duplicate ladder (closes #61)

Replace the 1 KiB end sampling with a ladder for same-size candidates:
SHA-256 of the first and last 64 KiB, then a content hash that is the
whole file below 50 MiB (proof of identity) and gigabyte-spaced 1 MiB
samples at or above (deliberately probabilistic). Two files are
duplicates only when size, head, tail, and content all agree.

The signature gains a content column; schema bumps to version 2, so a
version 1 database is rejected and must be rescanned (unavoidable — every
stored hash changed). Because report and trees group stored signatures
across separate scans, content is computed for every shared-size file,
not only within-run head/tail collisions; size remains the sole read
gate. README "Duplicate detection" documents each rung; tests cover the
window boundaries, the 50 MiB boundary, and a multi-gigabyte sampled
case with sparse temp files.

Model: opus-4-8

scan_test.go

@@ -53,7 +53,10 @@ func pattern(tag byte, n int) []byte {
 	return data
 }
 
-func TestHashHeadTail(t *testing.T) {
+// TestHashSignatureEnds exercises the head and tail rungs across the
+// window boundaries. Every file here is below wholeFileMax, so the
+// content rung is a whole-file hash.
+func TestHashSignatureEnds(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
@@ -64,11 +67,11 @@ func TestHashHeadTail(t *testing.T) {
 	}{
 		{"empty", nil},
 		{"one-byte", []byte("x")},
-		{"under-one-chunk", pattern(1, chunk-1)},
-		{"exactly-one-chunk", pattern(2, chunk)},
-		{"overlapping-reads", pattern(3, chunk+chunk/2)},
-		{"exactly-two-chunks", pattern(4, 2*chunk)},
-		{"beyond-two-chunks", pattern(5, 3*chunk)},
+		{"under-one-window", pattern(1, headTailWindow-1)},
+		{"exactly-one-window", pattern(2, headTailWindow)},
+		{"overlapping-windows", pattern(3, headTailWindow+headTailWindow/2)},
+		{"exactly-two-windows", pattern(4, 2*headTailWindow)},
+		{"beyond-two-windows", pattern(5, 3*headTailWindow)},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -76,12 +79,12 @@ func TestHashHeadTail(t *testing.T) {
 
 			p := writeFile(t, dir, c.name, c.data)
 
-			head, tail, err := hashHeadTail(p, int64(len(c.data)))
+			head, tail, content, err := hashSignature(p, int64(len(c.data)))
 			if err != nil {
-				t.Fatalf("hashHeadTail: %v", err)
+				t.Fatalf("hashSignature: %v", err)
 			}
 
-			n := min(chunk, len(c.data))
+			n := min(headTailWindow, len(c.data))
 			if want := hexSum(c.data[:n]); head != want {
 				t.Errorf("head = %s, want %s", head, want)
 			}
@@ -89,36 +92,179 @@ func TestHashHeadTail(t *testing.T) {
 			if want := hexSum(c.data[len(c.data)-n:]); tail != want {
 				t.Errorf("tail = %s, want %s", tail, want)
 			}
+
+			// Below wholeFileMax the content rung hashes the whole file.
+			if want := hexSum(c.data); content != want {
+				t.Errorf("content = %s, want whole-file %s", content, want)
+			}
 		})
 	}
 }
 
-func TestHashHeadTailErrors(t *testing.T) {
+func TestHashSignatureErrors(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
 
-	_, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1)
+	// A missing file: an error, and every hash left empty.
+	head, tail, content, err := hashSignature(filepath.Join(dir, "missing"), 1)
 	if err == nil {
 		t.Error("no error for a missing file")
 	}
 
+	if head != "" || tail != "" || content != "" {
+		t.Errorf("missing file returned hashes: %q %q %q", head, tail, content)
+	}
+
 	// A zero-length file has constant hashes and is never opened: even
 	// a missing path succeeds.
-	head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0)
-	if err != nil || head != emptyHash || tail != emptyHash {
-		t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes",
-			head, tail, err)
+	head, tail, content, err = hashSignature(filepath.Join(dir, "missing"), 0)
+	if err != nil ||
+		head != emptyHash || tail != emptyHash || content != emptyHash {
+		t.Errorf("empty: head=%q tail=%q content=%q err=%v, "+
+			"want constant hashes", head, tail, content, err)
 	}
 
 	// A file that shrank between the stat and hash passes: reading at
 	// the stat-reported size must fail rather than emit wrong hashes.
 	p := writeFile(t, dir, "shrunk", []byte("tiny"))
 
-	_, _, err = hashHeadTail(p, int64(2*chunk))
+	head, tail, content, err = hashSignature(p, int64(2*headTailWindow))
 	if err == nil {
 		t.Error("no error when the stat size exceeds the file size")
 	}
+
+	if head != "" || tail != "" || content != "" {
+		t.Errorf("shrunk file returned hashes: %q %q %q", head, tail, content)
+	}
+}
+
+// sparseFile creates a file that is logically size bytes long without
+// allocating blocks for the hole, so multi-gigabyte cases stay cheap.
+func sparseFile(t *testing.T, dir, name string, size int64) string {
+	t.Helper()
+
+	p := filepath.Join(dir, name)
+
+	f, err := os.Create(p) //nolint:gosec // test-controlled path
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = f.Truncate(size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = f.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return p
+}
+
+// pokeAt writes data into an existing file at off, leaving the rest of
+// the file (a sparse hole) untouched.
+func pokeAt(t *testing.T, path string, off int64, data []byte) {
+	t.Helper()
+
+	f, err := os.OpenFile(path, os.O_WRONLY, 0o600) //nolint:gosec // test path
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = f.WriteAt(data, off)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = f.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// contentHash returns just the content rung of a file's signature.
+func contentHash(t *testing.T, path string, size int64) string {
+	t.Helper()
+
+	_, _, content, err := hashSignature(path, size)
+	if err != nil {
+		t.Fatalf("hashSignature %s: %v", path, err)
+	}
+
+	return content
+}
+
+// TestContentRungBoundary checks the 50 MiB boundary between the two
+// content rungs: just below it the whole file is hashed and any byte
+// difference shows; at the boundary only the gigabyte-spaced samples are
+// hashed, so a difference outside a sample window is invisible.
+func TestContentRungBoundary(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	// A byte that lands outside the single [0, sampleWindow) sample a
+	// sub-gigabyte file has, but well inside the file.
+	const off = 10 * 1024 * 1024
+
+	// Just under the boundary: the whole-file rung sees the poked byte.
+	under := int64(wholeFileMax - 1)
+	underBase := sparseFile(t, dir, "under-base", under)
+	underPoked := sparseFile(t, dir, "under-poked", under)
+
+	pokeAt(t, underPoked, off, []byte{1})
+
+	if contentHash(t, underBase, under) == contentHash(t, underPoked, under) {
+		t.Error("whole-file rung ignored a byte difference below wholeFileMax")
+	}
+
+	// At the boundary: only [0, sampleWindow) is sampled, so the poked
+	// byte at off is invisible and the two content hashes match.
+	at := int64(wholeFileMax)
+	atBase := sparseFile(t, dir, "at-base", at)
+	atPoked := sparseFile(t, dir, "at-poked", at)
+
+	pokeAt(t, atPoked, off, []byte{1})
+
+	if contentHash(t, atBase, at) != contentHash(t, atPoked, at) {
+		t.Error("sampled rung saw a byte outside every sample window")
+	}
+}
+
+// TestContentRungMultiGigabyte exercises the sampled rung across several
+// gigabytes using sparse files: a difference inside the third sample
+// window (at offset 2*sampleStride) changes the hash, while a difference
+// in the gap after it does not.
+func TestContentRungMultiGigabyte(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	// Three sample windows (offsets 0, 1 GiB, 2 GiB) plus a trailing gap
+	// that no sample covers.
+	size := int64(2*sampleStride + 2*sampleWindow)
+	thirdSample := int64(2 * sampleStride)
+	gap := thirdSample + int64(sampleWindow)
+
+	base := sparseFile(t, dir, "g-base", size)
+	inSample := sparseFile(t, dir, "g-insample", size)
+	inGap := sparseFile(t, dir, "g-ingap", size)
+
+	pokeAt(t, inSample, thirdSample, []byte{1})
+	pokeAt(t, inGap, gap, []byte{1})
+
+	baseHash := contentHash(t, base, size)
+
+	if contentHash(t, inSample, size) == baseHash {
+		t.Error("sample at 2 GiB was not read: difference there was invisible")
+	}
+
+	if contentHash(t, inGap, size) != baseHash {
+		t.Error("a byte in an unsampled gap changed the content hash")
+	}
 }
 
 // collectWalk runs a walk over roots and returns the emitted records