Add 64 KiB head/tail and content-hash duplicate ladder (closes #61)

Replace the 1 KiB end sampling with a ladder for same-size candidates:
SHA-256 of the first and last 64 KiB, then a content hash that is the
whole file below 50 MiB (proof of identity) and gigabyte-spaced 1 MiB
samples at or above (deliberately probabilistic). Two files are
duplicates only when size, head, tail, and content all agree.

The signature gains a content column; schema bumps to version 2, so a
version 1 database is rejected and must be rescanned (unavoidable — every
stored hash changed). Because report and trees group stored signatures
across separate scans, content is computed for every shared-size file,
not only within-run head/tail collisions; size remains the sole read
gate. README "Duplicate detection" documents each rung; tests cover the
window boundaries, the 50 MiB boundary, and a multi-gigabyte sampled
case with sparse temp files.

Model: opus-4-8

scan.go

@@ -7,6 +7,7 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -16,8 +17,28 @@ import (
 	"syscall"
 )
 
-// chunk is the number of bytes hashed from each end of a file.
-const chunk = 1024
+// The duplicate ladder (see hashSignature and README "Duplicate
+// detection"). Same-size candidates are separated first by the hashes
+// of their end windows, then by a content hash that is exact for
+// smaller files and deliberately sampled for large ones.
+
+// headTailWindow is the number of bytes hashed from each end of a file
+// (the head and tail rungs). A file no larger than one window has head
+// and tail equal to the hash of its whole content.
+const headTailWindow = 64 * 1024
+
+// wholeFileMax is the size boundary between the two content rungs: a
+// file strictly smaller than this is content-hashed in full; a file
+// this size or larger is content-hashed by sampling.
+const wholeFileMax = 50 * 1024 * 1024
+
+// sampleStride is the spacing between content samples for large files:
+// one window is read at each gigabyte-aligned offset (0, 1 GiB, ...).
+const sampleStride = 1024 * 1024 * 1024
+
+// sampleWindow is the number of bytes read at each large-file sample
+// offset, truncated at end of file.
+const sampleWindow = 1024 * 1024
 
 // workQueueDepth bounds the job and result channels feeding the walk
 // and hash worker pools.
@@ -439,11 +460,12 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
 		s.resolve(rec.path)
 
 		s.batch = append(s.batch, scanRec{
-			size:  rec.size,
-			mtime: rec.mtime,
-			head:  r.head,
-			tail:  r.tail,
-			path:  rec.path,
+			size:    rec.size,
+			mtime:   rec.mtime,
+			head:    r.head,
+			tail:    r.tail,
+			content: r.content,
+			path:    rec.path,
 		})
 	}
 
@@ -834,13 +856,15 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
 	return statDev(st), st.Ino
 }
 
-// hashResult carries one inode run's head/tail hashes (or the error
-// that prevented hashing it) from the hash workers to the hash phase.
+// hashResult carries one inode run's signature hashes — head, tail, and
+// content — (or the error that prevented hashing it) from the hash
+// workers to the hash phase.
 type hashResult struct {
-	run  []fileRec
-	head string
-	tail string
-	err  error
+	run     []fileRec
+	head    string
+	tail    string
+	content string
+	err     error
 }
 
 // hashPool owns every goroutine of the hash worker pool: the feeder
@@ -930,11 +954,11 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 			continue
 		}
 
-		head, tail, err := hashHeadTail(run[0].path, run[0].size)
+		head, tail, content, err := hashSignature(run[0].path, run[0].size)
 
 		select {
 		case results <- hashResult{
-			run: run, head: head, tail: tail, err: err,
+			run: run, head: head, tail: tail, content: content, err: err,
 		}:
 		case <-ctx.Done():
 			return
@@ -942,47 +966,66 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 	}
 }
 
-// emptyHash is the lowercase-hex SHA-256 of the empty input: the head
-// and tail hash of every zero-length file.
+// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
+// tail, and content hash of every zero-length file.
 const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
 	"27ae41e4649b934ca495991b7852b855"
 
-// hashHeadTail returns the lowercase-hex SHA-256 of the first
-// min(chunk, size) bytes and of the last min(chunk, size) bytes of the
-// file at path. The two reads overlap when size < 2*chunk. size is the
-// value recorded when the file was statted; a zero-length file's
-// hashes are constant, so it is never even opened.
-func hashHeadTail(path string, size int64) (string, string, error) {
+// hashSignature computes the three content hashes that, with the file
+// size, form its duplicate signature: the SHA-256 of the first and last
+// headTailWindow bytes (the head and tail rungs), and a content hash
+// that is the SHA-256 of the whole file below wholeFileMax (the exact
+// rung) or of gigabyte-spaced samples at or above it (the sampled,
+// deliberately probabilistic rung). Two files are duplicates only when
+// all four agree; any mismatch means not a duplicate. size is the value
+// recorded when the file was statted; a zero-length file has constant
+// hashes and is never opened.
+func hashSignature(path string, size int64) (string, string, string, error) {
 	if size == 0 {
-		return emptyHash, emptyHash, nil
+		return emptyHash, emptyHash, emptyHash, nil
 	}
 
 	//nolint:gosec // hashing operator-supplied paths is the tool's purpose
 	f, err := os.Open(path)
 	if err != nil {
-		return "", "", err
+		return "", "", "", err
 	}
 
 	defer func() { _ = f.Close() }()
 
-	n := min(int64(chunk), size)
+	head, tail, err := hashEnds(f, size)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	content, err := hashContent(f, size)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	return head, tail, content, nil
+}
+
+// hashEnds returns the SHA-256 of the first and last headTailWindow
+// bytes of f. The two windows overlap when the file is between one and
+// two windows in size; when it is no larger than one window they
+// coincide, so the head hash is reused as the tail and only one read is
+// issued.
+func hashEnds(f *os.File, size int64) (string, string, error) {
+	n := min(int64(headTailWindow), size)
 
 	buf := make([]byte, n)
 
-	_, err = f.ReadAt(buf, 0)
+	_, err := f.ReadAt(buf, 0)
 	if err != nil {
 		return "", "", err
 	}
 
 	h := sha256.Sum256(buf)
+	head := hex.EncodeToString(h[:])
 
-	// When the whole file fits in one chunk the tail window is exactly
-	// the bytes just read: reuse the head hash instead of issuing a
-	// second read for every small file.
-	if size <= int64(chunk) {
-		hh := hex.EncodeToString(h[:])
-
-		return hh, hh, nil
+	if size <= int64(headTailWindow) {
+		return head, head, nil
 	}
 
 	_, err = f.ReadAt(buf, size-n)
@@ -992,5 +1035,55 @@ func hashHeadTail(path string, size int64) (string, string, error) {
 
 	t := sha256.Sum256(buf)
 
-	return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
+	return head, hex.EncodeToString(t[:]), nil
+}
+
+// hashContent returns the content-rung hash of f: the SHA-256 of the
+// whole file when it is smaller than wholeFileMax, or of sampled
+// windows when it is that size or larger.
+func hashContent(f *os.File, size int64) (string, error) {
+	if size >= int64(wholeFileMax) {
+		return hashSamples(f, size)
+	}
+
+	return hashWhole(f, size)
+}
+
+// hashWhole returns the SHA-256 of the entire file. A SectionReader is
+// used so the read is independent of the offset left by the end-window
+// reads.
+func hashWhole(f *os.File, size int64) (string, error) {
+	h := sha256.New()
+
+	_, err := io.Copy(h, io.NewSectionReader(f, 0, size))
+	if err != nil {
+		return "", err
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset
+// (0, sampleStride, 2*sampleStride, ... while inside the file), in
+// order, into one hash, each window truncated at end of file. This is
+// the probabilistic large-file rung: two files of equal size agreeing
+// on every sample are reported as duplicates without every byte being
+// read. Because size is part of the signature, files of different sizes
+// never reach this comparison, so the sample boundaries always align.
+func hashSamples(f *os.File, size int64) (string, error) {
+	h := sha256.New()
+	buf := make([]byte, sampleWindow)
+
+	for off := int64(0); off < size; off += int64(sampleStride) {
+		n := min(int64(sampleWindow), size-off)
+
+		_, err := f.ReadAt(buf[:n], off)
+		if err != nil {
+			return "", err
+		}
+
+		h.Write(buf[:n])
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
 }