Hash files under 10 MiB in full, gate larger files on head/tail

Amends the issue 61 ladder per the owner's design change. A file under
10 MiB (new headTailMin) is now hashed in full and compared directly,
with no end-window step: its head, tail, and content all carry the
whole-file SHA-256, so its signature is decided by size and content
alone. A file at 10 MiB or above keeps the 64 KiB head/tail gate, then
the whole-file content hash below 50 MiB or gigabyte-spaced 1 MiB
samples at or above. hashEnds drops its now-unreachable single-window
branch, and hashWhole errors if the file shrank below its recorded
size (the only read for the sub-10-MiB range). README, TODO, tests,
and the schema-version note updated to match.

Model: opus-4-8

scan.go

@@ -18,13 +18,22 @@ import (
 )
 
 // The duplicate ladder (see hashSignature and README "Duplicate
-// detection"). Same-size candidates are separated first by the hashes
-// of their end windows, then by a content hash that is exact for
-// smaller files and deliberately sampled for large ones.
+// detection"). A same-size candidate below headTailMin is hashed in
+// full and compared directly; a larger one is separated first by the
+// hashes of its end windows, then by a content hash that is exact below
+// wholeFileMax and deliberately sampled at or above it.
+
+// headTailMin is the size threshold for the end-window gate. A file
+// smaller than this is hashed in full directly, with no separate head
+// and tail step: its head, tail, and content all carry the whole-file
+// hash. A file this size or larger is separated first by its end
+// windows.
+const headTailMin = 10 * 1024 * 1024
 
 // headTailWindow is the number of bytes hashed from each end of a file
-// (the head and tail rungs). A file no larger than one window has head
-// and tail equal to the hash of its whole content.
+// at or above headTailMin (the head and tail rungs). Because
+// headTailMin is far larger than two windows, the head and tail windows
+// never overlap.
 const headTailWindow = 64 * 1024
 
 // wholeFileMax is the size boundary between the two content rungs: a
@@ -972,14 +981,17 @@ const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
 	"27ae41e4649b934ca495991b7852b855"
 
 // hashSignature computes the three content hashes that, with the file
-// size, form its duplicate signature: the SHA-256 of the first and last
-// headTailWindow bytes (the head and tail rungs), and a content hash
-// that is the SHA-256 of the whole file below wholeFileMax (the exact
-// rung) or of gigabyte-spaced samples at or above it (the sampled,
-// deliberately probabilistic rung). Two files are duplicates only when
-// all four agree; any mismatch means not a duplicate. size is the value
-// recorded when the file was statted; a zero-length file has constant
-// hashes and is never opened.
+// size, form its duplicate signature. A file below headTailMin is
+// hashed in full and its whole-file SHA-256 is returned as head, tail,
+// and content alike — that range takes no separate end-window step. For
+// a file at or above headTailMin the head and tail are the SHA-256 of
+// its first and last headTailWindow bytes, and content is the SHA-256
+// of the whole file below wholeFileMax (the exact rung) or of
+// gigabyte-spaced samples at or above it (the sampled, deliberately
+// probabilistic rung). Two files are duplicates only when all four
+// agree; any mismatch means not a duplicate. size is the value recorded
+// when the file was statted; a zero-length file has constant hashes and
+// is never opened.
 func hashSignature(path string, size int64) (string, string, string, error) {
 	if size == 0 {
 		return emptyHash, emptyHash, emptyHash, nil
@@ -993,6 +1005,17 @@ func hashSignature(path string, size int64) (string, string, string, error) {
 
 	defer func() { _ = f.Close() }()
 
+	// Below the threshold the whole file is hashed directly, with no
+	// end-window step: head and tail both carry the whole-file hash.
+	if size < int64(headTailMin) {
+		content, err := hashWhole(f, size)
+		if err != nil {
+			return "", "", "", err
+		}
+
+		return content, content, content, nil
+	}
+
 	head, tail, err := hashEnds(f, size)
 	if err != nil {
 		return "", "", "", err
@@ -1007,14 +1030,11 @@ func hashSignature(path string, size int64) (string, string, string, error) {
 }
 
 // hashEnds returns the SHA-256 of the first and last headTailWindow
-// bytes of f. The two windows overlap when the file is between one and
-// two windows in size; when it is no larger than one window they
-// coincide, so the head hash is reused as the tail and only one read is
-// issued.
+// bytes of f. It is called only for files at least headTailMin, which
+// is far larger than two windows, so the windows never overlap and both
+// reads are always full.
 func hashEnds(f *os.File, size int64) (string, string, error) {
-	n := min(int64(headTailWindow), size)
-
-	buf := make([]byte, n)
+	buf := make([]byte, headTailWindow)
 
 	_, err := f.ReadAt(buf, 0)
 	if err != nil {
@@ -1024,11 +1044,7 @@ func hashEnds(f *os.File, size int64) (string, string, error) {
 	h := sha256.Sum256(buf)
 	head := hex.EncodeToString(h[:])
 
-	if size <= int64(headTailWindow) {
-		return head, head, nil
-	}
-
-	_, err = f.ReadAt(buf, size-n)
+	_, err = f.ReadAt(buf, size-int64(headTailWindow))
 	if err != nil {
 		return "", "", err
 	}
@@ -1050,16 +1066,23 @@ func hashContent(f *os.File, size int64) (string, error) {
 }
 
 // hashWhole returns the SHA-256 of the entire file. A SectionReader is
-// used so the read is independent of the offset left by the end-window
-// reads.
+// used so the read is independent of the offset left by any end-window
+// reads. Reading fewer than size bytes means the file shrank between
+// the stat and the hash; that is an error rather than a hash of content
+// that no longer matches the recorded size.
 func hashWhole(f *os.File, size int64) (string, error) {
 	h := sha256.New()
 
-	_, err := io.Copy(h, io.NewSectionReader(f, 0, size))
+	n, err := io.Copy(h, io.NewSectionReader(f, 0, size))
 	if err != nil {
 		return "", err
 	}
 
+	if n != size {
+		return "", fmt.Errorf("read %d of %d bytes: %w", n, size,
+			io.ErrUnexpectedEOF)
+	}
+
 	return hex.EncodeToString(h.Sum(nil)), nil
 }