Add 64 KiB head/tail and content-hash duplicate ladder (closes #61) (#62)

scan.go

@@ -7,6 +7,7 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -16,8 +17,37 @@ import (
 	"syscall"
 )
 
-// chunk is the number of bytes hashed from each end of a file.
-const chunk = 1024
+// The duplicate ladder (see hashSignature and README "Duplicate
+// detection"). A same-size candidate below headTailMin is hashed in
+// full and compared directly; a larger one is separated first by the
+// hashes of its end windows, then by a content hash that is exact below
+// wholeFileMax and deliberately sampled at or above it.
+
+// headTailMin is the size threshold for the end-window gate. A file
+// smaller than this is hashed in full directly, with no separate head
+// and tail step: its head, tail, and content all carry the whole-file
+// hash. A file this size or larger is separated first by its end
+// windows.
+const headTailMin = 10 * 1024 * 1024
+
+// headTailWindow is the number of bytes hashed from each end of a file
+// at or above headTailMin (the head and tail rungs). Because
+// headTailMin is far larger than two windows, the head and tail windows
+// never overlap.
+const headTailWindow = 64 * 1024
+
+// wholeFileMax is the size boundary between the two content rungs: a
+// file strictly smaller than this is content-hashed in full; a file
+// this size or larger is content-hashed by sampling.
+const wholeFileMax = 50 * 1024 * 1024
+
+// sampleStride is the spacing between content samples for large files:
+// one window is read at each gigabyte-aligned offset (0, 1 GiB, ...).
+const sampleStride = 1024 * 1024 * 1024
+
+// sampleWindow is the number of bytes read at each large-file sample
+// offset, truncated at end of file.
+const sampleWindow = 1024 * 1024
 
 // workQueueDepth bounds the job and result channels feeding the walk
 // and hash worker pools.
@@ -439,11 +469,12 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
 		s.resolve(rec.path)
 
 		s.batch = append(s.batch, scanRec{
-			size:  rec.size,
-			mtime: rec.mtime,
-			head:  r.head,
-			tail:  r.tail,
-			path:  rec.path,
+			size:    rec.size,
+			mtime:   rec.mtime,
+			head:    r.head,
+			tail:    r.tail,
+			content: r.content,
+			path:    rec.path,
 		})
 	}
 
@@ -834,13 +865,15 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
 	return statDev(st), st.Ino
 }
 
-// hashResult carries one inode run's head/tail hashes (or the error
-// that prevented hashing it) from the hash workers to the hash phase.
+// hashResult carries one inode run's signature hashes — head, tail, and
+// content — (or the error that prevented hashing it) from the hash
+// workers to the hash phase.
 type hashResult struct {
-	run  []fileRec
-	head string
-	tail string
-	err  error
+	run     []fileRec
+	head    string
+	tail    string
+	content string
+	err     error
 }
 
 // hashPool owns every goroutine of the hash worker pool: the feeder
@@ -930,11 +963,11 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 			continue
 		}
 
-		head, tail, err := hashHeadTail(run[0].path, run[0].size)
+		head, tail, content, err := hashSignature(run[0].path, run[0].size)
 
 		select {
 		case results <- hashResult{
-			run: run, head: head, tail: tail, err: err,
+			run: run, head: head, tail: tail, content: content, err: err,
 		}:
 		case <-ctx.Done():
 			return
@@ -942,55 +975,138 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 	}
 }
 
-// emptyHash is the lowercase-hex SHA-256 of the empty input: the head
-// and tail hash of every zero-length file.
+// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
+// tail, and content hash of every zero-length file.
 const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
 	"27ae41e4649b934ca495991b7852b855"
 
-// hashHeadTail returns the lowercase-hex SHA-256 of the first
-// min(chunk, size) bytes and of the last min(chunk, size) bytes of the
-// file at path. The two reads overlap when size < 2*chunk. size is the
-// value recorded when the file was statted; a zero-length file's
-// hashes are constant, so it is never even opened.
-func hashHeadTail(path string, size int64) (string, string, error) {
+// hashSignature computes the three content hashes that, with the file
+// size, form its duplicate signature. A file below headTailMin is
+// hashed in full and its whole-file SHA-256 is returned as head, tail,
+// and content alike — that range takes no separate end-window step. For
+// a file at or above headTailMin the head and tail are the SHA-256 of
+// its first and last headTailWindow bytes, and content is the SHA-256
+// of the whole file below wholeFileMax (the exact rung) or of
+// gigabyte-spaced samples at or above it (the sampled, deliberately
+// probabilistic rung). Two files are duplicates only when all four
+// agree; any mismatch means not a duplicate. size is the value recorded
+// when the file was statted; a zero-length file has constant hashes and
+// is never opened.
+func hashSignature(path string, size int64) (string, string, string, error) {
 	if size == 0 {
-		return emptyHash, emptyHash, nil
+		return emptyHash, emptyHash, emptyHash, nil
 	}
 
 	//nolint:gosec // hashing operator-supplied paths is the tool's purpose
 	f, err := os.Open(path)
 	if err != nil {
-		return "", "", err
+		return "", "", "", err
 	}
 
 	defer func() { _ = f.Close() }()
 
-	n := min(int64(chunk), size)
+	// Below the threshold the whole file is hashed directly, with no
+	// end-window step: head and tail both carry the whole-file hash.
+	if size < int64(headTailMin) {
+		content, err := hashWhole(f, size)
+		if err != nil {
+			return "", "", "", err
+		}
 
-	buf := make([]byte, n)
+		return content, content, content, nil
+	}
 
-	_, err = f.ReadAt(buf, 0)
+	head, tail, err := hashEnds(f, size)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	content, err := hashContent(f, size)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	return head, tail, content, nil
+}
+
+// hashEnds returns the SHA-256 of the first and last headTailWindow
+// bytes of f. It is called only for files at least headTailMin, which
+// is far larger than two windows, so the windows never overlap and both
+// reads are always full.
+func hashEnds(f *os.File, size int64) (string, string, error) {
+	buf := make([]byte, headTailWindow)
+
+	_, err := f.ReadAt(buf, 0)
 	if err != nil {
 		return "", "", err
 	}
 
 	h := sha256.Sum256(buf)
+	head := hex.EncodeToString(h[:])
 
-	// When the whole file fits in one chunk the tail window is exactly
-	// the bytes just read: reuse the head hash instead of issuing a
-	// second read for every small file.
-	if size <= int64(chunk) {
-		hh := hex.EncodeToString(h[:])
-
-		return hh, hh, nil
-	}
-
-	_, err = f.ReadAt(buf, size-n)
+	_, err = f.ReadAt(buf, size-int64(headTailWindow))
 	if err != nil {
 		return "", "", err
 	}
 
 	t := sha256.Sum256(buf)
 
-	return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
+	return head, hex.EncodeToString(t[:]), nil
+}
+
+// hashContent returns the content-rung hash of f: the SHA-256 of the
+// whole file when it is smaller than wholeFileMax, or of sampled
+// windows when it is that size or larger.
+func hashContent(f *os.File, size int64) (string, error) {
+	if size >= int64(wholeFileMax) {
+		return hashSamples(f, size)
+	}
+
+	return hashWhole(f, size)
+}
+
+// hashWhole returns the SHA-256 of the entire file. A SectionReader is
+// used so the read is independent of the offset left by any end-window
+// reads. Reading fewer than size bytes means the file shrank between
+// the stat and the hash; that is an error rather than a hash of content
+// that no longer matches the recorded size.
+func hashWhole(f *os.File, size int64) (string, error) {
+	h := sha256.New()
+
+	n, err := io.Copy(h, io.NewSectionReader(f, 0, size))
+	if err != nil {
+		return "", err
+	}
+
+	if n != size {
+		return "", fmt.Errorf("read %d of %d bytes: %w", n, size,
+			io.ErrUnexpectedEOF)
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset
+// (0, sampleStride, 2*sampleStride, ... while inside the file), in
+// order, into one hash, each window truncated at end of file. This is
+// the probabilistic large-file rung: two files of equal size agreeing
+// on every sample are reported as duplicates without every byte being
+// read. Because size is part of the signature, files of different sizes
+// never reach this comparison, so the sample boundaries always align.
+func hashSamples(f *os.File, size int64) (string, error) {
+	h := sha256.New()
+	buf := make([]byte, sampleWindow)
+
+	for off := int64(0); off < size; off += int64(sampleStride) {
+		n := min(int64(sampleWindow), size-off)
+
+		_, err := f.ReadAt(buf[:n], off)
+		if err != nil {
+			return "", err
+		}
+
+		h.Write(buf[:n])
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
 }