Hash every file's head, tail and content in the hash phase (closes #132)

Every file the walk records now gets three hashes: head and tail, of
its first and last 64 KiB, and content. The content hash covers the
whole file under 100 MiB, and 1 MiB at each gigabyte from 100 MiB up,
so a large file is never read in full. One 100 MiB constant replaces
the 10 MiB and 50 MiB thresholds. The size census, the records stored
without hashes and the content phase are gone, so a rescan reads only
new and changed files. report and trees no longer special-case a
missing content hash. README "Duplicate detection" and "Rationale" are
rewritten.

Judgement call: info drops its two hash-count lines, which now always
equal the record count.

Model: opus-5-5

exclude_test.go

@@ -164,9 +164,8 @@ func TestDefaultExcludesHelp(t *testing.T) {
 // TestScanDoesNotEnterDefaultExclude checks that the walk does not
 // enter a default exclude below an operand. It says so on a line of its
 // own and does not count it as skipped. The records below it count as
-// outside the operands: the record of a file gone from it is kept, and
-// its size joins the size census, while a file added to it is not
-// recorded.
+// outside the operands: the record of a file gone from it is kept,
+// while a file added to it is not recorded.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
@@ -186,10 +185,6 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
 
 	unseen := writeFile(t, excluded, "new", pattern(2, 10))
 
-	// The same size as gone, so it is hashed only if the census holds the
-	// size of gone's record.
-	added := writeFile(t, dir, "added", pattern(3, 100))
-
 	st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
 		walkRules{excludes: []string{excluded}})
 	if err != nil {
@@ -203,11 +198,6 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
 	recs := dbRecords(t, db)
 	recordByPath(t, recs, gone)
 
-	if recordByPath(t, recs, added).head == "" {
-		t.Errorf("%s not hashed: the census lacks the size of %s",
-			added, gone)
-	}
-
 	if slices.Contains(recordPaths(recs), unseen) {
 		t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
 	}
@@ -391,9 +381,8 @@ func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
 // TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
 // that is not persistent beside one on the tests' temporary directory.
 // The records an earlier scan left in it count as outside the operands:
-// the record of a file gone from it is kept, and its size joins the size
-// census, while a file added to it is not recorded, and its size does
-// not join the census.
+// the record of a file gone from it is kept, while a file added to it
+// is not recorded.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
@@ -413,11 +402,6 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
 
 	unseen := writeFile(t, shm, "new", pattern(2, 10))
 
-	// added has the size of gone, and lonely that of unseen: each is
-	// hashed only if the census holds that size.
-	added := writeFile(t, dir, "added", pattern(3, 100))
-	lonely := writeFile(t, dir, "lonely", pattern(4, 10))
-
 	st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
 	if err != nil {
 		t.Fatal(err)
@@ -430,15 +414,6 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
 	recs := dbRecords(t, db)
 	recordByPath(t, recs, gone)
 
-	if recordByPath(t, recs, added).head == "" {
-		t.Errorf("%s not hashed: the census lacks the size of %s",
-			added, gone)
-	}
-
-	if recordByPath(t, recs, lonely).head != "" {
-		t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen)
-	}
-
 	if slices.Contains(recordPaths(recs), unseen) {
 		t.Errorf("%s recorded", unseen)
 	}
@@ -450,51 +425,6 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
 	}
 }
 
-// TestScanContentSkipsNonPersistent checks that the content phase does
-// not read a file on a filesystem that is not persistent, though an
-// earlier scan left a record for it: it is treated as a changed file,
-// so its record is left as it is and its identical copy in another
-// tree, having no other match, is not read either.
-func TestScanContentSkipsNonPersistent(t *testing.T) {
-	t.Parallel()
-
-	shm, _, rules := shmTempDir(t)
-	db := openTestDB(t)
-
-	// A scan that, without the list of filesystems, records a with a
-	// head and tail but no content hash.
-	a := sparseFileWithoutMatch(t, shm, "a", headTailMin)
-
-	syncTree(t, db, shm)
-
-	before := recordByPath(t, dbRecords(t, db), a)
-	if before.head == "" || before.content != "" {
-		t.Fatalf("after scanning %s: %+v, want head and tail only", shm, before)
-	}
-
-	b := sparseFile(t, t.TempDir(), "b", headTailMin)
-
-	st, err := syncScan(t.Context(), db, []string{filepath.Dir(b)}, 1, false,
-		rules)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.skipped != 0 {
-		t.Errorf("skipped = %d, want 0", st.skipped)
-	}
-
-	recs := dbRecords(t, db)
-	if r := recordByPath(t, recs, a); r != before {
-		t.Errorf("record = %+v, want it left as %+v", r, before)
-	}
-
-	if r := recordByPath(t, recs, b); r.content != "" {
-		t.Errorf("%s: content = %q, want none: its only match is on %s",
-			b, r.content, shm)
-	}
-}
-
 // TestWalkLeavesOutNonPersistent starts the walk itself in a directory
 // on a filesystem that is not persistent, which a scan never does: it
 // drops such an operand first. Below a directory on a persistent