Add 64 KiB head/tail and content-hash duplicate ladder (closes #61)
check / check (push) Successful in 1m0s
check / check (push) Successful in 1m0s
Replace the 1 KiB end sampling with a ladder for same-size candidates: SHA-256 of the first and last 64 KiB, then a content hash that is the whole file below 50 MiB (proof of identity) and gigabyte-spaced 1 MiB samples at or above (deliberately probabilistic). Two files are duplicates only when size, head, tail, and content all agree. The signature gains a content column; schema bumps to version 2, so a version 1 database is rejected and must be rescanned (unavoidable — every stored hash changed). Because report and trees group stored signatures across separate scans, content is computed for every shared-size file, not only within-run head/tail collisions; size remains the sole read gate. README "Duplicate detection" documents each rung; tests cover the window boundaries, the 50 MiB boundary, and a multi-gigabyte sampled case with sparse temp files. Model: opus-4-8
This commit is contained in:
@@ -38,6 +38,28 @@ func TestCollectDupeGroups(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectDupeGroupsContentSeparates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Same size, head, and tail, but different content hashes: the final
|
||||
// rung keeps them apart, so no group forms. Matching content groups.
|
||||
recs := []scanRec{
|
||||
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
|
||||
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
|
||||
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
if len(groups) != 1 {
|
||||
t.Fatalf("len(groups) = %d, want 1 (only the matching content)",
|
||||
len(groups))
|
||||
}
|
||||
|
||||
if !slices.Equal(groups[0].paths, []string{"/a", "/c"}) {
|
||||
t.Errorf("group paths = %q, want /a /c", groups[0].paths)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user