Add 64 KiB head/tail and content-hash duplicate ladder (closes #61)
check / check (push) Successful in 1m0s
check / check (push) Successful in 1m0s
Replace the 1 KiB end sampling with a ladder for same-size candidates: SHA-256 of the first and last 64 KiB, then a content hash that is the whole file below 50 MiB (proof of identity) and gigabyte-spaced 1 MiB samples at or above (deliberately probabilistic). Two files are duplicates only when size, head, tail, and content all agree. The signature gains a content column; schema bumps to version 2, so a version 1 database is rejected and must be rescanned (unavoidable — every stored hash changed). Because report and trees group stored signatures across separate scans, content is computed for every shared-size file, not only within-run head/tail collisions; size remains the sole read gate. README "Duplicate detection" documents each rung; tests cover the window boundaries, the 50 MiB boundary, and a multi-gigabyte sampled case with sparse temp files. Model: opus-4-8
This commit is contained in:
@@ -13,9 +13,10 @@ import (
|
||||
|
||||
// fileSig is a file's duplicate signature; mtime is excluded.
|
||||
type fileSig struct {
|
||||
size int64
|
||||
head string
|
||||
tail string
|
||||
size int64
|
||||
head string
|
||||
tail string
|
||||
content string
|
||||
}
|
||||
|
||||
// treeNode is one directory reconstructed from the scan stream.
|
||||
@@ -122,7 +123,9 @@ func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
|
||||
node.files = make(map[string]fileSig)
|
||||
}
|
||||
|
||||
sig := fileSig{size: r.size, head: r.head, tail: r.tail}
|
||||
sig := fileSig{
|
||||
size: r.size, head: r.head, tail: r.tail, content: r.content,
|
||||
}
|
||||
|
||||
// An unhashed record (its size was unique when last scanned)
|
||||
// has unknown content: give it a signature no other file can
|
||||
@@ -188,7 +191,7 @@ func (n *treeNode) compute() {
|
||||
for name, sig := range n.files {
|
||||
entries = append(entries,
|
||||
"f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+
|
||||
"\x00"+sig.head+"\x00"+sig.tail)
|
||||
"\x00"+sig.head+"\x00"+sig.tail+"\x00"+sig.content)
|
||||
n.fileCount++
|
||||
n.totalSize += sig.size
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user