Hash every file's head, tail and content in the hash phase (closes #132)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Every file the walk records now gets three hashes: head and tail, of its first and last 64 KiB, and content. The content hash covers the whole file under 100 MiB, and 1 MiB at each gigabyte from 100 MiB up, so a large file is never read in full. One 100 MiB constant replaces the 10 MiB and 50 MiB thresholds. The size census, the records stored without hashes and the content phase are gone, so a rescan reads only new and changed files. report and trees no longer special-case a missing content hash. README "Duplicate detection" and "Rationale" are rewritten. Judgement call: info drops its two hash-count lines, which now always equal the record count. Model: opus-5-5
This commit is contained in:
-59
@@ -612,62 +612,3 @@ func TestApplyChangesBatching(t *testing.T) {
|
||||
t.Fatalf("records = %d, want 0", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCountContentCandidates checks that countContentCandidates counts
|
||||
// the records the content phase's search returns: those of each group
|
||||
// of two or more records of headTailMin or more with a head, where at
|
||||
// least one has no content hash.
|
||||
func TestCountContentCandidates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
|
||||
// Each group's records: size, head, and content, one per record.
|
||||
groups := []struct {
|
||||
size int64
|
||||
head string
|
||||
contents []string
|
||||
}{
|
||||
{headTailMin, "h", []string{"", "c", "c"}}, // returned
|
||||
{headTailMin + 1, "h", []string{"", ""}}, // returned
|
||||
{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
|
||||
{headTailMin + 3, "h", []string{""}}, // one record
|
||||
{headTailMin - 1, "h", []string{"", ""}}, // too small
|
||||
{headTailMin + 4, "", []string{"", ""}}, // no head
|
||||
}
|
||||
|
||||
var recs []scanRec
|
||||
|
||||
for _, g := range groups {
|
||||
for _, content := range g.contents {
|
||||
recs = append(recs, scanRec{
|
||||
size: g.size, mtime: time.Unix(1, 0), head: g.head,
|
||||
tail: "t", content: content,
|
||||
path: fmt.Sprintf("/r/%d", len(recs)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := applyChanges(t.Context(), db, recs, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("applyChanges: %v", err)
|
||||
}
|
||||
|
||||
got, err := countContentCandidates(t.Context(), db)
|
||||
if err != nil {
|
||||
t.Fatalf("countContentCandidates: %v", err)
|
||||
}
|
||||
|
||||
var returned int64
|
||||
|
||||
err = loadContentCandidates(t.Context(), db,
|
||||
func(scanRec, bool) { returned++ })
|
||||
if err != nil {
|
||||
t.Fatalf("loadContentCandidates: %v", err)
|
||||
}
|
||||
|
||||
if got != 5 || returned != 5 {
|
||||
t.Errorf("count = %d, records returned = %d, want 5 and 5",
|
||||
got, returned)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user