Compute the content hash only when head and tail match (closes #61)
check / check (push) Successful in 49s
check / check (push) Successful in 49s
A file of 10 MiB or more now gets only its 64 KiB head and tail in the hash phase, so its content is read only when it can be a duplicate. A new content phase after the update phase finds every group of records, anywhere in the database, that share size, head and tail and include one without a content hash. It checks every member with lstat and, when at least two pass, reads those without a content hash through the existing worker pool; a stale file does not count as a match. report and trees leave out records without a content hash. The README, help text and TODO entry describe the gate; the schema stays at version 1. Lint suppressed: gosec on the file open in hashContentOnly, as in hashSignature, and on one chmod in a test. Model: opus-5-5
This commit was merged in pull request #65.
This commit is contained in:
+20
-17
@@ -7,22 +7,25 @@ import (
|
||||
|
||||
// Signature hashes shared by the smoke-test records.
|
||||
const (
|
||||
f1Head = "f1h"
|
||||
f1Tail = "f1t"
|
||||
f2Head = "f2h"
|
||||
f2Tail = "f2t"
|
||||
f1Head = "f1h"
|
||||
f1Tail = "f1t"
|
||||
f1Content = "f1c"
|
||||
f2Head = "f2h"
|
||||
f2Tail = "f2t"
|
||||
f2Content = "f2c"
|
||||
)
|
||||
|
||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||
func smokeTreeRecs() []scanRec {
|
||||
return []scanRec{
|
||||
{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t1/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, path: "/d/t1/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t2/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, path: "/d/t2/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t3/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, path: "/d/t3/sub/f2renamed"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
|
||||
path: "/d/t3/sub/f2renamed"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,8 +117,8 @@ func TestTreeDigestContentSensitivity(t *testing.T) {
|
||||
const sharedTail = "same"
|
||||
|
||||
recs := []scanRec{
|
||||
{size: 10, head: sharedTail, tail: sharedTail, path: "/r/a/f"},
|
||||
{size: 10, head: "DIFF", tail: sharedTail, path: "/r/b/f"},
|
||||
{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
|
||||
{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
|
||||
}
|
||||
|
||||
super, dirs := buildHierarchy(recs)
|
||||
@@ -181,8 +184,8 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
|
||||
// Identical sibling dirs share a parent, so their group cannot be
|
||||
// implied by a parent group and must be reported.
|
||||
recs := []scanRec{
|
||||
{size: 10, head: "h", tail: "t", path: "/p/x1/f"},
|
||||
{size: 10, head: "h", tail: "t", path: "/p/x2/f"},
|
||||
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
|
||||
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
|
||||
}
|
||||
|
||||
super, dirs := buildHierarchy(recs)
|
||||
@@ -203,9 +206,9 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
||||
// extra file, so the parents' digests differ and the x group must
|
||||
// be reported.
|
||||
recs := []scanRec{
|
||||
{size: 10, head: "h", tail: "t", path: "/p/a/x/f"},
|
||||
{size: 99, head: "e", tail: "e", path: "/p/a/extra"},
|
||||
{size: 10, head: "h", tail: "t", path: "/q/b/x/f"},
|
||||
{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
|
||||
{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
|
||||
{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
|
||||
}
|
||||
|
||||
super, dirs := buildHierarchy(recs)
|
||||
|
||||
Reference in New Issue
Block a user