Compute the content hash only when head and tail match (closes #61)
check / check (push) Successful in 49s
check / check (push) Successful in 49s
A file of 10 MiB or more now gets only its 64 KiB head and tail in the hash phase, so its content is read only when it can be a duplicate. A new content phase after the update phase finds every group of records, anywhere in the database, that share size, head and tail and include one without a content hash. It checks every member with lstat and, when at least two pass, reads those without a content hash through the existing worker pool; a stale file does not count as a match. report and trees leave out records without a content hash. The README, help text and TODO entry describe the gate; the schema stays at version 1. Lint suppressed: gosec on the file open in hashContentOnly, as in hashSignature, and on one chmod in a test. Model: opus-5-5
This commit was merged in pull request #65.
This commit is contained in:
+21
-17
@@ -9,15 +9,15 @@ func TestCollectDupeGroups(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
recs := []scanRec{
|
||||
{size: 100, head: "h", tail: "t", path: "/z/b"},
|
||||
{size: 100, head: "h", tail: "t", path: "/z/a"},
|
||||
{size: 100, head: "h", tail: "t", path: "/z/c"},
|
||||
{size: 4000, head: "H", tail: "T", path: "/big/2"},
|
||||
{size: 4000, head: "H", tail: "T", path: "/big/1"},
|
||||
{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
|
||||
{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
|
||||
{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
|
||||
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
|
||||
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
|
||||
// Same size as the /z group but a different head hash.
|
||||
{size: 100, head: "other", tail: "t", path: "/z/d"},
|
||||
{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
|
||||
// A singleton signature must not form a group.
|
||||
{size: 7, head: "u", tail: "u", path: "/lonely"},
|
||||
{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
@@ -43,10 +43,14 @@ func TestCollectDupeGroupsContentSeparates(t *testing.T) {
|
||||
|
||||
// Same size, head, and tail, but different content hashes: the final
|
||||
// rung keeps them apart, so no group forms. Matching content groups.
|
||||
// Records without a content hash never group, not even with each
|
||||
// other.
|
||||
recs := []scanRec{
|
||||
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
|
||||
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
|
||||
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
|
||||
{size: 100, head: "h", tail: "t", path: "/d"},
|
||||
{size: 100, head: "h", tail: "t", path: "/e"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
@@ -66,8 +70,8 @@ func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
|
||||
// mtime is informational only; records differing only in mtime
|
||||
// still group together.
|
||||
recs := []scanRec{
|
||||
{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
|
||||
{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
|
||||
{size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"},
|
||||
{size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
@@ -80,10 +84,10 @@ func TestCollectDupeGroupsTieBreak(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
recs := []scanRec{
|
||||
{size: 50, head: "b", tail: "b", path: "/beta/2"},
|
||||
{size: 50, head: "b", tail: "b", path: "/beta/1"},
|
||||
{size: 50, head: "a", tail: "a", path: "/alpha/2"},
|
||||
{size: 50, head: "a", tail: "a", path: "/alpha/1"},
|
||||
{size: 50, head: "b", tail: "b", content: "b", path: "/beta/2"},
|
||||
{size: 50, head: "b", tail: "b", content: "b", path: "/beta/1"},
|
||||
{size: 50, head: "a", tail: "a", content: "a", path: "/alpha/2"},
|
||||
{size: 50, head: "a", tail: "a", content: "a", path: "/alpha/1"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
@@ -102,10 +106,10 @@ func TestCollectDupeGroupsDeterministic(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
recs := []scanRec{
|
||||
{size: 1, head: "a", tail: "a", path: "/p/1"},
|
||||
{size: 1, head: "a", tail: "a", path: "/p/2"},
|
||||
{size: 2, head: "b", tail: "b", path: "/q/1"},
|
||||
{size: 2, head: "b", tail: "b", path: "/q/2"},
|
||||
{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
|
||||
{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
|
||||
{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
|
||||
{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
|
||||
}
|
||||
|
||||
forward := collectDupeGroups(recs)
|
||||
|
||||
Reference in New Issue
Block a user