check / check (push) Successful in 49s
A file of 10 MiB or more now gets only its 64 KiB head and tail in the hash phase, so its content is read only when it can be a duplicate. A new content phase after the update phase finds every group of records, anywhere in the database, that share size, head and tail and include one without a content hash. It checks every member with lstat and, when at least two pass, reads those without a content hash through the existing worker pool; a stale file does not count as a match. report and trees leave out records without a content hash. The README, help text and TODO entry describe the gate; the schema stays at version 1. Lint suppressed: gosec on the file open in hashContentOnly, as in hashSignature, and on one chmod in a test. Model: opus-5-5
224 lines
5.7 KiB
Go
224 lines
5.7 KiB
Go
package main
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
)
|
|
|
|
// Signature hashes shared by the smoke-test records.
|
|
const (
|
|
f1Head = "f1h"
|
|
f1Tail = "f1t"
|
|
f1Content = "f1c"
|
|
f2Head = "f2h"
|
|
f2Tail = "f2t"
|
|
f2Content = "f2c"
|
|
)
|
|
|
|
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
|
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
|
func smokeTreeRecs() []scanRec {
|
|
return []scanRec{
|
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
|
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
|
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
|
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
|
|
path: "/d/t3/sub/f2renamed"},
|
|
}
|
|
}
|
|
|
|
// nodeByPath finds the directory node with the given path.
|
|
func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode {
|
|
t.Helper()
|
|
|
|
for _, d := range dirs {
|
|
if d.path == path {
|
|
return d
|
|
}
|
|
}
|
|
|
|
t.Fatalf("no directory node with path %q", path)
|
|
|
|
return nil
|
|
}
|
|
|
|
// groupPaths flattens tree groups into their member path lists.
|
|
func groupPaths(groups [][]*treeNode) [][]string {
|
|
out := make([][]string, 0, len(groups))
|
|
for _, g := range groups {
|
|
paths := make([]string, 0, len(g))
|
|
for _, n := range g {
|
|
paths = append(paths, n.path)
|
|
}
|
|
|
|
out = append(out, paths)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func TestBuildHierarchyCounts(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
super, dirs := buildHierarchy(smokeTreeRecs())
|
|
super.compute()
|
|
|
|
d := nodeByPath(t, dirs, "/d")
|
|
if d.fileCount != 6 || d.totalSize != 9300 {
|
|
t.Errorf("/d: fileCount %d size %d, want 6 9300",
|
|
d.fileCount, d.totalSize)
|
|
}
|
|
|
|
t1 := nodeByPath(t, dirs, "/d/t1")
|
|
if t1.fileCount != 2 || t1.totalSize != 3100 {
|
|
t.Errorf("/d/t1: fileCount %d size %d, want 2 3100",
|
|
t1.fileCount, t1.totalSize)
|
|
}
|
|
|
|
sub := nodeByPath(t, dirs, "/d/t1/sub")
|
|
if sub.fileCount != 1 || sub.totalSize != 100 {
|
|
t.Errorf("/d/t1/sub: fileCount %d size %d, want 1 100",
|
|
sub.fileCount, sub.totalSize)
|
|
}
|
|
}
|
|
|
|
func TestTreeDigests(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
super, dirs := buildHierarchy(smokeTreeRecs())
|
|
super.compute()
|
|
|
|
t1 := nodeByPath(t, dirs, "/d/t1")
|
|
t2 := nodeByPath(t, dirs, "/d/t2")
|
|
t3 := nodeByPath(t, dirs, "/d/t3")
|
|
|
|
if t1.digest != t2.digest {
|
|
t.Error("identical trees /d/t1 and /d/t2 have different digests")
|
|
}
|
|
|
|
// t3 differs only in a filename; names are part of the digest.
|
|
if t1.digest == t3.digest {
|
|
t.Error("/d/t3 digest equals /d/t1 despite a renamed file")
|
|
}
|
|
|
|
sub1 := nodeByPath(t, dirs, "/d/t1/sub")
|
|
sub3 := nodeByPath(t, dirs, "/d/t3/sub")
|
|
|
|
if sub1.digest == sub3.digest {
|
|
t.Error("subdirs with differently-named files share a digest")
|
|
}
|
|
}
|
|
|
|
func TestTreeDigestContentSensitivity(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const sharedTail = "same"
|
|
|
|
recs := []scanRec{
|
|
{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
|
|
{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
|
|
}
|
|
|
|
super, dirs := buildHierarchy(recs)
|
|
super.compute()
|
|
|
|
a := nodeByPath(t, dirs, "/r/a")
|
|
b := nodeByPath(t, dirs, "/r/b")
|
|
|
|
if a.digest == b.digest {
|
|
t.Error("trees with different file content share a digest")
|
|
}
|
|
}
|
|
|
|
func TestCollectTreeGroupsMaximal(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
super, dirs := buildHierarchy(smokeTreeRecs())
|
|
super.compute()
|
|
|
|
groups := collectTreeGroups(dirs, super)
|
|
|
|
want := [][]string{{"/d/t1", "/d/t2"}}
|
|
if got := groupPaths(groups); !slices.EqualFunc(got, want,
|
|
slices.Equal) {
|
|
t.Fatalf("groups = %v, want %v", got, want)
|
|
}
|
|
|
|
// The /d/t1/sub vs /d/t2/sub group must be suppressed as implied
|
|
// by its parents' group; the winning group reports one copy's
|
|
// recursive totals.
|
|
if groups[0][0].fileCount != 2 || groups[0][0].totalSize != 3100 {
|
|
t.Errorf("group totals: %d files %d bytes, want 2 3100",
|
|
groups[0][0].fileCount, groups[0][0].totalSize)
|
|
}
|
|
}
|
|
|
|
func TestCollectTreeGroupsDeterministic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
recs := smokeTreeRecs()
|
|
|
|
super, dirs := buildHierarchy(recs)
|
|
super.compute()
|
|
|
|
forward := groupPaths(collectTreeGroups(dirs, super))
|
|
|
|
reversed := slices.Clone(recs)
|
|
slices.Reverse(reversed)
|
|
|
|
superR, dirsR := buildHierarchy(reversed)
|
|
superR.compute()
|
|
|
|
backward := groupPaths(collectTreeGroups(dirsR, superR))
|
|
if !slices.EqualFunc(forward, backward, slices.Equal) {
|
|
t.Fatalf("output depends on record order: %v vs %v",
|
|
forward, backward)
|
|
}
|
|
}
|
|
|
|
func TestCollectTreeGroupsSiblings(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Identical sibling dirs share a parent, so their group cannot be
|
|
// implied by a parent group and must be reported.
|
|
recs := []scanRec{
|
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
|
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
|
|
}
|
|
|
|
super, dirs := buildHierarchy(recs)
|
|
super.compute()
|
|
|
|
got := groupPaths(collectTreeGroups(dirs, super))
|
|
|
|
want := [][]string{{"/p/x1", "/p/x2"}}
|
|
if !slices.EqualFunc(got, want, slices.Equal) {
|
|
t.Fatalf("groups = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// /p/a and /q/b contain identical x subtrees, but /p/a has an
|
|
// extra file, so the parents' digests differ and the x group must
|
|
// be reported.
|
|
recs := []scanRec{
|
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
|
|
{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
|
|
{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
|
|
}
|
|
|
|
super, dirs := buildHierarchy(recs)
|
|
super.compute()
|
|
|
|
got := groupPaths(collectTreeGroups(dirs, super))
|
|
|
|
want := [][]string{{"/p/a/x", "/q/b/x"}}
|
|
if !slices.EqualFunc(got, want, slices.Equal) {
|
|
t.Fatalf("groups = %v, want %v", got, want)
|
|
}
|
|
}
|