diff --git a/report_test.go b/report_test.go new file mode 100644 index 0000000..f75d228 --- /dev/null +++ b/report_test.go @@ -0,0 +1,204 @@ +package main + +import ( + "bufio" + "fmt" + "slices" + "strings" + "testing" +) + +// mkRecord serializes one scan record in the on-the-wire format. +func mkRecord(size, mtime int64, head, tail, path string) string { + return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00", + size, mtime, head, tail, path) +} + +func TestSplitNULScanner(t *testing.T) { + t.Parallel() + + sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail")) + sc.Split(splitNUL) + + var got []string + for sc.Scan() { + got = append(got, sc.Text()) + } + + err := sc.Err() + if err != nil { + t.Fatalf("scanner error: %v", err) + } + + want := []string{"a", "bb", "", "tail"} + if !slices.Equal(got, want) { + t.Fatalf("tokens = %q, want %q", got, want) + } +} + +func TestParseScanStream(t *testing.T) { + t.Parallel() + + stream := mkRecord(10, 1, "h1", "t1", "/a/x") + + "garbage-without-tabs\x00" + + "notanumber\t1\th\tt\t/a/bad\x00" + + mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") + + "30\t3\th3\tt3\t/trailing/no-nul" + + recs, malformed := parseScanStream(strings.NewReader(stream), "test") + + if malformed != 2 { + t.Errorf("malformed = %d, want 2", malformed) + } + + want := []scanRec{ + {size: 10, head: "h1", tail: "t1", path: "/a/x"}, + {size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"}, + {size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"}, + } + if !slices.Equal(recs, want) { + t.Fatalf("recs = %+v, want %+v", recs, want) + } +} + +func TestParseScanStreamPathWithNewline(t *testing.T) { + t.Parallel() + + in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline")) + + recs, malformed := parseScanStream(in, "test") + if malformed != 0 || len(recs) != 1 { + t.Fatalf("got %d recs, %d malformed, want 1, 0", + len(recs), malformed) + } + + if recs[0].path != "/a/new\nline" { + t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline") + } +} + +func TestParseScanStreamEmpty(t *testing.T) { + t.Parallel() + + recs, malformed := parseScanStream(strings.NewReader(""), "test") + if len(recs) != 0 || malformed != 0 { + t.Fatalf("got %d recs, %d malformed, want 0, 0", + len(recs), malformed) + } +} + +func TestCollectDupeGroups(t *testing.T) { + t.Parallel() + + recs := []scanRec{ + {size: 100, head: "h", tail: "t", path: "/z/b"}, + {size: 100, head: "h", tail: "t", path: "/z/a"}, + {size: 100, head: "h", tail: "t", path: "/z/c"}, + {size: 4000, head: "H", tail: "T", path: "/big/2"}, + {size: 4000, head: "H", tail: "T", path: "/big/1"}, + // Same size as the /z group but a different head hash. + {size: 100, head: "other", tail: "t", path: "/z/d"}, + // A singleton signature must not form a group. + {size: 7, head: "u", tail: "u", path: "/lonely"}, + } + + groups := collectDupeGroups(recs) + if len(groups) != 2 { + t.Fatalf("len(groups) = %d, want 2", len(groups)) + } + + if groups[0].size != 4000 || + !slices.Equal(groups[0].paths, []string{"/big/1", "/big/2"}) { + t.Errorf("groups[0] = %+v, want size 4000, paths /big/1 /big/2", + groups[0]) + } + + if groups[1].size != 100 || + !slices.Equal(groups[1].paths, []string{"/z/a", "/z/b", "/z/c"}) { + t.Errorf("groups[1] = %+v, want size 100, paths /z/a /z/b /z/c", + groups[1]) + } +} + +func TestCollectDupeGroupsTieBreak(t *testing.T) { + t.Parallel() + + recs := []scanRec{ + {size: 50, head: "b", tail: "b", path: "/beta/2"}, + {size: 50, head: "b", tail: "b", path: "/beta/1"}, + {size: 50, head: "a", tail: "a", path: "/alpha/2"}, + {size: 50, head: "a", tail: "a", path: "/alpha/1"}, + } + + groups := collectDupeGroups(recs) + if len(groups) != 2 { + t.Fatalf("len(groups) = %d, want 2", len(groups)) + } + + // Equal sizes: ordered by first path ascending. + if groups[0].paths[0] != "/alpha/1" || groups[1].paths[0] != "/beta/1" { + t.Fatalf("tie-break order wrong: %q then %q", + groups[0].paths[0], groups[1].paths[0]) + } +} + +func TestCollectDupeGroupsDeterministic(t *testing.T) { + t.Parallel() + + recs := []scanRec{ + {size: 1, head: "a", tail: "a", path: "/p/1"}, + {size: 1, head: "a", tail: "a", path: "/p/2"}, + {size: 2, head: "b", tail: "b", path: "/q/1"}, + {size: 2, head: "b", tail: "b", path: "/q/2"}, + } + + forward := collectDupeGroups(recs) + + reversed := slices.Clone(recs) + slices.Reverse(reversed) + + backward := collectDupeGroups(reversed) + if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool { + return a.size == b.size && slices.Equal(a.paths, b.paths) + }) { + t.Fatalf("output depends on record order: %+v vs %+v", + forward, backward) + } +} + +func TestHumanBytes(t *testing.T) { + t.Parallel() + + cases := []struct { + n int64 + want string + }{ + {0, "0 B"}, + {1, "1 B"}, + {1023, "1023 B"}, + {1024, "1.0 KiB"}, + {1536, "1.5 KiB"}, + {1 << 20, "1.0 MiB"}, + {5 << 30, "5.0 GiB"}, + {1 << 40, "1.0 TiB"}, + {1 << 50, "1.0 PiB"}, + {1 << 60, "1.0 EiB"}, + } + for _, c := range cases { + if got := humanBytes(c.n); got != c.want { + t.Errorf("humanBytes(%d) = %q, want %q", c.n, got, c.want) + } + } +} + +func TestMalformedNote(t *testing.T) { + t.Parallel() + + if got := malformedNote(0); got != "" { + t.Errorf("malformedNote(0) = %q, want empty", got) + } + + if got := malformedNote(3); got != " (3 malformed, skipped)" { + t.Errorf("malformedNote(3) = %q", got) + } +} diff --git a/scan_test.go b/scan_test.go new file mode 100644 index 0000000..da13854 --- /dev/null +++ b/scan_test.go @@ -0,0 +1,311 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// writeFile creates a file with the given content and returns its path. +func writeFile(t *testing.T, dir, name string, data []byte) string { + t.Helper() + + p := filepath.Join(dir, name) + + err := os.MkdirAll(filepath.Dir(p), 0o750) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(p, data, 0o600) + if err != nil { + t.Fatal(err) + } + + return p +} + +// hexSum returns the lowercase-hex SHA-256 of data. +func hexSum(data []byte) string { + s := sha256.Sum256(data) + + return hex.EncodeToString(s[:]) +} + +// pattern returns n bytes of deterministic content seeded by tag. +func pattern(tag byte, n int) []byte { + data := make([]byte, n) + for i := range data { + data[i] = tag ^ byte(i) + } + + return data +} + +func TestHashHeadTail(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + cases := []struct { + name string + data []byte + }{ + {"empty", nil}, + {"one-byte", []byte("x")}, + {"under-one-chunk", pattern(1, chunk-1)}, + {"exactly-one-chunk", pattern(2, chunk)}, + {"overlapping-reads", pattern(3, chunk+chunk/2)}, + {"exactly-two-chunks", pattern(4, 2*chunk)}, + {"beyond-two-chunks", pattern(5, 3*chunk)}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + p := writeFile(t, dir, c.name, c.data) + + head, tail, err := hashHeadTail(p, int64(len(c.data))) + if err != nil { + t.Fatalf("hashHeadTail: %v", err) + } + + n := min(chunk, len(c.data)) + if want := hexSum(c.data[:n]); head != want { + t.Errorf("head = %s, want %s", head, want) + } + + if want := hexSum(c.data[len(c.data)-n:]); tail != want { + t.Errorf("tail = %s, want %s", tail, want) + } + }) + } +} + +func TestHashHeadTailErrors(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + _, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1) + if err == nil { + t.Error("no error for a missing file") + } + + // A file that shrank between the stat and hash passes: reading at + // the stat-reported size must fail rather than emit wrong hashes. + p := writeFile(t, dir, "shrunk", []byte("tiny")) + + _, _, err = hashHeadTail(p, int64(2*chunk)) + if err == nil { + t.Error("no error when the stat size exceeds the file size") + } +} + +func TestWalkPass(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + want := []string{ + writeFile(t, dir, "a.txt", []byte("a")), + writeFile(t, dir, "sub/b.txt", []byte("b")), + writeFile(t, dir, "sub/deeper/c.txt", []byte("c")), + } + + slices.Sort(want) + + // Files under a .zfs directory must never be walked. + writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a")) + + // Symlinks are skipped, not followed. + err := os.Symlink(want[0], filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + + paths, errs := walkPass(dir) + if errs != 0 { + t.Fatalf("errs = %d, want 0", errs) + } + + slices.Sort(paths) + + if !slices.Equal(paths, want) { + t.Fatalf("paths = %q, want %q", paths, want) + } +} + +func TestStatPass(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + a := writeFile(t, dir, "a", pattern(1, 10)) + b := writeFile(t, dir, "b", pattern(2, 20)) + missing := filepath.Join(dir, "vanished") + + recs, errs := statPass([]string{a, b, missing}, 2) + if errs != 1 { + t.Fatalf("errs = %d, want 1 for the vanished file", errs) + } + + slices.SortFunc(recs, func(x, y fileRec) int { + return strings.Compare(x.path, y.path) + }) + + if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 { + t.Fatalf("recs = %+v, want sizes 10 and 20", recs) + } + + if recs[0].mtime <= 0 || recs[1].mtime <= 0 { + t.Fatalf("recs = %+v, want positive mtimes", recs) + } +} + +// captureStdout runs fn with os.Stdout redirected to a temp file and +// returns everything fn wrote to it. +func captureStdout(t *testing.T, fn func()) []byte { + t.Helper() + + f, err := os.CreateTemp(t.TempDir(), "stdout") + if err != nil { + t.Fatal(err) + } + + orig := os.Stdout + os.Stdout = f + + defer func() { os.Stdout = orig }() + + fn() + + _, err = f.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + _ = f.Close() + + return data +} + +// buildSmokeTree recreates the README smoke-test filesystem layout +// with deterministic content and returns the tree root. +func buildSmokeTree(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + one := pattern(10, 2000) + f1 := pattern(30, 3000) + f2 := pattern(40, 100) + + writeFile(t, dir, "a/one.bin", one) + writeFile(t, dir, "b/copy.bin", one) + writeFile(t, dir, "b/copy2.bin", one) + // Same size as one.bin, different content. + writeFile(t, dir, "a/unique.bin", pattern(20, 2000)) + writeFile(t, dir, "tiny1", []byte("x")) + writeFile(t, dir, "tiny2", []byte("x")) + writeFile(t, dir, "tiny3", []byte("y")) + writeFile(t, dir, "empty1", nil) + writeFile(t, dir, "empty2", nil) + writeFile(t, dir, "t1/f1", f1) + writeFile(t, dir, "t1/sub/f2", f2) + writeFile(t, dir, "t2/f1", f1) + writeFile(t, dir, "t2/sub/f2", f2) + writeFile(t, dir, "t3/f1", f1) + writeFile(t, dir, "t3/sub/f2renamed", f2) + + return dir +} + +// scanToRecords runs the walk, stat, and hash passes over dir and +// parses the emitted stream back into records. +func scanToRecords(t *testing.T, dir string) []scanRec { + t.Helper() + + paths, walkErrs := walkPass(dir) + if walkErrs != 0 { + t.Fatalf("walk errors: %d", walkErrs) + } + + recs, statErrs := statPass(paths, 4) + if statErrs != 0 { + t.Fatalf("stat errors: %d", statErrs) + } + + var emitted, hashErrs int + + out := captureStdout(t, func() { + emitted, hashErrs = hashPass(recs, 4) + }) + + if emitted != len(paths) || hashErrs != 0 { + t.Fatalf("emitted %d of %d, %d hash errors", + emitted, len(paths), hashErrs) + } + + parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe") + if malformed != 0 || len(parsed) != len(paths) { + t.Fatalf("parsed %d records, %d malformed, want %d, 0", + len(parsed), malformed, len(paths)) + } + + return parsed +} + +//nolint:paralleltest // redirects the process-wide os.Stdout +func TestScanPipeline(t *testing.T) { + dir := buildSmokeTree(t) + parsed := scanToRecords(t, dir) + + groups := collectDupeGroups(parsed) + if len(groups) != 5 { + t.Fatalf("len(groups) = %d, want 5", len(groups)) + } + + wantSizes := []int64{3000, 2000, 100, 1, 0} + for i, g := range groups { + if g.size != wantSizes[i] { + t.Errorf("groups[%d].size = %d, want %d", + i, g.size, wantSizes[i]) + } + } + + wantF1 := []string{ + filepath.Join(dir, "t1/f1"), + filepath.Join(dir, "t2/f1"), + filepath.Join(dir, "t3/f1"), + } + if !slices.Equal(groups[0].paths, wantF1) { + t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1) + } + + super, dirs := buildHierarchy(parsed) + super.compute() + + tg := collectTreeGroups(dirs, super) + if len(tg) != 1 { + t.Fatalf("len(tree groups) = %d, want 1", len(tg)) + } + + wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")} + if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) { + t.Fatalf("tree group = %q, want %q", got, wantTrees) + } + + if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 { + t.Fatalf("tree totals: %d files %d bytes, want 2 3100", + tg[0][0].fileCount, tg[0][0].totalSize) + } +} diff --git a/trees_test.go b/trees_test.go new file mode 100644 index 0000000..ab444c3 --- /dev/null +++ b/trees_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "slices" + "testing" +) + +// Signature hashes shared by the smoke-test records. +const ( + f1Head = "f1h" + f1Tail = "f1t" + f2Head = "f2h" + f2Tail = "f2t" +) + +// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and +// /d/t2 are identical, /d/t3 differs from them only by one filename. +func smokeTreeRecs() []scanRec { + return []scanRec{ + {size: 3000, head: f1Head, tail: f1Tail, path: "/d/t1/f1"}, + {size: 100, head: f2Head, tail: f2Tail, path: "/d/t1/sub/f2"}, + {size: 3000, head: f1Head, tail: f1Tail, path: "/d/t2/f1"}, + {size: 100, head: f2Head, tail: f2Tail, path: "/d/t2/sub/f2"}, + {size: 3000, head: f1Head, tail: f1Tail, path: "/d/t3/f1"}, + {size: 100, head: f2Head, tail: f2Tail, path: "/d/t3/sub/f2renamed"}, + } +} + +// nodeByPath finds the directory node with the given path. +func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode { + t.Helper() + + for _, d := range dirs { + if d.path == path { + return d + } + } + + t.Fatalf("no directory node with path %q", path) + + return nil +} + +// groupPaths flattens tree groups into their member path lists. +func groupPaths(groups [][]*treeNode) [][]string { + out := make([][]string, 0, len(groups)) + for _, g := range groups { + paths := make([]string, 0, len(g)) + for _, n := range g { + paths = append(paths, n.path) + } + + out = append(out, paths) + } + + return out +} + +func TestBuildHierarchyCounts(t *testing.T) { + t.Parallel() + + super, dirs := buildHierarchy(smokeTreeRecs()) + super.compute() + + d := nodeByPath(t, dirs, "/d") + if d.fileCount != 6 || d.totalSize != 9300 { + t.Errorf("/d: fileCount %d size %d, want 6 9300", + d.fileCount, d.totalSize) + } + + t1 := nodeByPath(t, dirs, "/d/t1") + if t1.fileCount != 2 || t1.totalSize != 3100 { + t.Errorf("/d/t1: fileCount %d size %d, want 2 3100", + t1.fileCount, t1.totalSize) + } + + sub := nodeByPath(t, dirs, "/d/t1/sub") + if sub.fileCount != 1 || sub.totalSize != 100 { + t.Errorf("/d/t1/sub: fileCount %d size %d, want 1 100", + sub.fileCount, sub.totalSize) + } +} + +func TestTreeDigests(t *testing.T) { + t.Parallel() + + super, dirs := buildHierarchy(smokeTreeRecs()) + super.compute() + + t1 := nodeByPath(t, dirs, "/d/t1") + t2 := nodeByPath(t, dirs, "/d/t2") + t3 := nodeByPath(t, dirs, "/d/t3") + + if t1.digest != t2.digest { + t.Error("identical trees /d/t1 and /d/t2 have different digests") + } + + // t3 differs only in a filename; names are part of the digest. + if t1.digest == t3.digest { + t.Error("/d/t3 digest equals /d/t1 despite a renamed file") + } + + sub1 := nodeByPath(t, dirs, "/d/t1/sub") + sub3 := nodeByPath(t, dirs, "/d/t3/sub") + + if sub1.digest == sub3.digest { + t.Error("subdirs with differently-named files share a digest") + } +} + +func TestTreeDigestContentSensitivity(t *testing.T) { + t.Parallel() + + const sharedTail = "same" + + recs := []scanRec{ + {size: 10, head: sharedTail, tail: sharedTail, path: "/r/a/f"}, + {size: 10, head: "DIFF", tail: sharedTail, path: "/r/b/f"}, + } + + super, dirs := buildHierarchy(recs) + super.compute() + + a := nodeByPath(t, dirs, "/r/a") + b := nodeByPath(t, dirs, "/r/b") + + if a.digest == b.digest { + t.Error("trees with different file content share a digest") + } +} + +func TestCollectTreeGroupsMaximal(t *testing.T) { + t.Parallel() + + super, dirs := buildHierarchy(smokeTreeRecs()) + super.compute() + + groups := collectTreeGroups(dirs, super) + + want := [][]string{{"/d/t1", "/d/t2"}} + if got := groupPaths(groups); !slices.EqualFunc(got, want, + slices.Equal) { + t.Fatalf("groups = %v, want %v", got, want) + } + + // The /d/t1/sub vs /d/t2/sub group must be suppressed as implied + // by its parents' group; the winning group reports one copy's + // recursive totals. + if groups[0][0].fileCount != 2 || groups[0][0].totalSize != 3100 { + t.Errorf("group totals: %d files %d bytes, want 2 3100", + groups[0][0].fileCount, groups[0][0].totalSize) + } +} + +func TestCollectTreeGroupsDeterministic(t *testing.T) { + t.Parallel() + + recs := smokeTreeRecs() + + super, dirs := buildHierarchy(recs) + super.compute() + + forward := groupPaths(collectTreeGroups(dirs, super)) + + reversed := slices.Clone(recs) + slices.Reverse(reversed) + + superR, dirsR := buildHierarchy(reversed) + superR.compute() + + backward := groupPaths(collectTreeGroups(dirsR, superR)) + if !slices.EqualFunc(forward, backward, slices.Equal) { + t.Fatalf("output depends on record order: %v vs %v", + forward, backward) + } +} + +func TestCollectTreeGroupsSiblings(t *testing.T) { + t.Parallel() + + // Identical sibling dirs share a parent, so their group cannot be + // implied by a parent group and must be reported. + recs := []scanRec{ + {size: 10, head: "h", tail: "t", path: "/p/x1/f"}, + {size: 10, head: "h", tail: "t", path: "/p/x2/f"}, + } + + super, dirs := buildHierarchy(recs) + super.compute() + + got := groupPaths(collectTreeGroups(dirs, super)) + + want := [][]string{{"/p/x1", "/p/x2"}} + if !slices.EqualFunc(got, want, slices.Equal) { + t.Fatalf("groups = %v, want %v", got, want) + } +} + +func TestCollectTreeGroupsDifferingParents(t *testing.T) { + t.Parallel() + + // /p/a and /q/b contain identical x subtrees, but /p/a has an + // extra file, so the parents' digests differ and the x group must + // be reported. + recs := []scanRec{ + {size: 10, head: "h", tail: "t", path: "/p/a/x/f"}, + {size: 99, head: "e", tail: "e", path: "/p/a/extra"}, + {size: 10, head: "h", tail: "t", path: "/q/b/x/f"}, + } + + super, dirs := buildHierarchy(recs) + super.compute() + + got := groupPaths(collectTreeGroups(dirs, super)) + + want := [][]string{{"/p/a/x", "/q/b/x"}} + if !slices.EqualFunc(got, want, slices.Equal) { + t.Fatalf("groups = %v, want %v", got, want) + } +}