Add test suite covering parsing, grouping, digests, and scan passes
Covers splitNUL framing, parseScanStream (malformed records, tabs and newlines in paths, unterminated trailing record), duplicate grouping with ordering/tie-break/determinism, humanBytes units, Merkle digest equality and name/content sensitivity, maximal-tree suppression (including sibling and differing-parent cases), hashHeadTail edge sizes and error paths, walkPass .zfs/symlink skipping, statPass vanished files, and an end-to-end walk/stat/hash pipeline test of the README smoke-test scenario. 64% statement coverage.
This commit is contained in:
311
scan_test.go
Normal file
311
scan_test.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// writeFile creates a file with the given content and returns its path.
|
||||
func writeFile(t *testing.T, dir, name string, data []byte) string {
|
||||
t.Helper()
|
||||
|
||||
p := filepath.Join(dir, name)
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(p), 0o750)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(p, data, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// hexSum returns the lowercase-hex SHA-256 of data.
|
||||
func hexSum(data []byte) string {
|
||||
s := sha256.Sum256(data)
|
||||
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
// pattern returns n bytes of deterministic content seeded by tag.
|
||||
func pattern(tag byte, n int) []byte {
|
||||
data := make([]byte, n)
|
||||
for i := range data {
|
||||
data[i] = tag ^ byte(i)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func TestHashHeadTail(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
data []byte
|
||||
}{
|
||||
{"empty", nil},
|
||||
{"one-byte", []byte("x")},
|
||||
{"under-one-chunk", pattern(1, chunk-1)},
|
||||
{"exactly-one-chunk", pattern(2, chunk)},
|
||||
{"overlapping-reads", pattern(3, chunk+chunk/2)},
|
||||
{"exactly-two-chunks", pattern(4, 2*chunk)},
|
||||
{"beyond-two-chunks", pattern(5, 3*chunk)},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
p := writeFile(t, dir, c.name, c.data)
|
||||
|
||||
head, tail, err := hashHeadTail(p, int64(len(c.data)))
|
||||
if err != nil {
|
||||
t.Fatalf("hashHeadTail: %v", err)
|
||||
}
|
||||
|
||||
n := min(chunk, len(c.data))
|
||||
if want := hexSum(c.data[:n]); head != want {
|
||||
t.Errorf("head = %s, want %s", head, want)
|
||||
}
|
||||
|
||||
if want := hexSum(c.data[len(c.data)-n:]); tail != want {
|
||||
t.Errorf("tail = %s, want %s", tail, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashHeadTailErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
_, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1)
|
||||
if err == nil {
|
||||
t.Error("no error for a missing file")
|
||||
}
|
||||
|
||||
// A file that shrank between the stat and hash passes: reading at
|
||||
// the stat-reported size must fail rather than emit wrong hashes.
|
||||
p := writeFile(t, dir, "shrunk", []byte("tiny"))
|
||||
|
||||
_, _, err = hashHeadTail(p, int64(2*chunk))
|
||||
if err == nil {
|
||||
t.Error("no error when the stat size exceeds the file size")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkPass(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
want := []string{
|
||||
writeFile(t, dir, "a.txt", []byte("a")),
|
||||
writeFile(t, dir, "sub/b.txt", []byte("b")),
|
||||
writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
|
||||
}
|
||||
|
||||
slices.Sort(want)
|
||||
|
||||
// Files under a .zfs directory must never be walked.
|
||||
writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a"))
|
||||
|
||||
// Symlinks are skipped, not followed.
|
||||
err := os.Symlink(want[0], filepath.Join(dir, "link"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
paths, errs := walkPass(dir)
|
||||
if errs != 0 {
|
||||
t.Fatalf("errs = %d, want 0", errs)
|
||||
}
|
||||
|
||||
slices.Sort(paths)
|
||||
|
||||
if !slices.Equal(paths, want) {
|
||||
t.Fatalf("paths = %q, want %q", paths, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatPass(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
a := writeFile(t, dir, "a", pattern(1, 10))
|
||||
b := writeFile(t, dir, "b", pattern(2, 20))
|
||||
missing := filepath.Join(dir, "vanished")
|
||||
|
||||
recs, errs := statPass([]string{a, b, missing}, 2)
|
||||
if errs != 1 {
|
||||
t.Fatalf("errs = %d, want 1 for the vanished file", errs)
|
||||
}
|
||||
|
||||
slices.SortFunc(recs, func(x, y fileRec) int {
|
||||
return strings.Compare(x.path, y.path)
|
||||
})
|
||||
|
||||
if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
|
||||
t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
|
||||
}
|
||||
|
||||
if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
|
||||
t.Fatalf("recs = %+v, want positive mtimes", recs)
|
||||
}
|
||||
}
|
||||
|
||||
// captureStdout runs fn with os.Stdout redirected to a temp file and
|
||||
// returns everything fn wrote to it.
|
||||
func captureStdout(t *testing.T, fn func()) []byte {
|
||||
t.Helper()
|
||||
|
||||
f, err := os.CreateTemp(t.TempDir(), "stdout")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
orig := os.Stdout
|
||||
os.Stdout = f
|
||||
|
||||
defer func() { os.Stdout = orig }()
|
||||
|
||||
fn()
|
||||
|
||||
_, err = f.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_ = f.Close()
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
one := pattern(10, 2000)
|
||||
f1 := pattern(30, 3000)
|
||||
f2 := pattern(40, 100)
|
||||
|
||||
writeFile(t, dir, "a/one.bin", one)
|
||||
writeFile(t, dir, "b/copy.bin", one)
|
||||
writeFile(t, dir, "b/copy2.bin", one)
|
||||
// Same size as one.bin, different content.
|
||||
writeFile(t, dir, "a/unique.bin", pattern(20, 2000))
|
||||
writeFile(t, dir, "tiny1", []byte("x"))
|
||||
writeFile(t, dir, "tiny2", []byte("x"))
|
||||
writeFile(t, dir, "tiny3", []byte("y"))
|
||||
writeFile(t, dir, "empty1", nil)
|
||||
writeFile(t, dir, "empty2", nil)
|
||||
writeFile(t, dir, "t1/f1", f1)
|
||||
writeFile(t, dir, "t1/sub/f2", f2)
|
||||
writeFile(t, dir, "t2/f1", f1)
|
||||
writeFile(t, dir, "t2/sub/f2", f2)
|
||||
writeFile(t, dir, "t3/f1", f1)
|
||||
writeFile(t, dir, "t3/sub/f2renamed", f2)
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// scanToRecords runs the walk, stat, and hash passes over dir and
|
||||
// parses the emitted stream back into records.
|
||||
func scanToRecords(t *testing.T, dir string) []scanRec {
|
||||
t.Helper()
|
||||
|
||||
paths, walkErrs := walkPass(dir)
|
||||
if walkErrs != 0 {
|
||||
t.Fatalf("walk errors: %d", walkErrs)
|
||||
}
|
||||
|
||||
recs, statErrs := statPass(paths, 4)
|
||||
if statErrs != 0 {
|
||||
t.Fatalf("stat errors: %d", statErrs)
|
||||
}
|
||||
|
||||
var emitted, hashErrs int
|
||||
|
||||
out := captureStdout(t, func() {
|
||||
emitted, hashErrs = hashPass(recs, 4)
|
||||
})
|
||||
|
||||
if emitted != len(paths) || hashErrs != 0 {
|
||||
t.Fatalf("emitted %d of %d, %d hash errors",
|
||||
emitted, len(paths), hashErrs)
|
||||
}
|
||||
|
||||
parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe")
|
||||
if malformed != 0 || len(parsed) != len(paths) {
|
||||
t.Fatalf("parsed %d records, %d malformed, want %d, 0",
|
||||
len(parsed), malformed, len(paths))
|
||||
}
|
||||
|
||||
return parsed
|
||||
}
|
||||
|
||||
//nolint:paralleltest // redirects the process-wide os.Stdout
|
||||
func TestScanPipeline(t *testing.T) {
|
||||
dir := buildSmokeTree(t)
|
||||
parsed := scanToRecords(t, dir)
|
||||
|
||||
groups := collectDupeGroups(parsed)
|
||||
if len(groups) != 5 {
|
||||
t.Fatalf("len(groups) = %d, want 5", len(groups))
|
||||
}
|
||||
|
||||
wantSizes := []int64{3000, 2000, 100, 1, 0}
|
||||
for i, g := range groups {
|
||||
if g.size != wantSizes[i] {
|
||||
t.Errorf("groups[%d].size = %d, want %d",
|
||||
i, g.size, wantSizes[i])
|
||||
}
|
||||
}
|
||||
|
||||
wantF1 := []string{
|
||||
filepath.Join(dir, "t1/f1"),
|
||||
filepath.Join(dir, "t2/f1"),
|
||||
filepath.Join(dir, "t3/f1"),
|
||||
}
|
||||
if !slices.Equal(groups[0].paths, wantF1) {
|
||||
t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
|
||||
}
|
||||
|
||||
super, dirs := buildHierarchy(parsed)
|
||||
super.compute()
|
||||
|
||||
tg := collectTreeGroups(dirs, super)
|
||||
if len(tg) != 1 {
|
||||
t.Fatalf("len(tree groups) = %d, want 1", len(tg))
|
||||
}
|
||||
|
||||
wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")}
|
||||
if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) {
|
||||
t.Fatalf("tree group = %q, want %q", got, wantTrees)
|
||||
}
|
||||
|
||||
if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 {
|
||||
t.Fatalf("tree totals: %d files %d bytes, want 2 3100",
|
||||
tg[0][0].fileCount, tg[0][0].totalSize)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user