package main import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "os" "path/filepath" "slices" "strings" "testing" "time" ) // writeFile creates a file with the given content and returns its path. func writeFile(t *testing.T, dir, name string, data []byte) string { t.Helper() p := filepath.Join(dir, name) err := os.MkdirAll(filepath.Dir(p), 0o750) if err != nil { t.Fatal(err) } err = os.WriteFile(p, data, 0o600) if err != nil { t.Fatal(err) } return p } // hexSum returns the lowercase-hex SHA-256 of data. func hexSum(data []byte) string { s := sha256.Sum256(data) return hex.EncodeToString(s[:]) } // pattern returns n bytes of deterministic content seeded by tag. func pattern(tag byte, n int) []byte { data := make([]byte, n) for i := range data { data[i] = tag ^ byte(i) } return data } func TestHashHeadTail(t *testing.T) { t.Parallel() dir := t.TempDir() cases := []struct { name string data []byte }{ {"empty", nil}, {"one-byte", []byte("x")}, {"under-one-chunk", pattern(1, chunk-1)}, {"exactly-one-chunk", pattern(2, chunk)}, {"overlapping-reads", pattern(3, chunk+chunk/2)}, {"exactly-two-chunks", pattern(4, 2*chunk)}, {"beyond-two-chunks", pattern(5, 3*chunk)}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { t.Parallel() p := writeFile(t, dir, c.name, c.data) head, tail, err := hashHeadTail(p, int64(len(c.data))) if err != nil { t.Fatalf("hashHeadTail: %v", err) } n := min(chunk, len(c.data)) if want := hexSum(c.data[:n]); head != want { t.Errorf("head = %s, want %s", head, want) } if want := hexSum(c.data[len(c.data)-n:]); tail != want { t.Errorf("tail = %s, want %s", tail, want) } }) } } func TestHashHeadTailErrors(t *testing.T) { t.Parallel() dir := t.TempDir() _, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1) if err == nil { t.Error("no error for a missing file") } // A file that shrank between the stat and hash passes: reading at // the stat-reported size must fail rather than emit wrong hashes. p := writeFile(t, dir, "shrunk", []byte("tiny")) _, _, err = hashHeadTail(p, int64(2*chunk)) if err == nil { t.Error("no error when the stat size exceeds the file size") } } func TestWalkPass(t *testing.T) { t.Parallel() dir := t.TempDir() want := []string{ writeFile(t, dir, "a.txt", []byte("a")), writeFile(t, dir, "sub/b.txt", []byte("b")), writeFile(t, dir, "sub/deeper/c.txt", []byte("c")), } slices.Sort(want) // Files under a .zfs directory must never be walked. writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a")) // Symlinks are skipped, not followed. err := os.Symlink(want[0], filepath.Join(dir, "link")) if err != nil { t.Fatal(err) } paths, errs := walkPass([]string{dir}, false) if errs != 0 { t.Fatalf("errs = %d, want 0", errs) } slices.Sort(paths) if !slices.Equal(paths, want) { t.Fatalf("paths = %q, want %q", paths, want) } } func TestWalkPassMultipleRoots(t *testing.T) { t.Parallel() rootA := t.TempDir() rootB := t.TempDir() want := []string{ writeFile(t, rootA, "a1", []byte("1")), writeFile(t, rootA, "sub/a2", []byte("2")), writeFile(t, rootB, "b1", []byte("3")), } paths, errs := walkPass([]string{rootA, rootB}, false) if errs != 0 { t.Fatalf("errs = %d, want 0", errs) } // Operands are walked in the order given. if !slices.Equal(paths, want) { t.Fatalf("paths = %q, want %q", paths, want) } } func TestWalkPassFileAndSymlinkOperands(t *testing.T) { t.Parallel() dir := t.TempDir() f := writeFile(t, dir, "plain", []byte("data")) link := filepath.Join(dir, "link") err := os.Symlink(f, link) if err != nil { t.Fatal(err) } // A regular-file operand is emitted as itself. paths, errs := walkPass([]string{f}, false) if errs != 0 || !slices.Equal(paths, []string{f}) { t.Fatalf("file operand: paths = %q, errs = %d", paths, errs) } // A symlink operand is not followed and yields nothing. paths, errs = walkPass([]string{link}, false) if errs != 0 || len(paths) != 0 { t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs) } } func TestWalkPassOneFilesystemSameFS(t *testing.T) { t.Parallel() // Everything in one filesystem: -x must not skip anything. dir := t.TempDir() want := []string{ writeFile(t, dir, "a", []byte("a")), writeFile(t, dir, "sub/deep/b", []byte("b")), } paths, errs := walkPass([]string{dir}, true) if errs != 0 { t.Fatalf("errs = %d, want 0", errs) } slices.Sort(paths) if !slices.Equal(paths, want) { t.Fatalf("paths = %q, want %q", paths, want) } } func TestDeviceOf(t *testing.T) { t.Parallel() dir := t.TempDir() dev1, ok1 := deviceOf(dir) dev2, ok2 := deviceOf(dir) if !ok1 || !ok2 || dev1 != dev2 { t.Fatalf("deviceOf unstable: %d/%v vs %d/%v", dev1, ok1, dev2, ok2) } if _, ok := deviceOf(filepath.Join(dir, "missing")); ok { t.Fatal("deviceOf reported ok for a missing path") } } func TestStatPass(t *testing.T) { t.Parallel() dir := t.TempDir() a := writeFile(t, dir, "a", pattern(1, 10)) b := writeFile(t, dir, "b", pattern(2, 20)) missing := filepath.Join(dir, "vanished") recs, errs := statPass([]string{a, b, missing}, 2) if errs != 1 { t.Fatalf("errs = %d, want 1 for the vanished file", errs) } slices.SortFunc(recs, func(x, y fileRec) int { return strings.Compare(x.path, y.path) }) if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 { t.Fatalf("recs = %+v, want sizes 10 and 20", recs) } if recs[0].mtime <= 0 || recs[1].mtime <= 0 { t.Fatalf("recs = %+v, want positive mtimes", recs) } } // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { t.Helper() dir := t.TempDir() one := pattern(10, 2000) f1 := pattern(30, 3000) f2 := pattern(40, 100) writeFile(t, dir, "a/one.bin", one) writeFile(t, dir, "b/copy.bin", one) writeFile(t, dir, "b/copy2.bin", one) // Same size as one.bin, different content. writeFile(t, dir, "a/unique.bin", pattern(20, 2000)) writeFile(t, dir, "tiny1", []byte("x")) writeFile(t, dir, "tiny2", []byte("x")) writeFile(t, dir, "tiny3", []byte("y")) writeFile(t, dir, "empty1", nil) writeFile(t, dir, "empty2", nil) writeFile(t, dir, "t1/f1", f1) writeFile(t, dir, "t1/sub/f2", f2) writeFile(t, dir, "t2/f1", f1) writeFile(t, dir, "t2/sub/f2", f2) writeFile(t, dir, "t3/f1", f1) writeFile(t, dir, "t3/sub/f2renamed", f2) return dir } // smokeTreeFiles is the number of regular files buildSmokeTree creates. const smokeTreeFiles = 15 // syncTree synchronizes the database with the given roots and returns // the scan stats. func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { t.Helper() st, err := syncScan(db, roots, 4, false) if err != nil { t.Fatalf("syncScan: %v", err) } return st } // dbRecords returns every record currently in the database. func dbRecords(t *testing.T, db *sql.DB) []scanRec { t.Helper() recs, err := loadFileRows(db) if err != nil { t.Fatal(err) } return recs } // recordByPath finds the record with the given path. func recordByPath(t *testing.T, recs []scanRec, path string) scanRec { t.Helper() for _, r := range recs { if r.path == path { return r } } t.Fatalf("no record for %q", path) return scanRec{} } // recordPaths returns the sorted paths of recs. func recordPaths(recs []scanRec) []string { paths := make([]string, 0, len(recs)) for _, r := range recs { paths = append(paths, r.path) } slices.Sort(paths) return paths } // assertSmokeDupeGroups checks the file-level duplicate groups for the // smoke tree rooted at dir. func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { t.Helper() groups := collectDupeGroups(parsed) if len(groups) != 5 { t.Fatalf("len(groups) = %d, want 5", len(groups)) } wantSizes := []int64{3000, 2000, 100, 1, 0} for i, g := range groups { if g.size != wantSizes[i] { t.Errorf("groups[%d].size = %d, want %d", i, g.size, wantSizes[i]) } } wantF1 := []string{ filepath.Join(dir, "t1/f1"), filepath.Join(dir, "t2/f1"), filepath.Join(dir, "t3/f1"), } if !slices.Equal(groups[0].paths, wantF1) { t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1) } } // assertSmokeTreeGroups checks the duplicate-tree groups for the smoke // tree rooted at dir. func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) { t.Helper() super, dirs := buildHierarchy(parsed) super.compute() tg := collectTreeGroups(dirs, super) if len(tg) != 1 { t.Fatalf("len(tree groups) = %d, want 1", len(tg)) } wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")} if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) { t.Fatalf("tree group = %q, want %q", got, wantTrees) } if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 { t.Fatalf("tree totals: %d files %d bytes, want 2 3100", tg[0][0].fileCount, tg[0][0].totalSize) } } func TestScanPipeline(t *testing.T) { t.Parallel() dir := buildSmokeTree(t) db := openTestDB(t) st := syncTree(t, db, dir) if st != (scanStats{added: smokeTreeFiles}) { t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles) } parsed := dbRecords(t, db) if len(parsed) != smokeTreeFiles { t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles) } assertSmokeDupeGroups(t, dir, parsed) assertSmokeTreeGroups(t, dir, parsed) } func TestSyncScanUnchangedReuse(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) a := writeFile(t, dir, "a.bin", pattern(1, 500)) writeFile(t, dir, "b.bin", pattern(2, 600)) st := syncTree(t, db, dir) if st != (scanStats{added: 2}) { t.Fatalf("first scan stats = %+v, want 2 added", st) } // An immediate rescan reuses every record without reading file // contents. Prove the files are not re-read by corrupting a stored // hash and observing that it survives the rescan. _, err := db.ExecContext(context.Background(), "UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a)) if err != nil { t.Fatal(err) } st = syncTree(t, db, dir) if st != (scanStats{unchanged: 2}) { t.Fatalf("rescan stats = %+v, want 2 unchanged", st) } if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" { t.Fatalf("head = %q, want sentinel (file must not be re-read)", r.head) } } func TestSyncScanMtimeBump(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) a := writeFile(t, dir, "a.bin", pattern(1, 500)) syncTree(t, db, dir) // Bump the mtime forward: the file must be re-hashed even though // its size is unchanged. future := time.Now().Add(time.Hour) err := os.Chtimes(a, future, future) if err != nil { t.Fatal(err) } st := syncTree(t, db, dir) if st != (scanStats{updated: 1}) { t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) } if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() { t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix()) } } func TestSyncScanAddRemove(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) a := writeFile(t, dir, "a.bin", pattern(1, 500)) writeFile(t, dir, "b.bin", pattern(2, 600)) syncTree(t, db, dir) // Add one file, remove another. c := writeFile(t, dir, "c.bin", pattern(3, 700)) err := os.Remove(a) if err != nil { t.Fatal(err) } st := syncTree(t, db, dir) if st != (scanStats{added: 1, removed: 1, unchanged: 1}) { t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged", st) } want := []string{filepath.Join(dir, "b.bin"), c} if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) { t.Fatalf("paths = %q, want %q", got, want) } } func TestSyncScanSizeChange(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) p := writeFile(t, dir, "f", pattern(1, 100)) syncTree(t, db, dir) // Rewrite with a different size but force the mtime back to the // recorded value: the size mismatch alone must trigger a re-hash. old := recordByPath(t, dbRecords(t, db), p) writeFile(t, dir, "f", pattern(1, 200)) mt := time.Unix(old.mtime, 0) err := os.Chtimes(p, mt, mt) if err != nil { t.Fatal(err) } st := syncTree(t, db, dir) if st.updated != 1 { t.Fatalf("stats = %+v, want 1 updated", st) } if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 { t.Fatalf("size = %d, want 200", got.size) } } func TestSyncScanScope(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) writeFile(t, dir, "a/keep", pattern(1, 10)) gone := writeFile(t, dir, "b/gone", pattern(2, 10)) syncTree(t, db, dir) // Deleting a file outside the rescanned root must not remove its // record: records outside the scanned operands are untouched. err := os.Remove(gone) if err != nil { t.Fatal(err) } st := syncTree(t, db, filepath.Join(dir, "a")) if st.removed != 0 || st.unchanged != 1 { t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st) } if got := recordPaths(dbRecords(t, db)); len(got) != 2 { t.Fatalf("records = %q, want both retained", got) } // Rescanning the parent now removes the vanished file's record. st = syncTree(t, db, dir) if st.removed != 1 { t.Fatalf("parent stats = %+v, want 1 removed", st) } } func TestSyncScanRemovesNonRegular(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) p := writeFile(t, dir, "f", pattern(1, 10)) keep := writeFile(t, dir, "g", pattern(2, 10)) syncTree(t, db, dir) // Replace the file with a symlink: it is no longer walked, so its // record must be deleted. err := os.Remove(p) if err != nil { t.Fatal(err) } err = os.Symlink(keep, p) if err != nil { t.Fatal(err) } st := syncTree(t, db, dir) if st.removed != 1 || st.unchanged != 1 { t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st) } } func TestSyncScanOverlappingRoots(t *testing.T) { t.Parallel() dir := t.TempDir() db := openTestDB(t) writeFile(t, dir, "sub/f", pattern(1, 10)) // A file reachable via two overlapping operands yields one record. st := syncTree(t, db, dir, filepath.Join(dir, "sub")) if st.added != 1 { t.Fatalf("stats = %+v, want 1 added", st) } } func TestReportsNeverTouchFilesystem(t *testing.T) { t.Parallel() dir := buildSmokeTree(t) db := openTestDB(t) syncTree(t, db, dir) // Remove the scanned tree entirely; the analysis must be // unaffected because it reads the database alone. err := os.RemoveAll(dir) if err != nil { t.Fatal(err) } recs := dbRecords(t, db) assertSmokeDupeGroups(t, dir, recs) assertSmokeTreeGroups(t, dir, recs) } func TestUnderRoot(t *testing.T) { t.Parallel() const abRoot = "/a/b" cases := []struct { path string root string want bool }{ {"/a/b/c", abRoot, true}, {abRoot, abRoot, true}, {"/a/bc", abRoot, false}, {"/a", abRoot, false}, {"/x/y", "/", true}, {"/", "/", true}, } for _, c := range cases { if got := underRoot(c.path, c.root); got != c.want { t.Errorf("underRoot(%q, %q) = %v, want %v", c.path, c.root, got, c.want) } } } func TestUniquePaths(t *testing.T) { t.Parallel() got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"}) want := []string{"/a", "/b", "/c"} if !slices.Equal(got, want) { t.Fatalf("uniquePaths = %q, want %q", got, want) } }