Files
sfdupes/scan_test.go
sneak 1e7a519608
All checks were successful
check / check (push) Successful in 4s
Split the stat pass back out of the walk
Phases are strictly sequential again — walk, stat, hash, update per
operand — with parallelism only inside each phase. The walk
enumerates paths with per-directory workers (no lstat of file
entries); the stat pass lstats every collected path with per-file
workers, restoring its exact-total/ETA progress bar and per-file
parallelism inside wide flat directories.
2026-07-24 08:12:47 +07:00

685 lines
15 KiB
Go

package main
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
)
// writeFile creates a file with the given content and returns its path.
func writeFile(t *testing.T, dir, name string, data []byte) string {
t.Helper()
p := filepath.Join(dir, name)
err := os.MkdirAll(filepath.Dir(p), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.WriteFile(p, data, 0o600)
if err != nil {
t.Fatal(err)
}
return p
}
// hexSum returns the lowercase-hex SHA-256 of data.
func hexSum(data []byte) string {
s := sha256.Sum256(data)
return hex.EncodeToString(s[:])
}
// pattern returns n bytes of deterministic content seeded by tag.
func pattern(tag byte, n int) []byte {
data := make([]byte, n)
for i := range data {
data[i] = tag ^ byte(i)
}
return data
}
func TestHashHeadTail(t *testing.T) {
t.Parallel()
dir := t.TempDir()
cases := []struct {
name string
data []byte
}{
{"empty", nil},
{"one-byte", []byte("x")},
{"under-one-chunk", pattern(1, chunk-1)},
{"exactly-one-chunk", pattern(2, chunk)},
{"overlapping-reads", pattern(3, chunk+chunk/2)},
{"exactly-two-chunks", pattern(4, 2*chunk)},
{"beyond-two-chunks", pattern(5, 3*chunk)},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
t.Parallel()
p := writeFile(t, dir, c.name, c.data)
head, tail, err := hashHeadTail(p, int64(len(c.data)))
if err != nil {
t.Fatalf("hashHeadTail: %v", err)
}
n := min(chunk, len(c.data))
if want := hexSum(c.data[:n]); head != want {
t.Errorf("head = %s, want %s", head, want)
}
if want := hexSum(c.data[len(c.data)-n:]); tail != want {
t.Errorf("tail = %s, want %s", tail, want)
}
})
}
}
func TestHashHeadTailErrors(t *testing.T) {
t.Parallel()
dir := t.TempDir()
_, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1)
if err == nil {
t.Error("no error for a missing file")
}
// A file that shrank between the stat and hash passes: reading at
// the stat-reported size must fail rather than emit wrong hashes.
p := writeFile(t, dir, "shrunk", []byte("tiny"))
_, _, err = hashHeadTail(p, int64(2*chunk))
if err == nil {
t.Error("no error when the stat size exceeds the file size")
}
}
func TestWalkPass(t *testing.T) {
t.Parallel()
dir := t.TempDir()
want := []string{
writeFile(t, dir, "a.txt", []byte("a")),
writeFile(t, dir, "sub/b.txt", []byte("b")),
writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
}
slices.Sort(want)
// Files under a .zfs directory must never be walked.
writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a"))
// Symlinks are skipped, not followed.
err := os.Symlink(want[0], filepath.Join(dir, "link"))
if err != nil {
t.Fatal(err)
}
paths, errs := walkPass(dir, false, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
}
}
func TestWalkPassDeepAndWide(t *testing.T) {
t.Parallel()
// Exercise the dispatcher with more directories than workers and
// with nesting deeper than the worker count.
dir := t.TempDir()
deep := "deep" + strings.Repeat("/d", 30)
want := make([]string, 0, 41)
want = append(want, writeFile(t, dir, deep+"/f", []byte("x")))
for i := range 40 {
want = append(want, writeFile(t, dir,
fmt.Sprintf("wide/%02d/f", i), []byte("y")))
}
slices.Sort(want)
paths, errs := walkPass(dir, false, 8)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("walked %d paths, want %d", len(paths), len(want))
}
}
func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
t.Parallel()
dir := t.TempDir()
f := writeFile(t, dir, "plain", []byte("data"))
link := filepath.Join(dir, "link")
err := os.Symlink(f, link)
if err != nil {
t.Fatal(err)
}
// A regular-file operand is emitted as itself.
paths, errs := walkPass(f, false, 2)
if errs != 0 || !slices.Equal(paths, []string{f}) {
t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
}
// A symlink operand is not followed and yields nothing.
paths, errs = walkPass(link, false, 2)
if errs != 0 || len(paths) != 0 {
t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
}
}
func TestWalkPassOneFilesystemSameFS(t *testing.T) {
t.Parallel()
// Everything in one filesystem: -x must not skip anything.
dir := t.TempDir()
want := []string{
writeFile(t, dir, "a", []byte("a")),
writeFile(t, dir, "sub/deep/b", []byte("b")),
}
paths, errs := walkPass(dir, true, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
}
}
func TestStatPass(t *testing.T) {
t.Parallel()
dir := t.TempDir()
a := writeFile(t, dir, "a", pattern(1, 10))
b := writeFile(t, dir, "b", pattern(2, 20))
missing := filepath.Join(dir, "vanished")
recs, errs := statPass([]string{a, b, missing}, 2)
if errs != 1 {
t.Fatalf("errs = %d, want 1 for the vanished file", errs)
}
slices.SortFunc(recs, func(x, y fileRec) int {
return strings.Compare(x.path, y.path)
})
if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
}
if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
t.Fatalf("recs = %+v, want positive mtimes", recs)
}
}
func TestDeviceOfInfo(t *testing.T) {
t.Parallel()
dir := t.TempDir()
fi1, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
fi2, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev1, ok1 := deviceOfInfo(fi1)
dev2, ok2 := deviceOfInfo(fi2)
if !ok1 || !ok2 || dev1 != dev2 {
t.Fatalf("deviceOfInfo unstable: %d/%v vs %d/%v",
dev1, ok1, dev2, ok2)
}
}
// buildSmokeTree recreates the README smoke-test filesystem layout
// with deterministic content and returns the tree root.
func buildSmokeTree(t *testing.T) string {
t.Helper()
dir := t.TempDir()
one := pattern(10, 2000)
f1 := pattern(30, 3000)
f2 := pattern(40, 100)
writeFile(t, dir, "a/one.bin", one)
writeFile(t, dir, "b/copy.bin", one)
writeFile(t, dir, "b/copy2.bin", one)
// Same size as one.bin, different content.
writeFile(t, dir, "a/unique.bin", pattern(20, 2000))
writeFile(t, dir, "tiny1", []byte("x"))
writeFile(t, dir, "tiny2", []byte("x"))
writeFile(t, dir, "tiny3", []byte("y"))
writeFile(t, dir, "empty1", nil)
writeFile(t, dir, "empty2", nil)
writeFile(t, dir, "t1/f1", f1)
writeFile(t, dir, "t1/sub/f2", f2)
writeFile(t, dir, "t2/f1", f1)
writeFile(t, dir, "t2/sub/f2", f2)
writeFile(t, dir, "t3/f1", f1)
writeFile(t, dir, "t3/sub/f2renamed", f2)
return dir
}
// smokeTreeFiles is the number of regular files buildSmokeTree creates.
const smokeTreeFiles = 15
// syncTree synchronizes the database with the given roots and returns
// the scan stats.
func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
t.Helper()
st, err := syncScan(db, roots, 4, false)
if err != nil {
t.Fatalf("syncScan: %v", err)
}
return st
}
// dbRecords returns every record currently in the database.
func dbRecords(t *testing.T, db *sql.DB) []scanRec {
t.Helper()
recs, err := loadFileRows(db)
if err != nil {
t.Fatal(err)
}
return recs
}
// recordByPath finds the record with the given path.
func recordByPath(t *testing.T, recs []scanRec, path string) scanRec {
t.Helper()
for _, r := range recs {
if r.path == path {
return r
}
}
t.Fatalf("no record for %q", path)
return scanRec{}
}
// recordPaths returns the sorted paths of recs.
func recordPaths(recs []scanRec) []string {
paths := make([]string, 0, len(recs))
for _, r := range recs {
paths = append(paths, r.path)
}
slices.Sort(paths)
return paths
}
// assertSmokeDupeGroups checks the file-level duplicate groups for the
// smoke tree rooted at dir.
func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) {
t.Helper()
groups := collectDupeGroups(parsed)
if len(groups) != 5 {
t.Fatalf("len(groups) = %d, want 5", len(groups))
}
wantSizes := []int64{3000, 2000, 100, 1, 0}
for i, g := range groups {
if g.size != wantSizes[i] {
t.Errorf("groups[%d].size = %d, want %d",
i, g.size, wantSizes[i])
}
}
wantF1 := []string{
filepath.Join(dir, "t1/f1"),
filepath.Join(dir, "t2/f1"),
filepath.Join(dir, "t3/f1"),
}
if !slices.Equal(groups[0].paths, wantF1) {
t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
}
}
// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke
// tree rooted at dir.
func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) {
t.Helper()
super, dirs := buildHierarchy(parsed)
super.compute()
tg := collectTreeGroups(dirs, super)
if len(tg) != 1 {
t.Fatalf("len(tree groups) = %d, want 1", len(tg))
}
wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")}
if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) {
t.Fatalf("tree group = %q, want %q", got, wantTrees)
}
if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 {
t.Fatalf("tree totals: %d files %d bytes, want 2 3100",
tg[0][0].fileCount, tg[0][0].totalSize)
}
}
func TestScanPipeline(t *testing.T) {
t.Parallel()
dir := buildSmokeTree(t)
db := openTestDB(t)
st := syncTree(t, db, dir)
if st != (scanStats{added: smokeTreeFiles}) {
t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles)
}
parsed := dbRecords(t, db)
if len(parsed) != smokeTreeFiles {
t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles)
}
assertSmokeDupeGroups(t, dir, parsed)
assertSmokeTreeGroups(t, dir, parsed)
}
func TestSyncScanUnchangedReuse(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
writeFile(t, dir, "b.bin", pattern(2, 600))
st := syncTree(t, db, dir)
if st != (scanStats{added: 2}) {
t.Fatalf("first scan stats = %+v, want 2 added", st)
}
// An immediate rescan reuses every record without reading file
// contents. Prove the files are not re-read by corrupting a stored
// hash and observing that it survives the rescan.
_, err := db.ExecContext(context.Background(),
"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
if err != nil {
t.Fatal(err)
}
st = syncTree(t, db, dir)
if st != (scanStats{unchanged: 2}) {
t.Fatalf("rescan stats = %+v, want 2 unchanged", st)
}
if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
t.Fatalf("head = %q, want sentinel (file must not be re-read)",
r.head)
}
}
func TestSyncScanMtimeBump(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
syncTree(t, db, dir)
// Bump the mtime forward: the file must be re-hashed even though
// its size is unchanged.
future := time.Now().Add(time.Hour)
err := os.Chtimes(a, future, future)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{updated: 1}) {
t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
}
if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
}
}
func TestSyncScanAddRemove(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
writeFile(t, dir, "b.bin", pattern(2, 600))
syncTree(t, db, dir)
// Add one file, remove another.
c := writeFile(t, dir, "c.bin", pattern(3, 700))
err := os.Remove(a)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{added: 1, removed: 1, unchanged: 1}) {
t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged",
st)
}
want := []string{filepath.Join(dir, "b.bin"), c}
if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) {
t.Fatalf("paths = %q, want %q", got, want)
}
}
func TestSyncScanSizeChange(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
p := writeFile(t, dir, "f", pattern(1, 100))
syncTree(t, db, dir)
// Rewrite with a different size but force the mtime back to the
// recorded value: the size mismatch alone must trigger a re-hash.
old := recordByPath(t, dbRecords(t, db), p)
writeFile(t, dir, "f", pattern(1, 200))
mt := time.Unix(old.mtime, 0)
err := os.Chtimes(p, mt, mt)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st.updated != 1 {
t.Fatalf("stats = %+v, want 1 updated", st)
}
if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 {
t.Fatalf("size = %d, want 200", got.size)
}
}
func TestSyncScanScope(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
writeFile(t, dir, "a/keep", pattern(1, 10))
gone := writeFile(t, dir, "b/gone", pattern(2, 10))
syncTree(t, db, dir)
// Deleting a file outside the rescanned root must not remove its
// record: records outside the scanned operands are untouched.
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, filepath.Join(dir, "a"))
if st.removed != 0 || st.unchanged != 1 {
t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st)
}
if got := recordPaths(dbRecords(t, db)); len(got) != 2 {
t.Fatalf("records = %q, want both retained", got)
}
// Rescanning the parent now removes the vanished file's record.
st = syncTree(t, db, dir)
if st.removed != 1 {
t.Fatalf("parent stats = %+v, want 1 removed", st)
}
}
func TestSyncScanRemovesNonRegular(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
p := writeFile(t, dir, "f", pattern(1, 10))
keep := writeFile(t, dir, "g", pattern(2, 10))
syncTree(t, db, dir)
// Replace the file with a symlink: it is no longer walked, so its
// record must be deleted.
err := os.Remove(p)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(keep, p)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st.removed != 1 || st.unchanged != 1 {
t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st)
}
}
func TestSyncScanOverlappingRoots(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
writeFile(t, dir, "sub/f", pattern(1, 10))
// A file reachable via two overlapping operands yields one
// record: the second operand sees the rows the first operand
// committed and reuses them unchanged, which also proves each
// operand commits before the next starts.
st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
if st != (scanStats{added: 1, unchanged: 1}) {
t.Fatalf("stats = %+v, want 1 added 1 unchanged", st)
}
if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
t.Fatalf("records = %q, want exactly one", got)
}
}
func TestReportsNeverTouchFilesystem(t *testing.T) {
t.Parallel()
dir := buildSmokeTree(t)
db := openTestDB(t)
syncTree(t, db, dir)
// Remove the scanned tree entirely; the analysis must be
// unaffected because it reads the database alone.
err := os.RemoveAll(dir)
if err != nil {
t.Fatal(err)
}
recs := dbRecords(t, db)
assertSmokeDupeGroups(t, dir, recs)
assertSmokeTreeGroups(t, dir, recs)
}
func TestUnderRoot(t *testing.T) {
t.Parallel()
const abRoot = "/a/b"
cases := []struct {
path string
root string
want bool
}{
{"/a/b/c", abRoot, true},
{abRoot, abRoot, true},
{"/a/bc", abRoot, false},
{"/a", abRoot, false},
{"/x/y", "/", true},
{"/", "/", true},
}
for _, c := range cases {
if got := underRoot(c.path, c.root); got != c.want {
t.Errorf("underRoot(%q, %q) = %v, want %v",
c.path, c.root, got, c.want)
}
}
}