sfdupes/scan_test.go

package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
)

// writeFile creates a file with the given content and returns its path.
func writeFile(t *testing.T, dir, name string, data []byte) string {
	t.Helper()

	p := filepath.Join(dir, name)

	err := os.MkdirAll(filepath.Dir(p), 0o750)
	if err != nil {
		t.Fatal(err)
	}

	err = os.WriteFile(p, data, 0o600)
	if err != nil {
		t.Fatal(err)
	}

	return p
}

// hexSum returns the lowercase-hex SHA-256 of data.
func hexSum(data []byte) string {
	s := sha256.Sum256(data)

	return hex.EncodeToString(s[:])
}

// pattern returns n bytes of deterministic content seeded by tag.
func pattern(tag byte, n int) []byte {
	data := make([]byte, n)
	for i := range data {
		data[i] = tag ^ byte(i)
	}

	return data
}

func TestHashHeadTail(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	cases := []struct {
		name string
		data []byte
	}{
		{"empty", nil},
		{"one-byte", []byte("x")},
		{"under-one-chunk", pattern(1, chunk-1)},
		{"exactly-one-chunk", pattern(2, chunk)},
		{"overlapping-reads", pattern(3, chunk+chunk/2)},
		{"exactly-two-chunks", pattern(4, 2*chunk)},
		{"beyond-two-chunks", pattern(5, 3*chunk)},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			t.Parallel()

			p := writeFile(t, dir, c.name, c.data)

			head, tail, err := hashHeadTail(p, int64(len(c.data)))
			if err != nil {
				t.Fatalf("hashHeadTail: %v", err)
			}

			n := min(chunk, len(c.data))
			if want := hexSum(c.data[:n]); head != want {
				t.Errorf("head = %s, want %s", head, want)
			}

			if want := hexSum(c.data[len(c.data)-n:]); tail != want {
				t.Errorf("tail = %s, want %s", tail, want)
			}
		})
	}
}

func TestHashHeadTailErrors(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	_, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1)
	if err == nil {
		t.Error("no error for a missing file")
	}

	// A file that shrank between the stat and hash passes: reading at
	// the stat-reported size must fail rather than emit wrong hashes.
	p := writeFile(t, dir, "shrunk", []byte("tiny"))

	_, _, err = hashHeadTail(p, int64(2*chunk))
	if err == nil {
		t.Error("no error when the stat size exceeds the file size")
	}
}

func TestWalkPass(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	want := []string{
		writeFile(t, dir, "a.txt", []byte("a")),
		writeFile(t, dir, "sub/b.txt", []byte("b")),
		writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
	}

	slices.Sort(want)

	// Files under a .zfs directory must never be walked.
	writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a"))

	// Symlinks are skipped, not followed.
	err := os.Symlink(want[0], filepath.Join(dir, "link"))
	if err != nil {
		t.Fatal(err)
	}

	paths, errs := walkPass([]string{dir}, false)
	if errs != 0 {
		t.Fatalf("errs = %d, want 0", errs)
	}

	slices.Sort(paths)

	if !slices.Equal(paths, want) {
		t.Fatalf("paths = %q, want %q", paths, want)
	}
}

func TestWalkPassMultipleRoots(t *testing.T) {
	t.Parallel()

	rootA := t.TempDir()
	rootB := t.TempDir()
	want := []string{
		writeFile(t, rootA, "a1", []byte("1")),
		writeFile(t, rootA, "sub/a2", []byte("2")),
		writeFile(t, rootB, "b1", []byte("3")),
	}

	paths, errs := walkPass([]string{rootA, rootB}, false)
	if errs != 0 {
		t.Fatalf("errs = %d, want 0", errs)
	}

	// Operands are walked in the order given.
	if !slices.Equal(paths, want) {
		t.Fatalf("paths = %q, want %q", paths, want)
	}
}

func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	f := writeFile(t, dir, "plain", []byte("data"))

	link := filepath.Join(dir, "link")

	err := os.Symlink(f, link)
	if err != nil {
		t.Fatal(err)
	}

	// A regular-file operand is emitted as itself.
	paths, errs := walkPass([]string{f}, false)
	if errs != 0 || !slices.Equal(paths, []string{f}) {
		t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
	}

	// A symlink operand is not followed and yields nothing.
	paths, errs = walkPass([]string{link}, false)
	if errs != 0 || len(paths) != 0 {
		t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
	}
}

func TestWalkPassOneFilesystemSameFS(t *testing.T) {
	t.Parallel()

	// Everything in one filesystem: -x must not skip anything.
	dir := t.TempDir()
	want := []string{
		writeFile(t, dir, "a", []byte("a")),
		writeFile(t, dir, "sub/deep/b", []byte("b")),
	}

	paths, errs := walkPass([]string{dir}, true)
	if errs != 0 {
		t.Fatalf("errs = %d, want 0", errs)
	}

	slices.Sort(paths)

	if !slices.Equal(paths, want) {
		t.Fatalf("paths = %q, want %q", paths, want)
	}
}

func TestDeviceOf(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	dev1, ok1 := deviceOf(dir)

	dev2, ok2 := deviceOf(dir)
	if !ok1 || !ok2 || dev1 != dev2 {
		t.Fatalf("deviceOf unstable: %d/%v vs %d/%v", dev1, ok1, dev2, ok2)
	}

	if _, ok := deviceOf(filepath.Join(dir, "missing")); ok {
		t.Fatal("deviceOf reported ok for a missing path")
	}
}

func TestStatPass(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	a := writeFile(t, dir, "a", pattern(1, 10))
	b := writeFile(t, dir, "b", pattern(2, 20))
	missing := filepath.Join(dir, "vanished")

	recs, errs := statPass([]string{a, b, missing}, 2)
	if errs != 1 {
		t.Fatalf("errs = %d, want 1 for the vanished file", errs)
	}

	slices.SortFunc(recs, func(x, y fileRec) int {
		return strings.Compare(x.path, y.path)
	})

	if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
		t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
	}

	if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
		t.Fatalf("recs = %+v, want positive mtimes", recs)
	}
}

// captureStdout runs fn with os.Stdout redirected to a temp file and
// returns everything fn wrote to it.
func captureStdout(t *testing.T, fn func()) []byte {
	t.Helper()

	f, err := os.CreateTemp(t.TempDir(), "stdout")
	if err != nil {
		t.Fatal(err)
	}

	orig := os.Stdout
	os.Stdout = f

	defer func() { os.Stdout = orig }()

	fn()

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		t.Fatal(err)
	}

	data, err := io.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	_ = f.Close()

	return data
}

// buildSmokeTree recreates the README smoke-test filesystem layout
// with deterministic content and returns the tree root.
func buildSmokeTree(t *testing.T) string {
	t.Helper()

	dir := t.TempDir()
	one := pattern(10, 2000)
	f1 := pattern(30, 3000)
	f2 := pattern(40, 100)

	writeFile(t, dir, "a/one.bin", one)
	writeFile(t, dir, "b/copy.bin", one)
	writeFile(t, dir, "b/copy2.bin", one)
	// Same size as one.bin, different content.
	writeFile(t, dir, "a/unique.bin", pattern(20, 2000))
	writeFile(t, dir, "tiny1", []byte("x"))
	writeFile(t, dir, "tiny2", []byte("x"))
	writeFile(t, dir, "tiny3", []byte("y"))
	writeFile(t, dir, "empty1", nil)
	writeFile(t, dir, "empty2", nil)
	writeFile(t, dir, "t1/f1", f1)
	writeFile(t, dir, "t1/sub/f2", f2)
	writeFile(t, dir, "t2/f1", f1)
	writeFile(t, dir, "t2/sub/f2", f2)
	writeFile(t, dir, "t3/f1", f1)
	writeFile(t, dir, "t3/sub/f2renamed", f2)

	return dir
}

// scanToRecords runs the walk, stat, and hash passes over dir and
// parses the emitted stream back into records.
func scanToRecords(t *testing.T, dir string) []scanRec {
	t.Helper()

	paths, walkErrs := walkPass([]string{dir}, false)
	if walkErrs != 0 {
		t.Fatalf("walk errors: %d", walkErrs)
	}

	recs, statErrs := statPass(paths, 4)
	if statErrs != 0 {
		t.Fatalf("stat errors: %d", statErrs)
	}

	var emitted, hashErrs int

	out := captureStdout(t, func() {
		emitted, hashErrs = hashPass(recs, 4)
	})

	if emitted != len(paths) || hashErrs != 0 {
		t.Fatalf("emitted %d of %d, %d hash errors",
			emitted, len(paths), hashErrs)
	}

	parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe")
	if malformed != 0 || len(parsed) != len(paths) {
		t.Fatalf("parsed %d records, %d malformed, want %d, 0",
			len(parsed), malformed, len(paths))
	}

	return parsed
}

//nolint:paralleltest // redirects the process-wide os.Stdout
func TestScanPipeline(t *testing.T) {
	dir := buildSmokeTree(t)
	parsed := scanToRecords(t, dir)

	groups := collectDupeGroups(parsed)
	if len(groups) != 5 {
		t.Fatalf("len(groups) = %d, want 5", len(groups))
	}

	wantSizes := []int64{3000, 2000, 100, 1, 0}
	for i, g := range groups {
		if g.size != wantSizes[i] {
			t.Errorf("groups[%d].size = %d, want %d",
				i, g.size, wantSizes[i])
		}
	}

	wantF1 := []string{
		filepath.Join(dir, "t1/f1"),
		filepath.Join(dir, "t2/f1"),
		filepath.Join(dir, "t3/f1"),
	}
	if !slices.Equal(groups[0].paths, wantF1) {
		t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
	}

	super, dirs := buildHierarchy(parsed)
	super.compute()

	tg := collectTreeGroups(dirs, super)
	if len(tg) != 1 {
		t.Fatalf("len(tree groups) = %d, want 1", len(tg))
	}

	wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")}
	if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) {
		t.Fatalf("tree group = %q, want %q", got, wantTrees)
	}

	if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 {
		t.Fatalf("tree totals: %d files %d bytes, want 2 3100",
			tg[0][0].fileCount, tg[0][0].totalSize)
	}
}