sfdupes/trees_test.go

package main

import (
	"bytes"
	"database/sql"
	"os"
	"slices"
	"strings"
	"testing"
	"time"
)

// treesOpenLine is the plain line of trees' open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
	"remain unknown"

// smokeTreeRecs mirrors the tree of the test under README "Definition
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
// by one filename. Each file takes one 4 KiB block on disk.
func smokeTreeRecs() []scanRec {
	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")

	return []scanRec{
		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
			content: f1Content, path: "/d/t1/f1"},
		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
			content: f2Content, path: "/d/t1/sub/f2"},
		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
			content: f1Content, path: "/d/t2/f1"},
		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
			content: f2Content, path: "/d/t2/sub/f2"},
		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
			content: f1Content, path: "/d/t3/f1"},
		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
			content: f2Content, path: "/d/t3/sub/f2renamed"},
	}
}

// dbTree builds the directory hierarchy from the records in db the way
// trees does, and returns the super-root and every directory.
func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
	t.Helper()

	tree := newTreeBuilder()

	err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
	if err != nil {
		t.Fatal(err)
	}

	return tree.finish()
}

// treeOf writes recs into a fresh database and builds the directory
// hierarchy from it the way trees does.
func treeOf(t *testing.T, recs []scanRec) (*treeNode, []*treeNode) {
	t.Helper()

	db := openTestDB(t)

	err := applyChanges(t.Context(), db, recs, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	return dbTree(t, db)
}

// nodeByPath finds the directory node with the given path.
func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode {
	t.Helper()

	for _, d := range dirs {
		if d.path == path {
			return d
		}
	}

	t.Fatalf("no directory node with path %q", path)

	return nil
}

// groupPaths flattens tree groups into their member path lists.
func groupPaths(groups [][]*treeNode) [][]string {
	out := make([][]string, 0, len(groups))
	for _, g := range groups {
		paths := make([]string, 0, len(g))
		for _, n := range g {
			paths = append(paths, n.path)
		}

		out = append(out, paths)
	}

	return out
}

func TestTreeCounts(t *testing.T) {
	t.Parallel()

	_, dirs := treeOf(t, smokeTreeRecs())

	d := nodeByPath(t, dirs, "/d")
	if d.fileCount != 6 || d.totalSize != 9300 {
		t.Errorf("/d: fileCount %d size %d, want 6 9300",
			d.fileCount, d.totalSize)
	}

	t1 := nodeByPath(t, dirs, "/d/t1")
	if t1.fileCount != 2 || t1.totalSize != 3100 {
		t.Errorf("/d/t1: fileCount %d size %d, want 2 3100",
			t1.fileCount, t1.totalSize)
	}

	sub := nodeByPath(t, dirs, "/d/t1/sub")
	if sub.fileCount != 1 || sub.totalSize != 100 {
		t.Errorf("/d/t1/sub: fileCount %d size %d, want 1 100",
			sub.fileCount, sub.totalSize)
	}
}

func TestTreeRootPath(t *testing.T) {
	t.Parallel()

	// The root directory's path is "/", never empty, and its
	// children's paths start with a single slash.
	_, dirs := treeOf(t, []scanRec{{path: "/f"}, {path: "/srv/g"}})

	got := make([]string, 0, len(dirs))
	for _, d := range dirs {
		got = append(got, d.path)
	}

	slices.Sort(got)

	want := []string{"/", "/srv"}
	if !slices.Equal(got, want) {
		t.Fatalf("directory paths = %q, want %q", got, want)
	}
}

func TestTreeNamesSortingBeforeSlash(t *testing.T) {
	t.Parallel()

	// In path order "/a/b-x/f" and "/a/b.txt" come between the file
	// "/a/b" and "/a/b/f", because "-" and "." sort before "/". Each
	// directory must still be built once, whole, so /a matches /c.
	recs := make([]scanRec, 0, 8)

	for _, top := range []string{"/a", "/c"} {
		for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
			content := testHash("c")
			if p == "/b-x/f" {
				content = testHash("other")
			}

			recs = append(recs, scanRec{
				size: 1, head: testHash("h"), tail: testHash("t"),
				content: content, path: top + p,
			})
		}
	}

	super, dirs := treeOf(t, recs)

	got := make([]string, 0, len(dirs))
	for _, d := range dirs {
		got = append(got, d.path)
	}

	slices.Sort(got)

	want := []string{"/", "/a", "/a/b", "/a/b-x", "/c", "/c/b", "/c/b-x"}
	if !slices.Equal(got, want) {
		t.Fatalf("directory paths = %q, want %q", got, want)
	}

	groups := collectTreeGroups(dirs, super, nil)

	gotGroups := groupPaths(groups)
	wantGroups := [][]string{{"/a", "/c"}}

	if !slices.EqualFunc(gotGroups, wantGroups, slices.Equal) {
		t.Fatalf("groups = %v, want %v", gotGroups, wantGroups)
	}

	if groups[0][0].fileCount != 4 || groups[0][0].totalSize != 4 {
		t.Errorf("group totals: %d files %d bytes, want 4 4",
			groups[0][0].fileCount, groups[0][0].totalSize)
	}
}

func TestRunTreesEscapesPaths(t *testing.T) {
	t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))

	var stdout, stderr bytes.Buffer

	code := run([]string{cmdTrees}, &stdout, &stderr)
	if code != exitOK {
		t.Fatalf("run(trees) = %d, want %d; stderr: %s",
			code, exitOK, stderr.String())
	}

	want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
		`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n"
	if got := stdout.String(); got != want {
		t.Errorf("stdout = %q, want %q", got, want)
	}
}

// TestRunTreesDiskSize runs trees on two copies of a tree whose files
// take different space on disk. The row's disk_size is the sum over the
// dupe copy's files, those in its subdirectory included.
func TestRunTreesDiskSize(t *testing.T) {
	f, g := testHash("f"), testHash("g")
	recs := []scanRec{
		{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
		{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
		{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
		{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
	}

	t.Setenv(databaseEnv, seedDatabase(t, recs))

	want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n"
	if got := runStdout(t, cmdTrees); got != want {
		t.Errorf("stdout = %q, want %q", got, want)
	}
}

// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the four as it starts and another as
// it ends. read counts the 6 records before it reads them, group counts
// the 8 directories, and write the 1 group. The summary comes last, and
// stdout holds the report alone.
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
	path := seedDatabase(t, smokeTreeRecs())
	t.Setenv(databaseEnv, path)

	stderr := captureStderr(t)

	var stdout bytes.Buffer

	code := run([]string{cmdTrees}, &stdout, os.Stderr)
	if code != exitOK {
		t.Fatalf("run(trees) = %d, want %d; stderr: %s",
			code, exitOK, stderr())
	}

	report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
		"/d/t1\t/d/t2\t2\t3100\t8192\n"
	if got := stdout.String(); got != report {
		t.Errorf("stdout = %q, want %q", got, report)
	}

	// How each line starts; the rate at the end of a phase varies.
	want := []string{
		"trees: database " + path + " (from " + databaseEnv + ")",
		treesOpenLine,
		treesOpenLine,
		"[2/4] read: counting the records, elapsed 0s, remain unknown",
		"[2/4] read: [6/6] 100%, ",
		"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
		"[3/4] group: [8/8] 100%, ",
		"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
		"[4/4] write: [1/1] 100%, ",
		"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
	}

	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
	if len(lines) != len(want) {
		t.Fatalf("stderr lines = %q, want %d lines starting %q",
			lines, len(want), want)
	}

	for i := range want {
		if !strings.HasPrefix(lines[i], want[i]) {
			t.Errorf("stderr line %d = %q, want it to start %q",
				i+1, lines[i], want[i])
		}
	}
}

// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
// terminal its write phase's display is drawn on, as a plain "sfdupes
// trees" does: the terminal must show the header and the row whole,
// each on a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
	stderr := captureStderr(t)
	super, dirs := treeOf(t, smokeTreeRecs())
	dupes := collectTreeGroups(dirs, super, nil)

	// No timed draws: the display is drawn as it starts and as it ends.
	p := startProgress(progressParams{
		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
	}, terminalWidth, time.Hour, time.Now)

	_, _, err := writeTreeGroups(os.Stderr, dupes, p)

	p.finish()

	if err != nil {
		t.Fatal(err)
	}

	// A terminal shows each line as the text after its last carriage
	// return.
	var shown []string

	for line := range strings.SplitSeq(stderr(), "\n") {
		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
	}

	want := []string{
		"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
	}
	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
		t.Errorf("terminal shows lines %q, want %q first", shown, want)
	}
}

func TestTreeDigests(t *testing.T) {
	t.Parallel()

	_, dirs := treeOf(t, smokeTreeRecs())

	t1 := nodeByPath(t, dirs, "/d/t1")
	t2 := nodeByPath(t, dirs, "/d/t2")
	t3 := nodeByPath(t, dirs, "/d/t3")

	if t1.digest != t2.digest {
		t.Error("identical trees /d/t1 and /d/t2 have different digests")
	}

	// t3 differs only in a filename; names are part of the digest.
	if t1.digest == t3.digest {
		t.Error("/d/t3 digest equals /d/t1 despite a renamed file")
	}

	sub1 := nodeByPath(t, dirs, "/d/t1/sub")
	sub3 := nodeByPath(t, dirs, "/d/t3/sub")

	if sub1.digest == sub3.digest {
		t.Error("subdirs with differently-named files share a digest")
	}
}

func TestTreeDigestContentSensitivity(t *testing.T) {
	t.Parallel()

	same, c := testHash("same"), testHash("c")
	recs := []scanRec{
		{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
		{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
	}

	_, dirs := treeOf(t, recs)

	a := nodeByPath(t, dirs, "/r/a")
	b := nodeByPath(t, dirs, "/r/b")

	if a.digest == b.digest {
		t.Error("trees with different file content share a digest")
	}
}

func TestCollectTreeGroupsMaximal(t *testing.T) {
	t.Parallel()

	super, dirs := treeOf(t, smokeTreeRecs())

	groups := collectTreeGroups(dirs, super, nil)

	want := [][]string{{"/d/t1", "/d/t2"}}
	if got := groupPaths(groups); !slices.EqualFunc(got, want,
		slices.Equal) {
		t.Fatalf("groups = %v, want %v", got, want)
	}

	// The /d/t1/sub vs /d/t2/sub group must be suppressed as implied
	// by its parents' group; the winning group reports one copy's
	// recursive totals.
	if groups[0][0].fileCount != 2 || groups[0][0].totalSize != 3100 {
		t.Errorf("group totals: %d files %d bytes, want 2 3100",
			groups[0][0].fileCount, groups[0][0].totalSize)
	}
}

func TestCollectTreeGroupsDeterministic(t *testing.T) {
	t.Parallel()

	recs := smokeTreeRecs()

	super, dirs := treeOf(t, recs)

	forward := groupPaths(collectTreeGroups(dirs, super, nil))

	reversed := slices.Clone(recs)
	slices.Reverse(reversed)

	superR, dirsR := treeOf(t, reversed)

	backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
	if !slices.EqualFunc(forward, backward, slices.Equal) {
		t.Fatalf("output depends on record order: %v vs %v",
			forward, backward)
	}
}

func TestCollectTreeGroupsSiblings(t *testing.T) {
	t.Parallel()

	// Identical sibling dirs share a parent, so their group cannot be
	// implied by a parent group and must be reported.
	h, tl, c := testHash("h"), testHash("t"), testHash("c")
	recs := []scanRec{
		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
	}

	super, dirs := treeOf(t, recs)

	got := groupPaths(collectTreeGroups(dirs, super, nil))

	want := [][]string{{"/p/x1", "/p/x2"}}
	if !slices.EqualFunc(got, want, slices.Equal) {
		t.Fatalf("groups = %v, want %v", got, want)
	}
}

func TestCollectTreeGroupsDifferingParents(t *testing.T) {
	t.Parallel()

	// /p/a and /q/b contain identical x subtrees, but /p/a has an
	// extra file, so the parents' digests differ and the x group must
	// be reported.
	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
	recs := []scanRec{
		{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
		{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
		{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
	}

	super, dirs := treeOf(t, recs)

	got := groupPaths(collectTreeGroups(dirs, super, nil))

	want := [][]string{{"/p/a/x", "/q/b/x"}}
	if !slices.EqualFunc(got, want, slices.Equal) {
		t.Fatalf("groups = %v, want %v", got, want)
	}
}