sfdupes/report_test.go

package main

import (
	"bytes"
	"crypto/sha256"
	"database/sql"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"testing"
	"time"
)

// awkwardDir is a directory name holding every byte the reports escape.
const awkwardDir = "/d/\tone\ntwo\rthree\\four"

// testHash returns a stand-in hash for a test record: the bytes of s,
// then zeros. Stand-ins sort as their strings do, so a test can choose
// the order in which report reads its records.
func testHash(s string) [sha256.Size]byte {
	var h [sha256.Size]byte

	copy(h[:], s)

	return h
}

// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
// sorts after it, so awkwardDir coming first shows that sorting uses
// the raw path.
func awkwardPairRecs() []scanRec {
	h, tl, c := testHash("h"), testHash("t"), testHash("c")

	return []scanRec{
		{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
		{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
	}
}

// seedDatabase writes recs into a fresh database, closes it as scan
// does, out of WAL mode, and returns its path.
func seedDatabase(t *testing.T, recs []scanRec) string {
	t.Helper()

	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	err = applyChanges(t.Context(), db, recs, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	closeScanDatabase(t.Context(), db, path)

	return path
}

// dupeGroup is one duplicate group as report reads it: the size, and
// the paths in report order, first path first.
type dupeGroup struct {
	size  int64
	paths []string
}

// dupeGroups returns the duplicate groups report reads from db, in
// report order, from the rows it writes: a group's rows come together,
// each naming the group's first path.
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
	t.Helper()

	var out bytes.Buffer

	_, err := readReport(t.Context(), db, "", &out, nil)
	if err != nil {
		t.Fatal(err)
	}

	var groups []dupeGroup

	rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
	for _, row := range rows[1:] { // after the header
		fields := strings.Split(row, "\t")
		if len(fields) != 3 {
			t.Fatalf("row %q, want first, dupe and size", row)
		}

		size, err := strconv.ParseInt(fields[2], 10, 64)
		if err != nil {
			t.Fatal(err)
		}

		if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] {
			groups = append(groups,
				dupeGroup{size: size, paths: []string{fields[0]}})
		}

		g := &groups[len(groups)-1]
		g.paths = append(g.paths, fields[1])
	}

	return groups
}

// dupeGroupsOf writes recs into a fresh database and returns the
// duplicate groups report reads from it.
func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup {
	t.Helper()

	db := openTestDB(t)

	err := applyChanges(t.Context(), db, recs, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	return dupeGroups(t, db)
}

func TestRunReportEscapesPaths(t *testing.T) {
	t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))

	var stdout, stderr bytes.Buffer

	code := run([]string{cmdReport}, &stdout, &stderr)
	if code != exitOK {
		t.Fatalf("run(report) = %d, want %d; stderr: %s",
			code, exitOK, stderr.String())
	}

	want := "first\tdupe\tsize\n" +
		`/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n"
	if got := stdout.String(); got != want {
		t.Errorf("stdout = %q, want %q", got, want)
	}
}

func TestReportStdoutFailsWhileReading(t *testing.T) {
	t.Parallel()

	// Every two records are a duplicate pair of a size of their own, and
	// a pair's row is written once a smaller size is read. Each row
	// holds two paths longer than dir, so the rows come to more than
	// twice the stdout buffer, and stdout fails while records are still
	// being read, not once every record is.
	dir := "/" + strings.Repeat("d", 4096)

	recs := make([]scanRec, 2*ioBufSize/len(dir))
	for i := range recs {
		recs[i] = scanRec{
			size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
			content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
		}
	}

	db := openTestDB(t)

	err := applyChanges(t.Context(), db, recs, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	// Counts the records read, and draws nothing.
	prog := &progress{now: time.Now}

	_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
	if !errors.Is(err, errWriteFailed) ||
		!strings.HasPrefix(err.Error(), "write stdout: ") {
		t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
	}

	if prog.count >= int64(len(recs)) {
		t.Errorf("stdout failed after all %d records were read, want it "+
			"to stop the read", prog.count)
	}
}

// reportOpenLine is the plain line of report's open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
	"remain unknown"

// smokeReport is report's stdout for the smoke-test records.
const smokeReport = "first\tdupe\tsize\n" +
	"/d/t1/f1\t/d/t2/f1\t3000\n" +
	"/d/t1/f1\t/d/t3/f1\t3000\n" +
	"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
	"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"

// TestRunReportProgressNamesEachPhase runs report on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the two as it starts and another as
// it ends; read counts the 6 records before it reads them. The summary
// comes last, and stdout holds the report alone.
func TestRunReportProgressNamesEachPhase(t *testing.T) {
	path := seedDatabase(t, smokeTreeRecs())
	t.Setenv(databaseEnv, path)

	stderr := captureStderr(t)

	var stdout bytes.Buffer

	code := run([]string{cmdReport}, &stdout, os.Stderr)
	if code != exitOK {
		t.Fatalf("run(report) = %d, want %d; stderr: %s",
			code, exitOK, stderr())
	}

	if got := stdout.String(); got != smokeReport {
		t.Errorf("stdout = %q, want %q", got, smokeReport)
	}

	// How each line starts; the rate at the end of a phase varies.
	want := []string{
		"report: database " + path + " (from " + databaseEnv + ")",
		reportOpenLine,
		reportOpenLine,
		"[2/2] read: counting the records, elapsed 0s, remain unknown",
		"[2/2] read: [6/6] 100%, ",
		"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
			"6.1 KiB reclaimable",
	}

	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
	if len(lines) != len(want) {
		t.Fatalf("stderr lines = %q, want %d lines starting %q",
			lines, len(want), want)
	}

	for i := range want {
		if !strings.HasPrefix(lines[i], want[i]) {
			t.Errorf("stderr line %d = %q, want it to start %q",
				i+1, lines[i], want[i])
		}
	}
}

// TestReadReportOnTerminal writes the smoke-test report to the terminal
// its read phase's display is drawn on, as a plain "sfdupes report"
// does: the terminal must show the header and every row whole, each on
// a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestReadReportOnTerminal(t *testing.T) {
	stderr := captureStderr(t)
	db := openTestDB(t)

	err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	// No timed draws: the display is drawn as it starts and as it ends.
	p := startProgress(progressParams{
		label: reportPhase("read"), unit: "records", total: -1,
		waitingFor: countingRecords,
	}, terminalWidth, time.Hour, time.Now)

	_, err = readReport(t.Context(), db, "", os.Stderr, p)

	p.finish()

	if err != nil {
		t.Fatal(err)
	}

	// A terminal shows each line as the text after its last carriage
	// return.
	var shown []string

	for line := range strings.SplitSeq(stderr(), "\n") {
		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
	}

	want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
		t.Errorf("terminal shows lines %q, want %q first", shown, want)
	}
}

func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
	// README §Constraints: identical database contents give identical
	// output, whatever order the records were inserted in.
	a, b := testHash("a"), testHash("b")

	recs := append(smokeTreeRecs(), awkwardPairRecs()...)
	recs = append(recs,
		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
	)

	reversed := slices.Clone(recs)
	slices.Reverse(reversed)

	for _, name := range []string{cmdReport, cmdTrees} {
		t.Run(name, func(t *testing.T) {
			t.Setenv(databaseEnv, seedDatabase(t, recs))

			forward := runStdout(t, name)

			t.Setenv(databaseEnv, seedDatabase(t, reversed))

			backward := runStdout(t, name)

			if strings.Count(forward, "\n") < 3 {
				t.Errorf("stdout = %q, want at least two rows", forward)
			}

			if forward != backward {
				t.Errorf("stdout depends on insertion order: %q vs %q",
					forward, backward)
			}
		})
	}
}

// runStdout runs the subcommand name and returns its stdout, failing
// the test unless it succeeds.
func runStdout(t *testing.T, name string) string {
	t.Helper()

	var stdout, stderr bytes.Buffer

	code := run([]string{name}, &stdout, &stderr)
	if code != exitOK {
		t.Fatalf("run(%s) = %d, want %d; stderr: %s",
			name, code, exitOK, stderr.String())
	}

	return stdout.String()
}

func TestEscapePath(t *testing.T) {
	t.Parallel()

	cases := map[string]string{
		"/srv/plain":    "/srv/plain",
		"/a\tb":         `/a\tb`,
		"/a\nb":         `/a\nb`,
		"/a\rb":         `/a\rb`,
		`/a\b`:          `/a\\b`,
		`/a\tb`:         `/a\\tb`,
		"/not-utf8\xff": "/not-utf8\xff",
	}
	for in, want := range cases {
		if got := escapePath(in); got != want {
			t.Errorf("escapePath(%q) = %q, want %q", in, got, want)
		}
	}
}

// TestWarnfEscapes checks that a warning naming a path that holds a
// newline is still one line.
//
//nolint:paralleltest // replaces the process-wide os.Stderr
func TestWarnfEscapes(t *testing.T) {
	f, err := os.Create(filepath.Join(t.TempDir(), "stderr"))
	if err != nil {
		t.Fatal(err)
	}

	saved := os.Stderr
	os.Stderr = f

	t.Cleanup(func() {
		os.Stderr = saved

		_ = f.Close()
	})

	(&progress{}).warnf("stat %s: %s", "/d/a\nb", "gone")

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		t.Fatal(err)
	}

	got, err := io.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	want := `stat /d/a\nb: gone` + "\n"
	if string(got) != want {
		t.Errorf("warning = %q, want %q", got, want)
	}
}

func TestDupeGroups(t *testing.T) {
	t.Parallel()

	h, tl, c := testHash("h"), testHash("t"), testHash("c")
	bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
	u := testHash("u")
	recs := []scanRec{
		{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
		{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
		{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
		// Same size as the /z group but a different head hash.
		{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
		// A singleton signature must not form a group.
		{size: 7, head: u, tail: u, content: u, path: "/lonely"},
	}

	groups := dupeGroupsOf(t, recs)
	if len(groups) != 2 {
		t.Fatalf("len(groups) = %d, want 2", len(groups))
	}

	if groups[0].size != 4000 ||
		!slices.Equal(groups[0].paths, []string{"/big/1", "/big/2"}) {
		t.Errorf("groups[0] = %+v, want size 4000, paths /big/1 /big/2",
			groups[0])
	}

	if groups[1].size != 100 ||
		!slices.Equal(groups[1].paths, []string{"/z/a", "/z/b", "/z/c"}) {
		t.Errorf("groups[1] = %+v, want size 100, paths /z/a /z/b /z/c",
			groups[1])
	}
}

func TestDupeGroupsContentSeparates(t *testing.T) {
	t.Parallel()

	// Same size, head, and tail: /b differs from the others only in its
	// content hash, so it joins no group, while /a and /c group.
	h, tl := testHash("h"), testHash("t")
	recs := []scanRec{
		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
		{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
	}

	groups := dupeGroupsOf(t, recs)
	if len(groups) != 1 {
		t.Fatalf("len(groups) = %d, want 1 (only the matching content)",
			len(groups))
	}

	if !slices.Equal(groups[0].paths, []string{"/a", "/c"}) {
		t.Errorf("group paths = %q, want /a /c", groups[0].paths)
	}
}

func TestDupeGroupsMtimeExcluded(t *testing.T) {
	t.Parallel()

	// mtime is informational only; records differing only in mtime
	// still group together.
	h, tl, c := testHash("h"), testHash("t"), testHash("c")
	recs := []scanRec{
		{
			size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
			content: c, path: "/m/1",
		},
		{
			size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
			content: c, path: "/m/2",
		},
	}

	groups := dupeGroupsOf(t, recs)
	if len(groups) != 1 {
		t.Fatalf("len(groups) = %d, want 1", len(groups))
	}
}

func TestDupeGroupsTieBreak(t *testing.T) {
	t.Parallel()

	// The hashes sort opposite to the first paths, so ordering the
	// groups by hash instead of by first path fails this test.
	a, b := testHash("a"), testHash("b")
	recs := []scanRec{
		{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
		{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
		{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
		{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
	}

	groups := dupeGroupsOf(t, recs)
	if len(groups) != 2 {
		t.Fatalf("len(groups) = %d, want 2", len(groups))
	}

	// Equal sizes: ordered by first path ascending.
	if groups[0].paths[0] != "/alpha/1" || groups[1].paths[0] != "/beta/1" {
		t.Fatalf("tie-break order wrong: %q then %q",
			groups[0].paths[0], groups[1].paths[0])
	}
}

func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
	t.Parallel()

	// report reads the groups of size 50 in signature order, here /a,
	// /z, /k, which is neither the order of their first paths nor its
	// reverse, and must write them ordered by first path. The group of
	// size 7 comes after them all, though its first path sorts first.
	a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
	recs := []scanRec{
		{size: 50, head: a, tail: a, content: a, path: "/k/2"},
		{size: 50, head: a, tail: a, content: a, path: "/k/1"},
		{size: 50, head: b, tail: b, content: b, path: "/z/1"},
		{size: 50, head: b, tail: b, content: b, path: "/z/2"},
		{size: 50, head: c, tail: c, content: c, path: "/a/2"},
		{size: 50, head: c, tail: c, content: c, path: "/a/1"},
		{size: 7, head: d, tail: d, content: d, path: "/0/2"},
		{size: 7, head: d, tail: d, content: d, path: "/0/1"},
	}

	groups := dupeGroupsOf(t, recs)

	got := make([]string, 0, len(groups))
	for _, g := range groups {
		got = append(got, g.paths[0])
	}

	want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
	if !slices.Equal(got, want) {
		t.Errorf("groups' first paths = %q, want %q", got, want)
	}
}

func TestDupeGroupsDeterministic(t *testing.T) {
	t.Parallel()

	a, b := testHash("a"), testHash("b")
	recs := []scanRec{
		{size: 1, head: a, tail: a, content: a, path: "/p/1"},
		{size: 1, head: a, tail: a, content: a, path: "/p/2"},
		{size: 2, head: b, tail: b, content: b, path: "/q/1"},
		{size: 2, head: b, tail: b, content: b, path: "/q/2"},
	}

	forward := dupeGroupsOf(t, recs)

	reversed := slices.Clone(recs)
	slices.Reverse(reversed)

	backward := dupeGroupsOf(t, reversed)
	if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool {
		return a.size == b.size && slices.Equal(a.paths, b.paths)
	}) {
		t.Fatalf("output depends on record order: %+v vs %+v",
			forward, backward)
	}
}

func TestHumanBytes(t *testing.T) {
	t.Parallel()

	cases := []struct {
		n    int64
		want string
	}{
		{0, "0 B"},
		{1, "1 B"},
		{1023, "1023 B"},
		{1024, "1.0 KiB"},
		{1536, "1.5 KiB"},
		{1 << 20, "1.0 MiB"},
		{5 << 30, "5.0 GiB"},
		{1 << 40, "1.0 TiB"},
		{1 << 50, "1.0 PiB"},
		{1 << 60, "1.0 EiB"},
	}
	for _, c := range cases {
		if got := humanBytes(c.n); got != c.want {
			t.Errorf("humanBytes(%d) = %q, want %q", c.n, got, c.want)
		}
	}
}