sfdupes/main_test.go

package main

import (
	"bytes"
	"context"
	"errors"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

// usageMarker is the first line of cobra's usage text, which a usage
// error prints and a runtime failure must not.
const usageMarker = "Usage:"

// walSuffixes are the SQLite sidecar files a WAL-mode database keeps
// while it is open. A clean close checkpoints the WAL and removes
// both; finding either afterwards means the database was never closed.
//
//nolint:gochecknoglobals // a constant list, immutable by convention
var walSuffixes = []string{"-wal", "-shm"}

// assertNoSidecars fails when a WAL sidecar is still present beside the
// database at path.
func assertNoSidecars(t *testing.T, path string) {
	t.Helper()

	for _, suffix := range walSuffixes {
		_, err := os.Stat(path + suffix)
		if err == nil {
			t.Errorf("%s%s still present: the database was not closed",
				path, suffix)

			continue
		}

		if !errors.Is(err, fs.ErrNotExist) {
			t.Fatal(err)
		}
	}
}

// makeReadOnly takes write permission away from the database at path,
// from any WAL sidecar beside it, and from their directory, as for a
// user reading a database that a root cron scan keeps. Root ignores
// file permissions, so it skips the test when run as root.
func makeReadOnly(t *testing.T, path string) {
	t.Helper()

	if os.Geteuid() == 0 {
		t.Skip("root ignores file permissions")
	}

	err := os.Chmod(path, 0o400)
	if err != nil {
		t.Fatal(err)
	}

	for _, suffix := range walSuffixes {
		err = os.Chmod(path+suffix, 0o400)
		if err != nil && !errors.Is(err, fs.ErrNotExist) {
			t.Fatal(err)
		}
	}

	dir := filepath.Dir(path)

	//nolint:gosec // reaching the database needs the search bit
	err = os.Chmod(dir, 0o500)
	if err != nil {
		t.Fatal(err)
	}

	// Runs before t.TempDir's own cleanup, which must delete the files.
	t.Cleanup(func() {
		//nolint:gosec // removing the directory needs its search bit back
		_ = os.Chmod(dir, 0o700)
	})
}

// captureStderr redirects os.Stderr to a file for the rest of the test
// and returns a function reading back everything written to it. scan
// writes its warnings and summary straight to os.Stderr, not to the
// stderr writer run is given.
func captureStderr(t *testing.T) func() string {
	t.Helper()

	f, err := os.Create(filepath.Join(t.TempDir(), "stderr"))
	if err != nil {
		t.Fatal(err)
	}

	saved := os.Stderr
	os.Stderr = f

	t.Cleanup(func() {
		os.Stderr = saved

		_ = f.Close()
	})

	return func() string {
		// Read what has been written without disturbing the write
		// offset, so the capture can be inspected more than once.
		size, err := f.Seek(0, io.SeekCurrent)
		if err != nil {
			t.Fatal(err)
		}

		if size == 0 {
			return ""
		}

		b := make([]byte, size)

		_, err = f.ReadAt(b, 0)
		if err != nil {
			t.Fatal(err)
		}

		return string(b)
	}
}

// brokenDatabase writes a database that opens cleanly and passes the
// schema-version check but has no files table, so the first query
// fails with the database already open: a fatal error on a path that
// owns an open database. It closes the database the way scan does.
func brokenDatabase(t *testing.T) string {
	t.Helper()

	path := testDBPath(t)

	db, err := openDB(path, scanParams)
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.ExecContext(context.Background(),
		"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
	if err != nil {
		t.Fatal(err)
	}

	closeScanDatabase(t.Context(), db, path)

	return path
}

func TestOpenDatabaseKeepsWALWhileOpen(t *testing.T) {
	t.Parallel()

	// The premise of the fatal-path tests below: an open database has
	// a -wal sidecar, so its absence afterwards is evidence that the
	// database was closed and its WAL checkpointed.
	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	_, err = os.Stat(path + "-wal")
	if err != nil {
		t.Fatalf("no -wal beside an open database: %v", err)
	}

	err = db.Close()
	if err != nil {
		t.Fatal(err)
	}

	assertNoSidecars(t, path)
}

func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
	// Every subcommand that owns an open database must close it when
	// it fails: no os.Exit between the open and the return. The
	// sidecar check is evidence of the close only for scan: report and
	// trees only read a database that is out of WAL mode, which leaves
	// nothing on disk whether they close it or not.
	cases := map[string][]string{
		cmdScan:   {cmdScan},
		cmdReport: {cmdReport},
		cmdTrees:  {cmdTrees},
	}

	for name, args := range cases {
		t.Run(name, func(t *testing.T) {
			path := brokenDatabase(t)
			t.Setenv(databaseEnv, path)

			if name == cmdScan {
				args = append(args, t.TempDir())
			}

			var stdout, stderr bytes.Buffer

			code := run(args, &stdout, &stderr)
			if code != exitFatal {
				t.Errorf("run(%v) = %d, want %d", args, code, exitFatal)
			}

			assertNoSidecars(t, path)
			assertFatalOutput(t, stderr.String(), stdout.String())

			// Proof that the failure happened after the open: only a
			// query against the opened database can report this.
			if !strings.Contains(stderr.String(), "no such table: files") {
				t.Errorf("stderr = %q, want the failure to come from a "+
					"query on the open database", stderr.String())
			}
		})
	}
}

func TestRunMissingOperandIsFatalNotUsage(t *testing.T) {
	// README §Error handling: a PATH operand that does not exist is a
	// fatal error (1), not a usage error (2) — and a runtime failure
	// must not dump the usage text.
	t.Setenv(databaseEnv, testDBPath(t))

	var stdout, stderr bytes.Buffer

	missing := filepath.Join(t.TempDir(), "nope")

	code := run([]string{cmdScan, missing}, &stdout, &stderr)
	if code != exitFatal {
		t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal)
	}

	assertFatalOutput(t, stderr.String(), stdout.String())
}

// assertFatalOutput checks that a fatal error was reported the way
// README §Error handling and design goal 4 require: the message on
// stderr, prefixed with the program name, no usage text, and nothing
// at all on stdout.
func assertFatalOutput(t *testing.T, stderr, stdout string) {
	t.Helper()

	if !strings.Contains(stderr, "sfdupes: ") {
		t.Errorf("stderr = %q, want a \"sfdupes: \" error report", stderr)
	}

	if strings.Contains(stderr, usageMarker) {
		t.Errorf("stderr = %q, want no usage text for a runtime failure",
			stderr)
	}

	if stdout != "" {
		t.Errorf("stdout = %q, want nothing (data only)", stdout)
	}
}

func TestRunUsageErrors(t *testing.T) {
	// Usage errors keep exiting 2 with cobra's own report on stderr.
	cases := map[string]struct {
		args []string
		want string
	}{
		"no subcommand":      {[]string{}, usageMarker},
		"scan without paths": {[]string{cmdScan}, usageMarker},
		"report with args":   {[]string{cmdReport, "x"}, usageMarker},
		"trees with args":    {[]string{cmdTrees, "x"}, usageMarker},
		"unknown flag":       {[]string{cmdScan, "--nope", "/"}, usageMarker},
		"unknown subcommand": {[]string{"nope"}, "unknown command"},
	}

	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			// No usage error may reach the database, so point it at a
			// path that does not exist.
			t.Setenv(databaseEnv, testDBPath(t))

			var stdout, stderr bytes.Buffer

			code := run(tc.args, &stdout, &stderr)
			if code != exitUsage {
				t.Errorf("run(%v) = %d, want %d", tc.args, code, exitUsage)
			}

			if !strings.Contains(stderr.String(), tc.want) {
				t.Errorf("stderr = %q, want %q", stderr.String(), tc.want)
			}

			if got := stdout.String(); got != "" {
				t.Errorf("stdout = %q, want nothing (data only)", got)
			}
		})
	}
}

func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
	// README §scan mode: --workers below 1 is a usage error reported in
	// one line on stderr, before the scan opens the database.
	for _, workers := range []string{"0", "-1"} {
		t.Run(workers, func(t *testing.T) {
			dbPath := testDBPath(t)
			t.Setenv(databaseEnv, dbPath)

			var stdout, stderr bytes.Buffer

			args := []string{cmdScan, "--workers", workers, t.TempDir()}

			code := run(args, &stdout, &stderr)
			if code != exitUsage {
				t.Errorf("run(%v) = %d, want %d", args, code, exitUsage)
			}

			want := "Error: --workers must be at least 1, got " + workers +
				"\n"
			if got := stderr.String(); got != want {
				t.Errorf("stderr = %q, want %q", got, want)
			}

			if got := stdout.String(); got != "" {
				t.Errorf("stdout = %q, want nothing (data only)", got)
			}

			_, err := os.Stat(dbPath)
			if !errors.Is(err, fs.ErrNotExist) {
				t.Errorf("stat %s: %v, want the database never created",
					dbPath, err)
			}
		})
	}
}

func TestRunHelp(t *testing.T) {
	t.Parallel()

	// README §Subcommands: help goes to stderr, exits 0, and leaves
	// stdout empty.
	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}

	for _, args := range cases {
		var stdout, stderr bytes.Buffer

		code := run(args, &stdout, &stderr)
		if code != exitOK {
			t.Errorf("run(%v) = %d, want %d", args, code, exitOK)
		}

		if !strings.Contains(stderr.String(), usageMarker) {
			t.Errorf("run(%v) stderr = %q, want the help text",
				args, stderr.String())
		}

		if got := stdout.String(); got != "" {
			t.Errorf("run(%v) stdout = %q, want nothing (data only)",
				args, got)
		}
	}
}

func TestRunVersion(t *testing.T) {
	t.Parallel()

	// README §Subcommands: the version is one line on stdout, with
	// nothing on stderr, and exits 0.
	for _, arg := range []string{"--version", "-v"} {
		var stdout, stderr bytes.Buffer

		code := run([]string{arg}, &stdout, &stderr)
		if code != exitOK {
			t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
		}

		want := "sfdupes " + Version + "\n"
		if got := stdout.String(); got != want {
			t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
		}

		if got := stderr.String(); got != "" {
			t.Errorf("run(%s) stderr = %q, want nothing", arg, got)
		}
	}
}

func TestRunVersionWriteFailureIsFatal(t *testing.T) {
	t.Parallel()

	// README §Error handling: a stdout write failure exits 1, reported
	// in one line on stderr.
	var stderr bytes.Buffer

	code := run([]string{"--version"}, failingWriter{}, &stderr)
	if code != exitFatal {
		t.Errorf("run(--version) = %d, want %d", code, exitFatal)
	}

	want := "sfdupes: write stdout: " + errWriteFailed.Error() + "\n"
	if got := stderr.String(); got != want {
		t.Errorf("stderr = %q, want %q", got, want)
	}
}

// scanFixture builds a small tree holding one duplicate pair and one
// unreadable file and scans it into the database the caller has
// pointed SFDUPES_DATABASE at. The unreadable file makes the scan warn
// and skip, which README §Error handling still calls a successful run.
// It returns the duplicate pair's paths.
func scanFixture(t *testing.T) []string {
	t.Helper()

	dir := t.TempDir()

	dupes := []string{
		writeFile(t, dir, "one/a.bin", pattern(1, 300)),
		writeFile(t, dir, "two/a.bin", pattern(1, 300)),
	}

	// Same size as the pair, so the scan queues it for hashing and the
	// read fails.
	unreadable := writeFile(t, dir, "unreadable.bin", pattern(2, 300))

	err := os.Chmod(unreadable, 0)
	if err != nil {
		t.Fatal(err)
	}

	scanOK(t, dir)

	return dupes
}

// scanOK runs scan over operands, fails the test unless it exits 0 with
// nothing on stdout, and returns everything it printed to stderr.
func scanOK(t *testing.T, operands ...string) string {
	t.Helper()

	var stdout bytes.Buffer

	stderr := captureStderr(t)

	code := run(append([]string{cmdScan}, operands...), &stdout, os.Stderr)
	if code != exitOK {
		t.Fatalf("run(scan %q) = %d, want %d; stderr: %s",
			operands, code, exitOK, stderr())
	}

	if got := stdout.String(); got != "" {
		t.Errorf("scan stdout = %q, want nothing (data only)", got)
	}

	return stderr()
}

func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	scanFixture(t)
	assertNoSidecars(t, path)
}

func TestRunScanSkipsSymlinkOperand(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dir := t.TempDir()
	writeFile(t, dir, "target/sub/f", pattern(1, 10))

	link := filepath.Join(dir, "link")

	err := os.Symlink(filepath.Join(dir, "target"), link)
	if err != nil {
		t.Fatal(err)
	}

	// Scanning a directory through the symlink stores a record beneath
	// the symlink's own path for a file beneath its target.
	scanOK(t, filepath.Join(link, "sub"))

	assertOperandSkipped(t, path, link, "symlink",
		filepath.Join(link, "sub", "f"))
}

func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dir := t.TempDir()
	writeFile(t, dir, "target/sub/f", pattern(1, 10))

	link := filepath.Join(dir, "link")

	err := os.Symlink(filepath.Join(dir, "target"), link)
	if err != nil {
		t.Fatal(err)
	}

	// link is dropped as a symlink, but link/sub must still be scanned,
	// not dropped as lying under link.
	scanOK(t, link, filepath.Join(link, "sub"))

	db, err := openDB(path, reportParams)
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() { _ = db.Close() })

	recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
}

func TestRunScanSkipsZFSOperand(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	zfs := filepath.Join(t.TempDir(), ".zfs")
	snapshot := filepath.Join(zfs, "snapshot", "hourly")
	f := writeFile(t, snapshot, "f", pattern(1, 10))

	// An operand beneath a .zfs directory is walked, because it is not
	// itself named .zfs.
	scanOK(t, snapshot)

	assertOperandSkipped(t, path, zfs, ".zfs directory", f)
}

// assertOperandSkipped scans operand alone and checks that it is skipped
// as kind: a warning naming it, one skip in the summary, exit 0, and the
// record for kept, which an earlier scan stored beneath operand, still
// in the database at dbPath.
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
	kept string,
) {
	t.Helper()

	stderr := scanOK(t, operand)

	warning := "walk " + operand + ": skipping " + kind + " operand\n"
	if !strings.Contains(stderr, warning) {
		t.Errorf("stderr = %q, want %q", stderr, warning)
	}

	summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
		"0 unchanged), 1 skipped\n"
	if !strings.Contains(stderr, summary) {
		t.Errorf("stderr = %q, want %q", stderr, summary)
	}

	db, err := openDB(dbPath, reportParams)
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() { _ = db.Close() })

	recordByPath(t, dbRecords(t, db), kept)
}

func TestRunReportSucceeds(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dupes := scanFixture(t)

	var stdout, stderr bytes.Buffer

	code := run([]string{cmdReport}, &stdout, &stderr)
	if code != exitOK {
		t.Fatalf("run(report) = %d, want %d; stderr: %s",
			code, exitOK, stderr.String())
	}

	want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n"
	if got := stdout.String(); got != want {
		t.Errorf("stdout = %q, want %q", got, want)
	}

	assertNoSidecars(t, path)
}

func TestRunTreesSucceeds(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dupes := scanFixture(t)

	var stdout, stderr bytes.Buffer

	code := run([]string{cmdTrees}, &stdout, &stderr)
	if code != exitOK {
		t.Fatalf("run(trees) = %d, want %d; stderr: %s",
			code, exitOK, stderr.String())
	}

	// The two directories holding the duplicate pair are duplicate
	// trees of each other.
	want := "first\tdupe\tfiles\tsize\n" +
		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
	if got := stdout.String(); got != want {
		t.Errorf("stdout = %q, want %q", got, want)
	}

	assertNoSidecars(t, path)
}

func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
	// README §Database: report and trees need only read access to the
	// database file. With its directory read-only as well, SQLite
	// cannot create any file beside it.
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dupes := scanFixture(t)
	assertNoSidecars(t, path)
	makeReadOnly(t, path)

	cases := map[string]string{
		cmdReport: "first\tdupe\tsize\n" +
			dupes[0] + "\t" + dupes[1] + "\t300\n",
		cmdTrees: "first\tdupe\tfiles\tsize\n" +
			filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
			"\t1\t300\n",
	}

	for name, want := range cases {
		var stdout, stderr bytes.Buffer

		code := run([]string{name}, &stdout, &stderr)
		if code != exitOK {
			t.Errorf("run(%s) = %d, want %d; stderr: %s",
				name, code, exitOK, stderr.String())

			continue
		}

		if got := stdout.String(); got != want {
			t.Errorf("%s stdout = %q, want %q", name, got, want)
		}
	}
}

// holdScanLock takes the lock on the database at path, as a running
// scan does, and holds it until the test ends. It fails the test when
// the lock is already held.
func holdScanLock(t *testing.T, path string) {
	t.Helper()

	lock, err := lockScanDatabase(path)
	if err != nil {
		t.Fatalf("lock %s: %v", path, err)
	}

	t.Cleanup(func() { _ = lock.Close() })
}

func TestRunSecondScanFails(t *testing.T) {
	// README §Database: while one scan holds the lock, a second scan
	// fails at once, naming the lock file, without creating the
	// database.
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	holdScanLock(t, path)

	var stdout, stderr bytes.Buffer

	code := run([]string{cmdScan, t.TempDir()}, &stdout, &stderr)
	if code != exitFatal {
		t.Errorf("run(scan) = %d, want %d", code, exitFatal)
	}

	want := "sfdupes: another scan is running (lock held on " +
		path + ".lock)\n"
	if got := stderr.String(); got != want {
		t.Errorf("stderr = %q, want %q", got, want)
	}

	if got := stdout.String(); got != "" {
		t.Errorf("stdout = %q, want nothing (data only)", got)
	}

	_, err := os.Stat(path)
	if !errors.Is(err, fs.ErrNotExist) {
		t.Errorf("stat %s = %v, want the database not created", path, err)
	}
}

func TestRunScanReleasesLock(t *testing.T) {
	// README §Database: a scan releases the lock however it ends.
	t.Run("success", func(t *testing.T) {
		path := testDBPath(t)
		t.Setenv(databaseEnv, path)

		scanFixture(t)
		holdScanLock(t, path)
	})

	t.Run("fatal error", func(t *testing.T) {
		path := brokenDatabase(t)
		t.Setenv(databaseEnv, path)

		code := run([]string{cmdScan, t.TempDir()}, io.Discard, io.Discard)
		if code != exitFatal {
			t.Fatalf("run(scan) = %d, want %d", code, exitFatal)
		}

		holdScanLock(t, path)
	})
}

func TestRunReportsDuringScan(t *testing.T) {
	// README §Database: report and trees never take the lock, so they
	// run while a scan holds it.
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	scanFixture(t)
	holdScanLock(t, path)

	for _, name := range []string{cmdReport, cmdTrees} {
		var stderr bytes.Buffer

		code := run([]string{name}, io.Discard, &stderr)
		if code != exitOK {
			t.Errorf("run(%s) = %d, want %d; stderr: %s",
				name, code, exitOK, stderr.String())
		}
	}
}

func TestRunStdoutClosedIsFatal(t *testing.T) {
	// README §Error handling: a stdout write failure exits 1, reported
	// in one line on stderr.
	for _, name := range []string{cmdReport, cmdTrees} {
		t.Run(name, func(t *testing.T) {
			t.Setenv(databaseEnv, testDBPath(t))

			scanFixture(t)

			stdout, err := os.Create(filepath.Join(t.TempDir(), "stdout"))
			if err != nil {
				t.Fatal(err)
			}

			err = stdout.Close()
			if err != nil {
				t.Fatal(err)
			}

			var stderr bytes.Buffer

			code := run([]string{name}, stdout, &stderr)
			if code != exitFatal {
				t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
			}

			got := stderr.String()
			if !strings.HasPrefix(got, "sfdupes: write stdout: ") ||
				!strings.Contains(got, os.ErrClosed.Error()) ||
				strings.Count(got, "\n") != 1 {
				t.Errorf("stderr = %q, want one line reporting the "+
					"failed stdout write", got)
			}
		})
	}
}

// errWriteFailed is the error failingWriter returns.
var errWriteFailed = errors.New("write failed")

// failingWriter is a stdout that fails every write.
type failingWriter struct{}

func (failingWriter) Write([]byte) (int, error) { return 0, errWriteFailed }

func TestStdoutWriteErrorPropagates(t *testing.T) {
	t.Setenv(databaseEnv, testDBPath(t))

	scanFixture(t)

	cases := map[string]func(context.Context, io.Writer) error{
		cmdReport: runReport,
		cmdTrees:  runTrees,
	}

	for name, fn := range cases {
		err := fn(t.Context(), failingWriter{})
		if !errors.Is(err, errWriteFailed) {
			t.Errorf("%s: error = %v, want %v", name, err, errWriteFailed)
		}
	}
}