sfdupes/exclude_test.go

package main

import (
	"bytes"
	"errors"
	"maps"
	"os"
	"path/filepath"
	"runtime"
	"slices"
	"strings"
	"testing"
)

func TestDefaultExcludes(t *testing.T) {
	t.Parallel()

	// README "Rules for the walk": each system's list.
	cases := map[string][]string{
		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
		goosDarwin: {
			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
			"/System/Volumes/VM", "/System/Volumes/Data",
		},
		goosWindows: {
			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
			"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
			`Users\*\AppData\Local\Temp`,
		},
		"freebsd": nil,
	}

	for goos, want := range cases {
		if got := defaultExcludes(goos); !slices.Equal(got, want) {
			t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
		}
	}
}

func TestIsDefaultExclude(t *testing.T) {
	t.Parallel()

	// A * matches one path component, no more and no fewer.
	excludes := []string{"/scratch", "/home/*/cache"}

	cases := map[string]bool{
		"/scratch":        true,
		"/scratch/x":      false,
		"/home/u/cache":   true,
		"/home/u/v/cache": false,
		"/home/cache":     false,
	}

	for path, want := range cases {
		if got := isDefaultExclude(path, excludes); got != want {
			t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
		}
	}
}

func TestReadMountInfo(t *testing.T) {
	t.Parallel()

	// The second-to-last line has no optional fields. /mnt/over has an
	// ext4 filesystem mounted over its tmpfs.
	mountinfo := strings.Join([]string{
		"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
		"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
		"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
		`25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`,
		"26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw",
		"27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw",
		"28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
	}, "\n")

	got, err := readMountInfo(strings.NewReader(mountinfo))
	if err != nil {
		t.Fatal(err)
	}

	want := map[string]string{
		"/": "ext4", "/proc": "proc", "/sys": "sysfs",
		"/mnt/ram disk": "tmpfs", "/mnt/over": "ext4", "/srv/auto": "autofs",
	}
	if !maps.Equal(got, want) {
		t.Errorf("readMountInfo = %q, want %q", got, want)
	}

	_, err = readMountInfo(strings.NewReader("22 1 8:1 / / rw,relatime\n"))
	if !errors.Is(err, errMountInfoLine) {
		t.Errorf("readMountInfo of a line without a type = %v, want %v",
			err, errMountInfoLine)
	}
}

func TestNonPersistentFilesystems(t *testing.T) {
	t.Parallel()

	if runtime.GOOS != goosLinux {
		t.Skip("the test knows a mount only on Linux: /proc")
	}

	nonPersistent, err := nonPersistentFilesystems()
	if err != nil {
		t.Fatal(err)
	}

	dev, ok := deviceOfPath("/proc")
	if !ok {
		t.Fatal("no device for /proc")
	}

	if got := nonPersistent[dev]; got != "proc" {
		t.Errorf("filesystem of /proc = %q, want %q", got, "proc")
	}
}

func TestDefaultExcludesHelp(t *testing.T) {
	t.Parallel()

	for _, goos := range []string{goosLinux, goosDarwin, goosWindows} {
		help := defaultExcludesHelp(goos)

		for _, dir := range defaultExcludes(goos) {
			if !strings.Contains(help, "\n  "+dir+"\n") {
				t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
			}
		}

		// Only Linux and macOS list their mounted filesystems.
		for _, fsType := range nonPersistentFilesystemTypes() {
			listed := strings.Contains(help, " "+fsType+",") ||
				strings.Contains(help, " "+fsType+"\n")
			if listed != (goos != goosWindows) {
				t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
			}
		}
	}

	want := "This system has no default excludes.\n"
	if got := defaultExcludesHelp("freebsd"); got != want {
		t.Errorf("help for freebsd = %q, want %q", got, want)
	}

	// scan's help shows those of the system it runs on.
	var stdout, stderr bytes.Buffer

	run([]string{cmdScan, "-h"}, &stdout, &stderr)

	help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
	if !strings.Contains(stderr.String(), help) {
		t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
	}
}

// TestScanDoesNotEnterDefaultExclude checks that the walk does not
// enter a default exclude below an operand. It says so on a line of its
// own and does not count it as skipped. The records below it count as
// outside the operands: the record of a file gone from it is kept, and
// its size joins the size census, while a file added to it is not
// recorded.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
	stderr := captureStderr(t)
	dir := t.TempDir()
	db := openTestDB(t)
	excluded := filepath.Join(dir, "tmp")

	// A scan with no default excludes records the file in excluded.
	gone := writeFile(t, excluded, "gone", pattern(1, 100))
	syncTree(t, db, dir)

	err := os.Remove(gone)
	if err != nil {
		t.Fatal(err)
	}

	unseen := writeFile(t, excluded, "new", pattern(2, 10))

	// The same size as gone, so it is hashed only if the census holds the
	// size of gone's record.
	added := writeFile(t, dir, "added", pattern(3, 100))

	st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
		walkRules{excludes: []string{excluded}})
	if err != nil {
		t.Fatal(err)
	}

	if st.skipped != 0 {
		t.Errorf("skipped = %d, want 0", st.skipped)
	}

	recs := dbRecords(t, db)
	recordByPath(t, recs, gone)

	if recordByPath(t, recs, added).head == "" {
		t.Errorf("%s not hashed: the census lacks the size of %s",
			added, gone)
	}

	if slices.Contains(recordPaths(recs), unseen) {
		t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
	}

	line := "\nscan: skipped " + excluded + " (default exclude)\n"
	if !strings.Contains(stderr(), line) {
		t.Errorf("stderr = %q, want %q", stderr(), line)
	}
}

// TestScanWalksOperandInDefaultExclude checks that an operand at or
// inside a default exclude is walked, also when it lies under another
// operand, whose walk does not enter the default exclude.
func TestScanWalksOperandInDefaultExclude(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	excluded := filepath.Join(dir, "tmp")
	f := writeFile(t, excluded, "sub/f", pattern(1, 10))
	sub := filepath.Dir(f)

	cases := map[string][]string{
		"at it":                      {excluded},
		"inside it":                  {sub},
		"inside it under an operand": {dir, sub},
	}

	for name, roots := range cases {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			db := openTestDB(t)

			st, err := syncScan(t.Context(), db, roots, 1, false,
				walkRules{excludes: []string{excluded}})
			if err != nil {
				t.Fatal(err)
			}

			if st.added != 1 {
				t.Errorf("added = %d, want %s recorded once", st.added, f)
			}

			recordByPath(t, dbRecords(t, db), f)
		})
	}
}

// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default
// exclude given as an operand under another operand, which the walk of
// that other operand does not enter, gets no line saying it was
// skipped: it is walked as an operand of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
	stderr := captureStderr(t)
	dir := t.TempDir()
	db := openTestDB(t)
	excluded := filepath.Join(dir, "tmp")
	f := writeFile(t, excluded, "f", pattern(1, 10))

	st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
		walkRules{excludes: []string{excluded}})
	if err != nil {
		t.Fatal(err)
	}

	if st.added != 1 {
		t.Errorf("added = %d, want %s recorded once", st.added, f)
	}

	recordByPath(t, dbRecords(t, db), f)

	if strings.Contains(stderr(), "scan: skipped") {
		t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
	}
}

// TestRunScanTempDirOperand scans a tree in the tests' temporary
// directory, which on Linux lies inside the default exclude /tmp unless
// TMPDIR names another place, with the system's own default excludes:
// the scan walks the whole tree and reports no default exclude.
func TestRunScanTempDirOperand(t *testing.T) {
	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	dir := t.TempDir()
	if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
		t.Skip(dir + " is not inside a default exclude")
	}

	f := writeFile(t, dir, "sub/f", pattern(1, 10))

	stderr := scanOK(t, dir)
	if strings.Contains(stderr, "scan: skipped") {
		t.Errorf("stderr = %q, want no line saying a path was skipped",
			stderr)
	}

	assertRecorded(t, path, f, true)
}

// TestRunScanNoDefaultExcludes scans /var through the command. Its
// /var/tmp, on a persistent filesystem, is a default exclude on Linux:
// without --no-default-excludes the walk does not enter it and says so,
// and with that flag the walk records a file the test puts there.
func TestRunScanNoDefaultExcludes(t *testing.T) {
	if runtime.GOOS != goosLinux {
		t.Skip("only Linux has /var/tmp among its default excludes")
	}

	nonPersistent, err := nonPersistentFilesystems()
	if err != nil {
		t.Fatal(err)
	}

	rules := walkRules{nonPersistent: nonPersistent}

	for _, dir := range []string{"/var", "/var/tmp"} {
		fi, lstatErr := os.Lstat(dir)
		if lstatErr != nil || !fi.IsDir() ||
			rules.nonPersistentLine(dir, fi) != "" {
			t.Skip(dir + " is not a directory on a persistent filesystem here")
		}
	}

	//nolint:usetesting // the directory must be inside /var/tmp
	tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-")
	if err != nil {
		t.Skip("cannot write to /var/tmp here: " + err.Error())
	}

	t.Cleanup(func() { _ = os.RemoveAll(tmp) })

	f := writeFile(t, tmp, "f", pattern(1, 10))

	path := testDBPath(t)
	t.Setenv(databaseEnv, path)

	line := "scan: skipped /var/tmp (default exclude)\n"

	if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
		t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
	}

	assertRecorded(t, path, f, false)

	stderr := scanOK(t, "--no-default-excludes", "/var")
	if strings.Contains(stderr, line) {
		t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q",
			stderr, line)
	}

	assertRecorded(t, path, f, true)
}

// TestRunScanSkipsNonPersistentOperand scans, through the command, a
// directory on a filesystem that is not persistent, with and without
// --no-default-excludes: either way it is skipped, with a line saying
// so, and nothing in it is recorded.
func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
	shm, fsType, _ := shmTempDir(t)
	writeFile(t, shm, "f", pattern(1, 10))

	t.Setenv(databaseEnv, testDBPath(t))

	line := "scan: skipped " + shm + " (" + fsType +
		", not a persistent filesystem)\n"
	summary := "scan: 0 files seen"

	for _, flags := range [][]string{nil, {"--no-default-excludes"}} {
		stderr := scanOK(t, append(flags, shm)...)
		if !strings.Contains(stderr, line) ||
			!strings.Contains(stderr, summary) {
			t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr,
				line, summary)
		}
	}
}

// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
// that is not persistent beside one on the tests' temporary directory.
// The records an earlier scan left in it count as outside the operands:
// the record of a file gone from it is kept, and its size joins the size
// census, while a file added to it is not recorded, and its size does
// not join the census.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
	shm, fsType, rules := shmTempDir(t)
	stderr := captureStderr(t)
	dir := t.TempDir()
	db := openTestDB(t)

	// A scan that, without the list of filesystems, records gone.
	gone := writeFile(t, shm, "gone", pattern(1, 100))
	syncTree(t, db, shm)

	err := os.Remove(gone)
	if err != nil {
		t.Fatal(err)
	}

	unseen := writeFile(t, shm, "new", pattern(2, 10))

	// added has the size of gone, and lonely that of unseen: each is
	// hashed only if the census holds that size.
	added := writeFile(t, dir, "added", pattern(3, 100))
	lonely := writeFile(t, dir, "lonely", pattern(4, 10))

	st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
	if err != nil {
		t.Fatal(err)
	}

	if st.skipped != 0 {
		t.Errorf("skipped = %d, want 0", st.skipped)
	}

	recs := dbRecords(t, db)
	recordByPath(t, recs, gone)

	if recordByPath(t, recs, added).head == "" {
		t.Errorf("%s not hashed: the census lacks the size of %s",
			added, gone)
	}

	if recordByPath(t, recs, lonely).head != "" {
		t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen)
	}

	if slices.Contains(recordPaths(recs), unseen) {
		t.Errorf("%s recorded", unseen)
	}

	line := "scan: skipped " + shm + " (" + fsType +
		", not a persistent filesystem)\n"
	if !strings.Contains(stderr(), line) {
		t.Errorf("stderr = %q, want %q", stderr(), line)
	}
}

// TestWalkLeavesOutNonPersistent starts the walk itself in a directory
// on a filesystem that is not persistent, which a scan never does: it
// drops such an operand first. Below a directory on a persistent
// filesystem, one on another takes a mount, which a test cannot make.
// The walk enters no directory and records no file there, and names
// each.
func TestWalkLeavesOutNonPersistent(t *testing.T) {
	t.Parallel()

	shm, fsType, rules := shmTempDir(t)
	f := writeFile(t, shm, "f", pattern(1, 10))
	sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10)))

	var leftOut []string

	for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) {
		if ev.rec.path != "" {
			t.Errorf("walk recorded %s", ev.rec.path)
		}

		if ev.leftOut == "" {
			continue
		}

		leftOut = append(leftOut, ev.leftOut)

		want := "scan: skipped " + ev.leftOut + " (" + fsType +
			", not a persistent filesystem)"
		if ev.warn != want {
			t.Errorf("line = %q, want %q", ev.warn, want)
		}
	}

	slices.Sort(leftOut)

	if want := []string{f, sub}; !slices.Equal(leftOut, want) {
		t.Errorf("walk left out %q, want %q", leftOut, want)
	}
}

// shmTempDir returns a new directory in /dev/shm, removed when the test
// ends, the type of its filesystem, and the rules holding the real
// filesystems that are not persistent. It skips the test where /dev/shm
// is on a persistent filesystem, or cannot be written to.
func shmTempDir(t *testing.T) (string, string, walkRules) {
	t.Helper()

	nonPersistent, err := nonPersistentFilesystems()
	if err != nil {
		t.Fatal(err)
	}

	dev, ok := deviceOfPath("/dev/shm")

	fsType, found := nonPersistent[dev]
	if !ok || !found {
		t.Skip("/dev/shm is not a memory-backed filesystem here")
	}

	//nolint:usetesting // the directory must be inside /dev/shm
	dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-")
	if err != nil {
		t.Skip("cannot write to /dev/shm here: " + err.Error())
	}

	t.Cleanup(func() { _ = os.RemoveAll(dir) })

	return dir, fsType, walkRules{nonPersistent: nonPersistent}
}

// assertRecorded checks that the database at dbPath holds a record for
// path when want is set, and none when it is not.
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
	t.Helper()

	db, err := openDB(dbPath, reportParams)
	if err != nil {
		t.Fatal(err)
	}

	defer func() { _ = db.Close() }()

	if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want {
		t.Errorf("%s recorded: %v, want %v", path, got, want)
	}
}