scan: skip OS-appropriate default excludes (/proc, /sys, /tmp, /run, /var/tmp on Linux) (closes #124)

Below an operand, the walk no longer enters the default excludes: on
Linux /proc, /sys, /dev, /run, /tmp, /var/tmp and every mount point of a
kernel or memory-backed filesystem, read once from /proc/self/mountinfo;
on macOS /dev, /private/tmp, /private/var/tmp, /private/var/vm,
/System/Volumes/VM and /System/Volumes/Data. The lists live in the new
exclude.go. An operand at or inside one is still walked, also under
another operand, as with -x. Each skipped directory gets one stderr line
through the warning path, is not counted as skipped, and its records
count as outside the scanned operands. --no-default-excludes turns them
off; scan --help lists them.

Judgement call: an unreadable /proc/self/mountinfo fails the scan.
Judgement call: where filesystems are mounted over one another, the last decides.

Model: opus-5-5

exclude_test.go

@@ -0,0 +1,266 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// defaultExcludeWhy is what the scan says about a default exclude from
+// the fixed list when its walk does not enter it.
+const defaultExcludeWhy = "default exclude"
+
+func TestDefaultExcludes(t *testing.T) {
+	t.Parallel()
+
+	// README "Rules for the walk": each system's list.
+	cases := map[string][]string{
+		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
+		goosDarwin: {
+			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
+			"/System/Volumes/VM", "/System/Volumes/Data",
+		},
+		"freebsd": nil,
+	}
+
+	for goos, want := range cases {
+		if got := defaultExcludes(goos); !slices.Equal(got, want) {
+			t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
+		}
+	}
+}
+
+func TestKernelMounts(t *testing.T) {
+	t.Parallel()
+
+	// The second-to-last line has no optional fields. /mnt/over has an
+	// ext4 filesystem mounted over its tmpfs, so it holds files.
+	mountinfo := strings.Join([]string{
+		"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
+		"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
+		"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
+		`25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`,
+		"26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw",
+		"27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw",
+		"28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
+	}, "\n")
+
+	got, err := kernelMounts(strings.NewReader(mountinfo))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := map[string]string{
+		"/proc": "proc", "/sys": "sysfs", "/mnt/ram disk": "tmpfs",
+		"/srv/auto": "autofs",
+	}
+	if !maps.Equal(got, want) {
+		t.Errorf("kernelMounts = %q, want %q", got, want)
+	}
+
+	_, err = kernelMounts(strings.NewReader("22 1 8:1 / / rw,relatime\n"))
+	if !errors.Is(err, errMountInfoLine) {
+		t.Errorf("kernelMounts of a line without a type = %v, want %v",
+			err, errMountInfoLine)
+	}
+}
+
+func TestFindDefaultExcludesReadsMounts(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS != goosLinux {
+		t.Skip("only Linux lists its mounts in " + mountInfoPath)
+	}
+
+	excludes, err := findDefaultExcludes(goosLinux)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, dir := range defaultExcludes(goosLinux) {
+		if _, ok := excludes[dir]; !ok {
+			t.Errorf("%s is not a default exclude", dir)
+		}
+	}
+
+	want := defaultExcludeWhy + ", proc filesystem"
+	if got := excludes["/proc"]; got != want {
+		t.Errorf("/proc is %q, want %q", got, want)
+	}
+}
+
+func TestDefaultExcludesHelp(t *testing.T) {
+	t.Parallel()
+
+	for _, goos := range []string{goosLinux, goosDarwin} {
+		help := defaultExcludesHelp(goos)
+
+		for _, dir := range defaultExcludes(goos) {
+			if !strings.Contains(help, "\n  "+dir+"\n") {
+				t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
+			}
+		}
+
+		// Only Linux has filesystem types among its default excludes.
+		for _, fsType := range kernelFilesystemTypes() {
+			listed := strings.Contains(help, " "+fsType+",") ||
+				strings.Contains(help, " "+fsType+"\n")
+			if listed != (goos == goosLinux) {
+				t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
+			}
+		}
+	}
+
+	want := "This system has no default excludes."
+	if got := defaultExcludesHelp("freebsd"); got != want {
+		t.Errorf("help for freebsd = %q, want %q", got, want)
+	}
+
+	// scan's help shows those of the system it runs on.
+	var stdout, stderr bytes.Buffer
+
+	run([]string{cmdScan, "-h"}, &stdout, &stderr)
+
+	help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
+	if !strings.Contains(stderr.String(), help) {
+		t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
+	}
+}
+
+// TestScanDoesNotEnterDefaultExclude checks that the walk does not
+// enter a default exclude below an operand. It says so on a line of its
+// own and does not count it as skipped. The records below it count as
+// outside the operands: the record of a file gone from it is kept, and
+// its size joins the size census, while a file added to it is not
+// recorded.
+//
+//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
+func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
+	stderr := captureStderr(t)
+	dir := t.TempDir()
+	db := openTestDB(t)
+	excluded := filepath.Join(dir, "tmp")
+
+	// A scan with no default excludes records the file in excluded.
+	gone := writeFile(t, excluded, "gone", pattern(1, 100))
+	syncTree(t, db, dir)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unseen := writeFile(t, excluded, "new", pattern(2, 10))
+
+	// The same size as gone, so it is hashed only if the census holds the
+	// size of gone's record.
+	added := writeFile(t, dir, "added", pattern(3, 100))
+
+	st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
+		map[string]string{excluded: defaultExcludeWhy})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if st.skipped != 0 {
+		t.Errorf("skipped = %d, want 0", st.skipped)
+	}
+
+	recs := dbRecords(t, db)
+	recordByPath(t, recs, gone)
+
+	if recordByPath(t, recs, added).head == "" {
+		t.Errorf("%s not hashed: the census lacks the size of %s",
+			added, gone)
+	}
+
+	if slices.Contains(recordPaths(recs), unseen) {
+		t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
+	}
+
+	line := "\nscan: skipped " + excluded + " (" + defaultExcludeWhy + ")\n"
+	if !strings.Contains(stderr(), line) {
+		t.Errorf("stderr = %q, want %q", stderr(), line)
+	}
+}
+
+// TestScanWalksOperandInDefaultExclude checks that an operand at or
+// inside a default exclude is walked, also when it lies under another
+// operand, whose walk does not enter the default exclude.
+func TestScanWalksOperandInDefaultExclude(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	excluded := filepath.Join(dir, "tmp")
+	f := writeFile(t, excluded, "sub/f", pattern(1, 10))
+	sub := filepath.Dir(f)
+
+	cases := map[string][]string{
+		"at it":                      {excluded},
+		"inside it":                  {sub},
+		"inside it under an operand": {dir, sub},
+	}
+
+	for name, roots := range cases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			db := openTestDB(t)
+
+			st, err := syncScan(t.Context(), db, roots, 1, false,
+				map[string]string{excluded: defaultExcludeWhy})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if st.added != 1 {
+				t.Errorf("added = %d, want %s recorded once", st.added, f)
+			}
+
+			recordByPath(t, dbRecords(t, db), f)
+		})
+	}
+}
+
+// TestRunScanTempDirOperand scans a tree in the tests' temporary
+// directory, which on Linux lies inside the default exclude /tmp unless
+// TMPDIR names another place, with the system's own default excludes and
+// then with --no-default-excludes: each scan walks the whole tree and
+// reports no default exclude.
+func TestRunScanTempDirOperand(t *testing.T) {
+	path := testDBPath(t)
+	t.Setenv(databaseEnv, path)
+
+	dir := t.TempDir()
+	if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
+		t.Skip(dir + " is not inside a default exclude")
+	}
+
+	for i, flags := range [][]string{nil, {"--no-default-excludes"}} {
+		// A file added before each scan shows that the scan walked the
+		// tree.
+		f := writeFile(t, dir, fmt.Sprintf("sub/%d", i), pattern(1, 10))
+
+		stderr := scanOK(t, append(flags, dir)...)
+		if strings.Contains(stderr, defaultExcludeWhy) {
+			t.Errorf("scan %q: stderr = %q, want no default exclude",
+				flags, stderr)
+		}
+
+		db, err := openDB(path, reportParams)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		recordByPath(t, dbRecords(t, db), f)
+
+		_ = db.Close()
+	}
+}