package main import ( "bytes" "errors" "maps" "os" "path/filepath" "runtime" "slices" "strings" "testing" ) func TestDefaultExcludes(t *testing.T) { t.Parallel() // README "Rules for the walk": each system's list. cases := map[string][]string{ goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, goosDarwin: { "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", "/System/Volumes/VM", "/System/Volumes/Data", }, goosWindows: { "pagefile.sys", "swapfile.sys", "hiberfil.sys", "System Volume Information", "$Recycle.Bin", `Windows\Temp`, `Users\*\AppData\Local\Temp`, }, "freebsd": nil, } for goos, want := range cases { if got := defaultExcludes(goos); !slices.Equal(got, want) { t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want) } } } func TestIsDefaultExclude(t *testing.T) { t.Parallel() // A * matches one path component, no more and no fewer. excludes := []string{"/scratch", "/home/*/cache"} cases := map[string]bool{ "/scratch": true, "/scratch/x": false, "/home/u/cache": true, "/home/u/v/cache": false, "/home/cache": false, } for path, want := range cases { if got := isDefaultExclude(path, excludes); got != want { t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want) } } } func TestReadMountInfo(t *testing.T) { t.Parallel() // The second-to-last line has no optional fields. /mnt/over has an // ext4 filesystem mounted over its tmpfs. mountinfo := strings.Join([]string{ "22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw", "23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw", "24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw", `25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`, "26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw", "27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw", "28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw", }, "\n") got, err := readMountInfo(strings.NewReader(mountinfo)) if err != nil { t.Fatal(err) } want := map[string]string{ "/": "ext4", "/proc": "proc", "/sys": "sysfs", "/mnt/ram disk": "tmpfs", "/mnt/over": "ext4", "/srv/auto": "autofs", } if !maps.Equal(got, want) { t.Errorf("readMountInfo = %q, want %q", got, want) } _, err = readMountInfo(strings.NewReader("22 1 8:1 / / rw,relatime\n")) if !errors.Is(err, errMountInfoLine) { t.Errorf("readMountInfo of a line without a type = %v, want %v", err, errMountInfoLine) } } func TestNonPersistentFilesystems(t *testing.T) { t.Parallel() if runtime.GOOS != goosLinux { t.Skip("the test knows a mount only on Linux: /proc") } nonPersistent, err := nonPersistentFilesystems() if err != nil { t.Fatal(err) } dev, ok := deviceOfPath("/proc") if !ok { t.Fatal("no device for /proc") } if got := nonPersistent[dev]; got != "proc" { t.Errorf("filesystem of /proc = %q, want %q", got, "proc") } } func TestDefaultExcludesHelp(t *testing.T) { t.Parallel() for _, goos := range []string{goosLinux, goosDarwin, goosWindows} { help := defaultExcludesHelp(goos) for _, dir := range defaultExcludes(goos) { if !strings.Contains(help, "\n "+dir+"\n") { t.Errorf("help for %s does not list %s:\n%s", goos, dir, help) } } // Only Linux and macOS list their mounted filesystems. for _, fsType := range nonPersistentFilesystemTypes() { listed := strings.Contains(help, " "+fsType+",") || strings.Contains(help, " "+fsType+"\n") if listed != (goos != goosWindows) { t.Errorf("help for %s lists %s: %v", goos, fsType, listed) } } } want := "This system has no default excludes.\n" if got := defaultExcludesHelp("freebsd"); got != want { t.Errorf("help for freebsd = %q, want %q", got, want) } // scan's help shows those of the system it runs on. var stdout, stderr bytes.Buffer run([]string{cmdScan, "-h"}, &stdout, &stderr) help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS)) if !strings.Contains(stderr.String(), help) { t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help) } } // TestScanDoesNotEnterDefaultExclude checks that the walk does not // enter a default exclude below an operand. It says so on a line of its // own and does not count it as skipped. The records below it count as // outside the operands: the record of a file gone from it is kept, and // its size joins the size census, while a file added to it is not // recorded. // //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr func TestScanDoesNotEnterDefaultExclude(t *testing.T) { stderr := captureStderr(t) dir := t.TempDir() db := openTestDB(t) excluded := filepath.Join(dir, "tmp") // A scan with no default excludes records the file in excluded. gone := writeFile(t, excluded, "gone", pattern(1, 100)) syncTree(t, db, dir) err := os.Remove(gone) if err != nil { t.Fatal(err) } unseen := writeFile(t, excluded, "new", pattern(2, 10)) // The same size as gone, so it is hashed only if the census holds the // size of gone's record. added := writeFile(t, dir, "added", pattern(3, 100)) st, err := syncScan(t.Context(), db, []string{dir}, 1, false, walkRules{excludes: []string{excluded}}) if err != nil { t.Fatal(err) } if st.skipped != 0 { t.Errorf("skipped = %d, want 0", st.skipped) } recs := dbRecords(t, db) recordByPath(t, recs, gone) if recordByPath(t, recs, added).head == "" { t.Errorf("%s not hashed: the census lacks the size of %s", added, gone) } if slices.Contains(recordPaths(recs), unseen) { t.Errorf("%s recorded: the walk entered %s", unseen, excluded) } line := "\nscan: skipped " + excluded + " (default exclude)\n" if !strings.Contains(stderr(), line) { t.Errorf("stderr = %q, want %q", stderr(), line) } } // TestScanWalksOperandInDefaultExclude checks that an operand at or // inside a default exclude is walked, also when it lies under another // operand, whose walk does not enter the default exclude. func TestScanWalksOperandInDefaultExclude(t *testing.T) { t.Parallel() dir := t.TempDir() excluded := filepath.Join(dir, "tmp") f := writeFile(t, excluded, "sub/f", pattern(1, 10)) sub := filepath.Dir(f) cases := map[string][]string{ "at it": {excluded}, "inside it": {sub}, "inside it under an operand": {dir, sub}, } for name, roots := range cases { t.Run(name, func(t *testing.T) { t.Parallel() db := openTestDB(t) st, err := syncScan(t.Context(), db, roots, 1, false, walkRules{excludes: []string{excluded}}) if err != nil { t.Fatal(err) } if st.added != 1 { t.Errorf("added = %d, want %s recorded once", st.added, f) } recordByPath(t, dbRecords(t, db), f) }) } } // TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default // exclude given as an operand under another operand, which the walk of // that other operand does not enter, gets no line saying it was // skipped: it is walked as an operand of its own. // //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) { stderr := captureStderr(t) dir := t.TempDir() db := openTestDB(t) excluded := filepath.Join(dir, "tmp") f := writeFile(t, excluded, "f", pattern(1, 10)) st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false, walkRules{excludes: []string{excluded}}) if err != nil { t.Fatal(err) } if st.added != 1 { t.Errorf("added = %d, want %s recorded once", st.added, f) } recordByPath(t, dbRecords(t, db), f) if strings.Contains(stderr(), "scan: skipped") { t.Errorf("stderr = %q, want no line for %s", stderr(), excluded) } } // TestRunScanTempDirOperand scans a tree in the tests' temporary // directory, which on Linux lies inside the default exclude /tmp unless // TMPDIR names another place, with the system's own default excludes: // the scan walks the whole tree and reports no default exclude. func TestRunScanTempDirOperand(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) dir := t.TempDir() if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { t.Skip(dir + " is not inside a default exclude") } f := writeFile(t, dir, "sub/f", pattern(1, 10)) stderr := scanOK(t, dir) if strings.Contains(stderr, "scan: skipped") { t.Errorf("stderr = %q, want no line saying a path was skipped", stderr) } assertRecorded(t, path, f, true) } // TestRunScanNoDefaultExcludes scans /var through the command. Its // /var/tmp, on a persistent filesystem, is a default exclude on Linux: // without --no-default-excludes the walk does not enter it and says so, // and with that flag the walk records a file the test puts there. func TestRunScanNoDefaultExcludes(t *testing.T) { if runtime.GOOS != goosLinux { t.Skip("only Linux has /var/tmp among its default excludes") } nonPersistent, err := nonPersistentFilesystems() if err != nil { t.Fatal(err) } rules := walkRules{nonPersistent: nonPersistent} for _, dir := range []string{"/var", "/var/tmp"} { fi, lstatErr := os.Lstat(dir) if lstatErr != nil || !fi.IsDir() || rules.nonPersistentLine(dir, fi) != "" { t.Skip(dir + " is not a directory on a persistent filesystem here") } } //nolint:usetesting // the directory must be inside /var/tmp tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-") if err != nil { t.Skip("cannot write to /var/tmp here: " + err.Error()) } t.Cleanup(func() { _ = os.RemoveAll(tmp) }) f := writeFile(t, tmp, "f", pattern(1, 10)) path := testDBPath(t) t.Setenv(databaseEnv, path) line := "scan: skipped /var/tmp (default exclude)\n" if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) { t.Errorf("scan /var: stderr = %q, want %q", stderr, line) } assertRecorded(t, path, f, false) stderr := scanOK(t, "--no-default-excludes", "/var") if strings.Contains(stderr, line) { t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q", stderr, line) } assertRecorded(t, path, f, true) } // TestRunScanSkipsNonPersistentOperand scans, through the command, a // directory on a filesystem that is not persistent, with and without // --no-default-excludes: either way it is skipped, with a line saying // so, and nothing in it is recorded. func TestRunScanSkipsNonPersistentOperand(t *testing.T) { shm, fsType, _ := shmTempDir(t) writeFile(t, shm, "f", pattern(1, 10)) t.Setenv(databaseEnv, testDBPath(t)) line := "scan: skipped " + shm + " (" + fsType + ", not a persistent filesystem)\n" summary := "scan: 0 files seen" for _, flags := range [][]string{nil, {"--no-default-excludes"}} { stderr := scanOK(t, append(flags, shm)...) if !strings.Contains(stderr, line) || !strings.Contains(stderr, summary) { t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr, line, summary) } } } // TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem // that is not persistent beside one on the tests' temporary directory. // The records an earlier scan left in it count as outside the operands: // the record of a file gone from it is kept, and its size joins the size // census, while a file added to it is not recorded, and its size does // not join the census. // //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr func TestScanKeepsRecordsOnNonPersistent(t *testing.T) { shm, fsType, rules := shmTempDir(t) stderr := captureStderr(t) dir := t.TempDir() db := openTestDB(t) // A scan that, without the list of filesystems, records gone. gone := writeFile(t, shm, "gone", pattern(1, 100)) syncTree(t, db, shm) err := os.Remove(gone) if err != nil { t.Fatal(err) } unseen := writeFile(t, shm, "new", pattern(2, 10)) // added has the size of gone, and lonely that of unseen: each is // hashed only if the census holds that size. added := writeFile(t, dir, "added", pattern(3, 100)) lonely := writeFile(t, dir, "lonely", pattern(4, 10)) st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules) if err != nil { t.Fatal(err) } if st.skipped != 0 { t.Errorf("skipped = %d, want 0", st.skipped) } recs := dbRecords(t, db) recordByPath(t, recs, gone) if recordByPath(t, recs, added).head == "" { t.Errorf("%s not hashed: the census lacks the size of %s", added, gone) } if recordByPath(t, recs, lonely).head != "" { t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen) } if slices.Contains(recordPaths(recs), unseen) { t.Errorf("%s recorded", unseen) } line := "scan: skipped " + shm + " (" + fsType + ", not a persistent filesystem)\n" if !strings.Contains(stderr(), line) { t.Errorf("stderr = %q, want %q", stderr(), line) } } // TestWalkLeavesOutNonPersistent starts the walk itself in a directory // on a filesystem that is not persistent, which a scan never does: it // drops such an operand first. Below a directory on a persistent // filesystem, one on another takes a mount, which a test cannot make. // The walk enters no directory and records no file there, and names // each. func TestWalkLeavesOutNonPersistent(t *testing.T) { t.Parallel() shm, fsType, rules := shmTempDir(t) f := writeFile(t, shm, "f", pattern(1, 10)) sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10))) var leftOut []string for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) { if ev.rec.path != "" { t.Errorf("walk recorded %s", ev.rec.path) } if ev.leftOut == "" { continue } leftOut = append(leftOut, ev.leftOut) want := "scan: skipped " + ev.leftOut + " (" + fsType + ", not a persistent filesystem)" if ev.warn != want { t.Errorf("line = %q, want %q", ev.warn, want) } } slices.Sort(leftOut) if want := []string{f, sub}; !slices.Equal(leftOut, want) { t.Errorf("walk left out %q, want %q", leftOut, want) } } // shmTempDir returns a new directory in /dev/shm, removed when the test // ends, the type of its filesystem, and the rules holding the real // filesystems that are not persistent. It skips the test where /dev/shm // is on a persistent filesystem, or cannot be written to. func shmTempDir(t *testing.T) (string, string, walkRules) { t.Helper() nonPersistent, err := nonPersistentFilesystems() if err != nil { t.Fatal(err) } dev, ok := deviceOfPath("/dev/shm") fsType, found := nonPersistent[dev] if !ok || !found { t.Skip("/dev/shm is not a memory-backed filesystem here") } //nolint:usetesting // the directory must be inside /dev/shm dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-") if err != nil { t.Skip("cannot write to /dev/shm here: " + err.Error()) } t.Cleanup(func() { _ = os.RemoveAll(dir) }) return dir, fsType, walkRules{nonPersistent: nonPersistent} } // assertRecorded checks that the database at dbPath holds a record for // path when want is set, and none when it is not. func assertRecorded(t *testing.T, dbPath, path string, want bool) { t.Helper() db, err := openDB(dbPath, reportParams) if err != nil { t.Fatal(err) } defer func() { _ = db.Close() }() if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want { t.Errorf("%s recorded: %v, want %v", path, got, want) } }