sneak/sfdupes
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions 10 Packages Projects Releases Wiki Activity

scan: skip OS default excludes, and record nothing on filesystems that are not persistent (closes #124)
check / check (push) Waiting to run
Details

Browse Source 
scan never records or reads anything on a filesystem that is not
persistent (tmpfs, proc, sysfs and the like), however it reaches it; no
flag turns this off. It notes each such filesystem's device ID as it
starts: on Linux from /proc/self/mountinfo, on macOS by lstat of each
listed mount point. Below an operand the walk also skips the default
excludes, a path list per system (Linux, macOS, and Windows, not used
yet); --no-default-excludes turns off only those. A default exclude that
is itself an operand gets no skip line. The tests stop at once, saying
why, when TMPDIR is not on a persistent filesystem.

Judgement call: an unreadable mount list fails the scan, and no flag gets round it.
Judgement call: a file on such a filesystem below an operand gets the same line as a directory.
Judgement call: such an operand that another operand's walk also reaches is named twice.
Unverified: whether macOS's mount list gives the device lstat reports, so macOS keeps lstat, and a hidden mount there can still add another filesystem's device.
Unverified: the macOS code is never built or run here; bind mounts need root to test.

Model: opus-5-5
This commit is contained in:
sneak
2026-10-17 01:11:21 +00:00
parent 68f6bde5ee
commit 08323879ec
11 changed files with 1294 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scan.go
+210 -109
View File
@@ -13,6 +13,7 @@ import (
"os"
"os/signal"
"path/filepath"
"runtime"
"slices"
"strings"
"sync"
@@ -114,9 +115,12 @@ func scanPhase(name string) string {
// SIGTERM that interruptContext catches, the scan keeps what it has
// hashed (see syncScan), prints how many files its walk reached, and
// returns errInterrupted. workers must be at least 1; the scan command
// rejects anything less.
// rejects anything less. Before the lock is taken, the scan lists the
// filesystems that are not persistent, on which the walk records
// nothing; unless noDefaultExcludes is set, the walk does not enter the
// default excludes either.
func runScan(ctx context.Context, roots []string, workers int,
oneFS bool,
oneFS, noDefaultExcludes bool,
) error {
dbPath, source, err := databasePath()
if err != nil {
@@ -139,6 +143,16 @@ func runScan(ctx context.Context, roots []string, workers int,
return err
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
return fmt.Errorf("list the mounted filesystems: %w", err)
}
rules := walkRules{nonPersistent: nonPersistent}
if !noDefaultExcludes {
rules.excludes = defaultExcludes(runtime.GOOS)
}
lock, err := lockScanDatabase(dbPath)
if err != nil {
return err
@@ -159,7 +173,7 @@ func runScan(ctx context.Context, roots []string, workers int,
defer closeScanDatabase(ctx, db, dbPath)
st, err := syncScan(ctx, db, roots, workers, oneFS)
st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
if errors.Is(err, context.Canceled) {
return interrupted(st.walked)
}
@@ -231,14 +245,12 @@ func resolveRoots(roots []string) ([]string, error) {
}
// pruneRoots drops operands already covered by another operand:
// duplicates and any operand lying under another one. With -x, an
// operand under another one is kept when the walk of the deepest kept
// operand above it cannot reach it: when device puts any directory from
// it (from its directory, for a regular file) up to but not including
// that operand on a different filesystem than that operand. Every
// remaining file is then reachable through exactly one root, so walked
// paths are unique without keeping a scan-wide set of every path seen.
func pruneRoots(roots []string, oneFS bool,
// duplicates and any operand lying under another one. An operand under
// another one is kept, though, when the walk of the deepest kept operand
// above it does not reach it (see walkReaches). Every remaining file is
// then reachable through exactly one root, so walked paths are unique
// without keeping a scan-wide set of every path seen.
func pruneRoots(roots []string, oneFS bool, rules walkRules,
device func(path string) (uint64, bool),
) []string {
// Shorter-first ordering guarantees an ancestor is kept before any
@@ -262,40 +274,57 @@ func pruneRoots(roots []string, oneFS bool,
continue
}
// Without -x, and always for a duplicate, r is dropped.
if !oneFS || above == r {
continue
}
aboveDev, aboveOK := device(above)
if !aboveOK {
continue
}
// The walk of above enters a directory only when it is on
// above's filesystem, and then reaches every regular file in it
// whatever device the file reports, so for a regular-file
// operand the directories checked start at its directory.
dir := r
fi, err := os.Lstat(r)
if err == nil && fi.Mode().IsRegular() {
dir = filepath.Dir(r)
}
for ; dir != above; dir = filepath.Dir(dir) {
dev, ok := device(dir)
if ok && dev != aboveDev {
kept = append(kept, r)
break
}
// A duplicate is dropped, and so is an operand the walk of above
// reaches.
if above != r && !walkReaches(above, r, oneFS, rules, device) {
kept = append(kept, r)
}
}
return kept
}
// walkReaches reports whether the walk of operand above reaches r, an
// operand under it. That walk does not enter a default exclude in
// rules, nor a directory that device puts on a filesystem in rules that
// is not persistent, nor, with -x, one that device puts on a different
// filesystem than above, when device knows above's. It reaches every
// regular file in a directory it enters, so for a regular-file operand
// the directories checked start at its directory; they end below above.
func walkReaches(above, r string, oneFS bool, rules walkRules,
device func(path string) (uint64, bool),
) bool {
aboveDev, aboveDevOK := device(above)
dir := r
fi, err := os.Lstat(r)
if err == nil && fi.Mode().IsRegular() {
dir = filepath.Dir(r)
}
for ; dir != above; dir = filepath.Dir(dir) {
if isDefaultExclude(dir, rules.excludes) {
return false
}
dev, ok := device(dir)
if !ok {
continue
}
if _, found := rules.nonPersistent[dev]; found {
return false
}
if oneFS && aboveDevOK && dev != aboveDev {
return false
}
}
return true
}
// deviceOfPath returns the filesystem device ID of the file at path,
// when lstat succeeds and the platform exposes one.
func deviceOfPath(path string) (uint64, bool) {
@@ -337,11 +366,11 @@ type scanState struct {
// the hashed records still waiting in the batch, starts no other write
// or deletion, and returns the cancellation.
func syncScan(ctx context.Context, db *sql.DB, roots []string,
workers int, oneFS bool,
workers int, oneFS bool, rules walkRules,
) (scanStats, error) {
s := &scanState{db: db}
err := s.runPhases(ctx, roots, workers, oneFS)
err := s.runPhases(ctx, roots, workers, oneFS, rules)
if err == nil || ctx.Err() == nil {
return s.st, err
}
@@ -365,15 +394,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
// and content (fill in the content hash of every record of headTailMin
// or more whose size, head, and tail match another record's). Records
// outside the roots are never touched, except that the content phase
// fills in their content hash. Operands the walk cannot start from are
// dropped first, so the records beneath them are treated as if those
// operands had not been given.
// fills in their content hash. Operands the walk cannot start from, or
// that are not on a persistent filesystem, are dropped first, so the
// records beneath them are treated as if those operands had not been
// given. Below an operand, the walk follows rules.
func (s *scanState) runPhases(ctx context.Context, roots []string,
workers int, oneFS bool,
workers int, oneFS bool, rules walkRules,
) error {
// Types are checked before pruning so that an operand under a
// dropped one is still scanned, not dropped as lying under it.
roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
// Types and filesystems are checked before pruning so that an
// operand under a dropped one is still scanned, not dropped as lying
// under it.
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
deviceOfPath)
err := s.loadIndex(ctx, roots)
if err != nil {
@@ -381,7 +413,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
}
changed, unhashed := s.walkPhase(
startWalk(ctx, roots, oneFS, workers), roots)
startWalk(ctx, roots, oneFS, rules, workers), roots)
// A cancelled walk stops early, so its size census covers only part
// of the roots, and every file it never reached would look vanished
@@ -403,7 +435,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
return err
}
return s.contentPhase(ctx, workers)
return s.contentPhase(ctx, workers, rules)
}
// walkableRoots returns the operands the walk can start from: regular
@@ -411,9 +443,11 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
// about, counted as skipped, and dropped. A dropped operand is no
// longer a root, so the records stored beneath it are treated as if it
// had not been given: those under no other root count as outside the
// roots and are not deleted as unverified. An operand that fails lstat
// here is kept, and the walk warns about it.
func (s *scanState) walkableRoots(roots []string) []string {
// roots and are not deleted as unverified. An operand on a filesystem
// in rules that is not persistent is dropped the same way, with a line
// saying so that is not a warning, and is not counted as skipped. An
// operand that fails lstat here is kept, and the walk warns about it.
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
kept := make([]string, 0, len(roots))
for _, root := range roots {
@@ -427,6 +461,13 @@ func (s *scanState) walkableRoots(roots []string) []string {
continue
}
line := rules.nonPersistentLine(root, fi)
if line != "" {
fmt.Fprintln(os.Stderr, escapePath(line))
continue
}
}
kept = append(kept, root)
@@ -481,10 +522,12 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// (the content phase may still fill in its content hash). It returns
// the new-or-changed files and the unchanged files whose records lack
// hashes; both remain candidates until the census decides whether
// their sizes are shared. A record below a directory -x did not enter
// counts as outside the roots: it is not deleted, and its size joins
// the census. That does not hold below a root at or inside that
// directory, since that root's own walk verifies the records below it.
// their sizes are shared. A record at or below a path the walk left
// out, because -x stopped at it, it is a default exclude, or it is not
// on a persistent filesystem, counts as outside the roots: it is not
// deleted, and its size joins the census. That does not hold below a
// root at or inside that path, since that root's own walk verifies the
// records below it; nor is a line printed for such a root.
func (s *scanState) walkPhase(
events <-chan walkEvent, roots []string,
) ([]fileRec, []fileRec) {
@@ -502,12 +545,25 @@ func (s *scanState) walkPhase(
var (
changed, unhashed []fileRec
otherFSDirs []string
leftOut []string
)
for ev := range events {
if ev.otherFSDir != "" {
otherFSDirs = append(otherFSDirs, ev.otherFSDir)
leftOut = append(leftOut, ev.otherFSDir)
continue
}
// The line saying so is printed as a warning is, but what the
// walk leaves out is not counted as skipped. A default exclude
// that is itself a root is walked as one, so it gets no line.
if ev.leftOut != "" {
leftOut = append(leftOut, ev.leftOut)
if !slices.Contains(roots, ev.leftOut) {
prog.warnf("%s", ev.warn)
}
continue
}
@@ -542,12 +598,13 @@ func (s *scanState) walkPhase(
prog.finish()
for path, old := range s.existing {
// Kept only when a directory -x did not enter lies below the
// deepest root above the record. A root at or below that
// directory was walked on its own and did not find the file.
// Both contain the record, so the longer path is the deeper.
otherFSDir := deepestRoot(path, otherFSDirs)
if len(otherFSDir) > len(deepestRoot(path, roots)) {
// Kept only when a path the walk left out lies below the deepest
// root above the record, or is the record's own. A root at or
// below that path was walked on its own and did not find the
// file. Both contain the record, so the longer path is the
// deeper.
left := deepestRoot(path, leftOut)
if len(left) > len(deepestRoot(path, roots)) {
delete(s.existing, path)
s.sizes = append(s.sizes, old.size)
@@ -821,9 +878,12 @@ func (s *scanState) updatePhase(ctx context.Context) error {
// hash phase's worker pool and their records written back in batches. A
// failed read is warned about and counted as skipped; the record keeps
// its empty content, so it is never grouped, and a later scan tries
// again.
func (s *scanState) contentPhase(ctx context.Context, workers int) error {
toRead, recs, err := s.contentCandidates(ctx)
// again. No file on a filesystem in rules that is not persistent is
// read.
func (s *scanState) contentPhase(ctx context.Context, workers int,
rules walkRules,
) error {
toRead, recs, err := s.contentCandidates(ctx, rules)
if err != nil {
return err
}
@@ -858,18 +918,19 @@ func (s *scanState) contentPhase(ctx context.Context, workers int) error {
// contentCandidates returns the files the content phase reads, and
// their records by path. Every record contentCandidatesSQL returns has
// its file checked with lstat, whether or not it already has a content
// hash: a file that is gone, is no longer a regular file, or has
// changed by the walk's rule keeps its record as it is and does not
// count as a match for the others, and any other lstat error is warned
// about and counted as skipped, with the same result. If such a record
// has no content hash, it stays out of duplicate groups; if it has one,
// it is still reported until a scan covering its own tree updates or
// removes it. The files of a group that pass and have no content hash
// are read only if at least minGroupSize of the group's files pass, so
// a group whose other members are all stale costs no reads. Only the
// records to be read are kept.
// hash: a file that is gone, is no longer a regular file, has changed
// by the walk's rule, or is on a filesystem in rules that is not
// persistent keeps its record as it is and does not count as a match
// for the others, and any other lstat error is warned about and counted
// as skipped, with the same result. If such a record has no content
// hash, it stays out of duplicate groups; if it has one, it is still
// reported until a scan covering its own tree updates or removes it.
// The files of a group that pass and have no content hash are read only
// if at least minGroupSize of the group's files pass, so a group whose
// other members are all stale costs no reads. Only the records to be
// read are kept.
func (s *scanState) contentCandidates(
ctx context.Context,
ctx context.Context, rules walkRules,
) ([]fileRec, map[string]scanRec, error) {
// The search and the checks take real time on a large database;
// without a display the scan looks hung before the reads begin.
@@ -927,7 +988,7 @@ func (s *scanState) contentCandidates(
first = r
}
f, ok, err := unchangedFile(r)
f, ok, err := unchangedFile(r, rules)
if err != nil {
s.st.skipped++
@@ -961,9 +1022,10 @@ func (s *scanState) contentCandidates(
// unchangedFile lstats the file r names and returns it for reading if
// it is still the regular file r records: the same size, and an mtime
// no newer than recorded (the walk's change rule). A file that is gone
// or has changed reports false; any other lstat error is returned.
func unchangedFile(r scanRec) (fileRec, bool, error) {
// no newer than recorded (the walk's change rule). A file that is gone,
// has changed, or is on a filesystem in rules that is not persistent
// reports false; any other lstat error is returned.
func unchangedFile(r scanRec, rules walkRules) (fileRec, bool, error) {
fi, err := os.Lstat(r.path)
if errors.Is(err, fs.ErrNotExist) {
return fileRec{}, false, nil
@@ -980,6 +1042,12 @@ func unchangedFile(r scanRec) (fileRec, bool, error) {
dev, ino := inodeOfInfo(fi)
// The scan reads nothing on a filesystem that is not persistent,
// whatever record an earlier scan left for the file.
if _, found := rules.nonPersistent[dev]; found {
return fileRec{}, false, nil
}
return fileRec{
path: r.path, size: r.size, mtime: r.mtime, dev: dev, ino: ino,
}, true, nil
@@ -1046,27 +1114,31 @@ type dirJob struct {
}
// walkEvent is one walk result delivered to the walk phase: a regular
// file's statted record, a warning when fail is set, or, when
// otherFSDir is set, a directory -x did not enter because it is on
// another filesystem than its operand.
// file's statted record, a warning when fail is set, or a path the walk
// left out: when otherFSDir is set, a directory -x did not enter
// because it is on another filesystem than its operand, and when
// leftOut is set, a default exclude or a directory or file on a
// filesystem that is not persistent, with warn holding the line that
// says so.
type walkEvent struct {
rec fileRec
warn string
fail bool
otherFSDir string
leftOut string
}
// startWalk seeds every root into the shared walk worker pool and
// returns the event stream: one record per regular file, one warning
// event per per-path error, and one event per directory -x did not
// enter. The channel is closed when the walk completes, and also when
// ctx is cancelled — every goroutine in the pool abandons its blocking
// send in that case, so the consumer sees a truncated but properly
// terminated stream instead of a stalled one.
// event per per-path error, and one event per path -x or rules kept the
// walk from entering or recording. The channel is closed when the walk
// completes, and also when ctx is cancelled — every goroutine in the
// pool abandons its blocking send in that case, so the consumer sees a
// truncated but properly terminated stream instead of a stalled one.
func startWalk(ctx context.Context, roots []string, oneFS bool,
workers int,
rules walkRules, workers int,
) <-chan walkEvent {
jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS)
jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, rules)
go func() {
initial := make([]dirJob, 0, len(roots))
@@ -1177,7 +1249,7 @@ func seedRoot(ctx context.Context, root string,
// workers drop the directories still queued rather than stop reading
// jobs, so the range always runs out and the pool always tears down.
func startWalkWorkers(ctx context.Context, workers int,
oneFS bool,
oneFS bool, rules walkRules,
) (chan dirJob, chan []dirJob, chan walkEvent) {
jobs := make(chan dirJob, workQueueDepth)
subdirs := make(chan []dirJob, workers)
@@ -1193,7 +1265,7 @@ func startWalkWorkers(ctx context.Context, workers int,
}
select {
case subdirs <- walkOneDir(ctx, job, oneFS, events):
case subdirs <- walkOneDir(ctx, job, oneFS, rules, events):
case <-ctx.Done():
}
}
@@ -1251,7 +1323,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
// entry and a warning event per unreadable one, and returns the
// subdirectories to descend into.
func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
events chan<- walkEvent,
rules walkRules, events chan<- walkEvent,
) []dirJob {
entries, err := os.ReadDir(job.path)
if err != nil {
@@ -1275,7 +1347,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
p := filepath.Join(job.path, e.Name())
if e.IsDir() {
if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok {
sub, ok := subdirJob(ctx, p, e, job, oneFS, rules, events)
if ok {
subs = append(subs, sub)
}
@@ -1287,7 +1360,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
continue
}
emitFile(ctx, p, e, events)
emitFile(ctx, p, e, rules, events)
}
return subs
@@ -1297,9 +1370,11 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
// The lstat happens here in the walk worker, while the directory's
// metadata is still hot; a path that fails to stat (or stops being a
// regular file) between the directory read and the lstat is warned
// about and skipped.
// about and skipped. A file on a filesystem in rules that is not
// persistent, as through a bind mount of one file, is not recorded: it
// gets an event naming it instead.
func emitFile(ctx context.Context, p string, e fs.DirEntry,
events chan<- walkEvent,
rules walkRules, events chan<- walkEvent,
) {
info, err := e.Info()
if err != nil {
@@ -1315,6 +1390,13 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
return
}
line := rules.nonPersistentLine(p, info)
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
return
}
dev, ino := inodeOfInfo(info)
sendEvent(ctx, events, walkEvent{rec: fileRec{
@@ -1328,22 +1410,20 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
// subdirJob applies the descent rules to directory p: never enter
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
// snapshot), and with -x never enter a directory on a different
// filesystem than its operand, sending an event that names it so the
// scan keeps the records below it, except those below an operand at or
// snapshot), never enter a directory on a filesystem in rules that is
// not persistent, nor a default exclude in rules, and with -x never
// enter a directory on a different filesystem than its operand. For the
// last three it sends an event that names the directory, so the scan
// keeps the records below it, except those below an operand at or
// inside it.
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
parent dirJob, oneFS bool, events chan<- walkEvent,
parent dirJob, oneFS bool, rules walkRules,
events chan<- walkEvent,
) (dirJob, bool) {
if e.Name() == ".zfs" {
return dirJob{}, false
}
job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
if !oneFS || !parent.rootDevOK {
return job, true
}
info, err := e.Info()
if err != nil {
sendEvent(ctx, events, walkEvent{
@@ -1354,6 +1434,27 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
return dirJob{}, false
}
line := rules.nonPersistentLine(p, info)
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
return dirJob{}, false
}
if isDefaultExclude(p, rules.excludes) {
sendEvent(ctx, events, walkEvent{
leftOut: p,
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
})
return dirJob{}, false
}
job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
if !oneFS || !parent.rootDevOK {
return job, true
}
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
sendEvent(ctx, events, walkEvent{otherFSDir: p})