Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f2960116 |
@@ -399,8 +399,9 @@ scan and are enumerated concurrently: every operand seeds the shared walk worker
|
||||
pool. Overlapping operands are harmless — an operand that duplicates another or
|
||||
lies under another is dropped before walking, so every file is reached exactly
|
||||
once and produces one database record. With `-x`, though, an operand under
|
||||
another one is kept and walked on its own when it is on a different filesystem
|
||||
than the operand covering it, because that operand's walk does not enter it.
|
||||
another one is kept and walked on its own when it is a directory on a different
|
||||
filesystem than the operand covering it, or a regular file in such a directory,
|
||||
because the walk of the covering operand does not reach it.
|
||||
|
||||
An operand that is a symlink (never followed, not even as an operand), socket,
|
||||
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
||||
@@ -453,7 +454,7 @@ operands:
|
||||
directory is on another filesystem than its operand counts as outside the
|
||||
scanned operands: it is not deleted, and its size joins the size census. A
|
||||
filesystem mounted under an operand can therefore be scanned on its own
|
||||
schedule into the same database. Records below an operand inside such a
|
||||
schedule into the same database. Records below an operand at or inside such a
|
||||
directory are not kept this way, since that operand's own walk verifies them.
|
||||
|
||||
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
||||
|
||||
@@ -29,9 +29,10 @@
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- with `-x`, an operand under another operand but on a different filesystem is
|
||||
walked on its own instead of being dropped, and the records of files gone from
|
||||
below it are deleted (2026-10-14,
|
||||
- with `-x`, an operand under another operand is walked on its own instead of
|
||||
being dropped when it is a directory on a different filesystem than that
|
||||
operand, or a regular file in such a directory, and the records of files gone
|
||||
from below it are deleted (2026-10-14,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
||||
|
||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
||||
|
||||
@@ -204,11 +204,12 @@ func resolveRoots(roots []string) ([]string, error) {
|
||||
|
||||
// pruneRoots drops operands already covered by another operand:
|
||||
// duplicates and any operand lying under another one. With -x, an
|
||||
// operand under another one is kept when device puts it on a different
|
||||
// filesystem than the deepest kept operand above it, because that
|
||||
// operand's walk does not enter it. Every remaining file is then
|
||||
// reachable through exactly one root, so walked paths are unique
|
||||
// without keeping a scan-wide set of every path seen.
|
||||
// operand under another one is kept when device puts it, or for a
|
||||
// regular file its directory, on a different filesystem than the
|
||||
// deepest kept operand above it, because that operand's walk then does
|
||||
// not reach it. Every remaining file is then reachable through exactly
|
||||
// one root, so walked paths are unique without keeping a scan-wide set
|
||||
// of every path seen.
|
||||
func pruneRoots(roots []string, oneFS bool,
|
||||
device func(path string) (uint64, bool),
|
||||
) []string {
|
||||
@@ -233,11 +234,21 @@ func pruneRoots(roots []string, oneFS bool,
|
||||
continue
|
||||
}
|
||||
|
||||
if !oneFS {
|
||||
// Without -x, and always for a duplicate, r is dropped.
|
||||
if !oneFS || above == r {
|
||||
continue
|
||||
}
|
||||
|
||||
dev, ok := device(r)
|
||||
// The walk of above enters a directory only when it is on
|
||||
// above's filesystem, and then reaches every regular file in it
|
||||
// whatever device the file reports, so for a regular-file
|
||||
// operand its directory decides.
|
||||
dir := r
|
||||
if fi, err := os.Lstat(r); err == nil && fi.Mode().IsRegular() {
|
||||
dir = filepath.Dir(r)
|
||||
}
|
||||
|
||||
dev, ok := device(dir)
|
||||
aboveDev, aboveOK := device(above)
|
||||
|
||||
if ok && aboveOK && dev != aboveDev {
|
||||
@@ -424,8 +435,8 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
// hashes; both remain candidates until the census decides whether
|
||||
// their sizes are shared. A record below a directory -x did not enter
|
||||
// counts as outside the roots: it is not deleted, and its size joins
|
||||
// the census. That does not hold below a root inside that directory,
|
||||
// since that root's own walk verifies the records below it.
|
||||
// the census. That does not hold below a root at or inside that
|
||||
// directory, since that root's own walk verifies the records below it.
|
||||
func (s *scanState) walkPhase(
|
||||
events <-chan walkEvent, roots []string,
|
||||
) ([]fileRec, []fileRec) {
|
||||
|
||||
+50
-9
@@ -1135,13 +1135,14 @@ func TestSubdirJobStatError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// syncOneFilesystem scans roots into db with -x, taking each operand's
|
||||
// device from devices instead of from lstat, so a test can put an
|
||||
// operand on a filesystem that no directory here is on. It runs the
|
||||
// steps of runPhases except the content phase.
|
||||
// syncOneFilesystem scans roots into db with -x and returns the scan
|
||||
// stats, taking the device of each operand, and of a regular-file
|
||||
// operand's directory, from devices instead of from lstat, so a test
|
||||
// can put an operand on a filesystem that no directory here is on. It
|
||||
// runs the steps of runPhases except the content phase.
|
||||
func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
|
||||
devices map[string]uint64,
|
||||
) {
|
||||
) scanStats {
|
||||
t.Helper()
|
||||
|
||||
device := func(path string) (uint64, bool) {
|
||||
@@ -1160,13 +1161,19 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
initial := make([]dirJob, 0, len(roots))
|
||||
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
|
||||
|
||||
// seedRoot emits a regular-file operand itself and returns a job
|
||||
// only for a directory operand.
|
||||
var initial []dirJob
|
||||
|
||||
for _, root := range roots {
|
||||
initial = append(initial,
|
||||
dirJob{path: root, rootDev: devices[root], rootDevOK: true})
|
||||
for _, job := range seedRoot(ctx, root, events) {
|
||||
job.rootDev = devices[root]
|
||||
initial = append(initial, job)
|
||||
}
|
||||
}
|
||||
|
||||
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
|
||||
dispatchDirs(ctx, initial, jobs, subdirs)
|
||||
|
||||
changed, unhashed := s.walkPhase(events, roots)
|
||||
@@ -1181,6 +1188,8 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return s.st
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
|
||||
@@ -1293,6 +1302,38 @@ func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
|
||||
// regular-file operand under another operand is dropped when the file
|
||||
// reports a different device than that operand but its directory does
|
||||
// not, as a file bind-mounted into the tree does: the walk of the other
|
||||
// operand enters the directory and reaches the file, so the file is
|
||||
// recorded once.
|
||||
func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
file := writeFile(t, dir, "sub/f", pattern(1, 10))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
|
||||
dir: dev, filepath.Dir(file): dev, file: dev + 1,
|
||||
})
|
||||
|
||||
if st.added != 1 {
|
||||
t.Errorf("%s recorded %d times, want once", file, st.added)
|
||||
}
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
|
||||
Reference in New Issue
Block a user