Make phases scan-wide, walk operands concurrently, batch updates
All checks were successful
check / check (push) Successful in 5s

All PATH operands belong to a single scan: every operand seeds the
shared walk worker pool, and each pass (walk, stat, hash, update)
runs exactly once over the whole scan, so pass totals, percentages,
and ETAs are scan-global. The per-operand walk/hash/update cycles and
their stderr operand announcements are gone; duplicate paths from
overlapping operands are deduplicated before stat.

The update pass now commits in batched transactions (10k changes per
batch) instead of one scan-wide transaction: the filesystem is
authoritative and the database is an eventually-consistent reflection
of it, so scan-level atomicity buys nothing, while batches keep the
WAL small and let concurrent reports observe progress.
This commit is contained in:
2026-07-24 10:26:52 +07:00
parent 732fc351d7
commit 3ecf73c80a
6 changed files with 229 additions and 95 deletions

View File

@@ -131,7 +131,7 @@ func TestWalkPass(t *testing.T) {
t.Fatal(err)
}
paths, errs := walkPass(dir, false, 4)
paths, errs := walkPass([]string{dir}, false, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
@@ -161,7 +161,7 @@ func TestWalkPassDeepAndWide(t *testing.T) {
slices.Sort(want)
paths, errs := walkPass(dir, false, 8)
paths, errs := walkPass([]string{dir}, false, 8)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
@@ -173,6 +173,33 @@ func TestWalkPassDeepAndWide(t *testing.T) {
}
}
func TestWalkPassMultipleRoots(t *testing.T) {
t.Parallel()
rootA := t.TempDir()
rootB := t.TempDir()
want := []string{
writeFile(t, rootA, "a1", []byte("1")),
writeFile(t, rootA, "sub/a2", []byte("2")),
writeFile(t, rootB, "b1", []byte("3")),
}
slices.Sort(want)
// Operands are enumerated concurrently by the shared pool; order
// is unspecified.
paths, errs := walkPass([]string{rootA, rootB}, false, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
slices.Sort(paths)
if !slices.Equal(paths, want) {
t.Fatalf("paths = %q, want %q", paths, want)
}
}
func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
t.Parallel()
@@ -187,13 +214,13 @@ func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
}
// A regular-file operand is emitted as itself.
paths, errs := walkPass(f, false, 2)
paths, errs := walkPass([]string{f}, false, 2)
if errs != 0 || !slices.Equal(paths, []string{f}) {
t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
}
// A symlink operand is not followed and yields nothing.
paths, errs = walkPass(link, false, 2)
paths, errs = walkPass([]string{link}, false, 2)
if errs != 0 || len(paths) != 0 {
t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
}
@@ -209,7 +236,7 @@ func TestWalkPassOneFilesystemSameFS(t *testing.T) {
writeFile(t, dir, "sub/deep/b", []byte("b")),
}
paths, errs := walkPass(dir, true, 4)
paths, errs := walkPass([]string{dir}, true, 4)
if errs != 0 {
t.Fatalf("errs = %d, want 0", errs)
}
@@ -623,13 +650,11 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
writeFile(t, dir, "sub/f", pattern(1, 10))
// A file reachable via two overlapping operands yields one
// record: the second operand sees the rows the first operand
// committed and reuses them unchanged, which also proves each
// operand commits before the next starts.
// A file reachable via two overlapping operands is deduplicated
// by path in the shared walk and processed once.
st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
if st != (scanStats{added: 1, unchanged: 1}) {
t.Fatalf("stats = %+v, want 1 added 1 unchanged", st)
if st != (scanStats{added: 1}) {
t.Fatalf("stats = %+v, want 1 added", st)
}
if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
@@ -637,6 +662,17 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
}
}
func TestUniquePaths(t *testing.T) {
t.Parallel()
got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"})
want := []string{"/a", "/b", "/c"}
if !slices.Equal(got, want) {
t.Fatalf("uniquePaths = %q, want %q", got, want)
}
}
func TestReportsNeverTouchFilesystem(t *testing.T) {
t.Parallel()