Make phases scan-wide, walk operands concurrently, batch updates
All checks were successful
check / check (push) Successful in 5s
All checks were successful
check / check (push) Successful in 5s
All PATH operands belong to a single scan: every operand seeds the shared walk worker pool, and each pass (walk, stat, hash, update) runs exactly once over the whole scan, so pass totals, percentages, and ETAs are scan-global. The per-operand walk/hash/update cycles and their stderr operand announcements are gone; duplicate paths from overlapping operands are deduplicated before stat. The update pass now commits in batched transactions (10k changes per batch) instead of one scan-wide transaction: the filesystem is authoritative and the database is an eventually-consistent reflection of it, so scan-level atomicity buys nothing, while batches keep the WAL small and let concurrent reports observe progress.
This commit is contained in:
58
scan_test.go
58
scan_test.go
@@ -131,7 +131,7 @@ func TestWalkPass(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
paths, errs := walkPass(dir, false, 4)
|
||||
paths, errs := walkPass([]string{dir}, false, 4)
|
||||
if errs != 0 {
|
||||
t.Fatalf("errs = %d, want 0", errs)
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func TestWalkPassDeepAndWide(t *testing.T) {
|
||||
|
||||
slices.Sort(want)
|
||||
|
||||
paths, errs := walkPass(dir, false, 8)
|
||||
paths, errs := walkPass([]string{dir}, false, 8)
|
||||
if errs != 0 {
|
||||
t.Fatalf("errs = %d, want 0", errs)
|
||||
}
|
||||
@@ -173,6 +173,33 @@ func TestWalkPassDeepAndWide(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkPassMultipleRoots(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rootA := t.TempDir()
|
||||
rootB := t.TempDir()
|
||||
want := []string{
|
||||
writeFile(t, rootA, "a1", []byte("1")),
|
||||
writeFile(t, rootA, "sub/a2", []byte("2")),
|
||||
writeFile(t, rootB, "b1", []byte("3")),
|
||||
}
|
||||
|
||||
slices.Sort(want)
|
||||
|
||||
// Operands are enumerated concurrently by the shared pool; order
|
||||
// is unspecified.
|
||||
paths, errs := walkPass([]string{rootA, rootB}, false, 4)
|
||||
if errs != 0 {
|
||||
t.Fatalf("errs = %d, want 0", errs)
|
||||
}
|
||||
|
||||
slices.Sort(paths)
|
||||
|
||||
if !slices.Equal(paths, want) {
|
||||
t.Fatalf("paths = %q, want %q", paths, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -187,13 +214,13 @@ func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
|
||||
}
|
||||
|
||||
// A regular-file operand is emitted as itself.
|
||||
paths, errs := walkPass(f, false, 2)
|
||||
paths, errs := walkPass([]string{f}, false, 2)
|
||||
if errs != 0 || !slices.Equal(paths, []string{f}) {
|
||||
t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
|
||||
}
|
||||
|
||||
// A symlink operand is not followed and yields nothing.
|
||||
paths, errs = walkPass(link, false, 2)
|
||||
paths, errs = walkPass([]string{link}, false, 2)
|
||||
if errs != 0 || len(paths) != 0 {
|
||||
t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
|
||||
}
|
||||
@@ -209,7 +236,7 @@ func TestWalkPassOneFilesystemSameFS(t *testing.T) {
|
||||
writeFile(t, dir, "sub/deep/b", []byte("b")),
|
||||
}
|
||||
|
||||
paths, errs := walkPass(dir, true, 4)
|
||||
paths, errs := walkPass([]string{dir}, true, 4)
|
||||
if errs != 0 {
|
||||
t.Fatalf("errs = %d, want 0", errs)
|
||||
}
|
||||
@@ -623,13 +650,11 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
|
||||
|
||||
writeFile(t, dir, "sub/f", pattern(1, 10))
|
||||
|
||||
// A file reachable via two overlapping operands yields one
|
||||
// record: the second operand sees the rows the first operand
|
||||
// committed and reuses them unchanged, which also proves each
|
||||
// operand commits before the next starts.
|
||||
// A file reachable via two overlapping operands is deduplicated
|
||||
// by path in the shared walk and processed once.
|
||||
st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
|
||||
if st != (scanStats{added: 1, unchanged: 1}) {
|
||||
t.Fatalf("stats = %+v, want 1 added 1 unchanged", st)
|
||||
if st != (scanStats{added: 1}) {
|
||||
t.Fatalf("stats = %+v, want 1 added", st)
|
||||
}
|
||||
|
||||
if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
|
||||
@@ -637,6 +662,17 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniquePaths(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"})
|
||||
|
||||
want := []string{"/a", "/b", "/c"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("uniquePaths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportsNeverTouchFilesystem(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user