Make phases scan-wide, walk operands concurrently, batch updates
All checks were successful
check / check (push) Successful in 5s

All PATH operands belong to a single scan: every operand seeds the
shared walk worker pool, and each pass (walk, stat, hash, update)
runs exactly once over the whole scan, so pass totals, percentages,
and ETAs are scan-global. The per-operand walk/hash/update cycles and
their stderr operand announcements are gone; duplicate paths from
overlapping operands are deduplicated before stat.

The update pass now commits in batched transactions (10k changes per
batch) instead of one scan-wide transaction: the filesystem is
authoritative and the database is an eventually-consistent reflection
of it, so scan-level atomicity buys nothing, while batches keep the
WAL small and let concurrent reports observe progress.
This commit is contained in:
2026-07-24 10:26:52 +07:00
parent 732fc351d7
commit 3ecf73c80a
6 changed files with 229 additions and 95 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"slices"
"strings"
@@ -178,3 +179,46 @@ func TestApplyChangesRoundTrip(t *testing.T) {
t.Fatalf("rows = %+v, want just %+v", got, upd)
}
}
func TestApplyChangesBatching(t *testing.T) {
t.Parallel()
db := openTestDB(t)
// One more change than the batch size, so the update spans two
// transactions.
n := updateBatchSize + 1
recs := make([]scanRec, 0, n)
for i := range n {
recs = append(recs, scanRec{
size: int64(i), mtime: 1, head: "h", tail: "t",
path: fmt.Sprintf("/batch/%07d", i),
})
}
err := applyChanges(db, recs, nil, newProgress("update", int64(n)))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
got, err := loadFileRows(db)
if err != nil || len(got) != n {
t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n)
}
deletes := make([]string, 0, n)
for _, r := range recs {
deletes = append(deletes, r.path)
}
err = applyChanges(db, nil, deletes, newProgress("update", int64(n)))
if err != nil {
t.Fatalf("applyChanges deletes: %v", err)
}
got, err = loadFileRows(db)
if err != nil || len(got) != 0 {
t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err)
}
}