Make phases scan-wide, walk operands concurrently, batch updates
All checks were successful
check / check (push) Successful in 5s
All checks were successful
check / check (push) Successful in 5s
All PATH operands belong to a single scan: every operand seeds the shared walk worker pool, and each pass (walk, stat, hash, update) runs exactly once over the whole scan, so pass totals, percentages, and ETAs are scan-global. The per-operand walk/hash/update cycles and their stderr operand announcements are gone; duplicate paths from overlapping operands are deduplicated before stat. The update pass now commits in batched transactions (10k changes per batch) instead of one scan-wide transaction: the filesystem is authoritative and the database is an eventually-consistent reflection of it, so scan-level atomicity buys nothing, while batches keep the WAL small and let concurrent reports observe progress.
This commit is contained in:
44
db_test.go
44
db_test.go
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -178,3 +179,46 @@ func TestApplyChangesRoundTrip(t *testing.T) {
|
||||
t.Fatalf("rows = %+v, want just %+v", got, upd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyChangesBatching(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
|
||||
// One more change than the batch size, so the update spans two
|
||||
// transactions.
|
||||
n := updateBatchSize + 1
|
||||
|
||||
recs := make([]scanRec, 0, n)
|
||||
for i := range n {
|
||||
recs = append(recs, scanRec{
|
||||
size: int64(i), mtime: 1, head: "h", tail: "t",
|
||||
path: fmt.Sprintf("/batch/%07d", i),
|
||||
})
|
||||
}
|
||||
|
||||
err := applyChanges(db, recs, nil, newProgress("update", int64(n)))
|
||||
if err != nil {
|
||||
t.Fatalf("applyChanges: %v", err)
|
||||
}
|
||||
|
||||
got, err := loadFileRows(db)
|
||||
if err != nil || len(got) != n {
|
||||
t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n)
|
||||
}
|
||||
|
||||
deletes := make([]string, 0, n)
|
||||
for _, r := range recs {
|
||||
deletes = append(deletes, r.path)
|
||||
}
|
||||
|
||||
err = applyChanges(db, nil, deletes, newProgress("update", int64(n)))
|
||||
if err != nil {
|
||||
t.Fatalf("applyChanges deletes: %v", err)
|
||||
}
|
||||
|
||||
got, err = loadFileRows(db)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user