Show progress while loading the record index

On a database with tens of millions of records, indexing the existing
rows before the walk takes real single-core time with no output,
which is indistinguishable from a hang. Give the load its own
spinner, and render every phase display the moment the phase starts
instead of waiting for its first completed item.
This commit is contained in:
2026-07-25 14:28:11 +07:00
parent d4b43ebb30
commit 57efa64f18
3 changed files with 15 additions and 3 deletions

View File

@@ -395,9 +395,11 @@ all dupe rows) in human units.
Use the progress-bar library for all scan progress; rendering in the Use the progress-bar library for all scan progress; rendering in the
style of `pv` is the model. All progress goes to stderr. style of `pv` is the model. All progress goes to stderr.
Each phase gets its own display. The walk has no known total while Each phase gets its own display, rendered the moment the phase
running: show a live file count, rate, and elapsed time starts — a scan must never look hung. Loading the existing-record
(spinner-style, no percentage or ETA). The hash and update phases index (`load`) and the walk have no known totals while running: show
a live count, rate, and elapsed time (spinner-style, no percentage or
ETA). The hash and update phases
have exact totals — only files that actually need hashing appear in have exact totals — only files that actually need hashing appear in
the hash total, so its ETA is meaningful. Required elements for the the hash total, so its ETA is meaningful. Required elements for the
bars with known totals: bars with known totals:

View File

@@ -65,6 +65,9 @@ func newProgress(label string, total int64) *progress {
progressbar.OptionSetItsString("files"), progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true), progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(barThrottle), progressbar.OptionThrottle(barThrottle),
// Render at zero immediately: a phase must be visible the
// moment it starts, even before its first item completes.
progressbar.OptionSetRenderBlankState(true),
} }
if total >= 0 { if total >= 0 {
opts = append(opts, opts = append(opts,

View File

@@ -186,10 +186,17 @@ func syncScan(db *sql.DB, roots []string, workers int,
// can be recognized as a possible duplicate of a tree scanned // can be recognized as a possible duplicate of a tree scanned
// separately into the same database. // separately into the same database.
func (s *scanState) loadIndex(roots []string) error { func (s *scanState) loadIndex(roots []string) error {
// Indexing tens of millions of records takes real time; without a
// display the scan looks hung before the walk begins.
prog := newProgress("load", -1)
defer prog.finish()
s.existing = make(map[string]fileMeta) s.existing = make(map[string]fileMeta)
return loadFileMeta(s.db, return loadFileMeta(s.db,
func(path string, size, mtime int64, hashed bool) { func(path string, size, mtime int64, hashed bool) {
prog.increment()
if underAnyRoot(path, roots) { if underAnyRoot(path, roots) {
s.existing[path] = fileMeta{ s.existing[path] = fileMeta{
size: size, mtime: mtime, hashed: hashed, size: size, mtime: mtime, hashed: hashed,