On a database with tens of millions of records, indexing the existing rows before the walk takes real single-core time with no output, which is indistinguishable from a hang. Give the load its own spinner, and render every phase display the moment the phase starts instead of waiting for its first completed item.
167 lines
3.9 KiB
Go
167 lines
3.9 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/schollz/progressbar/v3"
|
|
)
|
|
|
|
// plainInterval is the minimum time between progress lines when stderr
|
|
// is not a TTY.
|
|
const plainInterval = 5 * time.Second
|
|
|
|
// barThrottle caps how often the TTY progress bar redraws.
|
|
const barThrottle = 100 * time.Millisecond
|
|
|
|
// walkSpinnerType selects the progressbar library's spinner style used
|
|
// for the walk pass, which has no known total.
|
|
const walkSpinnerType = 14
|
|
|
|
// percentScale converts a fraction to a percentage.
|
|
const percentScale = 100
|
|
|
|
// stderrIsTTY reports whether stderr is attached to a terminal.
|
|
func stderrIsTTY() bool {
|
|
fi, err := os.Stderr.Stat()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
return fi.Mode()&os.ModeCharDevice != 0
|
|
}
|
|
|
|
// progress renders one scan pass's progress on stderr. On a TTY it
|
|
// delegates to the progressbar library (spinner style when the total is
|
|
// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
|
|
// stderr is not a TTY it emits no ANSI redraws: it prints a plain
|
|
// one-line update no more often than every plainInterval.
|
|
//
|
|
// All methods must be called from the main goroutine only. A nil
|
|
// *progress is a valid no-display receiver: every method is a no-op,
|
|
// so batched database flushes during the streaming pass can reuse the
|
|
// update-pass helpers without rendering anything.
|
|
type progress struct {
|
|
label string
|
|
total int64 // -1 when unknown (walk pass)
|
|
bar *progressbar.ProgressBar
|
|
count int64
|
|
start time.Time
|
|
last time.Time
|
|
}
|
|
|
|
func newProgress(label string, total int64) *progress {
|
|
p := &progress{label: label, total: total, start: time.Now()}
|
|
if !stderrIsTTY() {
|
|
return p
|
|
}
|
|
|
|
opts := []progressbar.Option{
|
|
progressbar.OptionSetWriter(os.Stderr),
|
|
progressbar.OptionSetDescription(label),
|
|
progressbar.OptionShowCount(),
|
|
progressbar.OptionShowIts(),
|
|
progressbar.OptionSetItsString("files"),
|
|
progressbar.OptionSetElapsedTime(true),
|
|
progressbar.OptionThrottle(barThrottle),
|
|
// Render at zero immediately: a phase must be visible the
|
|
// moment it starts, even before its first item completes.
|
|
progressbar.OptionSetRenderBlankState(true),
|
|
}
|
|
if total >= 0 {
|
|
opts = append(opts,
|
|
progressbar.OptionSetPredictTime(true),
|
|
progressbar.OptionShowElapsedTimeOnFinish(),
|
|
)
|
|
} else {
|
|
opts = append(opts,
|
|
progressbar.OptionSetPredictTime(false),
|
|
progressbar.OptionSpinnerType(walkSpinnerType),
|
|
)
|
|
}
|
|
|
|
p.bar = progressbar.NewOptions64(total, opts...)
|
|
|
|
return p
|
|
}
|
|
|
|
// increment records one completed item and refreshes the display.
|
|
func (p *progress) increment() {
|
|
if p == nil {
|
|
return
|
|
}
|
|
|
|
p.count++
|
|
if p.bar != nil {
|
|
_ = p.bar.Add(1)
|
|
|
|
return
|
|
}
|
|
|
|
if time.Since(p.last) >= plainInterval {
|
|
p.last = time.Now()
|
|
fmt.Fprintln(os.Stderr, p.plainLine())
|
|
}
|
|
}
|
|
|
|
// warnf prints a one-line warning to stderr without corrupting the bar.
|
|
func (p *progress) warnf(format string, args ...any) {
|
|
if p == nil {
|
|
return
|
|
}
|
|
|
|
if p.bar != nil {
|
|
_ = p.bar.Clear()
|
|
}
|
|
|
|
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
|
}
|
|
|
|
// finish terminates the pass's display.
|
|
func (p *progress) finish() {
|
|
if p == nil {
|
|
return
|
|
}
|
|
|
|
if p.bar != nil {
|
|
_ = p.bar.Finish()
|
|
|
|
fmt.Fprintln(os.Stderr)
|
|
|
|
return
|
|
}
|
|
|
|
fmt.Fprintln(os.Stderr, p.plainLine())
|
|
}
|
|
|
|
// plainLine formats a non-TTY progress line.
|
|
func (p *progress) plainLine() string {
|
|
elapsed := time.Since(p.start)
|
|
if p.total < 0 {
|
|
return fmt.Sprintf("%s: %d files, elapsed %s",
|
|
p.label, p.count, elapsed.Round(time.Second))
|
|
}
|
|
|
|
pct := float64(percentScale)
|
|
if p.total > 0 {
|
|
pct = percentScale * float64(p.count) / float64(p.total)
|
|
}
|
|
|
|
rate := 0.0
|
|
if s := elapsed.Seconds(); s > 0 {
|
|
rate = float64(p.count) / s
|
|
}
|
|
|
|
eta := "?"
|
|
|
|
if rate > 0 && p.count <= p.total {
|
|
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
|
|
eta = remaining.Round(time.Second).String()
|
|
}
|
|
|
|
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
|
|
p.label, p.count, p.total, pct, rate,
|
|
elapsed.Round(time.Second), eta)
|
|
}
|