Files
sfdupes/progress.go
sneak b62b4f297f Stat in the walk, hash only shared sizes, flush batches mid-scan
Restructure scan into three phases: walk+stat, hash, update.

The stat pass is folded into the walk workers: each regular file is
lstatted as its directory is read, while the metadata is hot. The
walk builds a scan-wide size census (walked files plus records
outside the scan roots), and unchanged already-hashed files resolve
during the walk without further work.

Only files whose size at least one other file shares are ever read:
a size-unique file cannot be a duplicate, so it is recorded without
hashes (head and tail empty). When a later scan makes its size
shared, the file is hashed then, even if otherwise unchanged. report
excludes unhashed records; trees gives them a never-matching
signature so a tree containing one never compares equal to another.

Hashed records are committed in batched transactions while the hash
phase runs, so an interrupted scan keeps everything hashed so far
and the next run resumes cheaply. The hash phase total is exact,
giving a meaningful ETA.

Memory drops accordingly: the existing-record index holds only path,
size, mtime, and a hashed flag (no hash values); the walk carries one
small record per candidate file; overlapping operands are pruned up
front instead of deduplicating every walked path in a scan-wide set.
Files no bigger than one chunk are hashed with a single read.
2026-07-25 06:06:22 +07:00

164 lines
3.8 KiB
Go

package main
import (
"fmt"
"os"
"time"
"github.com/schollz/progressbar/v3"
)
// plainInterval is the minimum time between progress lines when stderr
// is not a TTY.
const plainInterval = 5 * time.Second
// barThrottle caps how often the TTY progress bar redraws.
const barThrottle = 100 * time.Millisecond
// walkSpinnerType selects the progressbar library's spinner style used
// for the walk pass, which has no known total.
const walkSpinnerType = 14
// percentScale converts a fraction to a percentage.
const percentScale = 100
// stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool {
fi, err := os.Stderr.Stat()
if err != nil {
return false
}
return fi.Mode()&os.ModeCharDevice != 0
}
// progress renders one scan pass's progress on stderr. On a TTY it
// delegates to the progressbar library (spinner style when the total is
// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
// stderr is not a TTY it emits no ANSI redraws: it prints a plain
// one-line update no more often than every plainInterval.
//
// All methods must be called from the main goroutine only. A nil
// *progress is a valid no-display receiver: every method is a no-op,
// so batched database flushes during the streaming pass can reuse the
// update-pass helpers without rendering anything.
type progress struct {
label string
total int64 // -1 when unknown (walk pass)
bar *progressbar.ProgressBar
count int64
start time.Time
last time.Time
}
func newProgress(label string, total int64) *progress {
p := &progress{label: label, total: total, start: time.Now()}
if !stderrIsTTY() {
return p
}
opts := []progressbar.Option{
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetDescription(label),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(barThrottle),
}
if total >= 0 {
opts = append(opts,
progressbar.OptionSetPredictTime(true),
progressbar.OptionShowElapsedTimeOnFinish(),
)
} else {
opts = append(opts,
progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(walkSpinnerType),
)
}
p.bar = progressbar.NewOptions64(total, opts...)
return p
}
// increment records one completed item and refreshes the display.
func (p *progress) increment() {
if p == nil {
return
}
p.count++
if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
}
}
// warnf prints a one-line warning to stderr without corrupting the bar.
func (p *progress) warnf(format string, args ...any) {
if p == nil {
return
}
if p.bar != nil {
_ = p.bar.Clear()
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
// finish terminates the pass's display.
func (p *progress) finish() {
if p == nil {
return
}
if p.bar != nil {
_ = p.bar.Finish()
fmt.Fprintln(os.Stderr)
return
}
fmt.Fprintln(os.Stderr, p.plainLine())
}
// plainLine formats a non-TTY progress line.
func (p *progress) plainLine() string {
elapsed := time.Since(p.start)
if p.total < 0 {
return fmt.Sprintf("%s: %d files, elapsed %s",
p.label, p.count, elapsed.Round(time.Second))
}
pct := float64(percentScale)
if p.total > 0 {
pct = percentScale * float64(p.count) / float64(p.total)
}
rate := 0.0
if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
}
eta := "?"
if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)
}