Skip removed paths silently and fix the progress line (closes #133)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A path removed while the scan runs gets no warning and no skipped count, in the walk or the hash phase. If the first of several hard links is gone, the next is read. Any other error is still warned about. On a terminal every display is the library's spinner, then the line. A phase with an exact total ends the padded line with its percent and a bar at a fixed column. sfdupes draws that bar: the library shows no spinner beside its own. The time remaining is labelled remain, not eta. Judgement call: an operand removed after the scan starts is passed over silently too. Deviation: README's 80-column promise for stalled lines is narrowed to its example. Model: opus-5-5
This commit is contained in:
@@ -418,7 +418,8 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
// roots and are not deleted as unverified. An operand on a filesystem
|
||||
// in rules that is not persistent is dropped the same way, with a line
|
||||
// saying so that is not a warning, and is not counted as skipped. An
|
||||
// operand that fails lstat here is kept, and the walk warns about it.
|
||||
// operand that fails lstat here is kept, and the walk warns about it
|
||||
// unless it no longer exists.
|
||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||
kept := make([]string, 0, len(roots))
|
||||
|
||||
@@ -629,8 +630,9 @@ func isDatabaseFile(f fileRec, rules walkRules) bool {
|
||||
// (an interrupted scan resumes cheaply: the next run skips everything
|
||||
// already recorded). A run that fails to hash is warned about and
|
||||
// counted as skipped; stale records for its paths, if any, are deleted
|
||||
// by the update phase. The total counts actual reads, so the display
|
||||
// shows a real ETA.
|
||||
// by the update phase. A run whose file no longer exists gets no record
|
||||
// and no warning, and is not counted as skipped. The total counts actual
|
||||
// reads, so the display's time remaining is meaningful.
|
||||
//
|
||||
// Returning early — a failed database write, or a cancelled scan — must
|
||||
// not strand the pool: the feeder would park forever on a full jobs
|
||||
@@ -662,6 +664,11 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
|
||||
|
||||
prog.increment()
|
||||
|
||||
// A file removed since the walk is no error: it gets no record.
|
||||
if errors.Is(r.err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
if r.err != nil {
|
||||
s.st.skipped += len(r.run)
|
||||
|
||||
@@ -829,10 +836,12 @@ type walkEvent struct {
|
||||
// startWalk seeds every root into the shared walk worker pool and
|
||||
// returns the event stream: one record per regular file, one warning
|
||||
// event per per-path error, and one event per path -x or rules kept the
|
||||
// walk from entering or recording. The channel is closed when the walk
|
||||
// completes, and also when ctx is cancelled — every goroutine in the
|
||||
// pool abandons its blocking send in that case, so the consumer sees a
|
||||
// truncated but properly terminated stream instead of a stalled one.
|
||||
// walk from entering or recording. A path that no longer exists when
|
||||
// the walk reaches it is no error, and gets no event. The channel is
|
||||
// closed when the walk completes, and also when ctx is cancelled —
|
||||
// every goroutine in the pool abandons its blocking send in that case,
|
||||
// so the consumer sees a truncated but properly terminated stream
|
||||
// instead of a stalled one.
|
||||
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
||||
rules walkRules, workers int,
|
||||
) <-chan walkEvent {
|
||||
@@ -899,13 +908,19 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
||||
// one of the scan's own database files in rules, and a directory
|
||||
// operand becomes an initial job. walkableRoots has already dropped
|
||||
// every other operand. One that has changed into something else since
|
||||
// is warned about and skipped here; it is still a root, so the records
|
||||
// stored beneath it, other than those under another root, are deleted
|
||||
// as unverified.
|
||||
// is warned about and skipped here, and one removed since is skipped
|
||||
// without a warning. Either is still a root, so the records stored
|
||||
// beneath it, other than those under another root, are deleted as
|
||||
// unverified.
|
||||
func seedRoot(ctx context.Context, root string, rules walkRules,
|
||||
events chan<- walkEvent,
|
||||
) []dirJob {
|
||||
fi, err := os.Lstat(root)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Removed since the scan started: no error, and no files.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
sendEvent(ctx, events, walkEvent{
|
||||
warn: fmt.Sprintf("walk %s: %v", root, err),
|
||||
@@ -1023,11 +1038,17 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
|
||||
|
||||
// walkOneDir reads one directory, emitting an event per regular-file
|
||||
// entry and a warning event per unreadable one, and returns the
|
||||
// subdirectories to descend into.
|
||||
// subdirectories to descend into. A directory or entry removed since
|
||||
// the walk found it gets no event.
|
||||
func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
||||
rules walkRules, events chan<- walkEvent,
|
||||
) []dirJob {
|
||||
entries, err := os.ReadDir(job.path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Removed since its parent was read: no error, and no files.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
sendEvent(ctx, events, walkEvent{
|
||||
warn: fmt.Sprintf("walk %s: %v", job.path, err),
|
||||
@@ -1070,16 +1091,22 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
||||
|
||||
// emitFile stats one regular directory entry and emits its record.
|
||||
// The lstat happens here in the walk worker, while the directory's
|
||||
// metadata is still hot; a path that fails to stat (or stops being a
|
||||
// regular file) between the directory read and the lstat is warned
|
||||
// about and skipped. A file on a filesystem in rules that is not
|
||||
// persistent, as through a bind mount of one file, is not recorded: it
|
||||
// gets an event naming it instead. One of the scan's own database files
|
||||
// in rules gets no event at all.
|
||||
// metadata is still hot; a path that fails to stat between the
|
||||
// directory read and the lstat is warned about and skipped, unless it
|
||||
// no longer exists, and one that has stopped being a regular file is
|
||||
// skipped. A file on a filesystem in rules that is not persistent, as
|
||||
// through a bind mount of one file, is not recorded: it gets an event
|
||||
// naming it instead. One of the scan's own database files in rules gets
|
||||
// no event at all.
|
||||
func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
||||
rules walkRules, events chan<- walkEvent,
|
||||
) {
|
||||
info, err := e.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Removed since its directory was read: no error.
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
sendEvent(ctx, events, walkEvent{
|
||||
warn: fmt.Sprintf("stat %s: %v", p, err),
|
||||
@@ -1131,6 +1158,11 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||
}
|
||||
|
||||
info, err := e.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Removed since its parent was read: no error, and no files.
|
||||
return dirJob{}, false
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
sendEvent(ctx, events, walkEvent{
|
||||
warn: fmt.Sprintf("walk %s: %v", p, err),
|
||||
@@ -1195,7 +1227,7 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
|
||||
|
||||
// hashResult carries the hashes computed for one inode run (or the
|
||||
// error that prevented computing them) from the pool's workers to the
|
||||
// hash phase.
|
||||
// hash phase. Its run leaves out the paths found removed.
|
||||
type hashResult struct {
|
||||
run []fileRec
|
||||
head [sha256.Size]byte
|
||||
@@ -1296,6 +1328,13 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
||||
|
||||
head, tail, content, err := hash(run[0].path, run[0].size)
|
||||
|
||||
// A path removed since the walk is dropped from its run. The
|
||||
// next path is a hard link to the same file, and is read instead.
|
||||
for errors.Is(err, fs.ErrNotExist) && len(run) > 1 {
|
||||
run = run[1:]
|
||||
head, tail, content, err = hash(run[0].path, run[0].size)
|
||||
}
|
||||
|
||||
select {
|
||||
case results <- hashResult{
|
||||
run: run, head: head, tail: tail, content: content, err: err,
|
||||
|
||||
Reference in New Issue
Block a user