Keep going past unreadable files with check --continue (closes #11) #15

Merged
clawbot merged 1 commits from check-continue-unreadable into next 2026-10-06 05:44:06 +02:00
3 changed files with 96 additions and 16 deletions
+3
View File
@@ -24,6 +24,9 @@ today) with lint, fmt, fmt-check, check, and hooks targets.
# Completed Steps
* 2026-10-06: `check --continue` keeps going past a file or directory
it cannot read: it counts it as failed, prints the error and the
path on stderr, and checks the rest of the tree
* 2026-10-05: golangci-lint settings take effect: canonical
`.golangci.yml` (v2 layout, settings under `linters.settings`),
golangci-lint pinned at v2.12.2 in `Dockerfile` and
+36 -16
View File
@@ -179,9 +179,9 @@ type processFunc func(
// countAndBar counts the files under paths and returns a progress bar sized
// to that total. It always returns either a non-nil bar or a non-nil error.
func countAndBar(
opts *options, paths []string, desc string,
opts *options, paths []string, desc string, cont bool,
) (*progressbar.ProgressBar, error) {
total, err := countFilesMultiple(opts, paths)
total, err := countFilesMultiple(opts, paths, cont)
if err != nil {
return nil, err
}
@@ -211,7 +211,7 @@ func runOverPaths(
var bar *progressbar.ProgressBar
if !opts.quiet {
bar, err = countAndBar(opts, paths, desc)
bar, err = countAndBar(opts, paths, desc, false)
if err != nil {
return err
}
@@ -268,7 +268,7 @@ func newSumCmd(opts *options) *cobra.Command {
func processSumAdd(
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
) error {
return walkAndProcess(opts, dir, stats, bar,
return walkAndProcess(opts, dir, false, stats, bar,
func(p string, info os.FileInfo, s *Stats) error {
if hasXattr(p, checksumKey) {
atomic.AddInt64(&s.FilesSkipped, 1)
@@ -290,7 +290,7 @@ func processSumAdd(
func processSumUpdate(
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
) error {
return walkAndProcess(opts, dir, stats, bar,
return walkAndProcess(opts, dir, false, stats, bar,
func(p string, info os.FileInfo, s *Stats) error {
t, err := readSumTime(p)
if err != nil || info.ModTime().After(t) {
@@ -386,7 +386,7 @@ func newClearCmd(opts *options) *cobra.Command {
func processClear(
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
) error {
return walkAndProcess(opts, dir, stats, bar,
return walkAndProcess(opts, dir, false, stats, bar,
func(p string, info os.FileInfo, s *Stats) error {
cleared, err := clearOne(opts, p)
if err != nil {
@@ -468,7 +468,7 @@ func runCheck(opts *options, args []string, cont bool) error {
var bar *progressbar.ProgressBar
if !opts.quiet {
bar, err = countAndBar(opts, paths, "Verifying checksums")
bar, err = countAndBar(opts, paths, "Verifying checksums", cont)
if err != nil {
return err
}
@@ -502,7 +502,7 @@ func processCheck(
// Track initial failed count to detect failures during this walk.
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
err := walkAndProcess(opts, dir, stats, bar,
err := walkAndProcess(opts, dir, cont, stats, bar,
func(p string, _ os.FileInfo, s *Stats) error {
return checkOne(opts, p, cont, s)
})
@@ -527,7 +527,7 @@ func checkOne(opts *options, p string, cont bool, s *Stats) error {
exp, err := xattr.Get(p, checksumKey)
if err != nil {
if !errors.Is(err, xattr.ENOATTR) {
return err
return unreadable(cont, s, err)
}
return missingChecksum(opts, p, cont, s)
@@ -535,9 +535,7 @@ func checkOne(opts *options, p string, cont bool, s *Stats) error {
act, bytesRead, err := fileMultihash(p)
if err != nil {
atomic.AddInt64(&s.FilesFailed, 1)
return err
return unreadable(cont, s, err)
}
ok := bytes.Equal(exp, act)
@@ -573,6 +571,21 @@ func missingChecksum(opts *options, p string, cont bool, s *Stats) error {
return errVerification
}
// unreadable counts a file or directory that could not be read as failed.
// With --continue it prints err, which names the path, to stderr and
// returns nil so the walk goes on; otherwise it returns err.
func unreadable(cont bool, s *Stats, err error) error {
atomic.AddInt64(&s.FilesFailed, 1)
if !cont {
return err
}
log.Print(err)
return nil
}
// reportCheck prints a per-file verification result when verbose output is on.
func reportCheck(opts *options, p, actual string, ok bool) {
if !opts.verbose || opts.quiet {
@@ -592,12 +605,18 @@ func reportCheck(opts *options, p, actual string, ok bool) {
///////////////////////////////////////////////////////////////////////////////
// countFiles counts the total number of regular files that will be processed.
func countFiles(opts *options, root string) (int64, error) {
// With cont, a path it cannot read is left out of the count instead of ending
// it; the walk that follows reports that path as failed.
func countFiles(opts *options, root string, cont bool) (int64, error) {
var count int64
root = filepath.Clean(root)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil && cont {
return nil
}
if err != nil {
return err
}
@@ -632,11 +651,11 @@ func countFiles(opts *options, root string) (int64, error) {
}
// countFilesMultiple counts files across multiple roots.
func countFilesMultiple(opts *options, roots []string) (int64, error) {
func countFilesMultiple(opts *options, roots []string, cont bool) (int64, error) {
var total int64
for _, root := range roots {
count, err := countFiles(opts, root)
count, err := countFiles(opts, root, cont)
if err != nil {
return total, err
}
@@ -672,6 +691,7 @@ func newProgressBar(total int64, description string) *progressbar.ProgressBar {
func walkAndProcess(
opts *options,
root string,
cont bool,
stats *Stats,
bar *progressbar.ProgressBar,
fn func(string, os.FileInfo, *Stats) error,
@@ -680,7 +700,7 @@ func walkAndProcess(
return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
return unreadable(cont, stats, err)
}
skip, skipErr := walkSkip(opts, root, p, info)
+57
View File
@@ -1,6 +1,7 @@
package main
import (
"errors"
"os"
"path/filepath"
"strings"
@@ -271,3 +272,59 @@ func TestPermissionErrors(t *testing.T) {
t.Fatalf("expected permission error on check, got nil")
}
}
func TestCheckContinuePastUnreadable(t *testing.T) {
t.Parallel()
opts := &options{}
dir := t.TempDir()
skipIfNoXattr(t, dir)
writeFile(t, dir, "a.txt", "one")
secret := writeFile(t, dir, "b.txt", "two")
writeFile(t, dir, "c/d.txt", "three")
writeFile(t, dir, "e.txt", "four")
err := processSumAdd(opts, dir, newTestStats(), nil)
if err != nil {
t.Fatalf("add: %v", err)
}
// An unreadable file and an unlistable directory sit between the
// readable files a.txt and e.txt.
sub := filepath.Join(dir, "c")
err = os.Chmod(secret, noPerm)
if err != nil {
t.Fatalf("chmod file: %v", err)
}
defer func() { _ = os.Chmod(secret, filePerm) }()
err = os.Chmod(sub, noPerm)
if err != nil {
t.Fatalf("chmod dir: %v", err)
}
defer func() { _ = os.Chmod(sub, dirPerm) }()
stats := newTestStats()
err = processCheck(opts, dir, true, stats, nil)
if !errors.Is(err, errVerification) {
t.Fatalf("expected verification error, got %v", err)
}
if stats.FilesProcessed != 2 || stats.FilesFailed != 2 {
t.Fatalf("expected 2 verified and 2 failed, got %d and %d",
stats.FilesProcessed, stats.FilesFailed)
}
// Without --quiet, runCheck counts the files for the progress bar
// before it checks any, so the count reaches the unlistable directory
// first.
err = runCheck(opts, []string{dir}, true)
if !errors.Is(err, errVerification) {
t.Fatalf("expected verification error from runCheck, got %v", err)
}
}