Keep going past unreadable files with check --continue (closes #11)
check / check (push) Successful in 2m1s
check / check (push) Successful in 2m1s
With --continue, check stopped at the first file whose content or checksum attribute it could not read, and at the first directory it could not list. Each of these now counts as failed, its error, which names the path, goes to stderr, and the walk goes on; the run still exits non-zero. The count that sizes the progress bar leaves such a path out, so the bar stays. The walk and the count, shared with sum and clear, take the continue setting; those commands pass false and still stop at the first error. Without --continue the first such error still stops the run, and the summary printed before it now counts an unreadable checksum attribute as failed, as it already did for unreadable content. Model: opus-5-5
This commit was merged in pull request #15.
This commit is contained in:
@@ -24,6 +24,9 @@ today) with lint, fmt, fmt-check, check, and hooks targets.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
* 2026-10-06: `check --continue` keeps going past a file or directory
|
||||||
|
it cannot read: it counts it as failed, prints the error and the
|
||||||
|
path on stderr, and checks the rest of the tree
|
||||||
* 2026-10-05: golangci-lint settings take effect: canonical
|
* 2026-10-05: golangci-lint settings take effect: canonical
|
||||||
`.golangci.yml` (v2 layout, settings under `linters.settings`),
|
`.golangci.yml` (v2 layout, settings under `linters.settings`),
|
||||||
golangci-lint pinned at v2.12.2 in `Dockerfile` and
|
golangci-lint pinned at v2.12.2 in `Dockerfile` and
|
||||||
|
|||||||
+36
-16
@@ -179,9 +179,9 @@ type processFunc func(
|
|||||||
// countAndBar counts the files under paths and returns a progress bar sized
|
// countAndBar counts the files under paths and returns a progress bar sized
|
||||||
// to that total. It always returns either a non-nil bar or a non-nil error.
|
// to that total. It always returns either a non-nil bar or a non-nil error.
|
||||||
func countAndBar(
|
func countAndBar(
|
||||||
opts *options, paths []string, desc string,
|
opts *options, paths []string, desc string, cont bool,
|
||||||
) (*progressbar.ProgressBar, error) {
|
) (*progressbar.ProgressBar, error) {
|
||||||
total, err := countFilesMultiple(opts, paths)
|
total, err := countFilesMultiple(opts, paths, cont)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -211,7 +211,7 @@ func runOverPaths(
|
|||||||
var bar *progressbar.ProgressBar
|
var bar *progressbar.ProgressBar
|
||||||
|
|
||||||
if !opts.quiet {
|
if !opts.quiet {
|
||||||
bar, err = countAndBar(opts, paths, desc)
|
bar, err = countAndBar(opts, paths, desc, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -268,7 +268,7 @@ func newSumCmd(opts *options) *cobra.Command {
|
|||||||
func processSumAdd(
|
func processSumAdd(
|
||||||
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
||||||
) error {
|
) error {
|
||||||
return walkAndProcess(opts, dir, stats, bar,
|
return walkAndProcess(opts, dir, false, stats, bar,
|
||||||
func(p string, info os.FileInfo, s *Stats) error {
|
func(p string, info os.FileInfo, s *Stats) error {
|
||||||
if hasXattr(p, checksumKey) {
|
if hasXattr(p, checksumKey) {
|
||||||
atomic.AddInt64(&s.FilesSkipped, 1)
|
atomic.AddInt64(&s.FilesSkipped, 1)
|
||||||
@@ -290,7 +290,7 @@ func processSumAdd(
|
|||||||
func processSumUpdate(
|
func processSumUpdate(
|
||||||
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
||||||
) error {
|
) error {
|
||||||
return walkAndProcess(opts, dir, stats, bar,
|
return walkAndProcess(opts, dir, false, stats, bar,
|
||||||
func(p string, info os.FileInfo, s *Stats) error {
|
func(p string, info os.FileInfo, s *Stats) error {
|
||||||
t, err := readSumTime(p)
|
t, err := readSumTime(p)
|
||||||
if err != nil || info.ModTime().After(t) {
|
if err != nil || info.ModTime().After(t) {
|
||||||
@@ -386,7 +386,7 @@ func newClearCmd(opts *options) *cobra.Command {
|
|||||||
func processClear(
|
func processClear(
|
||||||
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
opts *options, dir string, stats *Stats, bar *progressbar.ProgressBar,
|
||||||
) error {
|
) error {
|
||||||
return walkAndProcess(opts, dir, stats, bar,
|
return walkAndProcess(opts, dir, false, stats, bar,
|
||||||
func(p string, info os.FileInfo, s *Stats) error {
|
func(p string, info os.FileInfo, s *Stats) error {
|
||||||
cleared, err := clearOne(opts, p)
|
cleared, err := clearOne(opts, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -468,7 +468,7 @@ func runCheck(opts *options, args []string, cont bool) error {
|
|||||||
var bar *progressbar.ProgressBar
|
var bar *progressbar.ProgressBar
|
||||||
|
|
||||||
if !opts.quiet {
|
if !opts.quiet {
|
||||||
bar, err = countAndBar(opts, paths, "Verifying checksums")
|
bar, err = countAndBar(opts, paths, "Verifying checksums", cont)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -502,7 +502,7 @@ func processCheck(
|
|||||||
// Track initial failed count to detect failures during this walk.
|
// Track initial failed count to detect failures during this walk.
|
||||||
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
|
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
|
||||||
|
|
||||||
err := walkAndProcess(opts, dir, stats, bar,
|
err := walkAndProcess(opts, dir, cont, stats, bar,
|
||||||
func(p string, _ os.FileInfo, s *Stats) error {
|
func(p string, _ os.FileInfo, s *Stats) error {
|
||||||
return checkOne(opts, p, cont, s)
|
return checkOne(opts, p, cont, s)
|
||||||
})
|
})
|
||||||
@@ -527,7 +527,7 @@ func checkOne(opts *options, p string, cont bool, s *Stats) error {
|
|||||||
exp, err := xattr.Get(p, checksumKey)
|
exp, err := xattr.Get(p, checksumKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, xattr.ENOATTR) {
|
if !errors.Is(err, xattr.ENOATTR) {
|
||||||
return err
|
return unreadable(cont, s, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return missingChecksum(opts, p, cont, s)
|
return missingChecksum(opts, p, cont, s)
|
||||||
@@ -535,9 +535,7 @@ func checkOne(opts *options, p string, cont bool, s *Stats) error {
|
|||||||
|
|
||||||
act, bytesRead, err := fileMultihash(p)
|
act, bytesRead, err := fileMultihash(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
atomic.AddInt64(&s.FilesFailed, 1)
|
return unreadable(cont, s, err)
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ok := bytes.Equal(exp, act)
|
ok := bytes.Equal(exp, act)
|
||||||
@@ -573,6 +571,21 @@ func missingChecksum(opts *options, p string, cont bool, s *Stats) error {
|
|||||||
return errVerification
|
return errVerification
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unreadable counts a file or directory that could not be read as failed.
|
||||||
|
// With --continue it prints err, which names the path, to stderr and
|
||||||
|
// returns nil so the walk goes on; otherwise it returns err.
|
||||||
|
func unreadable(cont bool, s *Stats, err error) error {
|
||||||
|
atomic.AddInt64(&s.FilesFailed, 1)
|
||||||
|
|
||||||
|
if !cont {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Print(err)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// reportCheck prints a per-file verification result when verbose output is on.
|
// reportCheck prints a per-file verification result when verbose output is on.
|
||||||
func reportCheck(opts *options, p, actual string, ok bool) {
|
func reportCheck(opts *options, p, actual string, ok bool) {
|
||||||
if !opts.verbose || opts.quiet {
|
if !opts.verbose || opts.quiet {
|
||||||
@@ -592,12 +605,18 @@ func reportCheck(opts *options, p, actual string, ok bool) {
|
|||||||
///////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
// countFiles counts the total number of regular files that will be processed.
|
// countFiles counts the total number of regular files that will be processed.
|
||||||
func countFiles(opts *options, root string) (int64, error) {
|
// With cont, a path it cannot read is left out of the count instead of ending
|
||||||
|
// it; the walk that follows reports that path as failed.
|
||||||
|
func countFiles(opts *options, root string, cont bool) (int64, error) {
|
||||||
var count int64
|
var count int64
|
||||||
|
|
||||||
root = filepath.Clean(root)
|
root = filepath.Clean(root)
|
||||||
|
|
||||||
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil && cont {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -632,11 +651,11 @@ func countFiles(opts *options, root string) (int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// countFilesMultiple counts files across multiple roots.
|
// countFilesMultiple counts files across multiple roots.
|
||||||
func countFilesMultiple(opts *options, roots []string) (int64, error) {
|
func countFilesMultiple(opts *options, roots []string, cont bool) (int64, error) {
|
||||||
var total int64
|
var total int64
|
||||||
|
|
||||||
for _, root := range roots {
|
for _, root := range roots {
|
||||||
count, err := countFiles(opts, root)
|
count, err := countFiles(opts, root, cont)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return total, err
|
return total, err
|
||||||
}
|
}
|
||||||
@@ -672,6 +691,7 @@ func newProgressBar(total int64, description string) *progressbar.ProgressBar {
|
|||||||
func walkAndProcess(
|
func walkAndProcess(
|
||||||
opts *options,
|
opts *options,
|
||||||
root string,
|
root string,
|
||||||
|
cont bool,
|
||||||
stats *Stats,
|
stats *Stats,
|
||||||
bar *progressbar.ProgressBar,
|
bar *progressbar.ProgressBar,
|
||||||
fn func(string, os.FileInfo, *Stats) error,
|
fn func(string, os.FileInfo, *Stats) error,
|
||||||
@@ -680,7 +700,7 @@ func walkAndProcess(
|
|||||||
|
|
||||||
return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return unreadable(cont, stats, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
skip, skipErr := walkSkip(opts, root, p, info)
|
skip, skipErr := walkSkip(opts, root, p, info)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -271,3 +272,59 @@ func TestPermissionErrors(t *testing.T) {
|
|||||||
t.Fatalf("expected permission error on check, got nil")
|
t.Fatalf("expected permission error on check, got nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCheckContinuePastUnreadable(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
opts := &options{}
|
||||||
|
dir := t.TempDir()
|
||||||
|
skipIfNoXattr(t, dir)
|
||||||
|
|
||||||
|
writeFile(t, dir, "a.txt", "one")
|
||||||
|
secret := writeFile(t, dir, "b.txt", "two")
|
||||||
|
writeFile(t, dir, "c/d.txt", "three")
|
||||||
|
writeFile(t, dir, "e.txt", "four")
|
||||||
|
|
||||||
|
err := processSumAdd(opts, dir, newTestStats(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("add: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unreadable file and an unlistable directory sit between the
|
||||||
|
// readable files a.txt and e.txt.
|
||||||
|
sub := filepath.Join(dir, "c")
|
||||||
|
|
||||||
|
err = os.Chmod(secret, noPerm)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("chmod file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = os.Chmod(secret, filePerm) }()
|
||||||
|
|
||||||
|
err = os.Chmod(sub, noPerm)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("chmod dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = os.Chmod(sub, dirPerm) }()
|
||||||
|
|
||||||
|
stats := newTestStats()
|
||||||
|
|
||||||
|
err = processCheck(opts, dir, true, stats, nil)
|
||||||
|
if !errors.Is(err, errVerification) {
|
||||||
|
t.Fatalf("expected verification error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if stats.FilesProcessed != 2 || stats.FilesFailed != 2 {
|
||||||
|
t.Fatalf("expected 2 verified and 2 failed, got %d and %d",
|
||||||
|
stats.FilesProcessed, stats.FilesFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without --quiet, runCheck counts the files for the progress bar
|
||||||
|
// before it checks any, so the count reaches the unlistable directory
|
||||||
|
// first.
|
||||||
|
err = runCheck(opts, []string{dir}, true)
|
||||||
|
if !errors.Is(err, errVerification) {
|
||||||
|
t.Fatalf("expected verification error from runCheck, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user