Add quiet mode, progress bar, summary report, and stdin support
- Add -q/--quiet flag to suppress all output except errors - Add progress bar with 250ms refresh, file count, and ETA display - Print summary report to stderr on completion (files processed, skipped, failed, bytes, duration) - Support reading paths from stdin with "-" argument (e.g., find | attrsum sum add -) - Update README with new features and updated TODO section
This commit is contained in:
parent
c856ea25be
commit
86764abadf
13
README.md
13
README.md
@ -45,6 +45,12 @@ attrsum -v check --continue DIR1 DIR2
|
||||
|
||||
# remove checksum & timestamp xattrs
|
||||
attrsum clear DIR1 DIR2
|
||||
|
||||
# read paths from stdin (use - as argument)
|
||||
find /data -name "*.jpg" | attrsum sum add -
|
||||
|
||||
# quiet mode (suppress progress bar and summary)
|
||||
attrsum -q sum add DIR
|
||||
```
|
||||
|
||||
| xattr key | meaning |
|
||||
@ -55,9 +61,12 @@ attrsum clear DIR1 DIR2
|
||||
Flags:
|
||||
|
||||
* `-v, --verbose` — per-file log output
|
||||
* `-q, --quiet` — suppress all output except errors (no progress bar or summary)
|
||||
* `--exclude PATTERN` — skip paths matching rsync/Doublestar glob
|
||||
* `--exclude-dotfiles` — skip any path component that starts with `.`
|
||||
|
||||
All commands display a progress bar with ETA and print a summary report to stderr on completion (unless `--quiet` is specified).
|
||||
|
||||
`attrsum` **never follows symlinks** and skips non-regular files (sockets, devices, …).
|
||||
|
||||
---
|
||||
@ -75,13 +84,9 @@ required. Now you can trust a USB stick didn't eat your data.
|
||||
|
||||
Future improvements under consideration:
|
||||
|
||||
- **Quiet mode (`-q`)** — suppress all output except errors
|
||||
- **Dry-run mode (`--dry-run`, `-n`)** — show what would be done without making changes
|
||||
- **JSON output (`--json`)** — machine-readable output for scripting and integration
|
||||
- **Parallel processing (`-j N`)** — use multiple goroutines for faster checksumming on large trees
|
||||
- **Progress indicator** — show progress bar or spinner for long-running operations
|
||||
- **Summary report** — print statistics on completion (files processed, errors, bytes hashed)
|
||||
- **Read paths from stdin (`-`)** — support piping file lists, e.g. `find ... | attrsum sum add -`
|
||||
- **Exit code documentation** — formalize and document exit codes for scripting
|
||||
|
||||
---
|
||||
|
||||
728
attrsum.go
728
attrsum.go
@ -1,54 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
base58 "github.com/mr-tron/base58/base58"
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
base58 "github.com/mr-tron/base58/base58"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const (
|
||||
checksumKey = "berlin.sneak.app.attrsum.checksum"
|
||||
sumTimeKey = "berlin.sneak.app.attrsum.sumtime"
|
||||
checksumKey = "berlin.sneak.app.attrsum.checksum"
|
||||
sumTimeKey = "berlin.sneak.app.attrsum.sumtime"
|
||||
)
|
||||
|
||||
var (
|
||||
verbose bool
|
||||
excludePatterns []string
|
||||
excludeDotfiles bool
|
||||
verbose bool
|
||||
quiet bool
|
||||
excludePatterns []string
|
||||
excludeDotfiles bool
|
||||
)
|
||||
|
||||
// Stats tracks operation statistics for summary reporting
|
||||
type Stats struct {
|
||||
FilesProcessed int64
|
||||
FilesSkipped int64
|
||||
FilesFailed int64
|
||||
BytesProcessed int64
|
||||
StartTime time.Time
|
||||
}
|
||||
|
||||
func (s *Stats) Duration() time.Duration {
|
||||
return time.Since(s.StartTime)
|
||||
}
|
||||
|
||||
func (s *Stats) Print(operation string) {
|
||||
if quiet {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n%s complete: %d files processed, %d skipped, %d failed, %s bytes in %s\n",
|
||||
operation,
|
||||
s.FilesProcessed,
|
||||
s.FilesSkipped,
|
||||
s.FilesFailed,
|
||||
formatBytes(s.BytesProcessed),
|
||||
s.Duration().Round(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
func formatBytes(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
func main() {
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "attrsum",
|
||||
Short: "Compute and verify file checksums via xattrs",
|
||||
}
|
||||
rootCmd.SilenceUsage = true
|
||||
rootCmd.SilenceErrors = true
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "attrsum",
|
||||
Short: "Compute and verify file checksums via xattrs",
|
||||
}
|
||||
rootCmd.SilenceUsage = true
|
||||
rootCmd.SilenceErrors = true
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable verbose output")
|
||||
rootCmd.PersistentFlags().StringArrayVar(&excludePatterns, "exclude", nil, "exclude files/directories matching pattern (rsync-style, repeatable)")
|
||||
rootCmd.PersistentFlags().BoolVar(&excludeDotfiles, "exclude-dotfiles", false, "exclude any file or directory whose name starts with '.'")
|
||||
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable verbose output")
|
||||
rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "suppress all output except errors")
|
||||
rootCmd.PersistentFlags().StringArrayVar(&excludePatterns, "exclude", nil, "exclude files/directories matching pattern (rsync-style, repeatable)")
|
||||
rootCmd.PersistentFlags().BoolVar(&excludeDotfiles, "exclude-dotfiles", false, "exclude any file or directory whose name starts with '.'")
|
||||
|
||||
rootCmd.AddCommand(newSumCmd())
|
||||
rootCmd.AddCommand(newCheckCmd())
|
||||
rootCmd.AddCommand(newClearCmd())
|
||||
rootCmd.AddCommand(newSumCmd())
|
||||
rootCmd.AddCommand(newCheckCmd())
|
||||
rootCmd.AddCommand(newClearCmd())
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// expandPaths expands the given paths, reading from stdin if "-" is present
|
||||
func expandPaths(args []string) ([]string, error) {
|
||||
var paths []string
|
||||
for _, arg := range args {
|
||||
if arg == "-" {
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line != "" {
|
||||
paths = append(paths, line)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("reading stdin: %w", err)
|
||||
}
|
||||
} else {
|
||||
paths = append(paths, arg)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
@ -56,85 +123,115 @@ func main() {
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func newSumCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "sum",
|
||||
Short: "Checksum maintenance operations",
|
||||
}
|
||||
cmd := &cobra.Command{
|
||||
Use: "sum",
|
||||
Short: "Checksum maintenance operations",
|
||||
}
|
||||
|
||||
add := &cobra.Command{
|
||||
Use: "add <path>...",
|
||||
Short: "Write checksums for files missing them",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
for _, p := range a {
|
||||
if err := ProcessSumAdd(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
add := &cobra.Command{
|
||||
Use: "add <path>... (use - to read paths from stdin)",
|
||||
Short: "Write checksums for files missing them",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
paths, err := expandPaths(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
for _, p := range paths {
|
||||
if err := ProcessSumAdd(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stats.Print("sum add")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
upd := &cobra.Command{
|
||||
Use: "update <path>...",
|
||||
Short: "Recalculate checksum when file newer than stored sumtime",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
for _, p := range a {
|
||||
if err := ProcessSumUpdate(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
upd := &cobra.Command{
|
||||
Use: "update <path>... (use - to read paths from stdin)",
|
||||
Short: "Recalculate checksum when file newer than stored sumtime",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
paths, err := expandPaths(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
for _, p := range paths {
|
||||
if err := ProcessSumUpdate(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stats.Print("sum update")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cmd.AddCommand(add, upd)
|
||||
return cmd
|
||||
cmd.AddCommand(add, upd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func ProcessSumAdd(dir string) error {
|
||||
return walkAndProcess(dir, func(p string, _ os.FileInfo) error { return writeChecksumAndTime(p) })
|
||||
func ProcessSumAdd(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Adding checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
if hasXattr(p, checksumKey) {
|
||||
atomic.AddInt64(&s.FilesSkipped, 1)
|
||||
return nil
|
||||
}
|
||||
if err := writeChecksumAndTime(p, info, s); err != nil {
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func ProcessSumUpdate(dir string) error {
|
||||
return walkAndProcess(dir, func(p string, info os.FileInfo) error {
|
||||
t, err := readSumTime(p)
|
||||
if err != nil || info.ModTime().After(t) {
|
||||
return writeChecksumAndTime(p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
func ProcessSumUpdate(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Updating checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
t, err := readSumTime(p)
|
||||
if err != nil || info.ModTime().After(t) {
|
||||
if err := writeChecksumAndTime(p, info, s); err != nil {
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
atomic.AddInt64(&s.FilesSkipped, 1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func writeChecksumAndTime(path string) error {
|
||||
hash, err := fileMultihash(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := xattr.Set(path, checksumKey, hash); err != nil {
|
||||
return fmt.Errorf("set checksum attr: %w", err)
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("%s %s written\n", path, hash)
|
||||
}
|
||||
func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
|
||||
hash, err := fileMultihash(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := xattr.Set(path, checksumKey, hash); err != nil {
|
||||
return fmt.Errorf("set checksum attr: %w", err)
|
||||
}
|
||||
if verbose && !quiet {
|
||||
fmt.Printf("%s %s written\n", path, hash)
|
||||
}
|
||||
|
||||
ts := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
|
||||
return fmt.Errorf("set sumtime attr: %w", err)
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("%s %s written\n", path, ts)
|
||||
}
|
||||
return nil
|
||||
ts := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
|
||||
return fmt.Errorf("set sumtime attr: %w", err)
|
||||
}
|
||||
if verbose && !quiet {
|
||||
fmt.Printf("%s %s written\n", path, ts)
|
||||
}
|
||||
|
||||
atomic.AddInt64(&stats.FilesProcessed, 1)
|
||||
atomic.AddInt64(&stats.BytesProcessed, info.Size())
|
||||
return nil
|
||||
}
|
||||
|
||||
func readSumTime(path string) (time.Time, error) {
|
||||
b, err := xattr.Get(path, sumTimeKey)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.Parse(time.RFC3339Nano, string(b))
|
||||
b, err := xattr.Get(path, sumTimeKey)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.Parse(time.RFC3339Nano, string(b))
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
@ -142,40 +239,56 @@ func readSumTime(path string) (time.Time, error) {
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func newClearCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "clear <path>...",
|
||||
Short: "Remove checksum xattrs from tree",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
for _, p := range a {
|
||||
if err := ProcessClear(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return &cobra.Command{
|
||||
Use: "clear <path>... (use - to read paths from stdin)",
|
||||
Short: "Remove checksum xattrs from tree",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
paths, err := expandPaths(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
for _, p := range paths {
|
||||
if err := ProcessClear(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stats.Print("clear")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ProcessClear(dir string) error {
|
||||
return walkAndProcess(dir, func(p string, _ os.FileInfo) error {
|
||||
for _, k := range []string{checksumKey, sumTimeKey} {
|
||||
v, err := xattr.Get(p, k)
|
||||
if err != nil {
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("%s %s removed\n", p, string(v))
|
||||
}
|
||||
if err := xattr.Remove(p, k); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
func ProcessClear(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Clearing checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
cleared := false
|
||||
for _, k := range []string{checksumKey, sumTimeKey} {
|
||||
v, err := xattr.Get(p, k)
|
||||
if err != nil {
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
continue
|
||||
}
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
if verbose && !quiet {
|
||||
fmt.Printf("%s %s removed\n", p, string(v))
|
||||
}
|
||||
if err := xattr.Remove(p, k); err != nil {
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
cleared = true
|
||||
}
|
||||
if cleared {
|
||||
atomic.AddInt64(&s.FilesProcessed, 1)
|
||||
atomic.AddInt64(&s.BytesProcessed, info.Size())
|
||||
} else {
|
||||
atomic.AddInt64(&s.FilesSkipped, 1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
@ -183,158 +296,247 @@ func ProcessClear(dir string) error {
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func newCheckCmd() *cobra.Command {
|
||||
var cont bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "check <path>...",
|
||||
Short: "Verify stored checksums",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
for _, p := range a {
|
||||
if err := ProcessCheck(p, cont); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file")
|
||||
return cmd
|
||||
var cont bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "check <path>... (use - to read paths from stdin)",
|
||||
Short: "Verify stored checksums",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(_ *cobra.Command, a []string) error {
|
||||
paths, err := expandPaths(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
var finalErr error
|
||||
for _, p := range paths {
|
||||
if err := ProcessCheck(p, cont, stats); err != nil {
|
||||
if cont {
|
||||
finalErr = err
|
||||
} else {
|
||||
stats.Print("check")
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
stats.Print("check")
|
||||
return finalErr
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func ProcessCheck(dir string, cont bool) error {
|
||||
fail := errors.New("verification failed")
|
||||
bad := false
|
||||
func ProcessCheck(dir string, cont bool, stats *Stats) error {
|
||||
fail := errors.New("verification failed")
|
||||
bad := false
|
||||
|
||||
err := walkAndProcess(dir, func(p string, _ os.FileInfo) error {
|
||||
exp, err := xattr.Get(p, checksumKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
bad = true
|
||||
if verbose {
|
||||
fmt.Printf("%s <none> ERROR\n", p)
|
||||
}
|
||||
if cont {
|
||||
return nil
|
||||
}
|
||||
return fail
|
||||
}
|
||||
return err
|
||||
}
|
||||
err := walkAndProcess(dir, stats, "Verifying checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
exp, err := xattr.Get(p, checksumKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
bad = true
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
if verbose && !quiet {
|
||||
fmt.Printf("%s <none> ERROR\n", p)
|
||||
}
|
||||
if cont {
|
||||
return nil
|
||||
}
|
||||
return fail
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
act, err := fileMultihash(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ok := bytes.Equal(exp, act)
|
||||
if !ok {
|
||||
bad = true
|
||||
}
|
||||
if verbose {
|
||||
status := "OK"
|
||||
if !ok {
|
||||
status = "ERROR"
|
||||
}
|
||||
fmt.Printf("%s %s %s\n", p, act, status)
|
||||
}
|
||||
if !ok && !cont {
|
||||
return fail
|
||||
}
|
||||
return nil
|
||||
})
|
||||
act, err := fileMultihash(p)
|
||||
if err != nil {
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
ok := bytes.Equal(exp, act)
|
||||
if !ok {
|
||||
bad = true
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
} else {
|
||||
atomic.AddInt64(&s.FilesProcessed, 1)
|
||||
atomic.AddInt64(&s.BytesProcessed, info.Size())
|
||||
}
|
||||
if verbose && !quiet {
|
||||
status := "OK"
|
||||
if !ok {
|
||||
status = "ERROR"
|
||||
}
|
||||
fmt.Printf("%s %s %s\n", p, act, status)
|
||||
}
|
||||
if !ok && !cont {
|
||||
return fail
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, fail) {
|
||||
return fail
|
||||
}
|
||||
return err
|
||||
}
|
||||
if bad {
|
||||
return fail
|
||||
}
|
||||
return nil
|
||||
if err != nil {
|
||||
if errors.Is(err, fail) {
|
||||
return fail
|
||||
}
|
||||
return err
|
||||
}
|
||||
if bad {
|
||||
return fail
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Helpers
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func walkAndProcess(root string, fn func(string, os.FileInfo) error) error {
|
||||
root = filepath.Clean(root)
|
||||
return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// countFiles counts the total number of regular files that will be processed
|
||||
func countFiles(root string) int64 {
|
||||
var count int64
|
||||
root = filepath.Clean(root)
|
||||
filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
if shouldExclude(rel, info) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
// skip symlinks entirely
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if verbose {
|
||||
log.Printf("skip symlink %s", p)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func walkAndProcess(root string, stats *Stats, description string, fn func(string, os.FileInfo, *Stats) error) error {
|
||||
root = filepath.Clean(root)
|
||||
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
if shouldExclude(rel, info) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Count files first for progress bar
|
||||
total := countFiles(root)
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
if verbose {
|
||||
log.Printf("skip non-regular %s", p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fn(p, info)
|
||||
})
|
||||
// Create progress bar
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
bar = progressbar.NewOptions64(total,
|
||||
progressbar.OptionSetDescription(description),
|
||||
progressbar.OptionSetWriter(os.Stderr),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionSetItsString("files"),
|
||||
progressbar.OptionThrottle(250*time.Millisecond),
|
||||
progressbar.OptionShowElapsedTimeOnFinish(),
|
||||
progressbar.OptionSetPredictTime(true),
|
||||
progressbar.OptionFullWidth(),
|
||||
progressbar.OptionSetTheme(progressbar.Theme{
|
||||
Saucer: "=",
|
||||
SaucerHead: ">",
|
||||
SaucerPadding: " ",
|
||||
BarStart: "[",
|
||||
BarEnd: "]",
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// skip symlinks entirely
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if verbose && !quiet {
|
||||
log.Printf("skip symlink %s", p)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
if shouldExclude(rel, info) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
if verbose && !quiet {
|
||||
log.Printf("skip non-regular %s", p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fnErr := fn(p, info, stats)
|
||||
if bar != nil {
|
||||
bar.Add(1)
|
||||
}
|
||||
return fnErr
|
||||
})
|
||||
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func shouldExclude(rel string, info os.FileInfo) bool {
|
||||
if rel == "." || rel == "" {
|
||||
return false
|
||||
}
|
||||
if excludeDotfiles {
|
||||
for _, part := range strings.Split(rel, string(os.PathSeparator)) {
|
||||
if strings.HasPrefix(part, ".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, pat := range excludePatterns {
|
||||
if ok, _ := doublestar.PathMatch(pat, rel); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
if rel == "." || rel == "" {
|
||||
return false
|
||||
}
|
||||
if excludeDotfiles {
|
||||
for _, part := range strings.Split(rel, string(os.PathSeparator)) {
|
||||
if strings.HasPrefix(part, ".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, pat := range excludePatterns {
|
||||
if ok, _ := doublestar.PathMatch(pat, rel); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasXattr(path, key string) bool {
|
||||
_, err := xattr.Get(path, key)
|
||||
return err == nil
|
||||
_, err := xattr.Get(path, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func fileMultihash(path string) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(base58.Encode(mh)), nil
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(base58.Encode(mh)), nil
|
||||
}
|
||||
|
||||
@ -30,13 +30,17 @@ func writeFile(t *testing.T, root, name, content string) string {
|
||||
return p
|
||||
}
|
||||
|
||||
func newTestStats() *Stats {
|
||||
return &Stats{StartTime: time.Now()}
|
||||
}
|
||||
|
||||
func TestSumAddAndUpdate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
skipIfNoXattr(t, dir)
|
||||
|
||||
f := writeFile(t, dir, "a.txt", "hello")
|
||||
|
||||
if err := ProcessSumAdd(dir); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(f, checksumKey); err != nil {
|
||||
@ -49,7 +53,7 @@ func TestSumAddAndUpdate(t *testing.T) {
|
||||
now := time.Now().Add(2 * time.Second)
|
||||
os.Chtimes(f, now, now)
|
||||
|
||||
if err := ProcessSumUpdate(dir); err != nil {
|
||||
if err := ProcessSumUpdate(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
tsb2, _ := xattr.Get(f, sumTimeKey)
|
||||
@ -64,17 +68,17 @@ func TestProcessCheckIntegration(t *testing.T) {
|
||||
skipIfNoXattr(t, dir)
|
||||
|
||||
writeFile(t, dir, "b.txt", "world")
|
||||
if err := ProcessSumAdd(dir); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
if err := ProcessCheck(dir, false); err != nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err != nil {
|
||||
t.Fatalf("check ok: %v", err)
|
||||
}
|
||||
|
||||
f := filepath.Join(dir, "b.txt")
|
||||
os.WriteFile(f, []byte("corrupt"), 0o644)
|
||||
|
||||
if err := ProcessCheck(dir, false); err == nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
|
||||
t.Fatalf("expected mismatch error, got nil")
|
||||
}
|
||||
}
|
||||
@ -84,11 +88,11 @@ func TestClearRemovesAttrs(t *testing.T) {
|
||||
skipIfNoXattr(t, dir)
|
||||
|
||||
f := writeFile(t, dir, "c.txt", "data")
|
||||
if err := ProcessSumAdd(dir); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
|
||||
if err := ProcessClear(dir); err != nil {
|
||||
if err := ProcessClear(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("clear: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(f, checksumKey); err == nil {
|
||||
@ -113,7 +117,7 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
|
||||
excludePatterns = []string{"*.me"}
|
||||
defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
|
||||
|
||||
if err := ProcessSumAdd(dir); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add with excludes: %v", err)
|
||||
}
|
||||
|
||||
@ -139,7 +143,7 @@ func TestSkipBrokenSymlink(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not error and should not create xattrs on link
|
||||
if err := ProcessSumAdd(dir); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("ProcessSumAdd with symlink: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(link, checksumKey); err == nil {
|
||||
@ -155,13 +159,13 @@ func TestPermissionErrors(t *testing.T) {
|
||||
os.Chmod(secret, 0o000)
|
||||
defer os.Chmod(secret, 0o644)
|
||||
|
||||
if err := ProcessSumAdd(dir); err == nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error, got nil")
|
||||
}
|
||||
if err := ProcessSumUpdate(dir); err == nil {
|
||||
if err := ProcessSumUpdate(dir, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error on update, got nil")
|
||||
}
|
||||
if err := ProcessCheck(dir, false); err == nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error on check, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
6
go.mod
6
go.mod
@ -7,6 +7,7 @@ require (
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/multiformats/go-multihash v0.2.3
|
||||
github.com/pkg/xattr v0.4.10
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
)
|
||||
|
||||
@ -14,10 +15,13 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/multiformats/go-varint v0.0.6 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/sys v0.29.0 // indirect
|
||||
golang.org/x/term v0.28.0 // indirect
|
||||
lukechampine.com/blake3 v1.1.6 // indirect
|
||||
)
|
||||
|
||||
23
go.sum
23
go.sum
@ -1,13 +1,21 @@
|
||||
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
|
||||
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
||||
@ -16,19 +24,30 @@ github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2
|
||||
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
|
||||
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
|
||||
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
|
||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
|
||||
lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user