sneak/attrsum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: sneak:main
Branches Tags
sneak:main
sneak:improvement
sneak:1.0.0
..
compare: sneak:improvement
Branches Tags
sneak:main
sneak:improvement
sneak:1.0.0

No commits in common. "main" and "improvement" have entirely different histories.
main ... improvemen

2 changed files with 81 additions and 163 deletions
Show Stats Expand all files Collapse all files

186
attrsum.go
View File

@ -99,10 +99,8 @@ func main() {
// expandPaths expands the given paths, reading from stdin if "-" is present // expandPaths expands the given paths, reading from stdin if "-" is present
func expandPaths(args []string) ([]string, error) { func expandPaths(args []string) ([]string, error) {
var paths []string var paths []string
readFromStdin := false
for _, arg := range args { for _, arg := range args {
if arg == "-" { if arg == "-" {
readFromStdin = true
scanner := bufio.NewScanner(os.Stdin) scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() { for scanner.Scan() {
line := strings.TrimSpace(scanner.Text()) line := strings.TrimSpace(scanner.Text())
@ -117,12 +115,6 @@ func expandPaths(args []string) ([]string, error) {
paths = append(paths, arg) paths = append(paths, arg)
} }
} }
if len(paths) == 0 {
if readFromStdin {
return nil, errors.New("no paths provided on stdin")
}
return nil, errors.New("no paths provided")
}
return paths, nil return paths, nil
} }
@ -146,25 +138,11 @@ func newSumCmd() *cobra.Command {
return err return err
} }
stats := &Stats{StartTime: time.Now()} stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Adding checksums")
}
for _, p := range paths { for _, p := range paths {
if err := ProcessSumAdd(p, stats, bar); err != nil { if err := ProcessSumAdd(p, stats); err != nil {
if bar != nil {
bar.Finish()
}
return err return err
} }
} }
if bar != nil {
bar.Finish()
}
stats.Print("sum add") stats.Print("sum add")
return nil return nil
}, },
@ -180,25 +158,11 @@ func newSumCmd() *cobra.Command {
return err return err
} }
stats := &Stats{StartTime: time.Now()} stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Updating checksums")
}
for _, p := range paths { for _, p := range paths {
if err := ProcessSumUpdate(p, stats, bar); err != nil { if err := ProcessSumUpdate(p, stats); err != nil {
if bar != nil {
bar.Finish()
}
return err return err
} }
} }
if bar != nil {
bar.Finish()
}
stats.Print("sum update") stats.Print("sum update")
return nil return nil
}, },
@ -208,8 +172,8 @@ func newSumCmd() *cobra.Command {
return cmd return cmd
} }
func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error { func ProcessSumAdd(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error { return walkAndProcess(dir, stats, "Adding checksums", func(p string, info os.FileInfo, s *Stats) error {
if hasXattr(p, checksumKey) { if hasXattr(p, checksumKey) {
atomic.AddInt64(&s.FilesSkipped, 1) atomic.AddInt64(&s.FilesSkipped, 1)
return nil return nil
@ -222,8 +186,8 @@ func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error
}) })
} }
func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) error { func ProcessSumUpdate(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error { return walkAndProcess(dir, stats, "Updating checksums", func(p string, info os.FileInfo, s *Stats) error {
t, err := readSumTime(p) t, err := readSumTime(p)
if err != nil || info.ModTime().After(t) { if err != nil || info.ModTime().After(t) {
if err := writeChecksumAndTime(p, info, s); err != nil { if err := writeChecksumAndTime(p, info, s); err != nil {
@ -238,23 +202,10 @@ func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) er
} }
func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error { func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
// Record mtime before hashing to detect modifications during hash hash, err := fileMultihash(path)
mtimeBefore := info.ModTime()
hash, bytesRead, err := fileMultihash(path)
if err != nil { if err != nil {
return err return err
} }
// Check if file was modified during hashing
infoAfter, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("stat after hash: %w", err)
}
if !infoAfter.ModTime().Equal(mtimeBefore) {
return fmt.Errorf("%s: file modified during checksum calculation", path)
}
if err := xattr.Set(path, checksumKey, hash); err != nil { if err := xattr.Set(path, checksumKey, hash); err != nil {
return fmt.Errorf("set checksum attr: %w", err) return fmt.Errorf("set checksum attr: %w", err)
} }
@ -262,9 +213,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
fmt.Printf("%s %s written\n", path, hash) fmt.Printf("%s %s written\n", path, hash)
} }
// Store the file's mtime as sumtime (not wall-clock time) ts := time.Now().UTC().Format(time.RFC3339Nano)
// This makes update comparisons semantically correct
ts := mtimeBefore.UTC().Format(time.RFC3339Nano)
if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil { if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
return fmt.Errorf("set sumtime attr: %w", err) return fmt.Errorf("set sumtime attr: %w", err)
} }
@ -273,7 +222,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
} }
atomic.AddInt64(&stats.FilesProcessed, 1) atomic.AddInt64(&stats.FilesProcessed, 1)
atomic.AddInt64(&stats.BytesProcessed, bytesRead) atomic.AddInt64(&stats.BytesProcessed, info.Size())
return nil return nil
} }
@ -300,33 +249,19 @@ func newClearCmd() *cobra.Command {
return err return err
} }
stats := &Stats{StartTime: time.Now()} stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Clearing checksums")
}
for _, p := range paths { for _, p := range paths {
if err := ProcessClear(p, stats, bar); err != nil { if err := ProcessClear(p, stats); err != nil {
if bar != nil {
bar.Finish()
}
return err return err
} }
} }
if bar != nil {
bar.Finish()
}
stats.Print("clear") stats.Print("clear")
return nil return nil
}, },
} }
} }
func ProcessClear(dir string, stats *Stats, bar *progressbar.ProgressBar) error { func ProcessClear(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error { return walkAndProcess(dir, stats, "Clearing checksums", func(p string, info os.FileInfo, s *Stats) error {
cleared := false cleared := false
for _, k := range []string{checksumKey, sumTimeKey} { for _, k := range []string{checksumKey, sumTimeKey} {
v, err := xattr.Get(p, k) v, err := xattr.Get(p, k)
@ -372,31 +307,17 @@ func newCheckCmd() *cobra.Command {
return err return err
} }
stats := &Stats{StartTime: time.Now()} stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Verifying checksums")
}
var finalErr error var finalErr error
for _, p := range paths { for _, p := range paths {
if err := ProcessCheck(p, cont, stats, bar); err != nil { if err := ProcessCheck(p, cont, stats); err != nil {
if cont { if cont {
finalErr = err finalErr = err
} else { } else {
if bar != nil {
bar.Finish()
}
stats.Print("check") stats.Print("check")
return err return err
} }
} }
} }
if bar != nil {
bar.Finish()
}
stats.Print("check") stats.Print("check")
return finalErr return finalErr
}, },
@ -405,15 +326,15 @@ func newCheckCmd() *cobra.Command {
return cmd return cmd
} }
func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.ProgressBar) error { func ProcessCheck(dir string, cont bool, stats *Stats) error {
fail := errors.New("verification failed") fail := errors.New("verification failed")
// Track initial failed count to detect failures during this walk bad := false
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
err := walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error { err := walkAndProcess(dir, stats, "Verifying checksums", func(p string, info os.FileInfo, s *Stats) error {
exp, err := xattr.Get(p, checksumKey) exp, err := xattr.Get(p, checksumKey)
if err != nil { if err != nil {
if errors.Is(err, xattr.ENOATTR) { if errors.Is(err, xattr.ENOATTR) {
bad = true
atomic.AddInt64(&s.FilesFailed, 1) atomic.AddInt64(&s.FilesFailed, 1)
if verbose && !quiet { if verbose && !quiet {
fmt.Printf("%s <none> ERROR\n", p) fmt.Printf("%s <none> ERROR\n", p)
@ -426,17 +347,18 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
return err return err
} }
act, bytesRead, err := fileMultihash(p) act, err := fileMultihash(p)
if err != nil { if err != nil {
atomic.AddInt64(&s.FilesFailed, 1) atomic.AddInt64(&s.FilesFailed, 1)
return err return err
} }
ok := bytes.Equal(exp, act) ok := bytes.Equal(exp, act)
if !ok { if !ok {
bad = true
atomic.AddInt64(&s.FilesFailed, 1) atomic.AddInt64(&s.FilesFailed, 1)
} else { } else {
atomic.AddInt64(&s.FilesProcessed, 1) atomic.AddInt64(&s.FilesProcessed, 1)
atomic.AddInt64(&s.BytesProcessed, bytesRead) atomic.AddInt64(&s.BytesProcessed, info.Size())
} }
if verbose && !quiet { if verbose && !quiet {
status := "OK" status := "OK"
@ -457,8 +379,7 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
} }
return err return err
} }
// Check if any failures occurred during this walk if bad {
if atomic.LoadInt64(&stats.FilesFailed) > initialFailed {
return fail return fail
} }
return nil return nil
@ -469,16 +390,17 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// countFiles counts the total number of regular files that will be processed // countFiles counts the total number of regular files that will be processed
func countFiles(root string) (int64, error) { func countFiles(root string) int64 {
var count int64 var count int64
root = filepath.Clean(root) root = filepath.Clean(root)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return nil
} }
// Skip symlinks - note: filepath.Walk uses Lstat, so symlinks are
// reported as ModeSymlink, never as directories. Walk doesn't follow them.
if info.Mode()&os.ModeSymlink != 0 { if info.Mode()&os.ModeSymlink != 0 {
if info.IsDir() {
return filepath.SkipDir
}
return nil return nil
} }
rel, _ := filepath.Rel(root, p) rel, _ := filepath.Rel(root, p)
@ -497,25 +419,19 @@ func countFiles(root string) (int64, error) {
count++ count++
return nil return nil
}) })
return count, err return count
} }
// countFilesMultiple counts files across multiple roots func walkAndProcess(root string, stats *Stats, description string, fn func(string, os.FileInfo, *Stats) error) error {
func countFilesMultiple(roots []string) (int64, error) { root = filepath.Clean(root)
var total int64
for _, root := range roots {
count, err := countFiles(root)
if err != nil {
return total, err
}
total += count
}
return total, nil
}
// newProgressBar creates a new progress bar with standard options // Count files first for progress bar
func newProgressBar(total int64, description string) *progressbar.ProgressBar { total := countFiles(root)
return progressbar.NewOptions64(total,
// Create progress bar
var bar *progressbar.ProgressBar
if !quiet {
bar = progressbar.NewOptions64(total,
progressbar.OptionSetDescription(description), progressbar.OptionSetDescription(description),
progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowCount(), progressbar.OptionShowCount(),
@ -533,22 +449,21 @@ func newProgressBar(total int64, description string) *progressbar.ProgressBar {
BarEnd: "]", BarEnd: "]",
}), }),
) )
} }
func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn func(string, os.FileInfo, *Stats) error) error {
root = filepath.Clean(root)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }
// Skip symlinks - filepath.Walk uses Lstat, so symlinks are reported // skip symlinks entirely
// as ModeSymlink, never as directories. Walk doesn't follow them.
if info.Mode()&os.ModeSymlink != 0 { if info.Mode()&os.ModeSymlink != 0 {
if verbose && !quiet { if verbose && !quiet {
log.Printf("skip symlink %s", p) log.Printf("skip symlink %s", p)
} }
if info.IsDir() {
return filepath.SkipDir
}
return nil return nil
} }
@ -577,6 +492,10 @@ func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn
return fnErr return fnErr
}) })
if bar != nil {
bar.Finish()
}
return err return err
} }
@ -604,21 +523,20 @@ func hasXattr(path, key string) bool {
return err == nil return err == nil
} }
func fileMultihash(path string) (hash []byte, bytesRead int64, err error) { func fileMultihash(path string) ([]byte, error) {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return nil, 0, err return nil, err
} }
defer f.Close() defer f.Close()
h := sha256.New() h := sha256.New()
bytesRead, err = io.Copy(h, f) if _, err := io.Copy(h, f); err != nil {
if err != nil { return nil, err
return nil, bytesRead, err
} }
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil { if err != nil {
return nil, bytesRead, err return nil, err
} }
return []byte(base58.Encode(mh)), bytesRead, nil return []byte(base58.Encode(mh)), nil
} }

24
attrsum_test.go
View File

@ -40,7 +40,7 @@ func TestSumAddAndUpdate(t *testing.T) {
f := writeFile(t, dir, "a.txt", "hello") f := writeFile(t, dir, "a.txt", "hello")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil { if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
if _, err := xattr.Get(f, checksumKey); err != nil { if _, err := xattr.Get(f, checksumKey); err != nil {
@ -53,7 +53,7 @@ func TestSumAddAndUpdate(t *testing.T) {
now := time.Now().Add(2 * time.Second) now := time.Now().Add(2 * time.Second)
os.Chtimes(f, now, now) os.Chtimes(f, now, now)
if err := ProcessSumUpdate(dir, newTestStats(), nil); err != nil { if err := ProcessSumUpdate(dir, newTestStats()); err != nil {
t.Fatalf("update: %v", err) t.Fatalf("update: %v", err)
} }
tsb2, _ := xattr.Get(f, sumTimeKey) tsb2, _ := xattr.Get(f, sumTimeKey)
@ -68,17 +68,17 @@ func TestProcessCheckIntegration(t *testing.T) {
skipIfNoXattr(t, dir) skipIfNoXattr(t, dir)
writeFile(t, dir, "b.txt", "world") writeFile(t, dir, "b.txt", "world")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil { if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
if err := ProcessCheck(dir, false, newTestStats(), nil); err != nil { if err := ProcessCheck(dir, false, newTestStats()); err != nil {
t.Fatalf("check ok: %v", err) t.Fatalf("check ok: %v", err)
} }
f := filepath.Join(dir, "b.txt") f := filepath.Join(dir, "b.txt")
os.WriteFile(f, []byte("corrupt"), 0o644) os.WriteFile(f, []byte("corrupt"), 0o644)
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil { if err := ProcessCheck(dir, false, newTestStats()); err == nil {
t.Fatalf("expected mismatch error, got nil") t.Fatalf("expected mismatch error, got nil")
} }
} }
@ -88,11 +88,11 @@ func TestClearRemovesAttrs(t *testing.T) {
skipIfNoXattr(t, dir) skipIfNoXattr(t, dir)
f := writeFile(t, dir, "c.txt", "data") f := writeFile(t, dir, "c.txt", "data")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil { if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
if err := ProcessClear(dir, newTestStats(), nil); err != nil { if err := ProcessClear(dir, newTestStats()); err != nil {
t.Fatalf("clear: %v", err) t.Fatalf("clear: %v", err)
} }
if _, err := xattr.Get(f, checksumKey); err == nil { if _, err := xattr.Get(f, checksumKey); err == nil {
@ -117,7 +117,7 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
excludePatterns = []string{"*.me"} excludePatterns = []string{"*.me"}
defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }() defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil { if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add with excludes: %v", err) t.Fatalf("add with excludes: %v", err)
} }
@ -143,7 +143,7 @@ func TestSkipBrokenSymlink(t *testing.T) {
} }
// Should not error and should not create xattrs on link // Should not error and should not create xattrs on link
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil { if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("ProcessSumAdd with symlink: %v", err) t.Fatalf("ProcessSumAdd with symlink: %v", err)
} }
if _, err := xattr.Get(link, checksumKey); err == nil { if _, err := xattr.Get(link, checksumKey); err == nil {
@ -159,13 +159,13 @@ func TestPermissionErrors(t *testing.T) {
os.Chmod(secret, 0o000) os.Chmod(secret, 0o000)
defer os.Chmod(secret, 0o644) defer os.Chmod(secret, 0o644)
if err := ProcessSumAdd(dir, newTestStats(), nil); err == nil { if err := ProcessSumAdd(dir, newTestStats()); err == nil {
t.Fatalf("expected permission error, got nil") t.Fatalf("expected permission error, got nil")
} }
if err := ProcessSumUpdate(dir, newTestStats(), nil); err == nil { if err := ProcessSumUpdate(dir, newTestStats()); err == nil {
t.Fatalf("expected permission error on update, got nil") t.Fatalf("expected permission error on update, got nil")
} }
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil { if err := ProcessCheck(dir, false, newTestStats()); err == nil {
t.Fatalf("expected permission error on check, got nil") t.Fatalf("expected permission error on check, got nil")
} }
} }