Compare commits
No commits in common. "main" and "improvement" have entirely different histories.
main
...
improvemen
186
attrsum.go
186
attrsum.go
@ -99,10 +99,8 @@ func main() {
|
||||
// expandPaths expands the given paths, reading from stdin if "-" is present
|
||||
func expandPaths(args []string) ([]string, error) {
|
||||
var paths []string
|
||||
readFromStdin := false
|
||||
for _, arg := range args {
|
||||
if arg == "-" {
|
||||
readFromStdin = true
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
@ -117,12 +115,6 @@ func expandPaths(args []string) ([]string, error) {
|
||||
paths = append(paths, arg)
|
||||
}
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
if readFromStdin {
|
||||
return nil, errors.New("no paths provided on stdin")
|
||||
}
|
||||
return nil, errors.New("no paths provided")
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
@ -146,25 +138,11 @@ func newSumCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
total, err := countFilesMultiple(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bar = newProgressBar(total, "Adding checksums")
|
||||
}
|
||||
for _, p := range paths {
|
||||
if err := ProcessSumAdd(p, stats, bar); err != nil {
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
if err := ProcessSumAdd(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
stats.Print("sum add")
|
||||
return nil
|
||||
},
|
||||
@ -180,25 +158,11 @@ func newSumCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
total, err := countFilesMultiple(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bar = newProgressBar(total, "Updating checksums")
|
||||
}
|
||||
for _, p := range paths {
|
||||
if err := ProcessSumUpdate(p, stats, bar); err != nil {
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
if err := ProcessSumUpdate(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
stats.Print("sum update")
|
||||
return nil
|
||||
},
|
||||
@ -208,8 +172,8 @@ func newSumCmd() *cobra.Command {
|
||||
return cmd
|
||||
}
|
||||
|
||||
func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
|
||||
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
|
||||
func ProcessSumAdd(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Adding checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
if hasXattr(p, checksumKey) {
|
||||
atomic.AddInt64(&s.FilesSkipped, 1)
|
||||
return nil
|
||||
@ -222,8 +186,8 @@ func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error
|
||||
})
|
||||
}
|
||||
|
||||
func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
|
||||
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
|
||||
func ProcessSumUpdate(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Updating checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
t, err := readSumTime(p)
|
||||
if err != nil || info.ModTime().After(t) {
|
||||
if err := writeChecksumAndTime(p, info, s); err != nil {
|
||||
@ -238,23 +202,10 @@ func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) er
|
||||
}
|
||||
|
||||
func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
|
||||
// Record mtime before hashing to detect modifications during hash
|
||||
mtimeBefore := info.ModTime()
|
||||
|
||||
hash, bytesRead, err := fileMultihash(path)
|
||||
hash, err := fileMultihash(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if file was modified during hashing
|
||||
infoAfter, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat after hash: %w", err)
|
||||
}
|
||||
if !infoAfter.ModTime().Equal(mtimeBefore) {
|
||||
return fmt.Errorf("%s: file modified during checksum calculation", path)
|
||||
}
|
||||
|
||||
if err := xattr.Set(path, checksumKey, hash); err != nil {
|
||||
return fmt.Errorf("set checksum attr: %w", err)
|
||||
}
|
||||
@ -262,9 +213,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
|
||||
fmt.Printf("%s %s written\n", path, hash)
|
||||
}
|
||||
|
||||
// Store the file's mtime as sumtime (not wall-clock time)
|
||||
// This makes update comparisons semantically correct
|
||||
ts := mtimeBefore.UTC().Format(time.RFC3339Nano)
|
||||
ts := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
|
||||
return fmt.Errorf("set sumtime attr: %w", err)
|
||||
}
|
||||
@ -273,7 +222,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
|
||||
}
|
||||
|
||||
atomic.AddInt64(&stats.FilesProcessed, 1)
|
||||
atomic.AddInt64(&stats.BytesProcessed, bytesRead)
|
||||
atomic.AddInt64(&stats.BytesProcessed, info.Size())
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -300,33 +249,19 @@ func newClearCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
total, err := countFilesMultiple(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bar = newProgressBar(total, "Clearing checksums")
|
||||
}
|
||||
for _, p := range paths {
|
||||
if err := ProcessClear(p, stats, bar); err != nil {
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
if err := ProcessClear(p, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
stats.Print("clear")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ProcessClear(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
|
||||
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
|
||||
func ProcessClear(dir string, stats *Stats) error {
|
||||
return walkAndProcess(dir, stats, "Clearing checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
cleared := false
|
||||
for _, k := range []string{checksumKey, sumTimeKey} {
|
||||
v, err := xattr.Get(p, k)
|
||||
@ -372,31 +307,17 @@ func newCheckCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{StartTime: time.Now()}
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
total, err := countFilesMultiple(paths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bar = newProgressBar(total, "Verifying checksums")
|
||||
}
|
||||
var finalErr error
|
||||
for _, p := range paths {
|
||||
if err := ProcessCheck(p, cont, stats, bar); err != nil {
|
||||
if err := ProcessCheck(p, cont, stats); err != nil {
|
||||
if cont {
|
||||
finalErr = err
|
||||
} else {
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
stats.Print("check")
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
stats.Print("check")
|
||||
return finalErr
|
||||
},
|
||||
@ -405,15 +326,15 @@ func newCheckCmd() *cobra.Command {
|
||||
return cmd
|
||||
}
|
||||
|
||||
func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.ProgressBar) error {
|
||||
func ProcessCheck(dir string, cont bool, stats *Stats) error {
|
||||
fail := errors.New("verification failed")
|
||||
// Track initial failed count to detect failures during this walk
|
||||
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
|
||||
bad := false
|
||||
|
||||
err := walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
|
||||
err := walkAndProcess(dir, stats, "Verifying checksums", func(p string, info os.FileInfo, s *Stats) error {
|
||||
exp, err := xattr.Get(p, checksumKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
bad = true
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
if verbose && !quiet {
|
||||
fmt.Printf("%s <none> ERROR\n", p)
|
||||
@ -426,17 +347,18 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
|
||||
return err
|
||||
}
|
||||
|
||||
act, bytesRead, err := fileMultihash(p)
|
||||
act, err := fileMultihash(p)
|
||||
if err != nil {
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
return err
|
||||
}
|
||||
ok := bytes.Equal(exp, act)
|
||||
if !ok {
|
||||
bad = true
|
||||
atomic.AddInt64(&s.FilesFailed, 1)
|
||||
} else {
|
||||
atomic.AddInt64(&s.FilesProcessed, 1)
|
||||
atomic.AddInt64(&s.BytesProcessed, bytesRead)
|
||||
atomic.AddInt64(&s.BytesProcessed, info.Size())
|
||||
}
|
||||
if verbose && !quiet {
|
||||
status := "OK"
|
||||
@ -457,8 +379,7 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Check if any failures occurred during this walk
|
||||
if atomic.LoadInt64(&stats.FilesFailed) > initialFailed {
|
||||
if bad {
|
||||
return fail
|
||||
}
|
||||
return nil
|
||||
@ -469,16 +390,17 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// countFiles counts the total number of regular files that will be processed
|
||||
func countFiles(root string) (int64, error) {
|
||||
func countFiles(root string) int64 {
|
||||
var count int64
|
||||
root = filepath.Clean(root)
|
||||
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
// Skip symlinks - note: filepath.Walk uses Lstat, so symlinks are
|
||||
// reported as ModeSymlink, never as directories. Walk doesn't follow them.
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
@ -497,25 +419,19 @@ func countFiles(root string) (int64, error) {
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
return count, err
|
||||
return count
|
||||
}
|
||||
|
||||
// countFilesMultiple counts files across multiple roots
|
||||
func countFilesMultiple(roots []string) (int64, error) {
|
||||
var total int64
|
||||
for _, root := range roots {
|
||||
count, err := countFiles(root)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += count
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
func walkAndProcess(root string, stats *Stats, description string, fn func(string, os.FileInfo, *Stats) error) error {
|
||||
root = filepath.Clean(root)
|
||||
|
||||
// newProgressBar creates a new progress bar with standard options
|
||||
func newProgressBar(total int64, description string) *progressbar.ProgressBar {
|
||||
return progressbar.NewOptions64(total,
|
||||
// Count files first for progress bar
|
||||
total := countFiles(root)
|
||||
|
||||
// Create progress bar
|
||||
var bar *progressbar.ProgressBar
|
||||
if !quiet {
|
||||
bar = progressbar.NewOptions64(total,
|
||||
progressbar.OptionSetDescription(description),
|
||||
progressbar.OptionSetWriter(os.Stderr),
|
||||
progressbar.OptionShowCount(),
|
||||
@ -533,22 +449,21 @@ func newProgressBar(total int64, description string) *progressbar.ProgressBar {
|
||||
BarEnd: "]",
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn func(string, os.FileInfo, *Stats) error) error {
|
||||
root = filepath.Clean(root)
|
||||
}
|
||||
|
||||
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip symlinks - filepath.Walk uses Lstat, so symlinks are reported
|
||||
// as ModeSymlink, never as directories. Walk doesn't follow them.
|
||||
// skip symlinks entirely
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if verbose && !quiet {
|
||||
log.Printf("skip symlink %s", p)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -577,6 +492,10 @@ func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn
|
||||
return fnErr
|
||||
})
|
||||
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -604,21 +523,20 @@ func hasXattr(path, key string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func fileMultihash(path string) (hash []byte, bytesRead int64, err error) {
|
||||
func fileMultihash(path string) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
h := sha256.New()
|
||||
bytesRead, err = io.Copy(h, f)
|
||||
if err != nil {
|
||||
return nil, bytesRead, err
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return nil, bytesRead, err
|
||||
return nil, err
|
||||
}
|
||||
return []byte(base58.Encode(mh)), bytesRead, nil
|
||||
return []byte(base58.Encode(mh)), nil
|
||||
}
|
||||
|
||||
@ -40,7 +40,7 @@ func TestSumAddAndUpdate(t *testing.T) {
|
||||
|
||||
f := writeFile(t, dir, "a.txt", "hello")
|
||||
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(f, checksumKey); err != nil {
|
||||
@ -53,7 +53,7 @@ func TestSumAddAndUpdate(t *testing.T) {
|
||||
now := time.Now().Add(2 * time.Second)
|
||||
os.Chtimes(f, now, now)
|
||||
|
||||
if err := ProcessSumUpdate(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumUpdate(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
tsb2, _ := xattr.Get(f, sumTimeKey)
|
||||
@ -68,17 +68,17 @@ func TestProcessCheckIntegration(t *testing.T) {
|
||||
skipIfNoXattr(t, dir)
|
||||
|
||||
writeFile(t, dir, "b.txt", "world")
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
if err := ProcessCheck(dir, false, newTestStats(), nil); err != nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err != nil {
|
||||
t.Fatalf("check ok: %v", err)
|
||||
}
|
||||
|
||||
f := filepath.Join(dir, "b.txt")
|
||||
os.WriteFile(f, []byte("corrupt"), 0o644)
|
||||
|
||||
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
|
||||
t.Fatalf("expected mismatch error, got nil")
|
||||
}
|
||||
}
|
||||
@ -88,11 +88,11 @@ func TestClearRemovesAttrs(t *testing.T) {
|
||||
skipIfNoXattr(t, dir)
|
||||
|
||||
f := writeFile(t, dir, "c.txt", "data")
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add: %v", err)
|
||||
}
|
||||
|
||||
if err := ProcessClear(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessClear(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("clear: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(f, checksumKey); err == nil {
|
||||
@ -117,7 +117,7 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
|
||||
excludePatterns = []string{"*.me"}
|
||||
defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
|
||||
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("add with excludes: %v", err)
|
||||
}
|
||||
|
||||
@ -143,7 +143,7 @@ func TestSkipBrokenSymlink(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not error and should not create xattrs on link
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
|
||||
t.Fatalf("ProcessSumAdd with symlink: %v", err)
|
||||
}
|
||||
if _, err := xattr.Get(link, checksumKey); err == nil {
|
||||
@ -159,13 +159,13 @@ func TestPermissionErrors(t *testing.T) {
|
||||
os.Chmod(secret, 0o000)
|
||||
defer os.Chmod(secret, 0o644)
|
||||
|
||||
if err := ProcessSumAdd(dir, newTestStats(), nil); err == nil {
|
||||
if err := ProcessSumAdd(dir, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error, got nil")
|
||||
}
|
||||
if err := ProcessSumUpdate(dir, newTestStats(), nil); err == nil {
|
||||
if err := ProcessSumUpdate(dir, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error on update, got nil")
|
||||
}
|
||||
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
|
||||
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
|
||||
t.Fatalf("expected permission error on check, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user