Track actual bytes read instead of stale file size

fileMultihash now returns the number of bytes actually read during
hashing. This ensures BytesProcessed reflects the true amount of
data processed, not a potentially stale size from the initial walk.

attrsum.go

@@ -99,8 +99,10 @@ func main() {
 // expandPaths expands the given paths, reading from stdin if "-" is present
 func expandPaths(args []string) ([]string, error) {
 	var paths []string
+	readFromStdin := false
 	for _, arg := range args {
 		if arg == "-" {
+			readFromStdin = true
 			scanner := bufio.NewScanner(os.Stdin)
 			for scanner.Scan() {
 				line := strings.TrimSpace(scanner.Text())
@@ -115,6 +117,12 @@ func expandPaths(args []string) ([]string, error) {
 			paths = append(paths, arg)
 		}
 	}
+	if len(paths) == 0 {
+		if readFromStdin {
+			return nil, errors.New("no paths provided on stdin")
+		}
+		return nil, errors.New("no paths provided")
+	}
 	return paths, nil
 }
 
@@ -138,10 +146,24 @@ func newSumCmd() *cobra.Command {
 				return err
 			}
 			stats := &Stats{StartTime: time.Now()}
-			for _, p := range paths {
-				if err := ProcessSumAdd(p, stats); err != nil {
+			var bar *progressbar.ProgressBar
+			if !quiet {
+				total, err := countFilesMultiple(paths)
+				if err != nil {
 					return err
 				}
+				bar = newProgressBar(total, "Adding checksums")
+			}
+			for _, p := range paths {
+				if err := ProcessSumAdd(p, stats, bar); err != nil {
+					if bar != nil {
+						bar.Finish()
+					}
+					return err
+				}
+			}
+			if bar != nil {
+				bar.Finish()
 			}
 			stats.Print("sum add")
 			return nil
@@ -158,10 +180,24 @@ func newSumCmd() *cobra.Command {
 				return err
 			}
 			stats := &Stats{StartTime: time.Now()}
-			for _, p := range paths {
-				if err := ProcessSumUpdate(p, stats); err != nil {
+			var bar *progressbar.ProgressBar
+			if !quiet {
+				total, err := countFilesMultiple(paths)
+				if err != nil {
 					return err
 				}
+				bar = newProgressBar(total, "Updating checksums")
+			}
+			for _, p := range paths {
+				if err := ProcessSumUpdate(p, stats, bar); err != nil {
+					if bar != nil {
+						bar.Finish()
+					}
+					return err
+				}
+			}
+			if bar != nil {
+				bar.Finish()
 			}
 			stats.Print("sum update")
 			return nil
@@ -172,8 +208,8 @@ func newSumCmd() *cobra.Command {
 	return cmd
 }
 
-func ProcessSumAdd(dir string, stats *Stats) error {
-	return walkAndProcess(dir, stats, "Adding checksums", func(p string, info os.FileInfo, s *Stats) error {
+func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
+	return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
 		if hasXattr(p, checksumKey) {
 			atomic.AddInt64(&s.FilesSkipped, 1)
 			return nil
@@ -186,8 +222,8 @@ func ProcessSumAdd(dir string, stats *Stats) error {
 	})
 }
 
-func ProcessSumUpdate(dir string, stats *Stats) error {
-	return walkAndProcess(dir, stats, "Updating checksums", func(p string, info os.FileInfo, s *Stats) error {
+func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
+	return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
 		t, err := readSumTime(p)
 		if err != nil || info.ModTime().After(t) {
 			if err := writeChecksumAndTime(p, info, s); err != nil {
@@ -202,10 +238,23 @@ func ProcessSumUpdate(dir string, stats *Stats) error {
 }
 
 func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
-	hash, err := fileMultihash(path)
+	// Record mtime before hashing to detect modifications during hash
+	mtimeBefore := info.ModTime()
+
+	hash, bytesRead, err := fileMultihash(path)
 	if err != nil {
 		return err
 	}
+
+	// Check if file was modified during hashing
+	infoAfter, err := os.Lstat(path)
+	if err != nil {
+		return fmt.Errorf("stat after hash: %w", err)
+	}
+	if !infoAfter.ModTime().Equal(mtimeBefore) {
+		return fmt.Errorf("%s: file modified during checksum calculation", path)
+	}
+
 	if err := xattr.Set(path, checksumKey, hash); err != nil {
 		return fmt.Errorf("set checksum attr: %w", err)
 	}
@@ -213,7 +262,9 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
 		fmt.Printf("%s %s written\n", path, hash)
 	}
 
-	ts := time.Now().UTC().Format(time.RFC3339Nano)
+	// Store the file's mtime as sumtime (not wall-clock time)
+	// This makes update comparisons semantically correct
+	ts := mtimeBefore.UTC().Format(time.RFC3339Nano)
 	if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
 		return fmt.Errorf("set sumtime attr: %w", err)
 	}
@@ -222,7 +273,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
 	}
 
 	atomic.AddInt64(&stats.FilesProcessed, 1)
-	atomic.AddInt64(&stats.BytesProcessed, info.Size())
+	atomic.AddInt64(&stats.BytesProcessed, bytesRead)
 	return nil
 }
 
@@ -249,10 +300,24 @@ func newClearCmd() *cobra.Command {
 				return err
 			}
 			stats := &Stats{StartTime: time.Now()}
-			for _, p := range paths {
-				if err := ProcessClear(p, stats); err != nil {
+			var bar *progressbar.ProgressBar
+			if !quiet {
+				total, err := countFilesMultiple(paths)
+				if err != nil {
 					return err
 				}
+				bar = newProgressBar(total, "Clearing checksums")
+			}
+			for _, p := range paths {
+				if err := ProcessClear(p, stats, bar); err != nil {
+					if bar != nil {
+						bar.Finish()
+					}
+					return err
+				}
+			}
+			if bar != nil {
+				bar.Finish()
 			}
 			stats.Print("clear")
 			return nil
@@ -260,8 +325,8 @@ func newClearCmd() *cobra.Command {
 	}
 }
 
-func ProcessClear(dir string, stats *Stats) error {
-	return walkAndProcess(dir, stats, "Clearing checksums", func(p string, info os.FileInfo, s *Stats) error {
+func ProcessClear(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
+	return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
 		cleared := false
 		for _, k := range []string{checksumKey, sumTimeKey} {
 			v, err := xattr.Get(p, k)
@@ -307,17 +372,31 @@ func newCheckCmd() *cobra.Command {
 				return err
 			}
 			stats := &Stats{StartTime: time.Now()}
+			var bar *progressbar.ProgressBar
+			if !quiet {
+				total, err := countFilesMultiple(paths)
+				if err != nil {
+					return err
+				}
+				bar = newProgressBar(total, "Verifying checksums")
+			}
 			var finalErr error
 			for _, p := range paths {
-				if err := ProcessCheck(p, cont, stats); err != nil {
+				if err := ProcessCheck(p, cont, stats, bar); err != nil {
 					if cont {
 						finalErr = err
 					} else {
+						if bar != nil {
+							bar.Finish()
+						}
 						stats.Print("check")
 						return err
 					}
 				}
 			}
+			if bar != nil {
+				bar.Finish()
+			}
 			stats.Print("check")
 			return finalErr
 		},
@@ -326,15 +405,15 @@ func newCheckCmd() *cobra.Command {
 	return cmd
 }
 
-func ProcessCheck(dir string, cont bool, stats *Stats) error {
+func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.ProgressBar) error {
 	fail := errors.New("verification failed")
-	bad := false
+	// Track initial failed count to detect failures during this walk
+	initialFailed := atomic.LoadInt64(&stats.FilesFailed)
 
-	err := walkAndProcess(dir, stats, "Verifying checksums", func(p string, info os.FileInfo, s *Stats) error {
+	err := walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
 		exp, err := xattr.Get(p, checksumKey)
 		if err != nil {
 			if errors.Is(err, xattr.ENOATTR) {
-				bad = true
 				atomic.AddInt64(&s.FilesFailed, 1)
 				if verbose && !quiet {
 					fmt.Printf("%s <none> ERROR\n", p)
@@ -347,18 +426,17 @@ func ProcessCheck(dir string, cont bool, stats *Stats) error {
 			return err
 		}
 
-		act, err := fileMultihash(p)
+		act, bytesRead, err := fileMultihash(p)
 		if err != nil {
 			atomic.AddInt64(&s.FilesFailed, 1)
 			return err
 		}
 		ok := bytes.Equal(exp, act)
 		if !ok {
-			bad = true
 			atomic.AddInt64(&s.FilesFailed, 1)
 		} else {
 			atomic.AddInt64(&s.FilesProcessed, 1)
-			atomic.AddInt64(&s.BytesProcessed, info.Size())
+			atomic.AddInt64(&s.BytesProcessed, bytesRead)
 		}
 		if verbose && !quiet {
 			status := "OK"
@@ -379,7 +457,8 @@ func ProcessCheck(dir string, cont bool, stats *Stats) error {
 		}
 		return err
 	}
-	if bad {
+	// Check if any failures occurred during this walk
+	if atomic.LoadInt64(&stats.FilesFailed) > initialFailed {
 		return fail
 	}
 	return nil
@@ -390,17 +469,16 @@ func ProcessCheck(dir string, cont bool, stats *Stats) error {
 ///////////////////////////////////////////////////////////////////////////////
 
 // countFiles counts the total number of regular files that will be processed
-func countFiles(root string) int64 {
+func countFiles(root string) (int64, error) {
 	var count int64
 	root = filepath.Clean(root)
-	filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
 		if err != nil {
-			return nil
+			return err
 		}
+		// Skip symlinks - note: filepath.Walk uses Lstat, so symlinks are
+		// reported as ModeSymlink, never as directories. Walk doesn't follow them.
 		if info.Mode()&os.ModeSymlink != 0 {
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
 			return nil
 		}
 		rel, _ := filepath.Rel(root, p)
@@ -419,51 +497,58 @@ func countFiles(root string) int64 {
 		count++
 		return nil
 	})
-	return count
+	return count, err
 }
 
-func walkAndProcess(root string, stats *Stats, description string, fn func(string, os.FileInfo, *Stats) error) error {
-	root = filepath.Clean(root)
-
-	// Count files first for progress bar
-	total := countFiles(root)
-
-	// Create progress bar
-	var bar *progressbar.ProgressBar
-	if !quiet {
-		bar = progressbar.NewOptions64(total,
-			progressbar.OptionSetDescription(description),
-			progressbar.OptionSetWriter(os.Stderr),
-			progressbar.OptionShowCount(),
-			progressbar.OptionShowIts(),
-			progressbar.OptionSetItsString("files"),
-			progressbar.OptionThrottle(250*time.Millisecond),
-			progressbar.OptionShowElapsedTimeOnFinish(),
-			progressbar.OptionSetPredictTime(true),
-			progressbar.OptionFullWidth(),
-			progressbar.OptionSetTheme(progressbar.Theme{
-				Saucer:        "=",
-				SaucerHead:    ">",
-				SaucerPadding: " ",
-				BarStart:      "[",
-				BarEnd:        "]",
-			}),
-		)
+// countFilesMultiple counts files across multiple roots
+func countFilesMultiple(roots []string) (int64, error) {
+	var total int64
+	for _, root := range roots {
+		count, err := countFiles(root)
+		if err != nil {
+			return total, err
+		}
+		total += count
 	}
+	return total, nil
+}
+
+// newProgressBar creates a new progress bar with standard options
+func newProgressBar(total int64, description string) *progressbar.ProgressBar {
+	return progressbar.NewOptions64(total,
+		progressbar.OptionSetDescription(description),
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionShowCount(),
+		progressbar.OptionShowIts(),
+		progressbar.OptionSetItsString("files"),
+		progressbar.OptionThrottle(250*time.Millisecond),
+		progressbar.OptionShowElapsedTimeOnFinish(),
+		progressbar.OptionSetPredictTime(true),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetTheme(progressbar.Theme{
+			Saucer:        "=",
+			SaucerHead:    ">",
+			SaucerPadding: " ",
+			BarStart:      "[",
+			BarEnd:        "]",
+		}),
+	)
+}
+
+func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn func(string, os.FileInfo, *Stats) error) error {
+	root = filepath.Clean(root)
 
 	err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		// skip symlinks entirely
+		// Skip symlinks - filepath.Walk uses Lstat, so symlinks are reported
+		// as ModeSymlink, never as directories. Walk doesn't follow them.
 		if info.Mode()&os.ModeSymlink != 0 {
 			if verbose && !quiet {
 				log.Printf("skip symlink %s", p)
 			}
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
 			return nil
 		}
 
@@ -492,10 +577,6 @@ func walkAndProcess(root string, stats *Stats, description string, fn func(strin
 		return fnErr
 	})
 
-	if bar != nil {
-		bar.Finish()
-	}
-
 	return err
 }
 
@@ -523,20 +604,21 @@ func hasXattr(path, key string) bool {
 	return err == nil
 }
 
-func fileMultihash(path string) ([]byte, error) {
+func fileMultihash(path string) (hash []byte, bytesRead int64, err error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	defer f.Close()
 
 	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return nil, err
+	bytesRead, err = io.Copy(h, f)
+	if err != nil {
+		return nil, bytesRead, err
 	}
 	mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
 	if err != nil {
-		return nil, err
+		return nil, bytesRead, err
 	}
-	return []byte(base58.Encode(mh)), nil
+	return []byte(base58.Encode(mh)), bytesRead, nil
 }

attrsum_test.go

@@ -40,7 +40,7 @@ func TestSumAddAndUpdate(t *testing.T) {
 
 	f := writeFile(t, dir, "a.txt", "hello")
 
-	if err := ProcessSumAdd(dir, newTestStats()); err != nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("add: %v", err)
 	}
 	if _, err := xattr.Get(f, checksumKey); err != nil {
@@ -53,7 +53,7 @@ func TestSumAddAndUpdate(t *testing.T) {
 	now := time.Now().Add(2 * time.Second)
 	os.Chtimes(f, now, now)
 
-	if err := ProcessSumUpdate(dir, newTestStats()); err != nil {
+	if err := ProcessSumUpdate(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("update: %v", err)
 	}
 	tsb2, _ := xattr.Get(f, sumTimeKey)
@@ -68,17 +68,17 @@ func TestProcessCheckIntegration(t *testing.T) {
 	skipIfNoXattr(t, dir)
 
 	writeFile(t, dir, "b.txt", "world")
-	if err := ProcessSumAdd(dir, newTestStats()); err != nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("add: %v", err)
 	}
-	if err := ProcessCheck(dir, false, newTestStats()); err != nil {
+	if err := ProcessCheck(dir, false, newTestStats(), nil); err != nil {
 		t.Fatalf("check ok: %v", err)
 	}
 
 	f := filepath.Join(dir, "b.txt")
 	os.WriteFile(f, []byte("corrupt"), 0o644)
 
-	if err := ProcessCheck(dir, false, newTestStats()); err == nil {
+	if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
 		t.Fatalf("expected mismatch error, got nil")
 	}
 }
@@ -88,11 +88,11 @@ func TestClearRemovesAttrs(t *testing.T) {
 	skipIfNoXattr(t, dir)
 
 	f := writeFile(t, dir, "c.txt", "data")
-	if err := ProcessSumAdd(dir, newTestStats()); err != nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("add: %v", err)
 	}
 
-	if err := ProcessClear(dir, newTestStats()); err != nil {
+	if err := ProcessClear(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("clear: %v", err)
 	}
 	if _, err := xattr.Get(f, checksumKey); err == nil {
@@ -117,7 +117,7 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
 	excludePatterns = []string{"*.me"}
 	defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
 
-	if err := ProcessSumAdd(dir, newTestStats()); err != nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("add with excludes: %v", err)
 	}
 
@@ -143,7 +143,7 @@ func TestSkipBrokenSymlink(t *testing.T) {
 	}
 
 	// Should not error and should not create xattrs on link
-	if err := ProcessSumAdd(dir, newTestStats()); err != nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
 		t.Fatalf("ProcessSumAdd with symlink: %v", err)
 	}
 	if _, err := xattr.Get(link, checksumKey); err == nil {
@@ -159,13 +159,13 @@ func TestPermissionErrors(t *testing.T) {
 	os.Chmod(secret, 0o000)
 	defer os.Chmod(secret, 0o644)
 
-	if err := ProcessSumAdd(dir, newTestStats()); err == nil {
+	if err := ProcessSumAdd(dir, newTestStats(), nil); err == nil {
 		t.Fatalf("expected permission error, got nil")
 	}
-	if err := ProcessSumUpdate(dir, newTestStats()); err == nil {
+	if err := ProcessSumUpdate(dir, newTestStats(), nil); err == nil {
 		t.Fatalf("expected permission error on update, got nil")
 	}
-	if err := ProcessCheck(dir, false, newTestStats()); err == nil {
+	if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
 		t.Fatalf("expected permission error on check, got nil")
 	}
 }