sneak/attrsum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: sneak:main
Branches Tags
sneak:main
sneak:improvement
sneak:1.0.0
..
compare: sneak:improvement
Branches Tags
sneak:main
sneak:improvement
sneak:1.0.0

No commits in common. "main" and "improvement" have entirely different histories.
main ... improvemen

2 changed files with 81 additions and 163 deletions
Show Stats Expand all files Collapse all files

184
attrsum.go
View File

@ -99,10 +99,8 @@ func main() {
// expandPaths expands the given paths, reading from stdin if "-" is present
func expandPaths(args []string) ([]string, error) {
var paths []string
readFromStdin := false
for _, arg := range args {
if arg == "-" {
readFromStdin = true
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
@ -117,12 +115,6 @@ func expandPaths(args []string) ([]string, error) {
paths = append(paths, arg)
}
}
if len(paths) == 0 {
if readFromStdin {
return nil, errors.New("no paths provided on stdin")
}
return nil, errors.New("no paths provided")
}
return paths, nil
}
@ -146,25 +138,11 @@ func newSumCmd() *cobra.Command {
return err
}
stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Adding checksums")
}
for _, p := range paths {
if err := ProcessSumAdd(p, stats, bar); err != nil {
if bar != nil {
bar.Finish()
}
if err := ProcessSumAdd(p, stats); err != nil {
return err
}
}
if bar != nil {
bar.Finish()
}
stats.Print("sum add")
return nil
},
@ -180,25 +158,11 @@ func newSumCmd() *cobra.Command {
return err
}
stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Updating checksums")
}
for _, p := range paths {
if err := ProcessSumUpdate(p, stats, bar); err != nil {
if bar != nil {
bar.Finish()
}
if err := ProcessSumUpdate(p, stats); err != nil {
return err
}
}
if bar != nil {
bar.Finish()
}
stats.Print("sum update")
return nil
},
@ -208,8 +172,8 @@ func newSumCmd() *cobra.Command {
return cmd
}
func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
func ProcessSumAdd(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, "Adding checksums", func(p string, info os.FileInfo, s *Stats) error {
if hasXattr(p, checksumKey) {
atomic.AddInt64(&s.FilesSkipped, 1)
return nil
@ -222,8 +186,8 @@ func ProcessSumAdd(dir string, stats *Stats, bar *progressbar.ProgressBar) error
})
}
func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
func ProcessSumUpdate(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, "Updating checksums", func(p string, info os.FileInfo, s *Stats) error {
t, err := readSumTime(p)
if err != nil || info.ModTime().After(t) {
if err := writeChecksumAndTime(p, info, s); err != nil {
@ -238,23 +202,10 @@ func ProcessSumUpdate(dir string, stats *Stats, bar *progressbar.ProgressBar) er
}
func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
// Record mtime before hashing to detect modifications during hash
mtimeBefore := info.ModTime()
hash, bytesRead, err := fileMultihash(path)
hash, err := fileMultihash(path)
if err != nil {
return err
}
// Check if file was modified during hashing
infoAfter, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("stat after hash: %w", err)
}
if !infoAfter.ModTime().Equal(mtimeBefore) {
return fmt.Errorf("%s: file modified during checksum calculation", path)
}
if err := xattr.Set(path, checksumKey, hash); err != nil {
return fmt.Errorf("set checksum attr: %w", err)
}
@ -262,9 +213,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
fmt.Printf("%s %s written\n", path, hash)
}
// Store the file's mtime as sumtime (not wall-clock time)
// This makes update comparisons semantically correct
ts := mtimeBefore.UTC().Format(time.RFC3339Nano)
ts := time.Now().UTC().Format(time.RFC3339Nano)
if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
return fmt.Errorf("set sumtime attr: %w", err)
}
@ -273,7 +222,7 @@ func writeChecksumAndTime(path string, info os.FileInfo, stats *Stats) error {
}
atomic.AddInt64(&stats.FilesProcessed, 1)
atomic.AddInt64(&stats.BytesProcessed, bytesRead)
atomic.AddInt64(&stats.BytesProcessed, info.Size())
return nil
}
@ -300,33 +249,19 @@ func newClearCmd() *cobra.Command {
return err
}
stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Clearing checksums")
}
for _, p := range paths {
if err := ProcessClear(p, stats, bar); err != nil {
if bar != nil {
bar.Finish()
}
if err := ProcessClear(p, stats); err != nil {
return err
}
}
if bar != nil {
bar.Finish()
}
stats.Print("clear")
return nil
},
}
}
func ProcessClear(dir string, stats *Stats, bar *progressbar.ProgressBar) error {
return walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
func ProcessClear(dir string, stats *Stats) error {
return walkAndProcess(dir, stats, "Clearing checksums", func(p string, info os.FileInfo, s *Stats) error {
cleared := false
for _, k := range []string{checksumKey, sumTimeKey} {
v, err := xattr.Get(p, k)
@ -372,31 +307,17 @@ func newCheckCmd() *cobra.Command {
return err
}
stats := &Stats{StartTime: time.Now()}
var bar *progressbar.ProgressBar
if !quiet {
total, err := countFilesMultiple(paths)
if err != nil {
return err
}
bar = newProgressBar(total, "Verifying checksums")
}
var finalErr error
for _, p := range paths {
if err := ProcessCheck(p, cont, stats, bar); err != nil {
if err := ProcessCheck(p, cont, stats); err != nil {
if cont {
finalErr = err
} else {
if bar != nil {
bar.Finish()
}
stats.Print("check")
return err
}
}
}
if bar != nil {
bar.Finish()
}
stats.Print("check")
return finalErr
},
@ -405,15 +326,15 @@ func newCheckCmd() *cobra.Command {
return cmd
}
func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.ProgressBar) error {
func ProcessCheck(dir string, cont bool, stats *Stats) error {
fail := errors.New("verification failed")
// Track initial failed count to detect failures during this walk
initialFailed := atomic.LoadInt64(&stats.FilesFailed)
bad := false
err := walkAndProcess(dir, stats, bar, func(p string, info os.FileInfo, s *Stats) error {
err := walkAndProcess(dir, stats, "Verifying checksums", func(p string, info os.FileInfo, s *Stats) error {
exp, err := xattr.Get(p, checksumKey)
if err != nil {
if errors.Is(err, xattr.ENOATTR) {
bad = true
atomic.AddInt64(&s.FilesFailed, 1)
if verbose && !quiet {
fmt.Printf("%s <none> ERROR\n", p)
@ -426,17 +347,18 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
return err
}
act, bytesRead, err := fileMultihash(p)
act, err := fileMultihash(p)
if err != nil {
atomic.AddInt64(&s.FilesFailed, 1)
return err
}
ok := bytes.Equal(exp, act)
if !ok {
bad = true
atomic.AddInt64(&s.FilesFailed, 1)
} else {
atomic.AddInt64(&s.FilesProcessed, 1)
atomic.AddInt64(&s.BytesProcessed, bytesRead)
atomic.AddInt64(&s.BytesProcessed, info.Size())
}
if verbose && !quiet {
status := "OK"
@ -457,8 +379,7 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
}
return err
}
// Check if any failures occurred during this walk
if atomic.LoadInt64(&stats.FilesFailed) > initialFailed {
if bad {
return fail
}
return nil
@ -469,16 +390,17 @@ func ProcessCheck(dir string, cont bool, stats *Stats, bar *progressbar.Progress
///////////////////////////////////////////////////////////////////////////////
// countFiles counts the total number of regular files that will be processed
func countFiles(root string) (int64, error) {
func countFiles(root string) int64 {
var count int64
root = filepath.Clean(root)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
return nil
}
// Skip symlinks - note: filepath.Walk uses Lstat, so symlinks are
// reported as ModeSymlink, never as directories. Walk doesn't follow them.
if info.Mode()&os.ModeSymlink != 0 {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
rel, _ := filepath.Rel(root, p)
@ -497,25 +419,19 @@ func countFiles(root string) (int64, error) {
count++
return nil
})
return count, err
return count
}
// countFilesMultiple counts files across multiple roots
func countFilesMultiple(roots []string) (int64, error) {
var total int64
for _, root := range roots {
count, err := countFiles(root)
if err != nil {
return total, err
}
total += count
}
return total, nil
}
func walkAndProcess(root string, stats *Stats, description string, fn func(string, os.FileInfo, *Stats) error) error {
root = filepath.Clean(root)
// newProgressBar creates a new progress bar with standard options
func newProgressBar(total int64, description string) *progressbar.ProgressBar {
return progressbar.NewOptions64(total,
// Count files first for progress bar
total := countFiles(root)
// Create progress bar
var bar *progressbar.ProgressBar
if !quiet {
bar = progressbar.NewOptions64(total,
progressbar.OptionSetDescription(description),
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowCount(),
@ -535,20 +451,19 @@ func newProgressBar(total int64, description string) *progressbar.ProgressBar {
)
}
func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn func(string, os.FileInfo, *Stats) error) error {
root = filepath.Clean(root)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip symlinks - filepath.Walk uses Lstat, so symlinks are reported
// as ModeSymlink, never as directories. Walk doesn't follow them.
// skip symlinks entirely
if info.Mode()&os.ModeSymlink != 0 {
if verbose && !quiet {
log.Printf("skip symlink %s", p)
}
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
@ -577,6 +492,10 @@ func walkAndProcess(root string, stats *Stats, bar *progressbar.ProgressBar, fn
return fnErr
})
if bar != nil {
bar.Finish()
}
return err
}
@ -604,21 +523,20 @@ func hasXattr(path, key string) bool {
return err == nil
}
func fileMultihash(path string) (hash []byte, bytesRead int64, err error) {
func fileMultihash(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, 0, err
return nil, err
}
defer f.Close()
h := sha256.New()
bytesRead, err = io.Copy(h, f)
if err != nil {
return nil, bytesRead, err
if _, err := io.Copy(h, f); err != nil {
return nil, err
}
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil {
return nil, bytesRead, err
return nil, err
}
return []byte(base58.Encode(mh)), bytesRead, nil
return []byte(base58.Encode(mh)), nil
}

24
attrsum_test.go
View File

@ -40,7 +40,7 @@ func TestSumAddAndUpdate(t *testing.T) {
f := writeFile(t, dir, "a.txt", "hello")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err)
}
if _, err := xattr.Get(f, checksumKey); err != nil {
@ -53,7 +53,7 @@ func TestSumAddAndUpdate(t *testing.T) {
now := time.Now().Add(2 * time.Second)
os.Chtimes(f, now, now)
if err := ProcessSumUpdate(dir, newTestStats(), nil); err != nil {
if err := ProcessSumUpdate(dir, newTestStats()); err != nil {
t.Fatalf("update: %v", err)
}
tsb2, _ := xattr.Get(f, sumTimeKey)
@ -68,17 +68,17 @@ func TestProcessCheckIntegration(t *testing.T) {
skipIfNoXattr(t, dir)
writeFile(t, dir, "b.txt", "world")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err)
}
if err := ProcessCheck(dir, false, newTestStats(), nil); err != nil {
if err := ProcessCheck(dir, false, newTestStats()); err != nil {
t.Fatalf("check ok: %v", err)
}
f := filepath.Join(dir, "b.txt")
os.WriteFile(f, []byte("corrupt"), 0o644)
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
t.Fatalf("expected mismatch error, got nil")
}
}
@ -88,11 +88,11 @@ func TestClearRemovesAttrs(t *testing.T) {
skipIfNoXattr(t, dir)
f := writeFile(t, dir, "c.txt", "data")
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add: %v", err)
}
if err := ProcessClear(dir, newTestStats(), nil); err != nil {
if err := ProcessClear(dir, newTestStats()); err != nil {
t.Fatalf("clear: %v", err)
}
if _, err := xattr.Get(f, checksumKey); err == nil {
@ -117,7 +117,7 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
excludePatterns = []string{"*.me"}
defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("add with excludes: %v", err)
}
@ -143,7 +143,7 @@ func TestSkipBrokenSymlink(t *testing.T) {
}
// Should not error and should not create xattrs on link
if err := ProcessSumAdd(dir, newTestStats(), nil); err != nil {
if err := ProcessSumAdd(dir, newTestStats()); err != nil {
t.Fatalf("ProcessSumAdd with symlink: %v", err)
}
if _, err := xattr.Get(link, checksumKey); err == nil {
@ -159,13 +159,13 @@ func TestPermissionErrors(t *testing.T) {
os.Chmod(secret, 0o000)
defer os.Chmod(secret, 0o644)
if err := ProcessSumAdd(dir, newTestStats(), nil); err == nil {
if err := ProcessSumAdd(dir, newTestStats()); err == nil {
t.Fatalf("expected permission error, got nil")
}
if err := ProcessSumUpdate(dir, newTestStats(), nil); err == nil {
if err := ProcessSumUpdate(dir, newTestStats()); err == nil {
t.Fatalf("expected permission error on update, got nil")
}
if err := ProcessCheck(dir, false, newTestStats(), nil); err == nil {
if err := ProcessCheck(dir, false, newTestStats()); err == nil {
t.Fatalf("expected permission error on check, got nil")
}
}