From 1bb9528548f336274d2adcf2eedf75474bcd9911 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 8 May 2025 13:59:35 -0700 Subject: [PATCH] seems to work, tests pass. woo! --- Makefile | 23 ++- attrsum.go | 432 +++++++++++++++++++++++++++++------------------- attrsum_test.go | 88 +++++++--- go.mod | 3 +- go.sum | 2 + 5 files changed, 353 insertions(+), 195 deletions(-) diff --git a/Makefile b/Makefile index 69c2dc4..a2eac70 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,24 @@ +TESTDIR := $(HOME)/Documents/_SYSADMIN/cyberdyne + default: test test: - go test ./... -v + @go test ./... -v + +build: clean + @go build . + +clean: + @rm -f attrsum + +try: build + ./attrsum sum add -v $(TESTDIR) + ./attrsum check -v $(TESTDIR) + ./attrsum clear -v $(TESTDIR) + -./attrsum check -v $(TESTDIR) + ./attrsum sum add -v $(TESTDIR) + ./attrsum check -v $(TESTDIR) + touch $(TESTDIR)/* + ./attrsum sum update -v $(TESTDIR) + ./attrsum check -v $(TESTDIR) -build: - go build . diff --git a/attrsum.go b/attrsum.go index bfdc47e..0d6927b 100644 --- a/attrsum.go +++ b/attrsum.go @@ -1,46 +1,54 @@ package main import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "time" + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "time" - base58 "github.com/mr-tron/base58/base58" - "github.com/multiformats/go-multihash" - "github.com/pkg/xattr" - "github.com/spf13/cobra" + base58 "github.com/mr-tron/base58/base58" + "github.com/bmatcuk/doublestar/v4" + "github.com/multiformats/go-multihash" + "github.com/pkg/xattr" + "github.com/spf13/cobra" ) const ( - // Extended-attribute keys - checksumKey = "berlin.sneak.app.attrsum.checksum" - sumTimeKey = "berlin.sneak.app.attrsum.sumtime" + checksumKey = "berlin.sneak.app.attrsum.checksum" + sumTimeKey = "berlin.sneak.app.attrsum.sumtime" ) -var verbose bool +var ( + verbose bool + excludePatterns []string + excludeDotfiles bool +) func main() { - rootCmd := &cobra.Command{ - Use: "attrsum", - Short: "Compute and verify file checksums via xattrs", - } + rootCmd := &cobra.Command{ + Use: "attrsum", + Short: "Compute and verify file checksums via xattrs", + } + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true - rootCmd.PersistentFlags().BoolVarP( - &verbose, "verbose", "v", false, "enable verbose output", - ) + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable verbose output") + rootCmd.PersistentFlags().StringArrayVar(&excludePatterns, "exclude", nil, "exclude files/directories matching pattern (rsync-style, repeatable)") + rootCmd.PersistentFlags().BoolVar(&excludeDotfiles, "exclude-dotfiles", false, "exclude any file or directory whose name starts with '.'") - rootCmd.AddCommand(newSumCmd()) - rootCmd.AddCommand(newCheckCmd()) + rootCmd.AddCommand(newSumCmd()) + rootCmd.AddCommand(newCheckCmd()) + rootCmd.AddCommand(newClearCmd()) - if err := rootCmd.Execute(); err != nil { - log.Fatal(err) - } + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } } /////////////////////////////////////////////////////////////////////////////// @@ -48,86 +56,126 @@ func main() { /////////////////////////////////////////////////////////////////////////////// func newSumCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "sum", - Short: "Checksum maintenance operations", - } + cmd := &cobra.Command{ + Use: "sum", + Short: "Checksum maintenance operations", + } - addCmd := &cobra.Command{ - Use: "add ", - Short: "Write checksums for files missing them", - Args: cobra.ExactArgs(1), - RunE: func(_ *cobra.Command, args []string) error { - return ProcessSumAdd(args[0]) - }, - } + addCmd := &cobra.Command{ + Use: "add ", + Short: "Write checksums for files missing them", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return ProcessSumAdd(args[0]) + }, + } - updateCmd := &cobra.Command{ - Use: "update ", - Short: "Refresh checksums when file modified", - Args: cobra.ExactArgs(1), - RunE: func(_ *cobra.Command, args []string) error { - return ProcessSumUpdate(args[0]) - }, - } + updateCmd := &cobra.Command{ + Use: "update ", + Short: "Refresh checksums when file modified", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return ProcessSumUpdate(args[0]) + }, + } - cmd.AddCommand(addCmd, updateCmd) - return cmd + cmd.AddCommand(addCmd, updateCmd) + return cmd } -// ProcessSumAdd writes checksum & sumtime only when checksum is absent. func ProcessSumAdd(dir string) error { - return walkAndProcess(dir, func(path string, info os.FileInfo) error { - if hasXattr(path, checksumKey) { - if verbose { - log.Printf("skip existing %s", path) - } - return nil - } - return writeChecksumAndTime(path) - }) + return walkAndProcess(dir, func(path string, info os.FileInfo) error { + if hasXattr(path, checksumKey) { + if verbose { + log.Printf("skip existing %s", path) + } + return nil + } + return writeChecksumAndTime(path) + }) } -// ProcessSumUpdate recalculates checksum when file mtime exceeds stored -// sumtime (or when attributes are missing / malformed). func ProcessSumUpdate(dir string) error { - return walkAndProcess(dir, func(path string, info os.FileInfo) error { - needUpdate := false - t, err := readSumTime(path) - if err != nil || info.ModTime().After(t) { - needUpdate = true - } - if needUpdate { - if verbose { - log.Printf("update %s", path) - } - return writeChecksumAndTime(path) - } - return nil - }) + return walkAndProcess(dir, func(path string, info os.FileInfo) error { + needUpdate := false + t, err := readSumTime(path) + if err != nil || info.ModTime().After(t) { + needUpdate = true + } + if needUpdate { + if verbose { + log.Printf("update %s", path) + } + return writeChecksumAndTime(path) + } + return nil + }) } func writeChecksumAndTime(path string) error { - hash, err := fileMultihash(path) - if err != nil { - return err - } - if err := xattr.Set(path, checksumKey, hash); err != nil { - return fmt.Errorf("set checksum attr: %w", err) - } - nowStr := time.Now().UTC().Format(time.RFC3339Nano) - if err := xattr.Set(path, sumTimeKey, []byte(nowStr)); err != nil { - return fmt.Errorf("set sumtime attr: %w", err) - } - return nil + hash, err := fileMultihash(path) + if err != nil { + return err + } + if err := xattr.Set(path, checksumKey, hash); err != nil { + return fmt.Errorf("set checksum attr: %w", err) + } + if verbose { + fmt.Printf("%s %s written\n", path, string(hash)) + } + + nowStr := time.Now().UTC().Format(time.RFC3339Nano) + if err := xattr.Set(path, sumTimeKey, []byte(nowStr)); err != nil { + return fmt.Errorf("set sumtime attr: %w", err) + } + if verbose { + fmt.Printf("%s %s written\n", path, nowStr) + } + return nil } func readSumTime(path string) (time.Time, error) { - b, err := xattr.Get(path, sumTimeKey) - if err != nil { - return time.Time{}, err - } - return time.Parse(time.RFC3339Nano, string(b)) + b, err := xattr.Get(path, sumTimeKey) + if err != nil { + return time.Time{}, err + } + return time.Parse(time.RFC3339Nano, string(b)) +} + +/////////////////////////////////////////////////////////////////////////////// +// Clear operation +/////////////////////////////////////////////////////////////////////////////// + +func newClearCmd() *cobra.Command { + return &cobra.Command{ + Use: "clear ", + Short: "Remove checksum xattrs from files in tree", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return ProcessClear(args[0]) + }, + } +} + +func ProcessClear(dir string) error { + return walkAndProcess(dir, func(path string, info os.FileInfo) error { + for _, key := range []string{checksumKey, sumTimeKey} { + val, err := xattr.Get(path, key) + if err != nil { + if errors.Is(err, xattr.ENOATTR) { + continue + } + return err + } + if verbose { + fmt.Printf("%s %s removed\n", path, string(val)) + } + if err := xattr.Remove(path, key); err != nil { + return err + } + } + return nil + }) } /////////////////////////////////////////////////////////////////////////////// @@ -135,65 +183,77 @@ func readSumTime(path string) (time.Time, error) { /////////////////////////////////////////////////////////////////////////////// func newCheckCmd() *cobra.Command { - var cont bool - - cmd := &cobra.Command{ - Use: "check ", - Short: "Verify stored checksums against file contents", - Args: cobra.ExactArgs(1), - RunE: func(_ *cobra.Command, args []string) error { - return ProcessCheck(args[0], cont) - }, - } - - cmd.Flags().BoolVar(&cont, "continue", false, - "continue after errors and report each file") - return cmd + var cont bool + cmd := &cobra.Command{ + Use: "check ", + Short: "Verify stored checksums against file contents", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return ProcessCheck(args[0], cont) + }, + } + cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file") + return cmd } -// ProcessCheck verifies checksums and exits non-zero on first error unless -// --continue is supplied. func ProcessCheck(dir string, cont bool) error { - exitErr := errors.New("verification failed") + exitErr := errors.New("verification failed") + hadErr := false - err := walkAndProcess(dir, func(path string, info os.FileInfo) error { - exp, err := xattr.Get(path, checksumKey) - if err != nil { - if errors.Is(err, xattr.ENOATTR) { - log.Printf("ERROR missing xattr %s", path) - if cont { - return nil - } - return exitErr - } - return err - } + err := walkAndProcess(dir, func(path string, info os.FileInfo) error { + exp, err := xattr.Get(path, checksumKey) + if err != nil { + if errors.Is(err, xattr.ENOATTR) { + hadErr = true + if verbose { + fmt.Printf("%s ERROR\n", path) + } else { + log.Printf("ERROR missing xattr %s", path) + } + if cont { + return nil + } + return exitErr + } + return err + } - act, err := fileMultihash(path) - if err != nil { - return err - } + act, err := fileMultihash(path) + if err != nil { + return err + } - if !bytes.Equal(exp, act) { - log.Printf("ERROR checksum mismatch %s", path) - if cont { - return nil - } - return exitErr - } - if cont { - fmt.Printf("OK %s\n", path) - } - return nil - }) + ok := bytes.Equal(exp, act) + if !ok { + hadErr = true + } - if err != nil { - if errors.Is(err, exitErr) { - return exitErr - } - return err - } - return nil + if verbose { + res := "OK" + if !ok { + res = "ERROR" + } + fmt.Printf("%s %s %s\n", path, string(act), res) + } else if !ok { + log.Printf("ERROR checksum mismatch %s", path) + } + + if !ok && !cont { + return exitErr + } + return nil + }) + + if err != nil { + if errors.Is(err, exitErr) { + return exitErr + } + return err + } + if hadErr { + return exitErr + } + return nil } /////////////////////////////////////////////////////////////////////////////// @@ -201,38 +261,66 @@ func ProcessCheck(dir string, cont bool) error { /////////////////////////////////////////////////////////////////////////////// func walkAndProcess(root string, fn func(string, os.FileInfo) error) error { - return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - return fn(path, info) - }) + root = filepath.Clean(root) + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + + if shouldExclude(rel, info) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + if info.IsDir() { + return nil + } + return fn(path, info) + }) +} + +func shouldExclude(rel string, info os.FileInfo) bool { + if rel == "." || rel == "" { + return false + } + if excludeDotfiles { + for _, part := range strings.Split(rel, string(os.PathSeparator)) { + if strings.HasPrefix(part, ".") { + return true + } + } + } + for _, pat := range excludePatterns { + if ok, _ := doublestar.PathMatch(pat, rel); ok { + return true + } + } + return false } func hasXattr(path, key string) bool { - _, err := xattr.Get(path, key) - return err == nil + _, err := xattr.Get(path, key) + return err == nil } -// fileMultihash returns the base58-encoded SHA-2-256 multihash of the file. func fileMultihash(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return nil, err - } + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } - mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) - if err != nil { - return nil, err - } - return []byte(base58.Encode(mh)), nil + mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) + if err != nil { + return nil, err + } + return []byte(base58.Encode(mh)), nil } diff --git a/attrsum_test.go b/attrsum_test.go index cc75c0c..d575cc4 100644 --- a/attrsum_test.go +++ b/attrsum_test.go @@ -10,7 +10,7 @@ import ( "github.com/pkg/xattr" ) -// skipIfNoXattr skips tests when the underlying FS lacks xattr support. +// skipIfNoXattr skips tests when underlying FS lacks xattr support. func skipIfNoXattr(t *testing.T, path string) { if err := xattr.Set(path, "user.test", []byte("1")); err != nil { t.Skipf("skipping: xattr not supported: %v", err) @@ -21,14 +21,14 @@ func skipIfNoXattr(t *testing.T, path string) { func writeFile(t *testing.T, root, name, content string) string { t.Helper() - path := filepath.Join(root, name) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + p := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatalf("mkdir: %v", err) } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { t.Fatalf("write: %v", err) } - return path + return p } func TestSumAddAndUpdate(t *testing.T) { @@ -37,29 +37,20 @@ func TestSumAddAndUpdate(t *testing.T) { f := writeFile(t, dir, "a.txt", "hello") - // Add: write missing checksum. if err := ProcessSumAdd(dir); err != nil { t.Fatalf("add: %v", err) } - - // Attributes should exist. if _, err := xattr.Get(f, checksumKey); err != nil { t.Fatalf("checksum missing: %v", err) } - tsb, err := xattr.Get(f, sumTimeKey) - if err != nil { - t.Fatalf("sumtime missing: %v", err) - } + tsb, _ := xattr.Get(f, sumTimeKey) origTime, _ := time.Parse(time.RFC3339Nano, string(tsb)) - // Modify file and bump mtime. - if err := os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644); err != nil { - t.Fatalf("rewrite: %v", err) - } + // Modify file & bump mtime to force update. + os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644) now := time.Now().Add(2 * time.Second) os.Chtimes(f, now, now) - // Update should refresh checksum and time. if err := ProcessSumUpdate(dir); err != nil { t.Fatalf("update: %v", err) } @@ -78,12 +69,11 @@ func TestProcessCheckIntegration(t *testing.T) { if err := ProcessSumAdd(dir); err != nil { t.Fatalf("add: %v", err) } - if err := ProcessCheck(dir, false); err != nil { t.Fatalf("check ok: %v", err) } - // Corrupt file contents should produce an error. + // Corrupt -> should fail f := filepath.Join(dir, "b.txt") os.WriteFile(f, []byte("corrupt"), 0o644) @@ -92,6 +82,66 @@ func TestProcessCheckIntegration(t *testing.T) { } } +func TestClearRemovesAttrs(t *testing.T) { + dir := t.TempDir() + skipIfNoXattr(t, dir) + + f := writeFile(t, dir, "c.txt", "data") + if err := ProcessSumAdd(dir); err != nil { + t.Fatalf("add: %v", err) + } + // Ensure attrs exist + if _, err := xattr.Get(f, checksumKey); err != nil { + t.Fatalf("pre-clear checksum missing: %v", err) + } + + if err := ProcessClear(dir); err != nil { + t.Fatalf("clear: %v", err) + } + if _, err := xattr.Get(f, checksumKey); err == nil { + t.Fatalf("checksum still present after clear") + } + if _, err := xattr.Get(f, sumTimeKey); err == nil { + t.Fatalf("sumtime still present after clear") + } +} + +func TestExcludeDotfilesAndPatterns(t *testing.T) { + dir := t.TempDir() + skipIfNoXattr(t, dir) + + hidden := writeFile(t, dir, ".hidden", "dot") + keep := writeFile(t, dir, "keep.txt", "keep") + skip := writeFile(t, dir, "skip.me", "skip") + + // Save global state then set exclusions + oldDotfiles := excludeDotfiles + oldPatterns := excludePatterns + excludeDotfiles = true + excludePatterns = []string{"*.me"} + + defer func() { + excludeDotfiles = oldDotfiles + excludePatterns = oldPatterns + }() + + if err := ProcessSumAdd(dir); err != nil { + t.Fatalf("add with excludes: %v", err) + } + + // keep.txt should have xattrs + if _, err := xattr.Get(keep, checksumKey); err != nil { + t.Fatalf("expected xattr on keep.txt: %v", err) + } + // .hidden and skip.me should not + if _, err := xattr.Get(hidden, checksumKey); err == nil { + t.Fatalf(".hidden should have been excluded") + } + if _, err := xattr.Get(skip, checksumKey); err == nil { + t.Fatalf("skip.me should have been excluded by pattern") + } +} + func TestPermissionErrors(t *testing.T) { dir := t.TempDir() skipIfNoXattr(t, dir) diff --git a/go.mod b/go.mod index 864a668..3e178ce 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module git.eeqj.de/sneak/attrsum go 1.24.1 require ( + github.com/bmatcuk/doublestar/v4 v4.8.1 + github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multihash v0.2.3 github.com/pkg/xattr v0.4.10 github.com/spf13/cobra v1.9.1 @@ -12,7 +14,6 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/minio/sha256-simd v1.0.0 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/go.sum b/go.sum index 3e126fc..35e8990 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= +github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=