sneak/attrsum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

seems to work, tests pass. woo!

Browse Source
This commit is contained in:
Jeffrey Paul 2025-05-08 13:59:35 -07:00
parent 1497300850
commit 1bb9528548
5 changed files with 353 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Makefile
View File

@ -1,7 +1,24 @@
TESTDIR := $(HOME)/Documents/_SYSADMIN/cyberdyne
default: test default: test
test: test:
go test ./... -v @go test ./... -v
build: clean
@go build .
clean:
@rm -f attrsum
try: build
./attrsum sum add -v $(TESTDIR)
./attrsum check -v $(TESTDIR)
./attrsum clear -v $(TESTDIR)
-./attrsum check -v $(TESTDIR)
./attrsum sum add -v $(TESTDIR)
./attrsum check -v $(TESTDIR)
touch $(TESTDIR)/*
./attrsum sum update -v $(TESTDIR)
./attrsum check -v $(TESTDIR)
build:
go build .

432
attrsum.go
View File

@ -1,46 +1,54 @@
package main package main
import ( import (
"bytes" "bytes"
"crypto/sha256" "crypto/sha256"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log" "log"
"os" "os"
"path/filepath" "path/filepath"
"time" "strings"
"time"
base58 "github.com/mr-tron/base58/base58" base58 "github.com/mr-tron/base58/base58"
"github.com/multiformats/go-multihash" "github.com/bmatcuk/doublestar/v4"
"github.com/pkg/xattr" "github.com/multiformats/go-multihash"
"github.com/spf13/cobra" "github.com/pkg/xattr"
"github.com/spf13/cobra"
) )
const ( const (
// Extended-attribute keys checksumKey = "berlin.sneak.app.attrsum.checksum"
checksumKey = "berlin.sneak.app.attrsum.checksum" sumTimeKey = "berlin.sneak.app.attrsum.sumtime"
sumTimeKey = "berlin.sneak.app.attrsum.sumtime"
) )
var verbose bool var (
verbose bool
excludePatterns []string
excludeDotfiles bool
)
func main() { func main() {
rootCmd := &cobra.Command{ rootCmd := &cobra.Command{
Use: "attrsum", Use: "attrsum",
Short: "Compute and verify file checksums via xattrs", Short: "Compute and verify file checksums via xattrs",
} }
rootCmd.SilenceUsage = true
rootCmd.SilenceErrors = true
rootCmd.PersistentFlags().BoolVarP( rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable verbose output")
&verbose, "verbose", "v", false, "enable verbose output", rootCmd.PersistentFlags().StringArrayVar(&excludePatterns, "exclude", nil, "exclude files/directories matching pattern (rsync-style, repeatable)")
) rootCmd.PersistentFlags().BoolVar(&excludeDotfiles, "exclude-dotfiles", false, "exclude any file or directory whose name starts with '.'")
rootCmd.AddCommand(newSumCmd()) rootCmd.AddCommand(newSumCmd())
rootCmd.AddCommand(newCheckCmd()) rootCmd.AddCommand(newCheckCmd())
rootCmd.AddCommand(newClearCmd())
if err := rootCmd.Execute(); err != nil { if err := rootCmd.Execute(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -48,86 +56,126 @@ func main() {
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func newSumCmd() *cobra.Command { func newSumCmd() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "sum", Use: "sum",
Short: "Checksum maintenance operations", Short: "Checksum maintenance operations",
} }
addCmd := &cobra.Command{ addCmd := &cobra.Command{
Use: "add <directory>", Use: "add <directory>",
Short: "Write checksums for files missing them", Short: "Write checksums for files missing them",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, args []string) error {
return ProcessSumAdd(args[0]) return ProcessSumAdd(args[0])
}, },
} }
updateCmd := &cobra.Command{ updateCmd := &cobra.Command{
Use: "update <directory>", Use: "update <directory>",
Short: "Refresh checksums when file modified", Short: "Refresh checksums when file modified",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, args []string) error {
return ProcessSumUpdate(args[0]) return ProcessSumUpdate(args[0])
}, },
} }
cmd.AddCommand(addCmd, updateCmd) cmd.AddCommand(addCmd, updateCmd)
return cmd return cmd
} }
// ProcessSumAdd writes checksum & sumtime only when checksum is absent.
func ProcessSumAdd(dir string) error { func ProcessSumAdd(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error { return walkAndProcess(dir, func(path string, info os.FileInfo) error {
if hasXattr(path, checksumKey) { if hasXattr(path, checksumKey) {
if verbose { if verbose {
log.Printf("skip existing %s", path) log.Printf("skip existing %s", path)
} }
return nil return nil
} }
return writeChecksumAndTime(path) return writeChecksumAndTime(path)
}) })
} }
// ProcessSumUpdate recalculates checksum when file mtime exceeds stored
// sumtime (or when attributes are missing / malformed).
func ProcessSumUpdate(dir string) error { func ProcessSumUpdate(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error { return walkAndProcess(dir, func(path string, info os.FileInfo) error {
needUpdate := false needUpdate := false
t, err := readSumTime(path) t, err := readSumTime(path)
if err != nil || info.ModTime().After(t) { if err != nil || info.ModTime().After(t) {
needUpdate = true needUpdate = true
} }
if needUpdate { if needUpdate {
if verbose { if verbose {
log.Printf("update %s", path) log.Printf("update %s", path)
} }
return writeChecksumAndTime(path) return writeChecksumAndTime(path)
} }
return nil return nil
}) })
} }
func writeChecksumAndTime(path string) error { func writeChecksumAndTime(path string) error {
hash, err := fileMultihash(path) hash, err := fileMultihash(path)
if err != nil { if err != nil {
return err return err
} }
if err := xattr.Set(path, checksumKey, hash); err != nil { if err := xattr.Set(path, checksumKey, hash); err != nil {
return fmt.Errorf("set checksum attr: %w", err) return fmt.Errorf("set checksum attr: %w", err)
} }
nowStr := time.Now().UTC().Format(time.RFC3339Nano) if verbose {
if err := xattr.Set(path, sumTimeKey, []byte(nowStr)); err != nil { fmt.Printf("%s %s written\n", path, string(hash))
return fmt.Errorf("set sumtime attr: %w", err) }
}
return nil nowStr := time.Now().UTC().Format(time.RFC3339Nano)
if err := xattr.Set(path, sumTimeKey, []byte(nowStr)); err != nil {
return fmt.Errorf("set sumtime attr: %w", err)
}
if verbose {
fmt.Printf("%s %s written\n", path, nowStr)
}
return nil
} }
func readSumTime(path string) (time.Time, error) { func readSumTime(path string) (time.Time, error) {
b, err := xattr.Get(path, sumTimeKey) b, err := xattr.Get(path, sumTimeKey)
if err != nil { if err != nil {
return time.Time{}, err return time.Time{}, err
} }
return time.Parse(time.RFC3339Nano, string(b)) return time.Parse(time.RFC3339Nano, string(b))
}
///////////////////////////////////////////////////////////////////////////////
// Clear operation
///////////////////////////////////////////////////////////////////////////////
func newClearCmd() *cobra.Command {
return &cobra.Command{
Use: "clear <directory>",
Short: "Remove checksum xattrs from files in tree",
Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error {
return ProcessClear(args[0])
},
}
}
func ProcessClear(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error {
for _, key := range []string{checksumKey, sumTimeKey} {
val, err := xattr.Get(path, key)
if err != nil {
if errors.Is(err, xattr.ENOATTR) {
continue
}
return err
}
if verbose {
fmt.Printf("%s %s removed\n", path, string(val))
}
if err := xattr.Remove(path, key); err != nil {
return err
}
}
return nil
})
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -135,65 +183,77 @@ func readSumTime(path string) (time.Time, error) {
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func newCheckCmd() *cobra.Command { func newCheckCmd() *cobra.Command {
var cont bool var cont bool
cmd := &cobra.Command{
cmd := &cobra.Command{ Use: "check <directory>",
Use: "check <directory>", Short: "Verify stored checksums against file contents",
Short: "Verify stored checksums against file contents", Args: cobra.ExactArgs(1),
Args: cobra.ExactArgs(1), RunE: func(_ *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error { return ProcessCheck(args[0], cont)
return ProcessCheck(args[0], cont) },
}, }
} cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file")
return cmd
cmd.Flags().BoolVar(&cont, "continue", false,
"continue after errors and report each file")
return cmd
} }
// ProcessCheck verifies checksums and exits non-zero on first error unless
// --continue is supplied.
func ProcessCheck(dir string, cont bool) error { func ProcessCheck(dir string, cont bool) error {
exitErr := errors.New("verification failed") exitErr := errors.New("verification failed")
hadErr := false
err := walkAndProcess(dir, func(path string, info os.FileInfo) error { err := walkAndProcess(dir, func(path string, info os.FileInfo) error {
exp, err := xattr.Get(path, checksumKey) exp, err := xattr.Get(path, checksumKey)
if err != nil { if err != nil {
if errors.Is(err, xattr.ENOATTR) { if errors.Is(err, xattr.ENOATTR) {
log.Printf("ERROR missing xattr %s", path) hadErr = true
if cont { if verbose {
return nil fmt.Printf("%s <none> ERROR\n", path)
} } else {
return exitErr log.Printf("ERROR missing xattr %s", path)
} }
return err if cont {
} return nil
}
return exitErr
}
return err
}
act, err := fileMultihash(path) act, err := fileMultihash(path)
if err != nil { if err != nil {
return err return err
} }
if !bytes.Equal(exp, act) { ok := bytes.Equal(exp, act)
log.Printf("ERROR checksum mismatch %s", path) if !ok {
if cont { hadErr = true
return nil }
}
return exitErr
}
if cont {
fmt.Printf("OK %s\n", path)
}
return nil
})
if err != nil { if verbose {
if errors.Is(err, exitErr) { res := "OK"
return exitErr if !ok {
} res = "ERROR"
return err }
} fmt.Printf("%s %s %s\n", path, string(act), res)
return nil } else if !ok {
log.Printf("ERROR checksum mismatch %s", path)
}
if !ok && !cont {
return exitErr
}
return nil
})
if err != nil {
if errors.Is(err, exitErr) {
return exitErr
}
return err
}
if hadErr {
return exitErr
}
return nil
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -201,38 +261,66 @@ func ProcessCheck(dir string, cont bool) error {
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func walkAndProcess(root string, fn func(string, os.FileInfo) error) error { func walkAndProcess(root string, fn func(string, os.FileInfo) error) error {
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { root = filepath.Clean(root)
if err != nil { return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
return err if err != nil {
} return err
if info.IsDir() { }
return nil rel, _ := filepath.Rel(root, path)
}
return fn(path, info) if shouldExclude(rel, info) {
}) if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
return nil
}
return fn(path, info)
})
}
func shouldExclude(rel string, info os.FileInfo) bool {
if rel == "." || rel == "" {
return false
}
if excludeDotfiles {
for _, part := range strings.Split(rel, string(os.PathSeparator)) {
if strings.HasPrefix(part, ".") {
return true
}
}
}
for _, pat := range excludePatterns {
if ok, _ := doublestar.PathMatch(pat, rel); ok {
return true
}
}
return false
} }
func hasXattr(path, key string) bool { func hasXattr(path, key string) bool {
_, err := xattr.Get(path, key) _, err := xattr.Get(path, key)
return err == nil return err == nil
} }
// fileMultihash returns the base58-encoded SHA-2-256 multihash of the file.
func fileMultihash(path string) ([]byte, error) { func fileMultihash(path string) ([]byte, error) {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer f.Close() defer f.Close()
h := sha256.New() h := sha256.New()
if _, err := io.Copy(h, f); err != nil { if _, err := io.Copy(h, f); err != nil {
return nil, err return nil, err
} }
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return []byte(base58.Encode(mh)), nil return []byte(base58.Encode(mh)), nil
} }

88
attrsum_test.go
View File

@ -10,7 +10,7 @@ import (
"github.com/pkg/xattr" "github.com/pkg/xattr"
) )
// skipIfNoXattr skips tests when the underlying FS lacks xattr support. // skipIfNoXattr skips tests when underlying FS lacks xattr support.
func skipIfNoXattr(t *testing.T, path string) { func skipIfNoXattr(t *testing.T, path string) {
if err := xattr.Set(path, "user.test", []byte("1")); err != nil { if err := xattr.Set(path, "user.test", []byte("1")); err != nil {
t.Skipf("skipping: xattr not supported: %v", err) t.Skipf("skipping: xattr not supported: %v", err)
@ -21,14 +21,14 @@ func skipIfNoXattr(t *testing.T, path string) {
func writeFile(t *testing.T, root, name, content string) string { func writeFile(t *testing.T, root, name, content string) string {
t.Helper() t.Helper()
path := filepath.Join(root, name) p := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatalf("mkdir: %v", err) t.Fatalf("mkdir: %v", err)
} }
if err := os.WriteFile(path, []byte(content), 0o644); err != nil { if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatalf("write: %v", err) t.Fatalf("write: %v", err)
} }
return path return p
} }
func TestSumAddAndUpdate(t *testing.T) { func TestSumAddAndUpdate(t *testing.T) {
@ -37,29 +37,20 @@ func TestSumAddAndUpdate(t *testing.T) {
f := writeFile(t, dir, "a.txt", "hello") f := writeFile(t, dir, "a.txt", "hello")
// Add: write missing checksum.
if err := ProcessSumAdd(dir); err != nil { if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
// Attributes should exist.
if _, err := xattr.Get(f, checksumKey); err != nil { if _, err := xattr.Get(f, checksumKey); err != nil {
t.Fatalf("checksum missing: %v", err) t.Fatalf("checksum missing: %v", err)
} }
tsb, err := xattr.Get(f, sumTimeKey) tsb, _ := xattr.Get(f, sumTimeKey)
if err != nil {
t.Fatalf("sumtime missing: %v", err)
}
origTime, _ := time.Parse(time.RFC3339Nano, string(tsb)) origTime, _ := time.Parse(time.RFC3339Nano, string(tsb))
// Modify file and bump mtime. // Modify file & bump mtime to force update.
if err := os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644); err != nil { os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644)
t.Fatalf("rewrite: %v", err)
}
now := time.Now().Add(2 * time.Second) now := time.Now().Add(2 * time.Second)
os.Chtimes(f, now, now) os.Chtimes(f, now, now)
// Update should refresh checksum and time.
if err := ProcessSumUpdate(dir); err != nil { if err := ProcessSumUpdate(dir); err != nil {
t.Fatalf("update: %v", err) t.Fatalf("update: %v", err)
} }
@ -78,12 +69,11 @@ func TestProcessCheckIntegration(t *testing.T) {
if err := ProcessSumAdd(dir); err != nil { if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
if err := ProcessCheck(dir, false); err != nil { if err := ProcessCheck(dir, false); err != nil {
t.Fatalf("check ok: %v", err) t.Fatalf("check ok: %v", err)
} }
// Corrupt file contents should produce an error. // Corrupt -> should fail
f := filepath.Join(dir, "b.txt") f := filepath.Join(dir, "b.txt")
os.WriteFile(f, []byte("corrupt"), 0o644) os.WriteFile(f, []byte("corrupt"), 0o644)
@ -92,6 +82,66 @@ func TestProcessCheckIntegration(t *testing.T) {
} }
} }
func TestClearRemovesAttrs(t *testing.T) {
dir := t.TempDir()
skipIfNoXattr(t, dir)
f := writeFile(t, dir, "c.txt", "data")
if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add: %v", err)
}
// Ensure attrs exist
if _, err := xattr.Get(f, checksumKey); err != nil {
t.Fatalf("pre-clear checksum missing: %v", err)
}
if err := ProcessClear(dir); err != nil {
t.Fatalf("clear: %v", err)
}
if _, err := xattr.Get(f, checksumKey); err == nil {
t.Fatalf("checksum still present after clear")
}
if _, err := xattr.Get(f, sumTimeKey); err == nil {
t.Fatalf("sumtime still present after clear")
}
}
func TestExcludeDotfilesAndPatterns(t *testing.T) {
dir := t.TempDir()
skipIfNoXattr(t, dir)
hidden := writeFile(t, dir, ".hidden", "dot")
keep := writeFile(t, dir, "keep.txt", "keep")
skip := writeFile(t, dir, "skip.me", "skip")
// Save global state then set exclusions
oldDotfiles := excludeDotfiles
oldPatterns := excludePatterns
excludeDotfiles = true
excludePatterns = []string{"*.me"}
defer func() {
excludeDotfiles = oldDotfiles
excludePatterns = oldPatterns
}()
if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add with excludes: %v", err)
}
// keep.txt should have xattrs
if _, err := xattr.Get(keep, checksumKey); err != nil {
t.Fatalf("expected xattr on keep.txt: %v", err)
}
// .hidden and skip.me should not
if _, err := xattr.Get(hidden, checksumKey); err == nil {
t.Fatalf(".hidden should have been excluded")
}
if _, err := xattr.Get(skip, checksumKey); err == nil {
t.Fatalf("skip.me should have been excluded by pattern")
}
}
func TestPermissionErrors(t *testing.T) { func TestPermissionErrors(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
skipIfNoXattr(t, dir) skipIfNoXattr(t, dir)

3
go.mod
View File

@ -3,6 +3,8 @@ module git.eeqj.de/sneak/attrsum
go 1.24.1 go 1.24.1
require ( require (
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-multihash v0.2.3
github.com/pkg/xattr v0.4.10 github.com/pkg/xattr v0.4.10
github.com/spf13/cobra v1.9.1 github.com/spf13/cobra v1.9.1
@ -12,7 +14,6 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect github.com/multiformats/go-varint v0.0.6 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect github.com/spf13/pflag v1.0.6 // indirect

2
go.sum
View File

@ -1,3 +1,5 @@
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=