now skips all but regular files

This commit is contained in:
Jeffrey Paul 2025-05-08 14:10:38 -07:00
parent 1bb9528548
commit ebbe20dbdf
2 changed files with 90 additions and 99 deletions

View File

@ -52,7 +52,7 @@ func main() {
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Sum operations // Sum commands
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func newSumCmd() *cobra.Command { func newSumCmd() *cobra.Command {
@ -61,52 +61,33 @@ func newSumCmd() *cobra.Command {
Short: "Checksum maintenance operations", Short: "Checksum maintenance operations",
} }
addCmd := &cobra.Command{ add := &cobra.Command{
Use: "add <directory>", Use: "add <dir>",
Short: "Write checksums for files missing them", Short: "Write checksums for files missing them",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, a []string) error { return ProcessSumAdd(a[0]) },
return ProcessSumAdd(args[0])
},
} }
updateCmd := &cobra.Command{ upd := &cobra.Command{
Use: "update <directory>", Use: "update <dir>",
Short: "Refresh checksums when file modified", Short: "Recalculate checksum when file newer than stored sumtime",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, a []string) error { return ProcessSumUpdate(a[0]) },
return ProcessSumUpdate(args[0])
},
} }
cmd.AddCommand(addCmd, updateCmd) cmd.AddCommand(add, upd)
return cmd return cmd
} }
func ProcessSumAdd(dir string) error { func ProcessSumAdd(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error { return walkAndProcess(dir, func(p string, _ os.FileInfo) error { return writeChecksumAndTime(p) })
if hasXattr(path, checksumKey) {
if verbose {
log.Printf("skip existing %s", path)
}
return nil
}
return writeChecksumAndTime(path)
})
} }
func ProcessSumUpdate(dir string) error { func ProcessSumUpdate(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error { return walkAndProcess(dir, func(p string, info os.FileInfo) error {
needUpdate := false t, err := readSumTime(p)
t, err := readSumTime(path)
if err != nil || info.ModTime().After(t) { if err != nil || info.ModTime().After(t) {
needUpdate = true return writeChecksumAndTime(p)
}
if needUpdate {
if verbose {
log.Printf("update %s", path)
}
return writeChecksumAndTime(path)
} }
return nil return nil
}) })
@ -121,15 +102,15 @@ func writeChecksumAndTime(path string) error {
return fmt.Errorf("set checksum attr: %w", err) return fmt.Errorf("set checksum attr: %w", err)
} }
if verbose { if verbose {
fmt.Printf("%s %s written\n", path, string(hash)) fmt.Printf("%s %s written\n", path, hash)
} }
nowStr := time.Now().UTC().Format(time.RFC3339Nano) ts := time.Now().UTC().Format(time.RFC3339Nano)
if err := xattr.Set(path, sumTimeKey, []byte(nowStr)); err != nil { if err := xattr.Set(path, sumTimeKey, []byte(ts)); err != nil {
return fmt.Errorf("set sumtime attr: %w", err) return fmt.Errorf("set sumtime attr: %w", err)
} }
if verbose { if verbose {
fmt.Printf("%s %s written\n", path, nowStr) fmt.Printf("%s %s written\n", path, ts)
} }
return nil return nil
} }
@ -143,24 +124,22 @@ func readSumTime(path string) (time.Time, error) {
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Clear operation // Clear command
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func newClearCmd() *cobra.Command { func newClearCmd() *cobra.Command {
return &cobra.Command{ return &cobra.Command{
Use: "clear <directory>", Use: "clear <dir>",
Short: "Remove checksum xattrs from files in tree", Short: "Remove checksum xattrs from tree",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, a []string) error { return ProcessClear(a[0]) },
return ProcessClear(args[0])
},
} }
} }
func ProcessClear(dir string) error { func ProcessClear(dir string) error {
return walkAndProcess(dir, func(path string, info os.FileInfo) error { return walkAndProcess(dir, func(p string, _ os.FileInfo) error {
for _, key := range []string{checksumKey, sumTimeKey} { for _, k := range []string{checksumKey, sumTimeKey} {
val, err := xattr.Get(path, key) v, err := xattr.Get(p, k)
if err != nil { if err != nil {
if errors.Is(err, xattr.ENOATTR) { if errors.Is(err, xattr.ENOATTR) {
continue continue
@ -168,9 +147,9 @@ func ProcessClear(dir string) error {
return err return err
} }
if verbose { if verbose {
fmt.Printf("%s %s removed\n", path, string(val)) fmt.Printf("%s %s removed\n", p, string(v))
} }
if err := xattr.Remove(path, key); err != nil { if err := xattr.Remove(p, k); err != nil {
return err return err
} }
} }
@ -179,79 +158,70 @@ func ProcessClear(dir string) error {
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Check operation // Check command
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
func newCheckCmd() *cobra.Command { func newCheckCmd() *cobra.Command {
var cont bool var cont bool
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "check <directory>", Use: "check <dir>",
Short: "Verify stored checksums against file contents", Short: "Verify stored checksums",
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, a []string) error { return ProcessCheck(a[0], cont) },
return ProcessCheck(args[0], cont)
},
} }
cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file") cmd.Flags().BoolVar(&cont, "continue", false, "continue after errors and report each file")
return cmd return cmd
} }
func ProcessCheck(dir string, cont bool) error { func ProcessCheck(dir string, cont bool) error {
exitErr := errors.New("verification failed") fail := errors.New("verification failed")
hadErr := false bad := false
err := walkAndProcess(dir, func(path string, info os.FileInfo) error { err := walkAndProcess(dir, func(p string, _ os.FileInfo) error {
exp, err := xattr.Get(path, checksumKey) exp, err := xattr.Get(p, checksumKey)
if err != nil { if err != nil {
if errors.Is(err, xattr.ENOATTR) { if errors.Is(err, xattr.ENOATTR) {
hadErr = true bad = true
if verbose { if verbose {
fmt.Printf("%s <none> ERROR\n", path) fmt.Printf("%s <none> ERROR\n", p)
} else {
log.Printf("ERROR missing xattr %s", path)
} }
if cont { if cont {
return nil return nil
} }
return exitErr return fail
} }
return err return err
} }
act, err := fileMultihash(path) act, err := fileMultihash(p)
if err != nil { if err != nil {
return err return err
} }
ok := bytes.Equal(exp, act) ok := bytes.Equal(exp, act)
if !ok { if !ok {
hadErr = true bad = true
} }
if verbose { if verbose {
res := "OK" status := "OK"
if !ok { if !ok {
res = "ERROR" status = "ERROR"
} }
fmt.Printf("%s %s %s\n", path, string(act), res) fmt.Printf("%s %s %s\n", p, act, status)
} else if !ok {
log.Printf("ERROR checksum mismatch %s", path)
} }
if !ok && !cont { if !ok && !cont {
return exitErr return fail
} }
return nil return nil
}) })
if err != nil { if err != nil {
if errors.Is(err, exitErr) { if errors.Is(err, fail) {
return exitErr return fail
} }
return err return err
} }
if hadErr { if bad {
return exitErr return fail
} }
return nil return nil
} }
@ -262,12 +232,23 @@ func ProcessCheck(dir string, cont bool) error {
func walkAndProcess(root string, fn func(string, os.FileInfo) error) error { func walkAndProcess(root string, fn func(string, os.FileInfo) error) error {
root = filepath.Clean(root) root = filepath.Clean(root)
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { return filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }
rel, _ := filepath.Rel(root, path)
// skip symlinks entirely
if info.Mode()&os.ModeSymlink != 0 {
if verbose {
log.Printf("skip symlink %s", p)
}
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
rel, _ := filepath.Rel(root, p)
if shouldExclude(rel, info) { if shouldExclude(rel, info) {
if info.IsDir() { if info.IsDir() {
return filepath.SkipDir return filepath.SkipDir
@ -278,7 +259,13 @@ func walkAndProcess(root string, fn func(string, os.FileInfo) error) error {
if info.IsDir() { if info.IsDir() {
return nil return nil
} }
return fn(path, info) if !info.Mode().IsRegular() {
if verbose {
log.Printf("skip non-regular %s", p)
}
return nil
}
return fn(p, info)
}) })
} }
@ -317,7 +304,6 @@ func fileMultihash(path string) ([]byte, error) {
if _, err := io.Copy(h, f); err != nil { if _, err := io.Copy(h, f); err != nil {
return nil, err return nil, err
} }
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -10,7 +10,6 @@ import (
"github.com/pkg/xattr" "github.com/pkg/xattr"
) )
// skipIfNoXattr skips tests when underlying FS lacks xattr support.
func skipIfNoXattr(t *testing.T, path string) { func skipIfNoXattr(t *testing.T, path string) {
if err := xattr.Set(path, "user.test", []byte("1")); err != nil { if err := xattr.Set(path, "user.test", []byte("1")); err != nil {
t.Skipf("skipping: xattr not supported: %v", err) t.Skipf("skipping: xattr not supported: %v", err)
@ -46,7 +45,6 @@ func TestSumAddAndUpdate(t *testing.T) {
tsb, _ := xattr.Get(f, sumTimeKey) tsb, _ := xattr.Get(f, sumTimeKey)
origTime, _ := time.Parse(time.RFC3339Nano, string(tsb)) origTime, _ := time.Parse(time.RFC3339Nano, string(tsb))
// Modify file & bump mtime to force update.
os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644) os.WriteFile(f, []byte(strings.ToUpper("hello")), 0o644)
now := time.Now().Add(2 * time.Second) now := time.Now().Add(2 * time.Second)
os.Chtimes(f, now, now) os.Chtimes(f, now, now)
@ -73,7 +71,6 @@ func TestProcessCheckIntegration(t *testing.T) {
t.Fatalf("check ok: %v", err) t.Fatalf("check ok: %v", err)
} }
// Corrupt -> should fail
f := filepath.Join(dir, "b.txt") f := filepath.Join(dir, "b.txt")
os.WriteFile(f, []byte("corrupt"), 0o644) os.WriteFile(f, []byte("corrupt"), 0o644)
@ -90,10 +87,6 @@ func TestClearRemovesAttrs(t *testing.T) {
if err := ProcessSumAdd(dir); err != nil { if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add: %v", err) t.Fatalf("add: %v", err)
} }
// Ensure attrs exist
if _, err := xattr.Get(f, checksumKey); err != nil {
t.Fatalf("pre-clear checksum missing: %v", err)
}
if err := ProcessClear(dir); err != nil { if err := ProcessClear(dir); err != nil {
t.Fatalf("clear: %v", err) t.Fatalf("clear: %v", err)
@ -114,31 +107,43 @@ func TestExcludeDotfilesAndPatterns(t *testing.T) {
keep := writeFile(t, dir, "keep.txt", "keep") keep := writeFile(t, dir, "keep.txt", "keep")
skip := writeFile(t, dir, "skip.me", "skip") skip := writeFile(t, dir, "skip.me", "skip")
// Save global state then set exclusions oldDot := excludeDotfiles
oldDotfiles := excludeDotfiles oldPat := excludePatterns
oldPatterns := excludePatterns
excludeDotfiles = true excludeDotfiles = true
excludePatterns = []string{"*.me"} excludePatterns = []string{"*.me"}
defer func() { excludeDotfiles, excludePatterns = oldDot, oldPat }()
defer func() {
excludeDotfiles = oldDotfiles
excludePatterns = oldPatterns
}()
if err := ProcessSumAdd(dir); err != nil { if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("add with excludes: %v", err) t.Fatalf("add with excludes: %v", err)
} }
// keep.txt should have xattrs
if _, err := xattr.Get(keep, checksumKey); err != nil { if _, err := xattr.Get(keep, checksumKey); err != nil {
t.Fatalf("expected xattr on keep.txt: %v", err) t.Fatalf("expected xattr on keep.txt: %v", err)
} }
// .hidden and skip.me should not
if _, err := xattr.Get(hidden, checksumKey); err == nil { if _, err := xattr.Get(hidden, checksumKey); err == nil {
t.Fatalf(".hidden should have been excluded") t.Fatalf(".hidden should have been excluded")
} }
if _, err := xattr.Get(skip, checksumKey); err == nil { if _, err := xattr.Get(skip, checksumKey); err == nil {
t.Fatalf("skip.me should have been excluded by pattern") t.Fatalf("skip.me should have been excluded")
}
}
func TestSkipBrokenSymlink(t *testing.T) {
dir := t.TempDir()
skipIfNoXattr(t, dir)
// Create a dangling symlink
link := filepath.Join(dir, "dangling.lnk")
if err := os.Symlink(filepath.Join(dir, "nonexistent.txt"), link); err != nil {
t.Fatalf("symlink: %v", err)
}
// Should not error and should not create xattrs on link
if err := ProcessSumAdd(dir); err != nil {
t.Fatalf("ProcessSumAdd with symlink: %v", err)
}
if _, err := xattr.Get(link, checksumKey); err == nil {
t.Fatalf("symlink should not have xattr")
} }
} }