package main import ( "bufio" "crypto/sha256" "encoding/hex" "errors" "fmt" "io/fs" "os" "path/filepath" "syscall" ) // chunk is the number of bytes hashed from each end of a file. const chunk = 1024 // workQueueDepth bounds the job and result channels feeding the stat // and hash worker pools. const workQueueDepth = 1024 // errNotRegular reports a path that stopped being a regular file // between the walk and stat passes. var errNotRegular = errors.New("no longer a regular file") // fileRec carries one file between the stat and hash passes. type fileRec struct { path string size int64 mtime int64 } // runScan implements the scan subcommand: three sequential passes // (walk, stat, hash) over the trees named by the PATH operands, // emitting one NUL-terminated record per regular file on stdout. Flag // parsing and the at-least-one-operand check are done by cobra. func runScan(roots []string, workers int, oneFS bool) { if workers < 1 { workers = 1 } // A nonexistent operand is a fatal error before any scanning. for _, root := range roots { _, err := os.Lstat(root) if err != nil { fatalf("%v", err) } } paths, walkErrs := walkPass(roots, oneFS) recs, statErrs := statPass(paths, workers) emitted, hashErrs := hashPass(recs, workers) skipped := walkErrs + statErrs + hashErrs fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n", emitted, skipped) } // treeWalker carries the walk-pass state shared by all PATH operands. type treeWalker struct { prog *progress oneFS bool paths []string errs int } // walkPass enumerates every regular file under each root operand in // order. It never follows symlinks, never descends into directories // named .zfs, and warns and continues on any per-path error. With // oneFS set it never descends into a directory on a different // filesystem than its root operand. func walkPass(roots []string, oneFS bool) ([]string, int) { w := &treeWalker{prog: newProgress("walk", -1), oneFS: oneFS} for _, root := range roots { w.walkRoot(root) } w.prog.finish() return w.paths, w.errs } // walkRoot walks a single PATH operand, appending regular-file paths. func (w *treeWalker) walkRoot(root string) { rootDev, rootDevOK := deviceOf(root) walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil { w.errs++ w.prog.warnf("walk %s: %v", p, err) if d != nil && d.IsDir() { return filepath.SkipDir } return nil } if d.IsDir() { return w.dirAction(p, d, rootDev, rootDevOK) } // Regular files only: skip symlinks, sockets, FIFOs, and // device nodes. if !d.Type().IsRegular() { return nil } w.paths = append(w.paths, p) w.prog.increment() return nil }) if walkErr != nil { fatalf("walk %s: %v", root, walkErr) } } // dirAction decides whether the walk descends into directory p. func (w *treeWalker) dirAction(p string, d fs.DirEntry, rootDev uint64, rootDevOK bool) error { // ZFS snapshot pseudo-dirs would list every file once per // snapshot; never descend. if d.Name() == ".zfs" { return filepath.SkipDir } if !w.oneFS || !rootDevOK { return nil } info, err := d.Info() if err != nil { w.errs++ w.prog.warnf("walk %s: %v", p, err) return filepath.SkipDir } if dev, ok := deviceOfInfo(info); ok && dev != rootDev { return filepath.SkipDir } return nil } // deviceOf returns the filesystem device ID of path without following // symlinks. func deviceOf(path string) (uint64, bool) { fi, err := os.Lstat(path) if err != nil { return 0, false } return deviceOfInfo(fi) } // deviceOfInfo extracts the filesystem device ID from a FileInfo, when // the platform exposes one. func deviceOfInfo(fi fs.FileInfo) (uint64, bool) { st, ok := fi.Sys().(*syscall.Stat_t) if !ok { return 0, false } return statDev(st), true } // statPass lstats every collected path in a worker pool, recording size // and mtime. Paths that fail to stat (or are no longer regular files) // are warned about and dropped. func statPass(paths []string, workers int) ([]fileRec, int) { type result struct { rec fileRec err error } jobs := make(chan string, workQueueDepth) results := make(chan result, workQueueDepth) for range workers { go func() { for p := range jobs { fi, err := os.Lstat(p) switch { case err != nil: results <- result{rec: fileRec{path: p}, err: err} case !fi.Mode().IsRegular(): results <- result{ rec: fileRec{path: p}, err: errNotRegular, } default: results <- result{rec: fileRec{ path: p, size: fi.Size(), mtime: fi.ModTime().Unix(), }} } } }() } go func() { for _, p := range paths { jobs <- p } close(jobs) }() prog := newProgress("stat", int64(len(paths))) var errs int recs := make([]fileRec, 0, len(paths)) for range paths { r := <-results if r.err != nil { errs++ prog.warnf("stat %s: %v", r.rec.path, r.err) } else { recs = append(recs, r.rec) } prog.increment() } prog.finish() return recs, errs } // hashResult carries one file's head/tail hashes (or the error that // prevented hashing it) from the hash workers to the main goroutine. type hashResult struct { rec fileRec head string tail string err error } // startHashWorkers starts the hash worker pool over recs and returns // the channel its results arrive on (one per record, in completion // order). func startHashWorkers(recs []fileRec, workers int) <-chan hashResult { jobs := make(chan fileRec, workQueueDepth) results := make(chan hashResult, workQueueDepth) for range workers { go func() { for rec := range jobs { head, tail, err := hashHeadTail(rec.path, rec.size) results <- hashResult{rec: rec, head: head, tail: tail, err: err} } }() } go func() { for _, rec := range recs { jobs <- rec } close(jobs) }() return results } // hashPass hashes the first and last chunk bytes of every file in a // worker pool and emits the output records on stdout from the main // goroutine. Files that fail to open or read are warned about and // dropped. func hashPass(recs []fileRec, workers int) (int, int) { results := startHashWorkers(recs, workers) prog := newProgress("hash", int64(len(recs))) out := bufio.NewWriterSize(os.Stdout, ioBufSize) var emitted, errs int for range recs { r := <-results if r.err != nil { errs++ prog.warnf("hash %s: %v", r.rec.path, r.err) prog.increment() continue } _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path) if err != nil { fatalf("write stdout: %v", err) } emitted++ prog.increment() } prog.finish() err := out.Flush() if err != nil { fatalf("write stdout: %v", err) } return emitted, errs } // hashHeadTail returns the lowercase-hex SHA-256 of the first // min(chunk, size) bytes and of the last min(chunk, size) bytes of the // file at path. The two reads overlap when size < 2*chunk; for // size == 0 both hashes are of the empty input. size is the value // recorded by the stat pass. func hashHeadTail(path string, size int64) (string, string, error) { //nolint:gosec // hashing operator-supplied paths is the tool's purpose f, err := os.Open(path) if err != nil { return "", "", err } defer func() { _ = f.Close() }() n := min(int64(chunk), size) buf := make([]byte, n) if n > 0 { _, err = f.ReadAt(buf, 0) if err != nil { return "", "", err } } h := sha256.Sum256(buf) if n > 0 { _, err = f.ReadAt(buf, size-n) if err != nil { return "", "", err } } t := sha256.Sum256(buf) return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil }