sfdupes/scan.go

package main

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"syscall"
)

// chunk is the number of bytes hashed from each end of a file.
const chunk = 1024

// workQueueDepth bounds the job and result channels feeding the stat
// and hash worker pools.
const workQueueDepth = 1024

// errNotRegular reports a path that stopped being a regular file
// between the walk and stat passes.
var errNotRegular = errors.New("no longer a regular file")

// fileRec carries one file between the stat and hash passes.
type fileRec struct {
	path  string
	size  int64
	mtime int64
}

// runScan implements the scan subcommand: three sequential passes
// (walk, stat, hash) over the trees named by the PATH operands,
// emitting one NUL-terminated record per regular file on stdout. Flag
// parsing and the at-least-one-operand check are done by cobra.
func runScan(roots []string, workers int, oneFS bool) {
	if workers < 1 {
		workers = 1
	}

	// A nonexistent operand is a fatal error before any scanning.
	for _, root := range roots {
		_, err := os.Lstat(root)
		if err != nil {
			fatalf("%v", err)
		}
	}

	paths, walkErrs := walkPass(roots, oneFS)
	recs, statErrs := statPass(paths, workers)
	emitted, hashErrs := hashPass(recs, workers)

	skipped := walkErrs + statErrs + hashErrs
	fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n",
		emitted, skipped)
}

// treeWalker carries the walk-pass state shared by all PATH operands.
type treeWalker struct {
	prog  *progress
	oneFS bool
	paths []string
	errs  int
}

// walkPass enumerates every regular file under each root operand in
// order. It never follows symlinks, never descends into directories
// named .zfs, and warns and continues on any per-path error. With
// oneFS set it never descends into a directory on a different
// filesystem than its root operand.
func walkPass(roots []string, oneFS bool) ([]string, int) {
	w := &treeWalker{prog: newProgress("walk", -1), oneFS: oneFS}

	for _, root := range roots {
		w.walkRoot(root)
	}

	w.prog.finish()

	return w.paths, w.errs
}

// walkRoot walks a single PATH operand, appending regular-file paths.
func (w *treeWalker) walkRoot(root string) {
	rootDev, rootDevOK := deviceOf(root)

	walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			w.errs++

			w.prog.warnf("walk %s: %v", p, err)

			if d != nil && d.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if d.IsDir() {
			return w.dirAction(p, d, rootDev, rootDevOK)
		}
		// Regular files only: skip symlinks, sockets, FIFOs, and
		// device nodes.
		if !d.Type().IsRegular() {
			return nil
		}

		w.paths = append(w.paths, p)

		w.prog.increment()

		return nil
	})
	if walkErr != nil {
		fatalf("walk %s: %v", root, walkErr)
	}
}

// dirAction decides whether the walk descends into directory p.
func (w *treeWalker) dirAction(p string, d fs.DirEntry, rootDev uint64, rootDevOK bool) error {
	// ZFS snapshot pseudo-dirs would list every file once per
	// snapshot; never descend.
	if d.Name() == ".zfs" {
		return filepath.SkipDir
	}

	if !w.oneFS || !rootDevOK {
		return nil
	}

	info, err := d.Info()
	if err != nil {
		w.errs++

		w.prog.warnf("walk %s: %v", p, err)

		return filepath.SkipDir
	}

	if dev, ok := deviceOfInfo(info); ok && dev != rootDev {
		return filepath.SkipDir
	}

	return nil
}

// deviceOf returns the filesystem device ID of path without following
// symlinks.
func deviceOf(path string) (uint64, bool) {
	fi, err := os.Lstat(path)
	if err != nil {
		return 0, false
	}

	return deviceOfInfo(fi)
}

// deviceOfInfo extracts the filesystem device ID from a FileInfo, when
// the platform exposes one.
func deviceOfInfo(fi fs.FileInfo) (uint64, bool) {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return 0, false
	}

	return statDev(st), true
}

// statPass lstats every collected path in a worker pool, recording size
// and mtime. Paths that fail to stat (or are no longer regular files)
// are warned about and dropped.
func statPass(paths []string, workers int) ([]fileRec, int) {
	type result struct {
		rec fileRec
		err error
	}

	jobs := make(chan string, workQueueDepth)
	results := make(chan result, workQueueDepth)

	for range workers {
		go func() {
			for p := range jobs {
				fi, err := os.Lstat(p)
				switch {
				case err != nil:
					results <- result{rec: fileRec{path: p}, err: err}
				case !fi.Mode().IsRegular():
					results <- result{
						rec: fileRec{path: p},
						err: errNotRegular,
					}
				default:
					results <- result{rec: fileRec{
						path:  p,
						size:  fi.Size(),
						mtime: fi.ModTime().Unix(),
					}}
				}
			}
		}()
	}

	go func() {
		for _, p := range paths {
			jobs <- p
		}

		close(jobs)
	}()

	prog := newProgress("stat", int64(len(paths)))

	var errs int

	recs := make([]fileRec, 0, len(paths))
	for range paths {
		r := <-results
		if r.err != nil {
			errs++

			prog.warnf("stat %s: %v", r.rec.path, r.err)
		} else {
			recs = append(recs, r.rec)
		}

		prog.increment()
	}

	prog.finish()

	return recs, errs
}

// hashResult carries one file's head/tail hashes (or the error that
// prevented hashing it) from the hash workers to the main goroutine.
type hashResult struct {
	rec  fileRec
	head string
	tail string
	err  error
}

// startHashWorkers starts the hash worker pool over recs and returns
// the channel its results arrive on (one per record, in completion
// order).
func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
	jobs := make(chan fileRec, workQueueDepth)
	results := make(chan hashResult, workQueueDepth)

	for range workers {
		go func() {
			for rec := range jobs {
				head, tail, err := hashHeadTail(rec.path, rec.size)
				results <- hashResult{rec: rec, head: head, tail: tail, err: err}
			}
		}()
	}

	go func() {
		for _, rec := range recs {
			jobs <- rec
		}

		close(jobs)
	}()

	return results
}

// hashPass hashes the first and last chunk bytes of every file in a
// worker pool and emits the output records on stdout from the main
// goroutine. Files that fail to open or read are warned about and
// dropped.
func hashPass(recs []fileRec, workers int) (int, int) {
	results := startHashWorkers(recs, workers)
	prog := newProgress("hash", int64(len(recs)))
	out := bufio.NewWriterSize(os.Stdout, ioBufSize)

	var emitted, errs int

	for range recs {
		r := <-results
		if r.err != nil {
			errs++

			prog.warnf("hash %s: %v", r.rec.path, r.err)
			prog.increment()

			continue
		}

		_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
			r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
		if err != nil {
			fatalf("write stdout: %v", err)
		}

		emitted++

		prog.increment()
	}

	prog.finish()

	err := out.Flush()
	if err != nil {
		fatalf("write stdout: %v", err)
	}

	return emitted, errs
}

// hashHeadTail returns the lowercase-hex SHA-256 of the first
// min(chunk, size) bytes and of the last min(chunk, size) bytes of the
// file at path. The two reads overlap when size < 2*chunk; for
// size == 0 both hashes are of the empty input. size is the value
// recorded by the stat pass.
func hashHeadTail(path string, size int64) (string, string, error) {
	//nolint:gosec // hashing operator-supplied paths is the tool's purpose
	f, err := os.Open(path)
	if err != nil {
		return "", "", err
	}

	defer func() { _ = f.Close() }()

	n := min(int64(chunk), size)

	buf := make([]byte, n)
	if n > 0 {
		_, err = f.ReadAt(buf, 0)
		if err != nil {
			return "", "", err
		}
	}

	h := sha256.Sum256(buf)

	if n > 0 {
		_, err = f.ReadAt(buf, size-n)
		if err != nil {
			return "", "", err
		}
	}

	t := sha256.Sum256(buf)

	return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
}