Make code lint-clean under the canonical golangci-lint config
150 findings fixed: linter autofixes for whitespace style (wsl_v5, nlreturn, noinlineerr), named returns removed, magic numbers replaced with named constants, static errNotRegular error, explicit Close/Parse error handling, unused cobra params renamed to _, and runReport/ runTrees/hashPass split into helpers to satisfy cyclop, gocognit, and funlen. Behavior verified unchanged against the README smoke test.
This commit is contained in:
26
main.go
26
main.go
@@ -20,15 +20,23 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Exit codes: 0 is success (even with per-file warnings), exitFatal is
|
||||
// a fatal error, exitUsage is a usage error.
|
||||
const (
|
||||
exitFatal = 1
|
||||
exitUsage = 2
|
||||
)
|
||||
|
||||
func main() {
|
||||
root := &cobra.Command{
|
||||
Use: "sfdupes",
|
||||
Short: "Find candidate duplicate files by size and head/tail SHA-256",
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
// A missing subcommand prints usage and exits 2.
|
||||
_ = cmd.Usage()
|
||||
os.Exit(2)
|
||||
|
||||
os.Exit(exitUsage)
|
||||
},
|
||||
}
|
||||
// Everything on stdout is machine-readable data; all human-facing
|
||||
@@ -43,7 +51,7 @@ func main() {
|
||||
// The README specifies single-dash flags (-root, -workers);
|
||||
// parse them with the stdlib flag package inside runScan.
|
||||
DisableFlagParsing: true,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runScan(args)
|
||||
},
|
||||
}
|
||||
@@ -52,7 +60,7 @@ func main() {
|
||||
Use: "report [files.dat|-]",
|
||||
Short: "Read a scan stream and print the file-level duplicates report",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runReport(args)
|
||||
},
|
||||
}
|
||||
@@ -61,21 +69,23 @@ func main() {
|
||||
Use: "trees [files.dat|-]",
|
||||
Short: "Read a scan stream and print the duplicate-tree report",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runTrees(args)
|
||||
},
|
||||
}
|
||||
|
||||
root.AddCommand(scanCmd, reportCmd, treesCmd)
|
||||
if err := root.Execute(); err != nil {
|
||||
|
||||
err := root.Execute()
|
||||
if err != nil {
|
||||
// Cobra has already printed the error and usage to stderr;
|
||||
// an invalid subcommand or bad arguments is a usage error.
|
||||
os.Exit(2)
|
||||
os.Exit(exitUsage)
|
||||
}
|
||||
}
|
||||
|
||||
// fatalf reports a fatal error and exits 1.
|
||||
func fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
os.Exit(exitFatal)
|
||||
}
|
||||
|
||||
33
progress.go
33
progress.go
@@ -12,12 +12,23 @@ import (
|
||||
// is not a TTY.
|
||||
const plainInterval = 5 * time.Second
|
||||
|
||||
// barThrottle caps how often the TTY progress bar redraws.
|
||||
const barThrottle = 100 * time.Millisecond
|
||||
|
||||
// walkSpinnerType selects the progressbar library's spinner style used
|
||||
// for the walk pass, which has no known total.
|
||||
const walkSpinnerType = 14
|
||||
|
||||
// percentScale converts a fraction to a percentage.
|
||||
const percentScale = 100
|
||||
|
||||
// stderrIsTTY reports whether stderr is attached to a terminal.
|
||||
func stderrIsTTY() bool {
|
||||
fi, err := os.Stderr.Stat()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return fi.Mode()&os.ModeCharDevice != 0
|
||||
}
|
||||
|
||||
@@ -42,6 +53,7 @@ func newProgress(label string, total int64) *progress {
|
||||
if !stderrIsTTY() {
|
||||
return p
|
||||
}
|
||||
|
||||
opts := []progressbar.Option{
|
||||
progressbar.OptionSetWriter(os.Stderr),
|
||||
progressbar.OptionSetDescription(label),
|
||||
@@ -49,7 +61,7 @@ func newProgress(label string, total int64) *progress {
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionSetItsString("files"),
|
||||
progressbar.OptionSetElapsedTime(true),
|
||||
progressbar.OptionThrottle(100 * time.Millisecond),
|
||||
progressbar.OptionThrottle(barThrottle),
|
||||
}
|
||||
if total >= 0 {
|
||||
opts = append(opts,
|
||||
@@ -59,10 +71,12 @@ func newProgress(label string, total int64) *progress {
|
||||
} else {
|
||||
opts = append(opts,
|
||||
progressbar.OptionSetPredictTime(false),
|
||||
progressbar.OptionSpinnerType(14),
|
||||
progressbar.OptionSpinnerType(walkSpinnerType),
|
||||
)
|
||||
}
|
||||
|
||||
p.bar = progressbar.NewOptions64(total, opts...)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
@@ -71,8 +85,10 @@ func (p *progress) increment() {
|
||||
p.count++
|
||||
if p.bar != nil {
|
||||
_ = p.bar.Add(1)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if time.Since(p.last) >= plainInterval {
|
||||
p.last = time.Now()
|
||||
fmt.Fprintln(os.Stderr, p.plainLine())
|
||||
@@ -84,6 +100,7 @@ func (p *progress) warnf(format string, args ...any) {
|
||||
if p.bar != nil {
|
||||
_ = p.bar.Clear()
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
@@ -91,9 +108,12 @@ func (p *progress) warnf(format string, args ...any) {
|
||||
func (p *progress) finish() {
|
||||
if p.bar != nil {
|
||||
_ = p.bar.Finish()
|
||||
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stderr, p.plainLine())
|
||||
}
|
||||
|
||||
@@ -104,19 +124,24 @@ func (p *progress) plainLine() string {
|
||||
return fmt.Sprintf("%s: %d files, elapsed %s",
|
||||
p.label, p.count, elapsed.Round(time.Second))
|
||||
}
|
||||
pct := 100.0
|
||||
|
||||
pct := float64(percentScale)
|
||||
if p.total > 0 {
|
||||
pct = 100 * float64(p.count) / float64(p.total)
|
||||
pct = percentScale * float64(p.count) / float64(p.total)
|
||||
}
|
||||
|
||||
rate := 0.0
|
||||
if s := elapsed.Seconds(); s > 0 {
|
||||
rate = float64(p.count) / s
|
||||
}
|
||||
|
||||
eta := "?"
|
||||
|
||||
if rate > 0 && p.count <= p.total {
|
||||
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
|
||||
eta = remaining.Round(time.Second).String()
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
|
||||
p.label, p.count, p.total, pct, rate,
|
||||
elapsed.Round(time.Second), eta)
|
||||
|
||||
147
report.go
147
report.go
@@ -11,6 +11,26 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ioBufSize is the buffer size for the buffered scan-stream reader and
|
||||
// the buffered stdout writers.
|
||||
const ioBufSize = 1 << 20
|
||||
|
||||
// recordFieldCount is the number of tab-separated fields in a scan
|
||||
// record: size, mtime, head hash, tail hash, path.
|
||||
const recordFieldCount = 5
|
||||
|
||||
// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
|
||||
// to read scan records; a record longer than scanMaxRecordSize is a
|
||||
// fatal read error.
|
||||
const (
|
||||
scanInitBufSize = 64 << 10
|
||||
scanMaxRecordSize = 4 << 20
|
||||
)
|
||||
|
||||
// minGroupSize is the smallest number of members that makes a
|
||||
// duplicate group.
|
||||
const minGroupSize = 2
|
||||
|
||||
// scanRec is one well-formed record parsed from a scan stream. The
|
||||
// signature (size, head, tail) is the duplicate key; mtime is
|
||||
// informational and not retained.
|
||||
@@ -24,14 +44,16 @@ type scanRec struct {
|
||||
// openScanInput resolves the analysis-mode input: the file named by the
|
||||
// single optional positional argument, or stdin when it is absent or
|
||||
// "-". The returned closer must be called when reading is done.
|
||||
func openScanInput(args []string) (in io.Reader, name string, closer func()) {
|
||||
func openScanInput(args []string) (io.Reader, string, func()) {
|
||||
if len(args) == 1 && args[0] != "-" {
|
||||
f, err := os.Open(args[0])
|
||||
if err != nil {
|
||||
fatalf("open %s: %v", args[0], err)
|
||||
}
|
||||
return f, args[0], func() { f.Close() }
|
||||
|
||||
return f, args[0], func() { _ = f.Close() }
|
||||
}
|
||||
|
||||
return os.Stdin, "stdin", func() {}
|
||||
}
|
||||
|
||||
@@ -39,23 +61,33 @@ func openScanInput(args []string) (in io.Reader, name string, closer func()) {
|
||||
// that does not have exactly 5 fields or whose size is non-numeric is
|
||||
// counted as malformed and skipped. Total records read is
|
||||
// len(recs) + malformed.
|
||||
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
|
||||
sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
|
||||
sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
|
||||
func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
|
||||
sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
|
||||
sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
|
||||
sc.Split(splitNUL)
|
||||
|
||||
var (
|
||||
recs []scanRec
|
||||
malformed int
|
||||
)
|
||||
|
||||
for sc.Scan() {
|
||||
// The path is the last field and may itself contain tabs,
|
||||
// so split into at most 5 fields.
|
||||
fields := strings.SplitN(sc.Text(), "\t", 5)
|
||||
if len(fields) != 5 {
|
||||
// so split into at most recordFieldCount fields.
|
||||
fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
|
||||
if len(fields) != recordFieldCount {
|
||||
malformed++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
size, err := strconv.ParseInt(fields[0], 10, 64)
|
||||
if err != nil {
|
||||
malformed++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
recs = append(recs, scanRec{
|
||||
size: size,
|
||||
head: fields[2],
|
||||
@@ -63,9 +95,12 @@ func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int)
|
||||
path: fields[4],
|
||||
})
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
|
||||
err := sc.Err()
|
||||
if err != nil {
|
||||
fatalf("read %s: %v", name, err)
|
||||
}
|
||||
|
||||
return recs, malformed
|
||||
}
|
||||
|
||||
@@ -86,56 +121,37 @@ type dupeGroup struct {
|
||||
func runReport(args []string) {
|
||||
in, name, closer := openScanInput(args)
|
||||
defer closer()
|
||||
|
||||
recs, malformed := parseScanStream(in, name)
|
||||
records := len(recs) + malformed
|
||||
dupes := collectDupeGroups(recs)
|
||||
|
||||
type dupeKey struct {
|
||||
size int64
|
||||
head string
|
||||
tail string
|
||||
}
|
||||
groups := make(map[dupeKey][]string)
|
||||
for _, r := range recs {
|
||||
k := dupeKey{size: r.size, head: r.head, tail: r.tail}
|
||||
groups[k] = append(groups[k], r.path)
|
||||
}
|
||||
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||
|
||||
var dupes []dupeGroup
|
||||
for k, paths := range groups {
|
||||
if len(paths) < 2 {
|
||||
continue
|
||||
}
|
||||
slices.Sort(paths)
|
||||
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
|
||||
}
|
||||
// Biggest reclaimable space first; ties broken by first path.
|
||||
slices.SortFunc(dupes, func(a, b dupeGroup) int {
|
||||
if a.size != b.size {
|
||||
if a.size > b.size {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(a.paths[0], b.paths[0])
|
||||
})
|
||||
|
||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
||||
if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
|
||||
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
dupeFiles := 0
|
||||
|
||||
var reclaimable int64
|
||||
|
||||
for _, g := range dupes {
|
||||
for _, p := range g.paths[1:] {
|
||||
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
|
||||
g.paths[0], p, g.size); err != nil {
|
||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
|
||||
g.paths[0], p, g.size)
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
dupeFiles++
|
||||
reclaimable += g.size
|
||||
}
|
||||
}
|
||||
if err := out.Flush(); err != nil {
|
||||
|
||||
err = out.Flush()
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
@@ -145,24 +161,65 @@ func runReport(args []string) {
|
||||
humanBytes(reclaimable))
|
||||
}
|
||||
|
||||
// collectDupeGroups groups records by signature and returns every group
|
||||
// with two or more paths, each group's paths sorted lexicographically,
|
||||
// groups ordered by size descending then by first path ascending.
|
||||
func collectDupeGroups(recs []scanRec) []dupeGroup {
|
||||
groups := make(map[fileSig][]string)
|
||||
|
||||
for _, r := range recs {
|
||||
k := fileSig{size: r.size, head: r.head, tail: r.tail}
|
||||
groups[k] = append(groups[k], r.path)
|
||||
}
|
||||
|
||||
var dupes []dupeGroup
|
||||
|
||||
for k, paths := range groups {
|
||||
if len(paths) < minGroupSize {
|
||||
continue
|
||||
}
|
||||
|
||||
slices.Sort(paths)
|
||||
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
|
||||
}
|
||||
|
||||
// Biggest reclaimable space first; ties broken by first path.
|
||||
slices.SortFunc(dupes, func(a, b dupeGroup) int {
|
||||
if a.size != b.size {
|
||||
if a.size > b.size {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
return strings.Compare(a.paths[0], b.paths[0])
|
||||
})
|
||||
|
||||
return dupes
|
||||
}
|
||||
|
||||
// malformedNote formats the optional malformed-record note for the
|
||||
// stderr summaries.
|
||||
func malformedNote(malformed int) string {
|
||||
if malformed == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
|
||||
}
|
||||
|
||||
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
|
||||
// data without a terminator at EOF is returned as a final record.
|
||||
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
||||
func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
|
||||
if i := bytes.IndexByte(data, 0); i >= 0 {
|
||||
return i + 1, data[:i], nil
|
||||
}
|
||||
|
||||
if atEOF && len(data) > 0 {
|
||||
return len(data), data, nil
|
||||
}
|
||||
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
@@ -172,10 +229,12 @@ func humanBytes(n int64) string {
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
|
||||
div, exp := int64(unit), 0
|
||||
for m := n / unit; m >= unit; m /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
160
scan.go
160
scan.go
@@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -15,6 +16,14 @@ import (
|
||||
// chunk is the number of bytes hashed from each end of a file.
|
||||
const chunk = 1024
|
||||
|
||||
// workQueueDepth bounds the job and result channels feeding the stat
|
||||
// and hash worker pools.
|
||||
const workQueueDepth = 1024
|
||||
|
||||
// errNotRegular reports a path that stopped being a regular file
|
||||
// between the walk and stat passes.
|
||||
var errNotRegular = errors.New("no longer a regular file")
|
||||
|
||||
// fileRec carries one file between the stat and hash passes.
|
||||
type fileRec struct {
|
||||
path string
|
||||
@@ -30,11 +39,18 @@ func runScan(args []string) {
|
||||
root := fl.String("root", "/srv", "directory tree to scan")
|
||||
workers := fl.Int("workers", runtime.NumCPU(),
|
||||
"concurrent workers for the stat and hash passes")
|
||||
fl.Parse(args)
|
||||
// The flag set uses ExitOnError, so Parse cannot return a non-nil
|
||||
// error; the check keeps the error handled explicitly.
|
||||
err := fl.Parse(args)
|
||||
if err != nil {
|
||||
os.Exit(exitUsage)
|
||||
}
|
||||
|
||||
if fl.NArg() > 0 {
|
||||
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
|
||||
os.Exit(2)
|
||||
os.Exit(exitUsage)
|
||||
}
|
||||
|
||||
if *workers < 1 {
|
||||
*workers = 1
|
||||
}
|
||||
@@ -43,6 +59,7 @@ func runScan(args []string) {
|
||||
if err != nil {
|
||||
fatalf("root %s: %v", *root, err)
|
||||
}
|
||||
|
||||
if !st.IsDir() {
|
||||
fatalf("root %s: not a directory", *root)
|
||||
}
|
||||
@@ -59,23 +76,33 @@ func runScan(args []string) {
|
||||
// walkPass enumerates every regular file under root. It never follows
|
||||
// symlinks, never descends into directories named .zfs, and warns and
|
||||
// continues on any per-path error.
|
||||
func walkPass(root string) (paths []string, errs int) {
|
||||
func walkPass(root string) ([]string, int) {
|
||||
var (
|
||||
paths []string
|
||||
errs int
|
||||
)
|
||||
|
||||
prog := newProgress("walk", -1)
|
||||
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
errs++
|
||||
|
||||
prog.warnf("walk %s: %v", p, err)
|
||||
|
||||
if d != nil && d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// ZFS snapshot pseudo-dirs would list every file
|
||||
// once per snapshot; never descend.
|
||||
if d.Name() == ".zfs" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
// Regular files only: skip symlinks, sockets, FIFOs, and
|
||||
@@ -83,27 +110,35 @@ func walkPass(root string) (paths []string, errs int) {
|
||||
if !d.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
paths = append(paths, p)
|
||||
|
||||
prog.increment()
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
prog.finish()
|
||||
|
||||
if walkErr != nil {
|
||||
fatalf("walk %s: %v", root, walkErr)
|
||||
}
|
||||
|
||||
return paths, errs
|
||||
}
|
||||
|
||||
// statPass lstats every collected path in a worker pool, recording size
|
||||
// and mtime. Paths that fail to stat (or are no longer regular files)
|
||||
// are warned about and dropped.
|
||||
func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
||||
func statPass(paths []string, workers int) ([]fileRec, int) {
|
||||
type result struct {
|
||||
rec fileRec
|
||||
err error
|
||||
}
|
||||
jobs := make(chan string, 1024)
|
||||
results := make(chan result, 1024)
|
||||
|
||||
jobs := make(chan string, workQueueDepth)
|
||||
results := make(chan result, workQueueDepth)
|
||||
|
||||
for range workers {
|
||||
go func() {
|
||||
for p := range jobs {
|
||||
@@ -114,7 +149,7 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
||||
case !fi.Mode().IsRegular():
|
||||
results <- result{
|
||||
rec: fileRec{path: p},
|
||||
err: fmt.Errorf("no longer a regular file"),
|
||||
err: errNotRegular,
|
||||
}
|
||||
default:
|
||||
results <- result{rec: fileRec{
|
||||
@@ -126,78 +161,114 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
for _, p := range paths {
|
||||
jobs <- p
|
||||
}
|
||||
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
prog := newProgress("stat", int64(len(paths)))
|
||||
recs = make([]fileRec, 0, len(paths))
|
||||
|
||||
var errs int
|
||||
|
||||
recs := make([]fileRec, 0, len(paths))
|
||||
for range paths {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
errs++
|
||||
|
||||
prog.warnf("stat %s: %v", r.rec.path, r.err)
|
||||
} else {
|
||||
recs = append(recs, r.rec)
|
||||
}
|
||||
|
||||
prog.increment()
|
||||
}
|
||||
|
||||
prog.finish()
|
||||
|
||||
return recs, errs
|
||||
}
|
||||
|
||||
// hashResult carries one file's head/tail hashes (or the error that
|
||||
// prevented hashing it) from the hash workers to the main goroutine.
|
||||
type hashResult struct {
|
||||
rec fileRec
|
||||
head string
|
||||
tail string
|
||||
err error
|
||||
}
|
||||
|
||||
// startHashWorkers starts the hash worker pool over recs and returns
|
||||
// the channel its results arrive on (one per record, in completion
|
||||
// order).
|
||||
func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
|
||||
jobs := make(chan fileRec, workQueueDepth)
|
||||
results := make(chan hashResult, workQueueDepth)
|
||||
|
||||
for range workers {
|
||||
go func() {
|
||||
for rec := range jobs {
|
||||
head, tail, err := hashHeadTail(rec.path, rec.size)
|
||||
results <- hashResult{rec: rec, head: head, tail: tail, err: err}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
for _, rec := range recs {
|
||||
jobs <- rec
|
||||
}
|
||||
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// hashPass hashes the first and last chunk bytes of every file in a
|
||||
// worker pool and emits the output records on stdout from the main
|
||||
// goroutine. Files that fail to open or read are warned about and
|
||||
// dropped.
|
||||
func hashPass(recs []fileRec, workers int) (emitted, errs int) {
|
||||
type result struct {
|
||||
rec fileRec
|
||||
head string
|
||||
tail string
|
||||
err error
|
||||
}
|
||||
jobs := make(chan fileRec, 1024)
|
||||
results := make(chan result, 1024)
|
||||
for range workers {
|
||||
go func() {
|
||||
for rec := range jobs {
|
||||
head, tail, err := hashHeadTail(rec.path, rec.size)
|
||||
results <- result{rec: rec, head: head, tail: tail, err: err}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
for _, rec := range recs {
|
||||
jobs <- rec
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
func hashPass(recs []fileRec, workers int) (int, int) {
|
||||
results := startHashWorkers(recs, workers)
|
||||
prog := newProgress("hash", int64(len(recs)))
|
||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
||||
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||
|
||||
var emitted, errs int
|
||||
|
||||
for range recs {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
errs++
|
||||
|
||||
prog.warnf("hash %s: %v", r.rec.path, r.err)
|
||||
prog.increment()
|
||||
|
||||
continue
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
|
||||
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil {
|
||||
|
||||
_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
|
||||
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
emitted++
|
||||
|
||||
prog.increment()
|
||||
}
|
||||
|
||||
prog.finish()
|
||||
if err := out.Flush(); err != nil {
|
||||
|
||||
err := out.Flush()
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
return emitted, errs
|
||||
}
|
||||
|
||||
@@ -206,26 +277,35 @@ func hashPass(recs []fileRec, workers int) (emitted, errs int) {
|
||||
// file at path. The two reads overlap when size < 2*chunk; for
|
||||
// size == 0 both hashes are of the empty input. size is the value
|
||||
// recorded by the stat pass.
|
||||
func hashHeadTail(path string, size int64) (head, tail string, err error) {
|
||||
func hashHeadTail(path string, size int64) (string, string, error) {
|
||||
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
n := min(int64(chunk), size)
|
||||
|
||||
buf := make([]byte, n)
|
||||
if n > 0 {
|
||||
if _, err := f.ReadAt(buf, 0); err != nil {
|
||||
_, err = f.ReadAt(buf, 0)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
h := sha256.Sum256(buf)
|
||||
|
||||
if n > 0 {
|
||||
if _, err := f.ReadAt(buf, size-n); err != nil {
|
||||
_, err = f.ReadAt(buf, size-n)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
t := sha256.Sum256(buf)
|
||||
|
||||
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
|
||||
}
|
||||
|
||||
113
trees.go
113
trees.go
@@ -38,16 +38,64 @@ type treeNode struct {
|
||||
func runTrees(args []string) {
|
||||
in, name, closer := openScanInput(args)
|
||||
defer closer()
|
||||
|
||||
recs, malformed := parseScanStream(in, name)
|
||||
records := len(recs) + malformed
|
||||
|
||||
// Build the hierarchy under a synthetic super-root. Paths are
|
||||
// split on "/"; for absolute paths the first component is empty,
|
||||
// which simply becomes a top-level node representing "/".
|
||||
super, allDirs := buildHierarchy(recs)
|
||||
super.compute()
|
||||
|
||||
dupes := collectTreeGroups(allDirs, super)
|
||||
|
||||
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||
|
||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
dupeTrees := 0
|
||||
|
||||
var reclaimable int64
|
||||
|
||||
for _, g := range dupes {
|
||||
first := g[0]
|
||||
for _, n := range g[1:] {
|
||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||
first.path, n.path, first.fileCount, first.totalSize)
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
dupeTrees++
|
||||
reclaimable += first.totalSize
|
||||
}
|
||||
}
|
||||
|
||||
err = out.Flush()
|
||||
if err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
|
||||
records, malformedNote(malformed), len(dupes), dupeTrees,
|
||||
humanBytes(reclaimable))
|
||||
}
|
||||
|
||||
// buildHierarchy reconstructs the directory hierarchy from the record
|
||||
// paths under a synthetic super-root. Paths are split on "/"; for
|
||||
// absolute paths the first component is empty, which simply becomes a
|
||||
// top-level node representing "/". It returns the super-root and every
|
||||
// directory node created.
|
||||
func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
|
||||
super := &treeNode{}
|
||||
|
||||
var allDirs []*treeNode
|
||||
|
||||
for _, r := range recs {
|
||||
comps := strings.Split(r.path, "/")
|
||||
|
||||
node := super
|
||||
for _, c := range comps[:len(comps)-1] {
|
||||
child := node.dirs[c]
|
||||
@@ -56,76 +104,68 @@ func runTrees(args []string) {
|
||||
if node != super {
|
||||
childPath = node.path + "/" + c
|
||||
}
|
||||
|
||||
child = &treeNode{path: childPath, parent: node}
|
||||
if node.dirs == nil {
|
||||
node.dirs = make(map[string]*treeNode)
|
||||
}
|
||||
|
||||
node.dirs[c] = child
|
||||
allDirs = append(allDirs, child)
|
||||
}
|
||||
|
||||
node = child
|
||||
}
|
||||
|
||||
if node.files == nil {
|
||||
node.files = make(map[string]fileSig)
|
||||
}
|
||||
|
||||
node.files[comps[len(comps)-1]] = fileSig{
|
||||
size: r.size, head: r.head, tail: r.tail,
|
||||
}
|
||||
}
|
||||
super.compute()
|
||||
|
||||
// Group directories by digest; two or more dirs sharing a digest
|
||||
// are candidate duplicate trees.
|
||||
return super, allDirs
|
||||
}
|
||||
|
||||
// collectTreeGroups groups directories by digest and returns every
|
||||
// maximal group with two or more members, each group's members sorted
|
||||
// by path, groups ordered by tree size descending then by first path
|
||||
// ascending.
|
||||
func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
|
||||
groups := make(map[[sha256.Size]byte][]*treeNode)
|
||||
for _, d := range allDirs {
|
||||
groups[d.digest] = append(groups[d.digest], d)
|
||||
}
|
||||
|
||||
var dupes [][]*treeNode
|
||||
|
||||
for _, g := range groups {
|
||||
if len(g) < 2 || suppressed(g, super) {
|
||||
if len(g) < minGroupSize || suppressed(g, super) {
|
||||
continue
|
||||
}
|
||||
|
||||
slices.SortFunc(g, func(a, b *treeNode) int {
|
||||
return strings.Compare(a.path, b.path)
|
||||
})
|
||||
dupes = append(dupes, g)
|
||||
}
|
||||
|
||||
// Biggest reclaimable space first; ties broken by first path.
|
||||
slices.SortFunc(dupes, func(a, b []*treeNode) int {
|
||||
if a[0].totalSize != b[0].totalSize {
|
||||
if a[0].totalSize > b[0].totalSize {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
return strings.Compare(a[0].path, b[0].path)
|
||||
})
|
||||
|
||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
||||
if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
dupeTrees := 0
|
||||
var reclaimable int64
|
||||
for _, g := range dupes {
|
||||
first := g[0]
|
||||
for _, n := range g[1:] {
|
||||
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||
first.path, n.path, first.fileCount, first.totalSize); err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
dupeTrees++
|
||||
reclaimable += first.totalSize
|
||||
}
|
||||
}
|
||||
if err := out.Flush(); err != nil {
|
||||
fatalf("write stdout: %v", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
|
||||
records, malformedNote(malformed), len(dupes), dupeTrees,
|
||||
humanBytes(reclaimable))
|
||||
return dupes
|
||||
}
|
||||
|
||||
// compute fills in digest, fileCount, and totalSize for n and all of
|
||||
@@ -143,18 +183,22 @@ func (n *treeNode) compute() {
|
||||
n.fileCount++
|
||||
n.totalSize += sig.size
|
||||
}
|
||||
|
||||
for name, child := range n.dirs {
|
||||
child.compute()
|
||||
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
|
||||
n.fileCount += child.fileCount
|
||||
n.totalSize += child.totalSize
|
||||
}
|
||||
|
||||
slices.Sort(entries)
|
||||
|
||||
h := sha256.New()
|
||||
for _, e := range entries {
|
||||
h.Write([]byte(e))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
|
||||
copy(n.digest[:], h.Sum(nil))
|
||||
}
|
||||
|
||||
@@ -165,15 +209,19 @@ func (n *treeNode) compute() {
|
||||
// parent) or members whose parents differ are always reported.
|
||||
func suppressed(g []*treeNode, super *treeNode) bool {
|
||||
seen := make(map[*treeNode]bool, len(g))
|
||||
|
||||
var parentDigest [sha256.Size]byte
|
||||
|
||||
for i, n := range g {
|
||||
p := n.parent
|
||||
if p == nil || p == super {
|
||||
return false
|
||||
}
|
||||
|
||||
if seen[p] {
|
||||
return false // siblings: not implied by any parent group
|
||||
}
|
||||
|
||||
seen[p] = true
|
||||
if i == 0 {
|
||||
parentDigest = p.digest
|
||||
@@ -181,5 +229,6 @@ func suppressed(g []*treeNode, super *treeNode) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user