Make code lint-clean under the canonical golangci-lint config

150 findings fixed: linter autofixes for whitespace style (wsl_v5,
nlreturn, noinlineerr), named returns removed, magic numbers replaced
with named constants, static errNotRegular error, explicit Close/Parse
error handling, unused cobra params renamed to _, and runReport/
runTrees/hashPass split into helpers to satisfy cyclop, gocognit, and
funlen. Behavior verified unchanged against the README smoke test.
This commit is contained in:
2026-07-23 05:36:05 +07:00
parent 733b33d9d1
commit 065a533224
5 changed files with 351 additions and 128 deletions

26
main.go
View File

@@ -20,15 +20,23 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
// Exit codes: 0 is success (even with per-file warnings), exitFatal is
// a fatal error, exitUsage is a usage error.
const (
exitFatal = 1
exitUsage = 2
)
func main() { func main() {
root := &cobra.Command{ root := &cobra.Command{
Use: "sfdupes", Use: "sfdupes",
Short: "Find candidate duplicate files by size and head/tail SHA-256", Short: "Find candidate duplicate files by size and head/tail SHA-256",
Args: cobra.NoArgs, Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, _ []string) {
// A missing subcommand prints usage and exits 2. // A missing subcommand prints usage and exits 2.
_ = cmd.Usage() _ = cmd.Usage()
os.Exit(2)
os.Exit(exitUsage)
}, },
} }
// Everything on stdout is machine-readable data; all human-facing // Everything on stdout is machine-readable data; all human-facing
@@ -43,7 +51,7 @@ func main() {
// The README specifies single-dash flags (-root, -workers); // The README specifies single-dash flags (-root, -workers);
// parse them with the stdlib flag package inside runScan. // parse them with the stdlib flag package inside runScan.
DisableFlagParsing: true, DisableFlagParsing: true,
Run: func(cmd *cobra.Command, args []string) { Run: func(_ *cobra.Command, args []string) {
runScan(args) runScan(args)
}, },
} }
@@ -52,7 +60,7 @@ func main() {
Use: "report [files.dat|-]", Use: "report [files.dat|-]",
Short: "Read a scan stream and print the file-level duplicates report", Short: "Read a scan stream and print the file-level duplicates report",
Args: cobra.MaximumNArgs(1), Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) { Run: func(_ *cobra.Command, args []string) {
runReport(args) runReport(args)
}, },
} }
@@ -61,21 +69,23 @@ func main() {
Use: "trees [files.dat|-]", Use: "trees [files.dat|-]",
Short: "Read a scan stream and print the duplicate-tree report", Short: "Read a scan stream and print the duplicate-tree report",
Args: cobra.MaximumNArgs(1), Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) { Run: func(_ *cobra.Command, args []string) {
runTrees(args) runTrees(args)
}, },
} }
root.AddCommand(scanCmd, reportCmd, treesCmd) root.AddCommand(scanCmd, reportCmd, treesCmd)
if err := root.Execute(); err != nil {
err := root.Execute()
if err != nil {
// Cobra has already printed the error and usage to stderr; // Cobra has already printed the error and usage to stderr;
// an invalid subcommand or bad arguments is a usage error. // an invalid subcommand or bad arguments is a usage error.
os.Exit(2) os.Exit(exitUsage)
} }
} }
// fatalf reports a fatal error and exits 1. // fatalf reports a fatal error and exits 1.
func fatalf(format string, args ...any) { func fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...) fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
os.Exit(1) os.Exit(exitFatal)
} }

View File

@@ -12,12 +12,23 @@ import (
// is not a TTY. // is not a TTY.
const plainInterval = 5 * time.Second const plainInterval = 5 * time.Second
// barThrottle caps how often the TTY progress bar redraws.
const barThrottle = 100 * time.Millisecond
// walkSpinnerType selects the progressbar library's spinner style used
// for the walk pass, which has no known total.
const walkSpinnerType = 14
// percentScale converts a fraction to a percentage.
const percentScale = 100
// stderrIsTTY reports whether stderr is attached to a terminal. // stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool { func stderrIsTTY() bool {
fi, err := os.Stderr.Stat() fi, err := os.Stderr.Stat()
if err != nil { if err != nil {
return false return false
} }
return fi.Mode()&os.ModeCharDevice != 0 return fi.Mode()&os.ModeCharDevice != 0
} }
@@ -42,6 +53,7 @@ func newProgress(label string, total int64) *progress {
if !stderrIsTTY() { if !stderrIsTTY() {
return p return p
} }
opts := []progressbar.Option{ opts := []progressbar.Option{
progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetDescription(label), progressbar.OptionSetDescription(label),
@@ -49,7 +61,7 @@ func newProgress(label string, total int64) *progress {
progressbar.OptionShowIts(), progressbar.OptionShowIts(),
progressbar.OptionSetItsString("files"), progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true), progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(100 * time.Millisecond), progressbar.OptionThrottle(barThrottle),
} }
if total >= 0 { if total >= 0 {
opts = append(opts, opts = append(opts,
@@ -59,10 +71,12 @@ func newProgress(label string, total int64) *progress {
} else { } else {
opts = append(opts, opts = append(opts,
progressbar.OptionSetPredictTime(false), progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(14), progressbar.OptionSpinnerType(walkSpinnerType),
) )
} }
p.bar = progressbar.NewOptions64(total, opts...) p.bar = progressbar.NewOptions64(total, opts...)
return p return p
} }
@@ -71,8 +85,10 @@ func (p *progress) increment() {
p.count++ p.count++
if p.bar != nil { if p.bar != nil {
_ = p.bar.Add(1) _ = p.bar.Add(1)
return return
} }
if time.Since(p.last) >= plainInterval { if time.Since(p.last) >= plainInterval {
p.last = time.Now() p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine()) fmt.Fprintln(os.Stderr, p.plainLine())
@@ -84,6 +100,7 @@ func (p *progress) warnf(format string, args ...any) {
if p.bar != nil { if p.bar != nil {
_ = p.bar.Clear() _ = p.bar.Clear()
} }
fmt.Fprintf(os.Stderr, format+"\n", args...) fmt.Fprintf(os.Stderr, format+"\n", args...)
} }
@@ -91,9 +108,12 @@ func (p *progress) warnf(format string, args ...any) {
func (p *progress) finish() { func (p *progress) finish() {
if p.bar != nil { if p.bar != nil {
_ = p.bar.Finish() _ = p.bar.Finish()
fmt.Fprintln(os.Stderr) fmt.Fprintln(os.Stderr)
return return
} }
fmt.Fprintln(os.Stderr, p.plainLine()) fmt.Fprintln(os.Stderr, p.plainLine())
} }
@@ -104,19 +124,24 @@ func (p *progress) plainLine() string {
return fmt.Sprintf("%s: %d files, elapsed %s", return fmt.Sprintf("%s: %d files, elapsed %s",
p.label, p.count, elapsed.Round(time.Second)) p.label, p.count, elapsed.Round(time.Second))
} }
pct := 100.0
pct := float64(percentScale)
if p.total > 0 { if p.total > 0 {
pct = 100 * float64(p.count) / float64(p.total) pct = percentScale * float64(p.count) / float64(p.total)
} }
rate := 0.0 rate := 0.0
if s := elapsed.Seconds(); s > 0 { if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s rate = float64(p.count) / s
} }
eta := "?" eta := "?"
if rate > 0 && p.count <= p.total { if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second)) remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String() eta = remaining.Round(time.Second).String()
} }
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s", return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate, p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta) elapsed.Round(time.Second), eta)

147
report.go
View File

@@ -11,6 +11,26 @@ import (
"strings" "strings"
) )
// ioBufSize is the buffer size for the buffered scan-stream reader and
// the buffered stdout writers.
const ioBufSize = 1 << 20
// recordFieldCount is the number of tab-separated fields in a scan
// record: size, mtime, head hash, tail hash, path.
const recordFieldCount = 5
// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
// to read scan records; a record longer than scanMaxRecordSize is a
// fatal read error.
const (
scanInitBufSize = 64 << 10
scanMaxRecordSize = 4 << 20
)
// minGroupSize is the smallest number of members that makes a
// duplicate group.
const minGroupSize = 2
// scanRec is one well-formed record parsed from a scan stream. The // scanRec is one well-formed record parsed from a scan stream. The
// signature (size, head, tail) is the duplicate key; mtime is // signature (size, head, tail) is the duplicate key; mtime is
// informational and not retained. // informational and not retained.
@@ -24,14 +44,16 @@ type scanRec struct {
// openScanInput resolves the analysis-mode input: the file named by the // openScanInput resolves the analysis-mode input: the file named by the
// single optional positional argument, or stdin when it is absent or // single optional positional argument, or stdin when it is absent or
// "-". The returned closer must be called when reading is done. // "-". The returned closer must be called when reading is done.
func openScanInput(args []string) (in io.Reader, name string, closer func()) { func openScanInput(args []string) (io.Reader, string, func()) {
if len(args) == 1 && args[0] != "-" { if len(args) == 1 && args[0] != "-" {
f, err := os.Open(args[0]) f, err := os.Open(args[0])
if err != nil { if err != nil {
fatalf("open %s: %v", args[0], err) fatalf("open %s: %v", args[0], err)
} }
return f, args[0], func() { f.Close() }
return f, args[0], func() { _ = f.Close() }
} }
return os.Stdin, "stdin", func() {} return os.Stdin, "stdin", func() {}
} }
@@ -39,23 +61,33 @@ func openScanInput(args []string) (in io.Reader, name string, closer func()) {
// that does not have exactly 5 fields or whose size is non-numeric is // that does not have exactly 5 fields or whose size is non-numeric is
// counted as malformed and skipped. Total records read is // counted as malformed and skipped. Total records read is
// len(recs) + malformed. // len(recs) + malformed.
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) { func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20)) sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
sc.Buffer(make([]byte, 0, 64<<10), 4<<20) sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
sc.Split(splitNUL) sc.Split(splitNUL)
var (
recs []scanRec
malformed int
)
for sc.Scan() { for sc.Scan() {
// The path is the last field and may itself contain tabs, // The path is the last field and may itself contain tabs,
// so split into at most 5 fields. // so split into at most recordFieldCount fields.
fields := strings.SplitN(sc.Text(), "\t", 5) fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
if len(fields) != 5 { if len(fields) != recordFieldCount {
malformed++ malformed++
continue continue
} }
size, err := strconv.ParseInt(fields[0], 10, 64) size, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil { if err != nil {
malformed++ malformed++
continue continue
} }
recs = append(recs, scanRec{ recs = append(recs, scanRec{
size: size, size: size,
head: fields[2], head: fields[2],
@@ -63,9 +95,12 @@ func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int)
path: fields[4], path: fields[4],
}) })
} }
if err := sc.Err(); err != nil {
err := sc.Err()
if err != nil {
fatalf("read %s: %v", name, err) fatalf("read %s: %v", name, err)
} }
return recs, malformed return recs, malformed
} }
@@ -86,56 +121,37 @@ type dupeGroup struct {
func runReport(args []string) { func runReport(args []string) {
in, name, closer := openScanInput(args) in, name, closer := openScanInput(args)
defer closer() defer closer()
recs, malformed := parseScanStream(in, name) recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed records := len(recs) + malformed
dupes := collectDupeGroups(recs)
type dupeKey struct { out := bufio.NewWriterSize(os.Stdout, ioBufSize)
size int64
head string
tail string
}
groups := make(map[dupeKey][]string)
for _, r := range recs {
k := dupeKey{size: r.size, head: r.head, tail: r.tail}
groups[k] = append(groups[k], r.path)
}
var dupes []dupeGroup _, err := fmt.Fprintln(out, "first\tdupe\tsize")
for k, paths := range groups { if err != nil {
if len(paths) < 2 {
continue
}
slices.Sort(paths)
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b dupeGroup) int {
if a.size != b.size {
if a.size > b.size {
return -1
}
return 1
}
return strings.Compare(a.paths[0], b.paths[0])
})
out := bufio.NewWriterSize(os.Stdout, 1<<20)
if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
fatalf("write stdout: %v", err) fatalf("write stdout: %v", err)
} }
dupeFiles := 0 dupeFiles := 0
var reclaimable int64 var reclaimable int64
for _, g := range dupes { for _, g := range dupes {
for _, p := range g.paths[1:] { for _, p := range g.paths[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n", _, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
g.paths[0], p, g.size); err != nil { g.paths[0], p, g.size)
if err != nil {
fatalf("write stdout: %v", err) fatalf("write stdout: %v", err)
} }
dupeFiles++ dupeFiles++
reclaimable += g.size reclaimable += g.size
} }
} }
if err := out.Flush(); err != nil {
err = out.Flush()
if err != nil {
fatalf("write stdout: %v", err) fatalf("write stdout: %v", err)
} }
@@ -145,24 +161,65 @@ func runReport(args []string) {
humanBytes(reclaimable)) humanBytes(reclaimable))
} }
// collectDupeGroups groups records by signature and returns every group
// with two or more paths, each group's paths sorted lexicographically,
// groups ordered by size descending then by first path ascending.
func collectDupeGroups(recs []scanRec) []dupeGroup {
groups := make(map[fileSig][]string)
for _, r := range recs {
k := fileSig{size: r.size, head: r.head, tail: r.tail}
groups[k] = append(groups[k], r.path)
}
var dupes []dupeGroup
for k, paths := range groups {
if len(paths) < minGroupSize {
continue
}
slices.Sort(paths)
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b dupeGroup) int {
if a.size != b.size {
if a.size > b.size {
return -1
}
return 1
}
return strings.Compare(a.paths[0], b.paths[0])
})
return dupes
}
// malformedNote formats the optional malformed-record note for the // malformedNote formats the optional malformed-record note for the
// stderr summaries. // stderr summaries.
func malformedNote(malformed int) string { func malformedNote(malformed int) string {
if malformed == 0 { if malformed == 0 {
return "" return ""
} }
return fmt.Sprintf(" (%d malformed, skipped)", malformed) return fmt.Sprintf(" (%d malformed, skipped)", malformed)
} }
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing // splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
// data without a terminator at EOF is returned as a final record. // data without a terminator at EOF is returned as a final record.
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) { func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
if i := bytes.IndexByte(data, 0); i >= 0 { if i := bytes.IndexByte(data, 0); i >= 0 {
return i + 1, data[:i], nil return i + 1, data[:i], nil
} }
if atEOF && len(data) > 0 { if atEOF && len(data) > 0 {
return len(data), data, nil return len(data), data, nil
} }
return 0, nil, nil return 0, nil, nil
} }
@@ -172,10 +229,12 @@ func humanBytes(n int64) string {
if n < unit { if n < unit {
return fmt.Sprintf("%d B", n) return fmt.Sprintf("%d B", n)
} }
div, exp := int64(unit), 0 div, exp := int64(unit), 0
for m := n / unit; m >= unit; m /= unit { for m := n / unit; m >= unit; m /= unit {
div *= unit div *= unit
exp++ exp++
} }
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
} }

160
scan.go
View File

@@ -4,6 +4,7 @@ import (
"bufio" "bufio"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors"
"flag" "flag"
"fmt" "fmt"
"io/fs" "io/fs"
@@ -15,6 +16,14 @@ import (
// chunk is the number of bytes hashed from each end of a file. // chunk is the number of bytes hashed from each end of a file.
const chunk = 1024 const chunk = 1024
// workQueueDepth bounds the job and result channels feeding the stat
// and hash worker pools.
const workQueueDepth = 1024
// errNotRegular reports a path that stopped being a regular file
// between the walk and stat passes.
var errNotRegular = errors.New("no longer a regular file")
// fileRec carries one file between the stat and hash passes. // fileRec carries one file between the stat and hash passes.
type fileRec struct { type fileRec struct {
path string path string
@@ -30,11 +39,18 @@ func runScan(args []string) {
root := fl.String("root", "/srv", "directory tree to scan") root := fl.String("root", "/srv", "directory tree to scan")
workers := fl.Int("workers", runtime.NumCPU(), workers := fl.Int("workers", runtime.NumCPU(),
"concurrent workers for the stat and hash passes") "concurrent workers for the stat and hash passes")
fl.Parse(args) // The flag set uses ExitOnError, so Parse cannot return a non-nil
// error; the check keeps the error handled explicitly.
err := fl.Parse(args)
if err != nil {
os.Exit(exitUsage)
}
if fl.NArg() > 0 { if fl.NArg() > 0 {
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0)) fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
os.Exit(2) os.Exit(exitUsage)
} }
if *workers < 1 { if *workers < 1 {
*workers = 1 *workers = 1
} }
@@ -43,6 +59,7 @@ func runScan(args []string) {
if err != nil { if err != nil {
fatalf("root %s: %v", *root, err) fatalf("root %s: %v", *root, err)
} }
if !st.IsDir() { if !st.IsDir() {
fatalf("root %s: not a directory", *root) fatalf("root %s: not a directory", *root)
} }
@@ -59,23 +76,33 @@ func runScan(args []string) {
// walkPass enumerates every regular file under root. It never follows // walkPass enumerates every regular file under root. It never follows
// symlinks, never descends into directories named .zfs, and warns and // symlinks, never descends into directories named .zfs, and warns and
// continues on any per-path error. // continues on any per-path error.
func walkPass(root string) (paths []string, errs int) { func walkPass(root string) ([]string, int) {
var (
paths []string
errs int
)
prog := newProgress("walk", -1) prog := newProgress("walk", -1)
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
errs++ errs++
prog.warnf("walk %s: %v", p, err) prog.warnf("walk %s: %v", p, err)
if d != nil && d.IsDir() { if d != nil && d.IsDir() {
return filepath.SkipDir return filepath.SkipDir
} }
return nil return nil
} }
if d.IsDir() { if d.IsDir() {
// ZFS snapshot pseudo-dirs would list every file // ZFS snapshot pseudo-dirs would list every file
// once per snapshot; never descend. // once per snapshot; never descend.
if d.Name() == ".zfs" { if d.Name() == ".zfs" {
return filepath.SkipDir return filepath.SkipDir
} }
return nil return nil
} }
// Regular files only: skip symlinks, sockets, FIFOs, and // Regular files only: skip symlinks, sockets, FIFOs, and
@@ -83,27 +110,35 @@ func walkPass(root string) (paths []string, errs int) {
if !d.Type().IsRegular() { if !d.Type().IsRegular() {
return nil return nil
} }
paths = append(paths, p) paths = append(paths, p)
prog.increment() prog.increment()
return nil return nil
}) })
prog.finish() prog.finish()
if walkErr != nil { if walkErr != nil {
fatalf("walk %s: %v", root, walkErr) fatalf("walk %s: %v", root, walkErr)
} }
return paths, errs return paths, errs
} }
// statPass lstats every collected path in a worker pool, recording size // statPass lstats every collected path in a worker pool, recording size
// and mtime. Paths that fail to stat (or are no longer regular files) // and mtime. Paths that fail to stat (or are no longer regular files)
// are warned about and dropped. // are warned about and dropped.
func statPass(paths []string, workers int) (recs []fileRec, errs int) { func statPass(paths []string, workers int) ([]fileRec, int) {
type result struct { type result struct {
rec fileRec rec fileRec
err error err error
} }
jobs := make(chan string, 1024)
results := make(chan result, 1024) jobs := make(chan string, workQueueDepth)
results := make(chan result, workQueueDepth)
for range workers { for range workers {
go func() { go func() {
for p := range jobs { for p := range jobs {
@@ -114,7 +149,7 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
case !fi.Mode().IsRegular(): case !fi.Mode().IsRegular():
results <- result{ results <- result{
rec: fileRec{path: p}, rec: fileRec{path: p},
err: fmt.Errorf("no longer a regular file"), err: errNotRegular,
} }
default: default:
results <- result{rec: fileRec{ results <- result{rec: fileRec{
@@ -126,78 +161,114 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
} }
}() }()
} }
go func() { go func() {
for _, p := range paths { for _, p := range paths {
jobs <- p jobs <- p
} }
close(jobs) close(jobs)
}() }()
prog := newProgress("stat", int64(len(paths))) prog := newProgress("stat", int64(len(paths)))
recs = make([]fileRec, 0, len(paths))
var errs int
recs := make([]fileRec, 0, len(paths))
for range paths { for range paths {
r := <-results r := <-results
if r.err != nil { if r.err != nil {
errs++ errs++
prog.warnf("stat %s: %v", r.rec.path, r.err) prog.warnf("stat %s: %v", r.rec.path, r.err)
} else { } else {
recs = append(recs, r.rec) recs = append(recs, r.rec)
} }
prog.increment() prog.increment()
} }
prog.finish() prog.finish()
return recs, errs return recs, errs
} }
// hashResult carries one file's head/tail hashes (or the error that
// prevented hashing it) from the hash workers to the main goroutine.
type hashResult struct {
rec fileRec
head string
tail string
err error
}
// startHashWorkers starts the hash worker pool over recs and returns
// the channel its results arrive on (one per record, in completion
// order).
func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
jobs := make(chan fileRec, workQueueDepth)
results := make(chan hashResult, workQueueDepth)
for range workers {
go func() {
for rec := range jobs {
head, tail, err := hashHeadTail(rec.path, rec.size)
results <- hashResult{rec: rec, head: head, tail: tail, err: err}
}
}()
}
go func() {
for _, rec := range recs {
jobs <- rec
}
close(jobs)
}()
return results
}
// hashPass hashes the first and last chunk bytes of every file in a // hashPass hashes the first and last chunk bytes of every file in a
// worker pool and emits the output records on stdout from the main // worker pool and emits the output records on stdout from the main
// goroutine. Files that fail to open or read are warned about and // goroutine. Files that fail to open or read are warned about and
// dropped. // dropped.
func hashPass(recs []fileRec, workers int) (emitted, errs int) { func hashPass(recs []fileRec, workers int) (int, int) {
type result struct { results := startHashWorkers(recs, workers)
rec fileRec
head string
tail string
err error
}
jobs := make(chan fileRec, 1024)
results := make(chan result, 1024)
for range workers {
go func() {
for rec := range jobs {
head, tail, err := hashHeadTail(rec.path, rec.size)
results <- result{rec: rec, head: head, tail: tail, err: err}
}
}()
}
go func() {
for _, rec := range recs {
jobs <- rec
}
close(jobs)
}()
prog := newProgress("hash", int64(len(recs))) prog := newProgress("hash", int64(len(recs)))
out := bufio.NewWriterSize(os.Stdout, 1<<20) out := bufio.NewWriterSize(os.Stdout, ioBufSize)
var emitted, errs int
for range recs { for range recs {
r := <-results r := <-results
if r.err != nil { if r.err != nil {
errs++ errs++
prog.warnf("hash %s: %v", r.rec.path, r.err) prog.warnf("hash %s: %v", r.rec.path, r.err)
prog.increment() prog.increment()
continue continue
} }
if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil { _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
if err != nil {
fatalf("write stdout: %v", err) fatalf("write stdout: %v", err)
} }
emitted++ emitted++
prog.increment() prog.increment()
} }
prog.finish() prog.finish()
if err := out.Flush(); err != nil {
err := out.Flush()
if err != nil {
fatalf("write stdout: %v", err) fatalf("write stdout: %v", err)
} }
return emitted, errs return emitted, errs
} }
@@ -206,26 +277,35 @@ func hashPass(recs []fileRec, workers int) (emitted, errs int) {
// file at path. The two reads overlap when size < 2*chunk; for // file at path. The two reads overlap when size < 2*chunk; for
// size == 0 both hashes are of the empty input. size is the value // size == 0 both hashes are of the empty input. size is the value
// recorded by the stat pass. // recorded by the stat pass.
func hashHeadTail(path string, size int64) (head, tail string, err error) { func hashHeadTail(path string, size int64) (string, string, error) {
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
defer f.Close()
defer func() { _ = f.Close() }()
n := min(int64(chunk), size) n := min(int64(chunk), size)
buf := make([]byte, n) buf := make([]byte, n)
if n > 0 { if n > 0 {
if _, err := f.ReadAt(buf, 0); err != nil { _, err = f.ReadAt(buf, 0)
if err != nil {
return "", "", err return "", "", err
} }
} }
h := sha256.Sum256(buf) h := sha256.Sum256(buf)
if n > 0 { if n > 0 {
if _, err := f.ReadAt(buf, size-n); err != nil { _, err = f.ReadAt(buf, size-n)
if err != nil {
return "", "", err return "", "", err
} }
} }
t := sha256.Sum256(buf) t := sha256.Sum256(buf)
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
} }

113
trees.go
View File

@@ -38,16 +38,64 @@ type treeNode struct {
func runTrees(args []string) { func runTrees(args []string) {
in, name, closer := openScanInput(args) in, name, closer := openScanInput(args)
defer closer() defer closer()
recs, malformed := parseScanStream(in, name) recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed records := len(recs) + malformed
// Build the hierarchy under a synthetic super-root. Paths are super, allDirs := buildHierarchy(recs)
// split on "/"; for absolute paths the first component is empty, super.compute()
// which simply becomes a top-level node representing "/".
dupes := collectTreeGroups(allDirs, super)
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees := 0
var reclaimable int64
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
first.path, n.path, first.fileCount, first.totalSize)
if err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees++
reclaimable += first.totalSize
}
}
err = out.Flush()
if err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeTrees,
humanBytes(reclaimable))
}
// buildHierarchy reconstructs the directory hierarchy from the record
// paths under a synthetic super-root. Paths are split on "/"; for
// absolute paths the first component is empty, which simply becomes a
// top-level node representing "/". It returns the super-root and every
// directory node created.
func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
super := &treeNode{} super := &treeNode{}
var allDirs []*treeNode var allDirs []*treeNode
for _, r := range recs { for _, r := range recs {
comps := strings.Split(r.path, "/") comps := strings.Split(r.path, "/")
node := super node := super
for _, c := range comps[:len(comps)-1] { for _, c := range comps[:len(comps)-1] {
child := node.dirs[c] child := node.dirs[c]
@@ -56,76 +104,68 @@ func runTrees(args []string) {
if node != super { if node != super {
childPath = node.path + "/" + c childPath = node.path + "/" + c
} }
child = &treeNode{path: childPath, parent: node} child = &treeNode{path: childPath, parent: node}
if node.dirs == nil { if node.dirs == nil {
node.dirs = make(map[string]*treeNode) node.dirs = make(map[string]*treeNode)
} }
node.dirs[c] = child node.dirs[c] = child
allDirs = append(allDirs, child) allDirs = append(allDirs, child)
} }
node = child node = child
} }
if node.files == nil { if node.files == nil {
node.files = make(map[string]fileSig) node.files = make(map[string]fileSig)
} }
node.files[comps[len(comps)-1]] = fileSig{ node.files[comps[len(comps)-1]] = fileSig{
size: r.size, head: r.head, tail: r.tail, size: r.size, head: r.head, tail: r.tail,
} }
} }
super.compute()
// Group directories by digest; two or more dirs sharing a digest return super, allDirs
// are candidate duplicate trees. }
// collectTreeGroups groups directories by digest and returns every
// maximal group with two or more members, each group's members sorted
// by path, groups ordered by tree size descending then by first path
// ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode) groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs { for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d) groups[d.digest] = append(groups[d.digest], d)
} }
var dupes [][]*treeNode var dupes [][]*treeNode
for _, g := range groups { for _, g := range groups {
if len(g) < 2 || suppressed(g, super) { if len(g) < minGroupSize || suppressed(g, super) {
continue continue
} }
slices.SortFunc(g, func(a, b *treeNode) int { slices.SortFunc(g, func(a, b *treeNode) int {
return strings.Compare(a.path, b.path) return strings.Compare(a.path, b.path)
}) })
dupes = append(dupes, g) dupes = append(dupes, g)
} }
// Biggest reclaimable space first; ties broken by first path. // Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b []*treeNode) int { slices.SortFunc(dupes, func(a, b []*treeNode) int {
if a[0].totalSize != b[0].totalSize { if a[0].totalSize != b[0].totalSize {
if a[0].totalSize > b[0].totalSize { if a[0].totalSize > b[0].totalSize {
return -1 return -1
} }
return 1 return 1
} }
return strings.Compare(a[0].path, b[0].path) return strings.Compare(a[0].path, b[0].path)
}) })
out := bufio.NewWriterSize(os.Stdout, 1<<20) return dupes
if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees := 0
var reclaimable int64
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
first.path, n.path, first.fileCount, first.totalSize); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees++
reclaimable += first.totalSize
}
}
if err := out.Flush(); err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeTrees,
humanBytes(reclaimable))
} }
// compute fills in digest, fileCount, and totalSize for n and all of // compute fills in digest, fileCount, and totalSize for n and all of
@@ -143,18 +183,22 @@ func (n *treeNode) compute() {
n.fileCount++ n.fileCount++
n.totalSize += sig.size n.totalSize += sig.size
} }
for name, child := range n.dirs { for name, child := range n.dirs {
child.compute() child.compute()
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
n.fileCount += child.fileCount n.fileCount += child.fileCount
n.totalSize += child.totalSize n.totalSize += child.totalSize
} }
slices.Sort(entries) slices.Sort(entries)
h := sha256.New() h := sha256.New()
for _, e := range entries { for _, e := range entries {
h.Write([]byte(e)) h.Write([]byte(e))
h.Write([]byte{0}) h.Write([]byte{0})
} }
copy(n.digest[:], h.Sum(nil)) copy(n.digest[:], h.Sum(nil))
} }
@@ -165,15 +209,19 @@ func (n *treeNode) compute() {
// parent) or members whose parents differ are always reported. // parent) or members whose parents differ are always reported.
func suppressed(g []*treeNode, super *treeNode) bool { func suppressed(g []*treeNode, super *treeNode) bool {
seen := make(map[*treeNode]bool, len(g)) seen := make(map[*treeNode]bool, len(g))
var parentDigest [sha256.Size]byte var parentDigest [sha256.Size]byte
for i, n := range g { for i, n := range g {
p := n.parent p := n.parent
if p == nil || p == super { if p == nil || p == super {
return false return false
} }
if seen[p] { if seen[p] {
return false // siblings: not implied by any parent group return false // siblings: not implied by any parent group
} }
seen[p] = true seen[p] = true
if i == 0 { if i == 0 {
parentDigest = p.digest parentDigest = p.digest
@@ -181,5 +229,6 @@ func suppressed(g []*treeNode, super *treeNode) bool {
return false return false
} }
} }
return true return true
} }