From 065a533224b0756a17a7d7e5c9ed55be66ba4f8a Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 23 Jul 2026 05:36:05 +0700 Subject: [PATCH] Make code lint-clean under the canonical golangci-lint config 150 findings fixed: linter autofixes for whitespace style (wsl_v5, nlreturn, noinlineerr), named returns removed, magic numbers replaced with named constants, static errNotRegular error, explicit Close/Parse error handling, unused cobra params renamed to _, and runReport/ runTrees/hashPass split into helpers to satisfy cyclop, gocognit, and funlen. Behavior verified unchanged against the README smoke test. --- main.go | 26 ++++++--- progress.go | 33 +++++++++-- report.go | 147 ++++++++++++++++++++++++++++++++--------------- scan.go | 160 +++++++++++++++++++++++++++++++++++++++------------- trees.go | 113 ++++++++++++++++++++++++++----------- 5 files changed, 351 insertions(+), 128 deletions(-) diff --git a/main.go b/main.go index 51e2900..ea38a2c 100644 --- a/main.go +++ b/main.go @@ -20,15 +20,23 @@ import ( "github.com/spf13/cobra" ) +// Exit codes: 0 is success (even with per-file warnings), exitFatal is +// a fatal error, exitUsage is a usage error. +const ( + exitFatal = 1 + exitUsage = 2 +) + func main() { root := &cobra.Command{ Use: "sfdupes", Short: "Find candidate duplicate files by size and head/tail SHA-256", Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { // A missing subcommand prints usage and exits 2. _ = cmd.Usage() - os.Exit(2) + + os.Exit(exitUsage) }, } // Everything on stdout is machine-readable data; all human-facing @@ -43,7 +51,7 @@ func main() { // The README specifies single-dash flags (-root, -workers); // parse them with the stdlib flag package inside runScan. DisableFlagParsing: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { runScan(args) }, } @@ -52,7 +60,7 @@ func main() { Use: "report [files.dat|-]", Short: "Read a scan stream and print the file-level duplicates report", Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { runReport(args) }, } @@ -61,21 +69,23 @@ func main() { Use: "trees [files.dat|-]", Short: "Read a scan stream and print the duplicate-tree report", Args: cobra.MaximumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { runTrees(args) }, } root.AddCommand(scanCmd, reportCmd, treesCmd) - if err := root.Execute(); err != nil { + + err := root.Execute() + if err != nil { // Cobra has already printed the error and usage to stderr; // an invalid subcommand or bad arguments is a usage error. - os.Exit(2) + os.Exit(exitUsage) } } // fatalf reports a fatal error and exits 1. func fatalf(format string, args ...any) { fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...) - os.Exit(1) + os.Exit(exitFatal) } diff --git a/progress.go b/progress.go index 9aedbee..9ee2b2e 100644 --- a/progress.go +++ b/progress.go @@ -12,12 +12,23 @@ import ( // is not a TTY. const plainInterval = 5 * time.Second +// barThrottle caps how often the TTY progress bar redraws. +const barThrottle = 100 * time.Millisecond + +// walkSpinnerType selects the progressbar library's spinner style used +// for the walk pass, which has no known total. +const walkSpinnerType = 14 + +// percentScale converts a fraction to a percentage. +const percentScale = 100 + // stderrIsTTY reports whether stderr is attached to a terminal. func stderrIsTTY() bool { fi, err := os.Stderr.Stat() if err != nil { return false } + return fi.Mode()&os.ModeCharDevice != 0 } @@ -42,6 +53,7 @@ func newProgress(label string, total int64) *progress { if !stderrIsTTY() { return p } + opts := []progressbar.Option{ progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetDescription(label), @@ -49,7 +61,7 @@ func newProgress(label string, total int64) *progress { progressbar.OptionShowIts(), progressbar.OptionSetItsString("files"), progressbar.OptionSetElapsedTime(true), - progressbar.OptionThrottle(100 * time.Millisecond), + progressbar.OptionThrottle(barThrottle), } if total >= 0 { opts = append(opts, @@ -59,10 +71,12 @@ func newProgress(label string, total int64) *progress { } else { opts = append(opts, progressbar.OptionSetPredictTime(false), - progressbar.OptionSpinnerType(14), + progressbar.OptionSpinnerType(walkSpinnerType), ) } + p.bar = progressbar.NewOptions64(total, opts...) + return p } @@ -71,8 +85,10 @@ func (p *progress) increment() { p.count++ if p.bar != nil { _ = p.bar.Add(1) + return } + if time.Since(p.last) >= plainInterval { p.last = time.Now() fmt.Fprintln(os.Stderr, p.plainLine()) @@ -84,6 +100,7 @@ func (p *progress) warnf(format string, args ...any) { if p.bar != nil { _ = p.bar.Clear() } + fmt.Fprintf(os.Stderr, format+"\n", args...) } @@ -91,9 +108,12 @@ func (p *progress) warnf(format string, args ...any) { func (p *progress) finish() { if p.bar != nil { _ = p.bar.Finish() + fmt.Fprintln(os.Stderr) + return } + fmt.Fprintln(os.Stderr, p.plainLine()) } @@ -104,19 +124,24 @@ func (p *progress) plainLine() string { return fmt.Sprintf("%s: %d files, elapsed %s", p.label, p.count, elapsed.Round(time.Second)) } - pct := 100.0 + + pct := float64(percentScale) if p.total > 0 { - pct = 100 * float64(p.count) / float64(p.total) + pct = percentScale * float64(p.count) / float64(p.total) } + rate := 0.0 if s := elapsed.Seconds(); s > 0 { rate = float64(p.count) / s } + eta := "?" + if rate > 0 && p.count <= p.total { remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second)) eta = remaining.Round(time.Second).String() } + return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s", p.label, p.count, p.total, pct, rate, elapsed.Round(time.Second), eta) diff --git a/report.go b/report.go index 096c4aa..a66104e 100644 --- a/report.go +++ b/report.go @@ -11,6 +11,26 @@ import ( "strings" ) +// ioBufSize is the buffer size for the buffered scan-stream reader and +// the buffered stdout writers. +const ioBufSize = 1 << 20 + +// recordFieldCount is the number of tab-separated fields in a scan +// record: size, mtime, head hash, tail hash, path. +const recordFieldCount = 5 + +// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used +// to read scan records; a record longer than scanMaxRecordSize is a +// fatal read error. +const ( + scanInitBufSize = 64 << 10 + scanMaxRecordSize = 4 << 20 +) + +// minGroupSize is the smallest number of members that makes a +// duplicate group. +const minGroupSize = 2 + // scanRec is one well-formed record parsed from a scan stream. The // signature (size, head, tail) is the duplicate key; mtime is // informational and not retained. @@ -24,14 +44,16 @@ type scanRec struct { // openScanInput resolves the analysis-mode input: the file named by the // single optional positional argument, or stdin when it is absent or // "-". The returned closer must be called when reading is done. -func openScanInput(args []string) (in io.Reader, name string, closer func()) { +func openScanInput(args []string) (io.Reader, string, func()) { if len(args) == 1 && args[0] != "-" { f, err := os.Open(args[0]) if err != nil { fatalf("open %s: %v", args[0], err) } - return f, args[0], func() { f.Close() } + + return f, args[0], func() { _ = f.Close() } } + return os.Stdin, "stdin", func() {} } @@ -39,23 +61,33 @@ func openScanInput(args []string) (in io.Reader, name string, closer func()) { // that does not have exactly 5 fields or whose size is non-numeric is // counted as malformed and skipped. Total records read is // len(recs) + malformed. -func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) { - sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20)) - sc.Buffer(make([]byte, 0, 64<<10), 4<<20) +func parseScanStream(in io.Reader, name string) ([]scanRec, int) { + sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize)) + sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize) sc.Split(splitNUL) + + var ( + recs []scanRec + malformed int + ) + for sc.Scan() { // The path is the last field and may itself contain tabs, - // so split into at most 5 fields. - fields := strings.SplitN(sc.Text(), "\t", 5) - if len(fields) != 5 { + // so split into at most recordFieldCount fields. + fields := strings.SplitN(sc.Text(), "\t", recordFieldCount) + if len(fields) != recordFieldCount { malformed++ + continue } + size, err := strconv.ParseInt(fields[0], 10, 64) if err != nil { malformed++ + continue } + recs = append(recs, scanRec{ size: size, head: fields[2], @@ -63,9 +95,12 @@ func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) path: fields[4], }) } - if err := sc.Err(); err != nil { + + err := sc.Err() + if err != nil { fatalf("read %s: %v", name, err) } + return recs, malformed } @@ -86,56 +121,37 @@ type dupeGroup struct { func runReport(args []string) { in, name, closer := openScanInput(args) defer closer() + recs, malformed := parseScanStream(in, name) records := len(recs) + malformed + dupes := collectDupeGroups(recs) - type dupeKey struct { - size int64 - head string - tail string - } - groups := make(map[dupeKey][]string) - for _, r := range recs { - k := dupeKey{size: r.size, head: r.head, tail: r.tail} - groups[k] = append(groups[k], r.path) - } + out := bufio.NewWriterSize(os.Stdout, ioBufSize) - var dupes []dupeGroup - for k, paths := range groups { - if len(paths) < 2 { - continue - } - slices.Sort(paths) - dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) - } - // Biggest reclaimable space first; ties broken by first path. - slices.SortFunc(dupes, func(a, b dupeGroup) int { - if a.size != b.size { - if a.size > b.size { - return -1 - } - return 1 - } - return strings.Compare(a.paths[0], b.paths[0]) - }) - - out := bufio.NewWriterSize(os.Stdout, 1<<20) - if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil { + _, err := fmt.Fprintln(out, "first\tdupe\tsize") + if err != nil { fatalf("write stdout: %v", err) } + dupeFiles := 0 + var reclaimable int64 + for _, g := range dupes { for _, p := range g.paths[1:] { - if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n", - g.paths[0], p, g.size); err != nil { + _, err = fmt.Fprintf(out, "%s\t%s\t%d\n", + g.paths[0], p, g.size) + if err != nil { fatalf("write stdout: %v", err) } + dupeFiles++ reclaimable += g.size } } - if err := out.Flush(); err != nil { + + err = out.Flush() + if err != nil { fatalf("write stdout: %v", err) } @@ -145,24 +161,65 @@ func runReport(args []string) { humanBytes(reclaimable)) } +// collectDupeGroups groups records by signature and returns every group +// with two or more paths, each group's paths sorted lexicographically, +// groups ordered by size descending then by first path ascending. +func collectDupeGroups(recs []scanRec) []dupeGroup { + groups := make(map[fileSig][]string) + + for _, r := range recs { + k := fileSig{size: r.size, head: r.head, tail: r.tail} + groups[k] = append(groups[k], r.path) + } + + var dupes []dupeGroup + + for k, paths := range groups { + if len(paths) < minGroupSize { + continue + } + + slices.Sort(paths) + dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) + } + + // Biggest reclaimable space first; ties broken by first path. + slices.SortFunc(dupes, func(a, b dupeGroup) int { + if a.size != b.size { + if a.size > b.size { + return -1 + } + + return 1 + } + + return strings.Compare(a.paths[0], b.paths[0]) + }) + + return dupes +} + // malformedNote formats the optional malformed-record note for the // stderr summaries. func malformedNote(malformed int) string { if malformed == 0 { return "" } + return fmt.Sprintf(" (%d malformed, skipped)", malformed) } // splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing // data without a terminator at EOF is returned as a final record. -func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) { +func splitNUL(data []byte, atEOF bool) (int, []byte, error) { if i := bytes.IndexByte(data, 0); i >= 0 { return i + 1, data[:i], nil } + if atEOF && len(data) > 0 { return len(data), data, nil } + return 0, nil, nil } @@ -172,10 +229,12 @@ func humanBytes(n int64) string { if n < unit { return fmt.Sprintf("%d B", n) } + div, exp := int64(unit), 0 for m := n / unit; m >= unit; m /= unit { div *= unit exp++ } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) } diff --git a/scan.go b/scan.go index 818ee6c..50fd3f9 100644 --- a/scan.go +++ b/scan.go @@ -4,6 +4,7 @@ import ( "bufio" "crypto/sha256" "encoding/hex" + "errors" "flag" "fmt" "io/fs" @@ -15,6 +16,14 @@ import ( // chunk is the number of bytes hashed from each end of a file. const chunk = 1024 +// workQueueDepth bounds the job and result channels feeding the stat +// and hash worker pools. +const workQueueDepth = 1024 + +// errNotRegular reports a path that stopped being a regular file +// between the walk and stat passes. +var errNotRegular = errors.New("no longer a regular file") + // fileRec carries one file between the stat and hash passes. type fileRec struct { path string @@ -30,11 +39,18 @@ func runScan(args []string) { root := fl.String("root", "/srv", "directory tree to scan") workers := fl.Int("workers", runtime.NumCPU(), "concurrent workers for the stat and hash passes") - fl.Parse(args) + // The flag set uses ExitOnError, so Parse cannot return a non-nil + // error; the check keeps the error handled explicitly. + err := fl.Parse(args) + if err != nil { + os.Exit(exitUsage) + } + if fl.NArg() > 0 { fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0)) - os.Exit(2) + os.Exit(exitUsage) } + if *workers < 1 { *workers = 1 } @@ -43,6 +59,7 @@ func runScan(args []string) { if err != nil { fatalf("root %s: %v", *root, err) } + if !st.IsDir() { fatalf("root %s: not a directory", *root) } @@ -59,23 +76,33 @@ func runScan(args []string) { // walkPass enumerates every regular file under root. It never follows // symlinks, never descends into directories named .zfs, and warns and // continues on any per-path error. -func walkPass(root string) (paths []string, errs int) { +func walkPass(root string) ([]string, int) { + var ( + paths []string + errs int + ) + prog := newProgress("walk", -1) walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil { errs++ + prog.warnf("walk %s: %v", p, err) + if d != nil && d.IsDir() { return filepath.SkipDir } + return nil } + if d.IsDir() { // ZFS snapshot pseudo-dirs would list every file // once per snapshot; never descend. if d.Name() == ".zfs" { return filepath.SkipDir } + return nil } // Regular files only: skip symlinks, sockets, FIFOs, and @@ -83,27 +110,35 @@ func walkPass(root string) (paths []string, errs int) { if !d.Type().IsRegular() { return nil } + paths = append(paths, p) + prog.increment() + return nil }) + prog.finish() + if walkErr != nil { fatalf("walk %s: %v", root, walkErr) } + return paths, errs } // statPass lstats every collected path in a worker pool, recording size // and mtime. Paths that fail to stat (or are no longer regular files) // are warned about and dropped. -func statPass(paths []string, workers int) (recs []fileRec, errs int) { +func statPass(paths []string, workers int) ([]fileRec, int) { type result struct { rec fileRec err error } - jobs := make(chan string, 1024) - results := make(chan result, 1024) + + jobs := make(chan string, workQueueDepth) + results := make(chan result, workQueueDepth) + for range workers { go func() { for p := range jobs { @@ -114,7 +149,7 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) { case !fi.Mode().IsRegular(): results <- result{ rec: fileRec{path: p}, - err: fmt.Errorf("no longer a regular file"), + err: errNotRegular, } default: results <- result{rec: fileRec{ @@ -126,78 +161,114 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) { } }() } + go func() { for _, p := range paths { jobs <- p } + close(jobs) }() prog := newProgress("stat", int64(len(paths))) - recs = make([]fileRec, 0, len(paths)) + + var errs int + + recs := make([]fileRec, 0, len(paths)) for range paths { r := <-results if r.err != nil { errs++ + prog.warnf("stat %s: %v", r.rec.path, r.err) } else { recs = append(recs, r.rec) } + prog.increment() } + prog.finish() + return recs, errs } +// hashResult carries one file's head/tail hashes (or the error that +// prevented hashing it) from the hash workers to the main goroutine. +type hashResult struct { + rec fileRec + head string + tail string + err error +} + +// startHashWorkers starts the hash worker pool over recs and returns +// the channel its results arrive on (one per record, in completion +// order). +func startHashWorkers(recs []fileRec, workers int) <-chan hashResult { + jobs := make(chan fileRec, workQueueDepth) + results := make(chan hashResult, workQueueDepth) + + for range workers { + go func() { + for rec := range jobs { + head, tail, err := hashHeadTail(rec.path, rec.size) + results <- hashResult{rec: rec, head: head, tail: tail, err: err} + } + }() + } + + go func() { + for _, rec := range recs { + jobs <- rec + } + + close(jobs) + }() + + return results +} + // hashPass hashes the first and last chunk bytes of every file in a // worker pool and emits the output records on stdout from the main // goroutine. Files that fail to open or read are warned about and // dropped. -func hashPass(recs []fileRec, workers int) (emitted, errs int) { - type result struct { - rec fileRec - head string - tail string - err error - } - jobs := make(chan fileRec, 1024) - results := make(chan result, 1024) - for range workers { - go func() { - for rec := range jobs { - head, tail, err := hashHeadTail(rec.path, rec.size) - results <- result{rec: rec, head: head, tail: tail, err: err} - } - }() - } - go func() { - for _, rec := range recs { - jobs <- rec - } - close(jobs) - }() - +func hashPass(recs []fileRec, workers int) (int, int) { + results := startHashWorkers(recs, workers) prog := newProgress("hash", int64(len(recs))) - out := bufio.NewWriterSize(os.Stdout, 1<<20) + out := bufio.NewWriterSize(os.Stdout, ioBufSize) + + var emitted, errs int + for range recs { r := <-results if r.err != nil { errs++ + prog.warnf("hash %s: %v", r.rec.path, r.err) prog.increment() + continue } - if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", - r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil { + + _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", + r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path) + if err != nil { fatalf("write stdout: %v", err) } + emitted++ + prog.increment() } + prog.finish() - if err := out.Flush(); err != nil { + + err := out.Flush() + if err != nil { fatalf("write stdout: %v", err) } + return emitted, errs } @@ -206,26 +277,35 @@ func hashPass(recs []fileRec, workers int) (emitted, errs int) { // file at path. The two reads overlap when size < 2*chunk; for // size == 0 both hashes are of the empty input. size is the value // recorded by the stat pass. -func hashHeadTail(path string, size int64) (head, tail string, err error) { +func hashHeadTail(path string, size int64) (string, string, error) { + //nolint:gosec // hashing operator-supplied paths is the tool's purpose f, err := os.Open(path) if err != nil { return "", "", err } - defer f.Close() + + defer func() { _ = f.Close() }() n := min(int64(chunk), size) + buf := make([]byte, n) if n > 0 { - if _, err := f.ReadAt(buf, 0); err != nil { + _, err = f.ReadAt(buf, 0) + if err != nil { return "", "", err } } + h := sha256.Sum256(buf) + if n > 0 { - if _, err := f.ReadAt(buf, size-n); err != nil { + _, err = f.ReadAt(buf, size-n) + if err != nil { return "", "", err } } + t := sha256.Sum256(buf) + return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil } diff --git a/trees.go b/trees.go index 86848fc..5f068e6 100644 --- a/trees.go +++ b/trees.go @@ -38,16 +38,64 @@ type treeNode struct { func runTrees(args []string) { in, name, closer := openScanInput(args) defer closer() + recs, malformed := parseScanStream(in, name) records := len(recs) + malformed - // Build the hierarchy under a synthetic super-root. Paths are - // split on "/"; for absolute paths the first component is empty, - // which simply becomes a top-level node representing "/". + super, allDirs := buildHierarchy(recs) + super.compute() + + dupes := collectTreeGroups(allDirs, super) + + out := bufio.NewWriterSize(os.Stdout, ioBufSize) + + _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize") + if err != nil { + fatalf("write stdout: %v", err) + } + + dupeTrees := 0 + + var reclaimable int64 + + for _, g := range dupes { + first := g[0] + for _, n := range g[1:] { + _, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", + first.path, n.path, first.fileCount, first.totalSize) + if err != nil { + fatalf("write stdout: %v", err) + } + + dupeTrees++ + reclaimable += first.totalSize + } + } + + err = out.Flush() + if err != nil { + fatalf("write stdout: %v", err) + } + + fmt.Fprintf(os.Stderr, + "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", + records, malformedNote(malformed), len(dupes), dupeTrees, + humanBytes(reclaimable)) +} + +// buildHierarchy reconstructs the directory hierarchy from the record +// paths under a synthetic super-root. Paths are split on "/"; for +// absolute paths the first component is empty, which simply becomes a +// top-level node representing "/". It returns the super-root and every +// directory node created. +func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) { super := &treeNode{} + var allDirs []*treeNode + for _, r := range recs { comps := strings.Split(r.path, "/") + node := super for _, c := range comps[:len(comps)-1] { child := node.dirs[c] @@ -56,76 +104,68 @@ func runTrees(args []string) { if node != super { childPath = node.path + "/" + c } + child = &treeNode{path: childPath, parent: node} if node.dirs == nil { node.dirs = make(map[string]*treeNode) } + node.dirs[c] = child allDirs = append(allDirs, child) } + node = child } + if node.files == nil { node.files = make(map[string]fileSig) } + node.files[comps[len(comps)-1]] = fileSig{ size: r.size, head: r.head, tail: r.tail, } } - super.compute() - // Group directories by digest; two or more dirs sharing a digest - // are candidate duplicate trees. + return super, allDirs +} + +// collectTreeGroups groups directories by digest and returns every +// maximal group with two or more members, each group's members sorted +// by path, groups ordered by tree size descending then by first path +// ascending. +func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode { groups := make(map[[sha256.Size]byte][]*treeNode) for _, d := range allDirs { groups[d.digest] = append(groups[d.digest], d) } + var dupes [][]*treeNode + for _, g := range groups { - if len(g) < 2 || suppressed(g, super) { + if len(g) < minGroupSize || suppressed(g, super) { continue } + slices.SortFunc(g, func(a, b *treeNode) int { return strings.Compare(a.path, b.path) }) dupes = append(dupes, g) } + // Biggest reclaimable space first; ties broken by first path. slices.SortFunc(dupes, func(a, b []*treeNode) int { if a[0].totalSize != b[0].totalSize { if a[0].totalSize > b[0].totalSize { return -1 } + return 1 } + return strings.Compare(a[0].path, b[0].path) }) - out := bufio.NewWriterSize(os.Stdout, 1<<20) - if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil { - fatalf("write stdout: %v", err) - } - dupeTrees := 0 - var reclaimable int64 - for _, g := range dupes { - first := g[0] - for _, n := range g[1:] { - if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", - first.path, n.path, first.fileCount, first.totalSize); err != nil { - fatalf("write stdout: %v", err) - } - dupeTrees++ - reclaimable += first.totalSize - } - } - if err := out.Flush(); err != nil { - fatalf("write stdout: %v", err) - } - - fmt.Fprintf(os.Stderr, - "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", - records, malformedNote(malformed), len(dupes), dupeTrees, - humanBytes(reclaimable)) + return dupes } // compute fills in digest, fileCount, and totalSize for n and all of @@ -143,18 +183,22 @@ func (n *treeNode) compute() { n.fileCount++ n.totalSize += sig.size } + for name, child := range n.dirs { child.compute() entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) n.fileCount += child.fileCount n.totalSize += child.totalSize } + slices.Sort(entries) + h := sha256.New() for _, e := range entries { h.Write([]byte(e)) h.Write([]byte{0}) } + copy(n.digest[:], h.Sum(nil)) } @@ -165,15 +209,19 @@ func (n *treeNode) compute() { // parent) or members whose parents differ are always reported. func suppressed(g []*treeNode, super *treeNode) bool { seen := make(map[*treeNode]bool, len(g)) + var parentDigest [sha256.Size]byte + for i, n := range g { p := n.parent if p == nil || p == super { return false } + if seen[p] { return false // siblings: not implied by any parent group } + seen[p] = true if i == 0 { parentDigest = p.digest @@ -181,5 +229,6 @@ func suppressed(g []*treeNode, super *treeNode) bool { return false } } + return true }