Make code lint-clean under the canonical golangci-lint config

150 findings fixed: linter autofixes for whitespace style (wsl_v5,
nlreturn, noinlineerr), named returns removed, magic numbers replaced
with named constants, static errNotRegular error, explicit Close/Parse
error handling, unused cobra params renamed to _, and runReport/
runTrees/hashPass split into helpers to satisfy cyclop, gocognit, and
funlen. Behavior verified unchanged against the README smoke test.
This commit is contained in:
2026-07-23 05:36:05 +07:00
parent 733b33d9d1
commit 065a533224
5 changed files with 351 additions and 128 deletions

26
main.go
View File

@@ -20,15 +20,23 @@ import (
"github.com/spf13/cobra"
)
// Exit codes: 0 is success (even with per-file warnings), exitFatal is
// a fatal error, exitUsage is a usage error.
const (
exitFatal = 1
exitUsage = 2
)
func main() {
root := &cobra.Command{
Use: "sfdupes",
Short: "Find candidate duplicate files by size and head/tail SHA-256",
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
Run: func(cmd *cobra.Command, _ []string) {
// A missing subcommand prints usage and exits 2.
_ = cmd.Usage()
os.Exit(2)
os.Exit(exitUsage)
},
}
// Everything on stdout is machine-readable data; all human-facing
@@ -43,7 +51,7 @@ func main() {
// The README specifies single-dash flags (-root, -workers);
// parse them with the stdlib flag package inside runScan.
DisableFlagParsing: true,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, args []string) {
runScan(args)
},
}
@@ -52,7 +60,7 @@ func main() {
Use: "report [files.dat|-]",
Short: "Read a scan stream and print the file-level duplicates report",
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, args []string) {
runReport(args)
},
}
@@ -61,21 +69,23 @@ func main() {
Use: "trees [files.dat|-]",
Short: "Read a scan stream and print the duplicate-tree report",
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, args []string) {
runTrees(args)
},
}
root.AddCommand(scanCmd, reportCmd, treesCmd)
if err := root.Execute(); err != nil {
err := root.Execute()
if err != nil {
// Cobra has already printed the error and usage to stderr;
// an invalid subcommand or bad arguments is a usage error.
os.Exit(2)
os.Exit(exitUsage)
}
}
// fatalf reports a fatal error and exits 1.
func fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
os.Exit(1)
os.Exit(exitFatal)
}

View File

@@ -12,12 +12,23 @@ import (
// is not a TTY.
const plainInterval = 5 * time.Second
// barThrottle caps how often the TTY progress bar redraws.
const barThrottle = 100 * time.Millisecond
// walkSpinnerType selects the progressbar library's spinner style used
// for the walk pass, which has no known total.
const walkSpinnerType = 14
// percentScale converts a fraction to a percentage.
const percentScale = 100
// stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool {
fi, err := os.Stderr.Stat()
if err != nil {
return false
}
return fi.Mode()&os.ModeCharDevice != 0
}
@@ -42,6 +53,7 @@ func newProgress(label string, total int64) *progress {
if !stderrIsTTY() {
return p
}
opts := []progressbar.Option{
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetDescription(label),
@@ -49,7 +61,7 @@ func newProgress(label string, total int64) *progress {
progressbar.OptionShowIts(),
progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(100 * time.Millisecond),
progressbar.OptionThrottle(barThrottle),
}
if total >= 0 {
opts = append(opts,
@@ -59,10 +71,12 @@ func newProgress(label string, total int64) *progress {
} else {
opts = append(opts,
progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(14),
progressbar.OptionSpinnerType(walkSpinnerType),
)
}
p.bar = progressbar.NewOptions64(total, opts...)
return p
}
@@ -71,8 +85,10 @@ func (p *progress) increment() {
p.count++
if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
@@ -84,6 +100,7 @@ func (p *progress) warnf(format string, args ...any) {
if p.bar != nil {
_ = p.bar.Clear()
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
@@ -91,9 +108,12 @@ func (p *progress) warnf(format string, args ...any) {
func (p *progress) finish() {
if p.bar != nil {
_ = p.bar.Finish()
fmt.Fprintln(os.Stderr)
return
}
fmt.Fprintln(os.Stderr, p.plainLine())
}
@@ -104,19 +124,24 @@ func (p *progress) plainLine() string {
return fmt.Sprintf("%s: %d files, elapsed %s",
p.label, p.count, elapsed.Round(time.Second))
}
pct := 100.0
pct := float64(percentScale)
if p.total > 0 {
pct = 100 * float64(p.count) / float64(p.total)
pct = percentScale * float64(p.count) / float64(p.total)
}
rate := 0.0
if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
}
eta := "?"
if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)

147
report.go
View File

@@ -11,6 +11,26 @@ import (
"strings"
)
// ioBufSize is the buffer size for the buffered scan-stream reader and
// the buffered stdout writers.
const ioBufSize = 1 << 20
// recordFieldCount is the number of tab-separated fields in a scan
// record: size, mtime, head hash, tail hash, path.
const recordFieldCount = 5
// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
// to read scan records; a record longer than scanMaxRecordSize is a
// fatal read error.
const (
scanInitBufSize = 64 << 10
scanMaxRecordSize = 4 << 20
)
// minGroupSize is the smallest number of members that makes a
// duplicate group.
const minGroupSize = 2
// scanRec is one well-formed record parsed from a scan stream. The
// signature (size, head, tail) is the duplicate key; mtime is
// informational and not retained.
@@ -24,14 +44,16 @@ type scanRec struct {
// openScanInput resolves the analysis-mode input: the file named by the
// single optional positional argument, or stdin when it is absent or
// "-". The returned closer must be called when reading is done.
func openScanInput(args []string) (in io.Reader, name string, closer func()) {
func openScanInput(args []string) (io.Reader, string, func()) {
if len(args) == 1 && args[0] != "-" {
f, err := os.Open(args[0])
if err != nil {
fatalf("open %s: %v", args[0], err)
}
return f, args[0], func() { f.Close() }
return f, args[0], func() { _ = f.Close() }
}
return os.Stdin, "stdin", func() {}
}
@@ -39,23 +61,33 @@ func openScanInput(args []string) (in io.Reader, name string, closer func()) {
// that does not have exactly 5 fields or whose size is non-numeric is
// counted as malformed and skipped. Total records read is
// len(recs) + malformed.
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
sc.Split(splitNUL)
var (
recs []scanRec
malformed int
)
for sc.Scan() {
// The path is the last field and may itself contain tabs,
// so split into at most 5 fields.
fields := strings.SplitN(sc.Text(), "\t", 5)
if len(fields) != 5 {
// so split into at most recordFieldCount fields.
fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
if len(fields) != recordFieldCount {
malformed++
continue
}
size, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
malformed++
continue
}
recs = append(recs, scanRec{
size: size,
head: fields[2],
@@ -63,9 +95,12 @@ func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int)
path: fields[4],
})
}
if err := sc.Err(); err != nil {
err := sc.Err()
if err != nil {
fatalf("read %s: %v", name, err)
}
return recs, malformed
}
@@ -86,56 +121,37 @@ type dupeGroup struct {
func runReport(args []string) {
in, name, closer := openScanInput(args)
defer closer()
recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed
dupes := collectDupeGroups(recs)
type dupeKey struct {
size int64
head string
tail string
}
groups := make(map[dupeKey][]string)
for _, r := range recs {
k := dupeKey{size: r.size, head: r.head, tail: r.tail}
groups[k] = append(groups[k], r.path)
}
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
var dupes []dupeGroup
for k, paths := range groups {
if len(paths) < 2 {
continue
}
slices.Sort(paths)
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b dupeGroup) int {
if a.size != b.size {
if a.size > b.size {
return -1
}
return 1
}
return strings.Compare(a.paths[0], b.paths[0])
})
out := bufio.NewWriterSize(os.Stdout, 1<<20)
if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
fatalf("write stdout: %v", err)
}
dupeFiles := 0
var reclaimable int64
for _, g := range dupes {
for _, p := range g.paths[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
g.paths[0], p, g.size); err != nil {
_, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
g.paths[0], p, g.size)
if err != nil {
fatalf("write stdout: %v", err)
}
dupeFiles++
reclaimable += g.size
}
}
if err := out.Flush(); err != nil {
err = out.Flush()
if err != nil {
fatalf("write stdout: %v", err)
}
@@ -145,24 +161,65 @@ func runReport(args []string) {
humanBytes(reclaimable))
}
// collectDupeGroups groups records by signature and returns every group
// with two or more paths, each group's paths sorted lexicographically,
// groups ordered by size descending then by first path ascending.
func collectDupeGroups(recs []scanRec) []dupeGroup {
groups := make(map[fileSig][]string)
for _, r := range recs {
k := fileSig{size: r.size, head: r.head, tail: r.tail}
groups[k] = append(groups[k], r.path)
}
var dupes []dupeGroup
for k, paths := range groups {
if len(paths) < minGroupSize {
continue
}
slices.Sort(paths)
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b dupeGroup) int {
if a.size != b.size {
if a.size > b.size {
return -1
}
return 1
}
return strings.Compare(a.paths[0], b.paths[0])
})
return dupes
}
// malformedNote formats the optional malformed-record note for the
// stderr summaries.
func malformedNote(malformed int) string {
if malformed == 0 {
return ""
}
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
}
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
// data without a terminator at EOF is returned as a final record.
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
if i := bytes.IndexByte(data, 0); i >= 0 {
return i + 1, data[:i], nil
}
if atEOF && len(data) > 0 {
return len(data), data, nil
}
return 0, nil, nil
}
@@ -172,10 +229,12 @@ func humanBytes(n int64) string {
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
for m := n / unit; m >= unit; m /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

160
scan.go
View File

@@ -4,6 +4,7 @@ import (
"bufio"
"crypto/sha256"
"encoding/hex"
"errors"
"flag"
"fmt"
"io/fs"
@@ -15,6 +16,14 @@ import (
// chunk is the number of bytes hashed from each end of a file.
const chunk = 1024
// workQueueDepth bounds the job and result channels feeding the stat
// and hash worker pools.
const workQueueDepth = 1024
// errNotRegular reports a path that stopped being a regular file
// between the walk and stat passes.
var errNotRegular = errors.New("no longer a regular file")
// fileRec carries one file between the stat and hash passes.
type fileRec struct {
path string
@@ -30,11 +39,18 @@ func runScan(args []string) {
root := fl.String("root", "/srv", "directory tree to scan")
workers := fl.Int("workers", runtime.NumCPU(),
"concurrent workers for the stat and hash passes")
fl.Parse(args)
// The flag set uses ExitOnError, so Parse cannot return a non-nil
// error; the check keeps the error handled explicitly.
err := fl.Parse(args)
if err != nil {
os.Exit(exitUsage)
}
if fl.NArg() > 0 {
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
os.Exit(2)
os.Exit(exitUsage)
}
if *workers < 1 {
*workers = 1
}
@@ -43,6 +59,7 @@ func runScan(args []string) {
if err != nil {
fatalf("root %s: %v", *root, err)
}
if !st.IsDir() {
fatalf("root %s: not a directory", *root)
}
@@ -59,23 +76,33 @@ func runScan(args []string) {
// walkPass enumerates every regular file under root. It never follows
// symlinks, never descends into directories named .zfs, and warns and
// continues on any per-path error.
func walkPass(root string) (paths []string, errs int) {
func walkPass(root string) ([]string, int) {
var (
paths []string
errs int
)
prog := newProgress("walk", -1)
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
errs++
prog.warnf("walk %s: %v", p, err)
if d != nil && d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
// ZFS snapshot pseudo-dirs would list every file
// once per snapshot; never descend.
if d.Name() == ".zfs" {
return filepath.SkipDir
}
return nil
}
// Regular files only: skip symlinks, sockets, FIFOs, and
@@ -83,27 +110,35 @@ func walkPass(root string) (paths []string, errs int) {
if !d.Type().IsRegular() {
return nil
}
paths = append(paths, p)
prog.increment()
return nil
})
prog.finish()
if walkErr != nil {
fatalf("walk %s: %v", root, walkErr)
}
return paths, errs
}
// statPass lstats every collected path in a worker pool, recording size
// and mtime. Paths that fail to stat (or are no longer regular files)
// are warned about and dropped.
func statPass(paths []string, workers int) (recs []fileRec, errs int) {
func statPass(paths []string, workers int) ([]fileRec, int) {
type result struct {
rec fileRec
err error
}
jobs := make(chan string, 1024)
results := make(chan result, 1024)
jobs := make(chan string, workQueueDepth)
results := make(chan result, workQueueDepth)
for range workers {
go func() {
for p := range jobs {
@@ -114,7 +149,7 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
case !fi.Mode().IsRegular():
results <- result{
rec: fileRec{path: p},
err: fmt.Errorf("no longer a regular file"),
err: errNotRegular,
}
default:
results <- result{rec: fileRec{
@@ -126,78 +161,114 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
}
}()
}
go func() {
for _, p := range paths {
jobs <- p
}
close(jobs)
}()
prog := newProgress("stat", int64(len(paths)))
recs = make([]fileRec, 0, len(paths))
var errs int
recs := make([]fileRec, 0, len(paths))
for range paths {
r := <-results
if r.err != nil {
errs++
prog.warnf("stat %s: %v", r.rec.path, r.err)
} else {
recs = append(recs, r.rec)
}
prog.increment()
}
prog.finish()
return recs, errs
}
// hashResult carries one file's head/tail hashes (or the error that
// prevented hashing it) from the hash workers to the main goroutine.
type hashResult struct {
rec fileRec
head string
tail string
err error
}
// startHashWorkers starts the hash worker pool over recs and returns
// the channel its results arrive on (one per record, in completion
// order).
func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
jobs := make(chan fileRec, workQueueDepth)
results := make(chan hashResult, workQueueDepth)
for range workers {
go func() {
for rec := range jobs {
head, tail, err := hashHeadTail(rec.path, rec.size)
results <- hashResult{rec: rec, head: head, tail: tail, err: err}
}
}()
}
go func() {
for _, rec := range recs {
jobs <- rec
}
close(jobs)
}()
return results
}
// hashPass hashes the first and last chunk bytes of every file in a
// worker pool and emits the output records on stdout from the main
// goroutine. Files that fail to open or read are warned about and
// dropped.
func hashPass(recs []fileRec, workers int) (emitted, errs int) {
type result struct {
rec fileRec
head string
tail string
err error
}
jobs := make(chan fileRec, 1024)
results := make(chan result, 1024)
for range workers {
go func() {
for rec := range jobs {
head, tail, err := hashHeadTail(rec.path, rec.size)
results <- result{rec: rec, head: head, tail: tail, err: err}
}
}()
}
go func() {
for _, rec := range recs {
jobs <- rec
}
close(jobs)
}()
func hashPass(recs []fileRec, workers int) (int, int) {
results := startHashWorkers(recs, workers)
prog := newProgress("hash", int64(len(recs)))
out := bufio.NewWriterSize(os.Stdout, 1<<20)
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
var emitted, errs int
for range recs {
r := <-results
if r.err != nil {
errs++
prog.warnf("hash %s: %v", r.rec.path, r.err)
prog.increment()
continue
}
if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil {
_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
if err != nil {
fatalf("write stdout: %v", err)
}
emitted++
prog.increment()
}
prog.finish()
if err := out.Flush(); err != nil {
err := out.Flush()
if err != nil {
fatalf("write stdout: %v", err)
}
return emitted, errs
}
@@ -206,26 +277,35 @@ func hashPass(recs []fileRec, workers int) (emitted, errs int) {
// file at path. The two reads overlap when size < 2*chunk; for
// size == 0 both hashes are of the empty input. size is the value
// recorded by the stat pass.
func hashHeadTail(path string, size int64) (head, tail string, err error) {
func hashHeadTail(path string, size int64) (string, string, error) {
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
f, err := os.Open(path)
if err != nil {
return "", "", err
}
defer f.Close()
defer func() { _ = f.Close() }()
n := min(int64(chunk), size)
buf := make([]byte, n)
if n > 0 {
if _, err := f.ReadAt(buf, 0); err != nil {
_, err = f.ReadAt(buf, 0)
if err != nil {
return "", "", err
}
}
h := sha256.Sum256(buf)
if n > 0 {
if _, err := f.ReadAt(buf, size-n); err != nil {
_, err = f.ReadAt(buf, size-n)
if err != nil {
return "", "", err
}
}
t := sha256.Sum256(buf)
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
}

113
trees.go
View File

@@ -38,16 +38,64 @@ type treeNode struct {
func runTrees(args []string) {
in, name, closer := openScanInput(args)
defer closer()
recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed
// Build the hierarchy under a synthetic super-root. Paths are
// split on "/"; for absolute paths the first component is empty,
// which simply becomes a top-level node representing "/".
super, allDirs := buildHierarchy(recs)
super.compute()
dupes := collectTreeGroups(allDirs, super)
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees := 0
var reclaimable int64
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
first.path, n.path, first.fileCount, first.totalSize)
if err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees++
reclaimable += first.totalSize
}
}
err = out.Flush()
if err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeTrees,
humanBytes(reclaimable))
}
// buildHierarchy reconstructs the directory hierarchy from the record
// paths under a synthetic super-root. Paths are split on "/"; for
// absolute paths the first component is empty, which simply becomes a
// top-level node representing "/". It returns the super-root and every
// directory node created.
func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
super := &treeNode{}
var allDirs []*treeNode
for _, r := range recs {
comps := strings.Split(r.path, "/")
node := super
for _, c := range comps[:len(comps)-1] {
child := node.dirs[c]
@@ -56,76 +104,68 @@ func runTrees(args []string) {
if node != super {
childPath = node.path + "/" + c
}
child = &treeNode{path: childPath, parent: node}
if node.dirs == nil {
node.dirs = make(map[string]*treeNode)
}
node.dirs[c] = child
allDirs = append(allDirs, child)
}
node = child
}
if node.files == nil {
node.files = make(map[string]fileSig)
}
node.files[comps[len(comps)-1]] = fileSig{
size: r.size, head: r.head, tail: r.tail,
}
}
super.compute()
// Group directories by digest; two or more dirs sharing a digest
// are candidate duplicate trees.
return super, allDirs
}
// collectTreeGroups groups directories by digest and returns every
// maximal group with two or more members, each group's members sorted
// by path, groups ordered by tree size descending then by first path
// ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d)
}
var dupes [][]*treeNode
for _, g := range groups {
if len(g) < 2 || suppressed(g, super) {
if len(g) < minGroupSize || suppressed(g, super) {
continue
}
slices.SortFunc(g, func(a, b *treeNode) int {
return strings.Compare(a.path, b.path)
})
dupes = append(dupes, g)
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b []*treeNode) int {
if a[0].totalSize != b[0].totalSize {
if a[0].totalSize > b[0].totalSize {
return -1
}
return 1
}
return strings.Compare(a[0].path, b[0].path)
})
out := bufio.NewWriterSize(os.Stdout, 1<<20)
if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees := 0
var reclaimable int64
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
first.path, n.path, first.fileCount, first.totalSize); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees++
reclaimable += first.totalSize
}
}
if err := out.Flush(); err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeTrees,
humanBytes(reclaimable))
return dupes
}
// compute fills in digest, fileCount, and totalSize for n and all of
@@ -143,18 +183,22 @@ func (n *treeNode) compute() {
n.fileCount++
n.totalSize += sig.size
}
for name, child := range n.dirs {
child.compute()
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
n.fileCount += child.fileCount
n.totalSize += child.totalSize
}
slices.Sort(entries)
h := sha256.New()
for _, e := range entries {
h.Write([]byte(e))
h.Write([]byte{0})
}
copy(n.digest[:], h.Sum(nil))
}
@@ -165,15 +209,19 @@ func (n *treeNode) compute() {
// parent) or members whose parents differ are always reported.
func suppressed(g []*treeNode, super *treeNode) bool {
seen := make(map[*treeNode]bool, len(g))
var parentDigest [sha256.Size]byte
for i, n := range g {
p := n.parent
if p == nil || p == super {
return false
}
if seen[p] {
return false // siblings: not implied by any parent group
}
seen[p] = true
if i == 0 {
parentDigest = p.digest
@@ -181,5 +229,6 @@ func suppressed(g []*treeNode, super *treeNode) bool {
return false
}
}
return true
}