sneak/sfdupes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sfdupes implementation: scan, report, and trees subcommands

Browse Source
This commit is contained in:
Jeffrey Paul
2026-07-22 22:06:22 +07:00
parent dbc7f5a786
commit 1fad4a7d1c
8 changed files with 867 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Makefile Normal file
View File

@@ -0,0 +1,17 @@
# sfdupes is pure Go; disable cgo so builds never touch the host C
# toolchain.
export CGO_ENABLED = 0
.PHONY: all build check clean
all: build
build:
go build
check: build
test -z "$$(gofmt -l .)"
go vet ./...
clean:
rm -f sfdupes files.dat

17
go.mod Normal file
View File

@@ -0,0 +1,17 @@
module sneak.berlin/go/sfdupes
go 1.25.7
require (
github.com/schollz/progressbar/v3 v3.19.1
github.com/spf13/cobra v1.10.2
)
require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/spf13/pflag v1.0.9 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/term v0.44.0 // indirect
)

32
go.sum Normal file
View File

@@ -0,0 +1,32 @@
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/schollz/progressbar/v3 v3.19.1 h1:iv8BgwOvdML/S3p84uBpy/IMigv4U9594vPZYa2EdrU=
github.com/schollz/progressbar/v3 v3.19.1/go.mod h1:LFL7jqimKxfhero4K1eCkUr/6R39AgQeiPCJtlTWIW8=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

81
main.go Normal file
View File

@@ -0,0 +1,81 @@
// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading full file contents. Files are
// considered duplicates when they have identical size, identical SHA-256
// of their first 1024 bytes, and identical SHA-256 of their last 1024
// bytes.
//
// Usage:
//
// sfdupes scan [-root /srv] [-workers N] > files.dat
// sfdupes report [files.dat|-] > dupes.tsv
// sfdupes trees [files.dat|-] > dupetrees.tsv
//
// See README.md for the complete specification.
package main
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
func main() {
root := &cobra.Command{
Use: "sfdupes",
Short: "Find candidate duplicate files by size and head/tail SHA-256",
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
// A missing subcommand prints usage and exits 2.
_ = cmd.Usage()
os.Exit(2)
},
}
// Everything on stdout is machine-readable data; all human-facing
// output (help, usage, errors) goes to stderr.
root.SetOut(os.Stderr)
root.SetErr(os.Stderr)
root.CompletionOptions.DisableDefaultCmd = true
scanCmd := &cobra.Command{
Use: "scan [-root /srv] [-workers N]",
Short: "Walk a tree and emit one record per regular file on stdout",
// The README specifies single-dash flags (-root, -workers);
// parse them with the stdlib flag package inside runScan.
DisableFlagParsing: true,
Run: func(cmd *cobra.Command, args []string) {
runScan(args)
},
}
reportCmd := &cobra.Command{
Use: "report [files.dat|-]",
Short: "Read a scan stream and print the file-level duplicates report",
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
runReport(args)
},
}
treesCmd := &cobra.Command{
Use: "trees [files.dat|-]",
Short: "Read a scan stream and print the duplicate-tree report",
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
runTrees(args)
},
}
root.AddCommand(scanCmd, reportCmd, treesCmd)
if err := root.Execute(); err != nil {
// Cobra has already printed the error and usage to stderr;
// an invalid subcommand or bad arguments is a usage error.
os.Exit(2)
}
}
// fatalf reports a fatal error and exits 1.
func fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
os.Exit(1)
}

123
progress.go Normal file
View File

@@ -0,0 +1,123 @@
package main
import (
"fmt"
"os"
"time"
"github.com/schollz/progressbar/v3"
)
// plainInterval is the minimum time between progress lines when stderr
// is not a TTY.
const plainInterval = 5 * time.Second
// stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool {
fi, err := os.Stderr.Stat()
if err != nil {
return false
}
return fi.Mode()&os.ModeCharDevice != 0
}
// progress renders one scan pass's progress on stderr. On a TTY it
// delegates to the progressbar library (spinner style when the total is
// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
// stderr is not a TTY it emits no ANSI redraws: it prints a plain
// one-line update no more often than every plainInterval.
//
// All methods must be called from the main goroutine only.
type progress struct {
label string
total int64 // -1 when unknown (walk pass)
bar *progressbar.ProgressBar
count int64
start time.Time
last time.Time
}
func newProgress(label string, total int64) *progress {
p := &progress{label: label, total: total, start: time.Now()}
if !stderrIsTTY() {
return p
}
opts := []progressbar.Option{
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetDescription(label),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(100 * time.Millisecond),
}
if total >= 0 {
opts = append(opts,
progressbar.OptionSetPredictTime(true),
progressbar.OptionShowElapsedTimeOnFinish(),
)
} else {
opts = append(opts,
progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(14),
)
}
p.bar = progressbar.NewOptions64(total, opts...)
return p
}
// increment records one completed item and refreshes the display.
func (p *progress) increment() {
p.count++
if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
}
}
// warnf prints a one-line warning to stderr without corrupting the bar.
func (p *progress) warnf(format string, args ...any) {
if p.bar != nil {
_ = p.bar.Clear()
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
// finish terminates the pass's display.
func (p *progress) finish() {
if p.bar != nil {
_ = p.bar.Finish()
fmt.Fprintln(os.Stderr)
return
}
fmt.Fprintln(os.Stderr, p.plainLine())
}
// plainLine formats a non-TTY progress line.
func (p *progress) plainLine() string {
elapsed := time.Since(p.start)
if p.total < 0 {
return fmt.Sprintf("%s: %d files, elapsed %s",
p.label, p.count, elapsed.Round(time.Second))
}
pct := 100.0
if p.total > 0 {
pct = 100 * float64(p.count) / float64(p.total)
}
rate := 0.0
if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
}
eta := "?"
if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)
}

181
report.go Normal file
View File

@@ -0,0 +1,181 @@
package main
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"slices"
"strconv"
"strings"
)
// scanRec is one well-formed record parsed from a scan stream. The
// signature (size, head, tail) is the duplicate key; mtime is
// informational and not retained.
type scanRec struct {
size int64
head string
tail string
path string
}
// openScanInput resolves the analysis-mode input: the file named by the
// single optional positional argument, or stdin when it is absent or
// "-". The returned closer must be called when reading is done.
func openScanInput(args []string) (in io.Reader, name string, closer func()) {
if len(args) == 1 && args[0] != "-" {
f, err := os.Open(args[0])
if err != nil {
fatalf("open %s: %v", args[0], err)
}
return f, args[0], func() { f.Close() }
}
return os.Stdin, "stdin", func() {}
}
// parseScanStream reads every NUL-terminated record from in. A record
// that does not have exactly 5 fields or whose size is non-numeric is
// counted as malformed and skipped. Total records read is
// len(recs) + malformed.
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
sc.Split(splitNUL)
for sc.Scan() {
// The path is the last field and may itself contain tabs,
// so split into at most 5 fields.
fields := strings.SplitN(sc.Text(), "\t", 5)
if len(fields) != 5 {
malformed++
continue
}
size, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
malformed++
continue
}
recs = append(recs, scanRec{
size: size,
head: fields[2],
tail: fields[3],
path: fields[4],
})
}
if err := sc.Err(); err != nil {
fatalf("read %s: %v", name, err)
}
return recs, malformed
}
// dupeGroup is one set of candidate-duplicate files: identical size,
// head hash, and tail hash. paths is sorted lexicographically; the
// first entry is the group's "first", the rest are dupes.
type dupeGroup struct {
size int64
paths []string
}
// runReport implements the report subcommand: it reads a scan stream
// from the named file (or stdin when absent or "-") and prints the
// file-level duplicates report as TSV on stdout. It never touches the
// scanned filesystem; its only I/O is the scan input, stdout, and
// stderr. args holds the positional arguments already validated by
// cobra (at most one).
func runReport(args []string) {
in, name, closer := openScanInput(args)
defer closer()
recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed
type dupeKey struct {
size int64
head string
tail string
}
groups := make(map[dupeKey][]string)
for _, r := range recs {
k := dupeKey{size: r.size, head: r.head, tail: r.tail}
groups[k] = append(groups[k], r.path)
}
var dupes []dupeGroup
for k, paths := range groups {
if len(paths) < 2 {
continue
}
slices.Sort(paths)
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b dupeGroup) int {
if a.size != b.size {
if a.size > b.size {
return -1
}
return 1
}
return strings.Compare(a.paths[0], b.paths[0])
})
out := bufio.NewWriterSize(os.Stdout, 1<<20)
if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
fatalf("write stdout: %v", err)
}
dupeFiles := 0
var reclaimable int64
for _, g := range dupes {
for _, p := range g.paths[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
g.paths[0], p, g.size); err != nil {
fatalf("write stdout: %v", err)
}
dupeFiles++
reclaimable += g.size
}
}
if err := out.Flush(); err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeFiles,
humanBytes(reclaimable))
}
// malformedNote formats the optional malformed-record note for the
// stderr summaries.
func malformedNote(malformed int) string {
if malformed == 0 {
return ""
}
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
}
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
// data without a terminator at EOF is returned as a final record.
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
if i := bytes.IndexByte(data, 0); i >= 0 {
return i + 1, data[:i], nil
}
if atEOF && len(data) > 0 {
return len(data), data, nil
}
return 0, nil, nil
}
// humanBytes formats a byte count in human units (binary prefixes).
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
for m := n / unit; m >= unit; m /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

231
scan.go Normal file
View File

@@ -0,0 +1,231 @@
package main
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"flag"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
)
// chunk is the number of bytes hashed from each end of a file.
const chunk = 1024
// fileRec carries one file between the stat and hash passes.
type fileRec struct {
path string
size int64
mtime int64
}
// runScan implements the scan subcommand: three sequential passes (walk,
// stat, hash) over the tree under -root, emitting one NUL-terminated
// record per regular file on stdout.
func runScan(args []string) {
fl := flag.NewFlagSet("scan", flag.ExitOnError)
root := fl.String("root", "/srv", "directory tree to scan")
workers := fl.Int("workers", runtime.NumCPU(),
"concurrent workers for the stat and hash passes")
fl.Parse(args)
if fl.NArg() > 0 {
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
os.Exit(2)
}
if *workers < 1 {
*workers = 1
}
st, err := os.Stat(*root)
if err != nil {
fatalf("root %s: %v", *root, err)
}
if !st.IsDir() {
fatalf("root %s: not a directory", *root)
}
paths, walkErrs := walkPass(*root)
recs, statErrs := statPass(paths, *workers)
emitted, hashErrs := hashPass(recs, *workers)
skipped := walkErrs + statErrs + hashErrs
fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n",
emitted, skipped)
}
// walkPass enumerates every regular file under root. It never follows
// symlinks, never descends into directories named .zfs, and warns and
// continues on any per-path error.
func walkPass(root string) (paths []string, errs int) {
prog := newProgress("walk", -1)
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
errs++
prog.warnf("walk %s: %v", p, err)
if d != nil && d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
// ZFS snapshot pseudo-dirs would list every file
// once per snapshot; never descend.
if d.Name() == ".zfs" {
return filepath.SkipDir
}
return nil
}
// Regular files only: skip symlinks, sockets, FIFOs, and
// device nodes.
if !d.Type().IsRegular() {
return nil
}
paths = append(paths, p)
prog.increment()
return nil
})
prog.finish()
if walkErr != nil {
fatalf("walk %s: %v", root, walkErr)
}
return paths, errs
}
// statPass lstats every collected path in a worker pool, recording size
// and mtime. Paths that fail to stat (or are no longer regular files)
// are warned about and dropped.
func statPass(paths []string, workers int) (recs []fileRec, errs int) {
type result struct {
rec fileRec
err error
}
jobs := make(chan string, 1024)
results := make(chan result, 1024)
for range workers {
go func() {
for p := range jobs {
fi, err := os.Lstat(p)
switch {
case err != nil:
results <- result{rec: fileRec{path: p}, err: err}
case !fi.Mode().IsRegular():
results <- result{
rec: fileRec{path: p},
err: fmt.Errorf("no longer a regular file"),
}
default:
results <- result{rec: fileRec{
path: p,
size: fi.Size(),
mtime: fi.ModTime().Unix(),
}}
}
}
}()
}
go func() {
for _, p := range paths {
jobs <- p
}
close(jobs)
}()
prog := newProgress("stat", int64(len(paths)))
recs = make([]fileRec, 0, len(paths))
for range paths {
r := <-results
if r.err != nil {
errs++
prog.warnf("stat %s: %v", r.rec.path, r.err)
} else {
recs = append(recs, r.rec)
}
prog.increment()
}
prog.finish()
return recs, errs
}
// hashPass hashes the first and last chunk bytes of every file in a
// worker pool and emits the output records on stdout from the main
// goroutine. Files that fail to open or read are warned about and
// dropped.
func hashPass(recs []fileRec, workers int) (emitted, errs int) {
type result struct {
rec fileRec
head string
tail string
err error
}
jobs := make(chan fileRec, 1024)
results := make(chan result, 1024)
for range workers {
go func() {
for rec := range jobs {
head, tail, err := hashHeadTail(rec.path, rec.size)
results <- result{rec: rec, head: head, tail: tail, err: err}
}
}()
}
go func() {
for _, rec := range recs {
jobs <- rec
}
close(jobs)
}()
prog := newProgress("hash", int64(len(recs)))
out := bufio.NewWriterSize(os.Stdout, 1<<20)
for range recs {
r := <-results
if r.err != nil {
errs++
prog.warnf("hash %s: %v", r.rec.path, r.err)
prog.increment()
continue
}
if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil {
fatalf("write stdout: %v", err)
}
emitted++
prog.increment()
}
prog.finish()
if err := out.Flush(); err != nil {
fatalf("write stdout: %v", err)
}
return emitted, errs
}
// hashHeadTail returns the lowercase-hex SHA-256 of the first
// min(chunk, size) bytes and of the last min(chunk, size) bytes of the
// file at path. The two reads overlap when size < 2*chunk; for
// size == 0 both hashes are of the empty input. size is the value
// recorded by the stat pass.
func hashHeadTail(path string, size int64) (head, tail string, err error) {
f, err := os.Open(path)
if err != nil {
return "", "", err
}
defer f.Close()
n := min(int64(chunk), size)
buf := make([]byte, n)
if n > 0 {
if _, err := f.ReadAt(buf, 0); err != nil {
return "", "", err
}
}
h := sha256.Sum256(buf)
if n > 0 {
if _, err := f.ReadAt(buf, size-n); err != nil {
return "", "", err
}
}
t := sha256.Sum256(buf)
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
}

185
trees.go Normal file
View File

@@ -0,0 +1,185 @@
package main
import (
"bufio"
"crypto/sha256"
"fmt"
"os"
"slices"
"strconv"
"strings"
)
// fileSig is a file's duplicate signature; mtime is excluded.
type fileSig struct {
size int64
head string
tail string
}
// treeNode is one directory reconstructed from the scan stream.
type treeNode struct {
path string
parent *treeNode
dirs map[string]*treeNode
files map[string]fileSig
digest [sha256.Size]byte
fileCount int64
totalSize int64
}
// runTrees implements the trees subcommand: it reads a scan stream from
// the named file (or stdin when absent or "-"), reconstructs the
// directory hierarchy from the record paths, computes a Merkle-style
// digest per directory, and prints maximal duplicate-tree groups as TSV
// on stdout. It never touches the scanned filesystem; its only I/O is
// the scan input, stdout, and stderr. args holds the positional
// arguments already validated by cobra (at most one).
func runTrees(args []string) {
in, name, closer := openScanInput(args)
defer closer()
recs, malformed := parseScanStream(in, name)
records := len(recs) + malformed
// Build the hierarchy under a synthetic super-root. Paths are
// split on "/"; for absolute paths the first component is empty,
// which simply becomes a top-level node representing "/".
super := &treeNode{}
var allDirs []*treeNode
for _, r := range recs {
comps := strings.Split(r.path, "/")
node := super
for _, c := range comps[:len(comps)-1] {
child := node.dirs[c]
if child == nil {
childPath := c
if node != super {
childPath = node.path + "/" + c
}
child = &treeNode{path: childPath, parent: node}
if node.dirs == nil {
node.dirs = make(map[string]*treeNode)
}
node.dirs[c] = child
allDirs = append(allDirs, child)
}
node = child
}
if node.files == nil {
node.files = make(map[string]fileSig)
}
node.files[comps[len(comps)-1]] = fileSig{
size: r.size, head: r.head, tail: r.tail,
}
}
super.compute()
// Group directories by digest; two or more dirs sharing a digest
// are candidate duplicate trees.
groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d)
}
var dupes [][]*treeNode
for _, g := range groups {
if len(g) < 2 || suppressed(g, super) {
continue
}
slices.SortFunc(g, func(a, b *treeNode) int {
return strings.Compare(a.path, b.path)
})
dupes = append(dupes, g)
}
// Biggest reclaimable space first; ties broken by first path.
slices.SortFunc(dupes, func(a, b []*treeNode) int {
if a[0].totalSize != b[0].totalSize {
if a[0].totalSize > b[0].totalSize {
return -1
}
return 1
}
return strings.Compare(a[0].path, b[0].path)
})
out := bufio.NewWriterSize(os.Stdout, 1<<20)
if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees := 0
var reclaimable int64
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
first.path, n.path, first.fileCount, first.totalSize); err != nil {
fatalf("write stdout: %v", err)
}
dupeTrees++
reclaimable += first.totalSize
}
}
if err := out.Flush(); err != nil {
fatalf("write stdout: %v", err)
}
fmt.Fprintf(os.Stderr,
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
records, malformedNote(malformed), len(dupes), dupeTrees,
humanBytes(reclaimable))
}
// compute fills in digest, fileCount, and totalSize for n and all of
// its descendants. A directory's digest is the SHA-256 of its child
// entries — files serialized with name and signature, subdirectories
// with name and recursive digest — sorted byte-lexicographically.
// Filenames cannot contain NUL or "/", so NUL delimiters are
// unambiguous.
func (n *treeNode) compute() {
entries := make([]string, 0, len(n.dirs)+len(n.files))
for name, sig := range n.files {
entries = append(entries,
"f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+
"\x00"+sig.head+"\x00"+sig.tail)
n.fileCount++
n.totalSize += sig.size
}
for name, child := range n.dirs {
child.compute()
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
n.fileCount += child.fileCount
n.totalSize += child.totalSize
}
slices.Sort(entries)
h := sha256.New()
for _, e := range entries {
h.Write([]byte(e))
h.Write([]byte{0})
}
copy(n.digest[:], h.Sum(nil))
}
// suppressed reports whether a duplicate-tree group is non-maximal: its
// members' parents are pairwise distinct real directories that all
// share a single digest, so the group is wholly implied by its parents'
// (or a further ancestor's) group. Groups containing siblings (shared
// parent) or members whose parents differ are always reported.
func suppressed(g []*treeNode, super *treeNode) bool {
seen := make(map[*treeNode]bool, len(g))
var parentDigest [sha256.Size]byte
for i, n := range g {
p := n.parent
if p == nil || p == super {
return false
}
if seen[p] {
return false // siblings: not implied by any parent group
}
seen[p] = true
if i == 0 {
parentDigest = p.digest
} else if p.digest != parentDigest {
return false
}
}
return true
}