diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..80938d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +# sfdupes is pure Go; disable cgo so builds never touch the host C +# toolchain. +export CGO_ENABLED = 0 + +.PHONY: all build check clean + +all: build + +build: + go build + +check: build + test -z "$$(gofmt -l .)" + go vet ./... + +clean: + rm -f sfdupes files.dat diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..83bdfdb --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module sneak.berlin/go/sfdupes + +go 1.25.7 + +require ( + github.com/schollz/progressbar/v3 v3.19.1 + github.com/spf13/cobra v1.10.2 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cf50240 --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.19.1 h1:iv8BgwOvdML/S3p84uBpy/IMigv4U9594vPZYa2EdrU= +github.com/schollz/progressbar/v3 v3.19.1/go.mod h1:LFL7jqimKxfhero4K1eCkUr/6R39AgQeiPCJtlTWIW8= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..51e2900 --- /dev/null +++ b/main.go @@ -0,0 +1,81 @@ +// Command sfdupes quickly identifies candidate duplicate files across +// very large filesystems without reading full file contents. Files are +// considered duplicates when they have identical size, identical SHA-256 +// of their first 1024 bytes, and identical SHA-256 of their last 1024 +// bytes. +// +// Usage: +// +// sfdupes scan [-root /srv] [-workers N] > files.dat +// sfdupes report [files.dat|-] > dupes.tsv +// sfdupes trees [files.dat|-] > dupetrees.tsv +// +// See README.md for the complete specification. +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + root := &cobra.Command{ + Use: "sfdupes", + Short: "Find candidate duplicate files by size and head/tail SHA-256", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + // A missing subcommand prints usage and exits 2. + _ = cmd.Usage() + os.Exit(2) + }, + } + // Everything on stdout is machine-readable data; all human-facing + // output (help, usage, errors) goes to stderr. + root.SetOut(os.Stderr) + root.SetErr(os.Stderr) + root.CompletionOptions.DisableDefaultCmd = true + + scanCmd := &cobra.Command{ + Use: "scan [-root /srv] [-workers N]", + Short: "Walk a tree and emit one record per regular file on stdout", + // The README specifies single-dash flags (-root, -workers); + // parse them with the stdlib flag package inside runScan. + DisableFlagParsing: true, + Run: func(cmd *cobra.Command, args []string) { + runScan(args) + }, + } + + reportCmd := &cobra.Command{ + Use: "report [files.dat|-]", + Short: "Read a scan stream and print the file-level duplicates report", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + runReport(args) + }, + } + + treesCmd := &cobra.Command{ + Use: "trees [files.dat|-]", + Short: "Read a scan stream and print the duplicate-tree report", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + runTrees(args) + }, + } + + root.AddCommand(scanCmd, reportCmd, treesCmd) + if err := root.Execute(); err != nil { + // Cobra has already printed the error and usage to stderr; + // an invalid subcommand or bad arguments is a usage error. + os.Exit(2) + } +} + +// fatalf reports a fatal error and exits 1. +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...) + os.Exit(1) +} diff --git a/progress.go b/progress.go new file mode 100644 index 0000000..9aedbee --- /dev/null +++ b/progress.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/schollz/progressbar/v3" +) + +// plainInterval is the minimum time between progress lines when stderr +// is not a TTY. +const plainInterval = 5 * time.Second + +// stderrIsTTY reports whether stderr is attached to a terminal. +func stderrIsTTY() bool { + fi, err := os.Stderr.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// progress renders one scan pass's progress on stderr. On a TTY it +// delegates to the progressbar library (spinner style when the total is +// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When +// stderr is not a TTY it emits no ANSI redraws: it prints a plain +// one-line update no more often than every plainInterval. +// +// All methods must be called from the main goroutine only. +type progress struct { + label string + total int64 // -1 when unknown (walk pass) + bar *progressbar.ProgressBar + count int64 + start time.Time + last time.Time +} + +func newProgress(label string, total int64) *progress { + p := &progress{label: label, total: total, start: time.Now()} + if !stderrIsTTY() { + return p + } + opts := []progressbar.Option{ + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetDescription(label), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetItsString("files"), + progressbar.OptionSetElapsedTime(true), + progressbar.OptionThrottle(100 * time.Millisecond), + } + if total >= 0 { + opts = append(opts, + progressbar.OptionSetPredictTime(true), + progressbar.OptionShowElapsedTimeOnFinish(), + ) + } else { + opts = append(opts, + progressbar.OptionSetPredictTime(false), + progressbar.OptionSpinnerType(14), + ) + } + p.bar = progressbar.NewOptions64(total, opts...) + return p +} + +// increment records one completed item and refreshes the display. +func (p *progress) increment() { + p.count++ + if p.bar != nil { + _ = p.bar.Add(1) + return + } + if time.Since(p.last) >= plainInterval { + p.last = time.Now() + fmt.Fprintln(os.Stderr, p.plainLine()) + } +} + +// warnf prints a one-line warning to stderr without corrupting the bar. +func (p *progress) warnf(format string, args ...any) { + if p.bar != nil { + _ = p.bar.Clear() + } + fmt.Fprintf(os.Stderr, format+"\n", args...) +} + +// finish terminates the pass's display. +func (p *progress) finish() { + if p.bar != nil { + _ = p.bar.Finish() + fmt.Fprintln(os.Stderr) + return + } + fmt.Fprintln(os.Stderr, p.plainLine()) +} + +// plainLine formats a non-TTY progress line. +func (p *progress) plainLine() string { + elapsed := time.Since(p.start) + if p.total < 0 { + return fmt.Sprintf("%s: %d files, elapsed %s", + p.label, p.count, elapsed.Round(time.Second)) + } + pct := 100.0 + if p.total > 0 { + pct = 100 * float64(p.count) / float64(p.total) + } + rate := 0.0 + if s := elapsed.Seconds(); s > 0 { + rate = float64(p.count) / s + } + eta := "?" + if rate > 0 && p.count <= p.total { + remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second)) + eta = remaining.Round(time.Second).String() + } + return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s", + p.label, p.count, p.total, pct, rate, + elapsed.Round(time.Second), eta) +} diff --git a/report.go b/report.go new file mode 100644 index 0000000..096c4aa --- /dev/null +++ b/report.go @@ -0,0 +1,181 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" +) + +// scanRec is one well-formed record parsed from a scan stream. The +// signature (size, head, tail) is the duplicate key; mtime is +// informational and not retained. +type scanRec struct { + size int64 + head string + tail string + path string +} + +// openScanInput resolves the analysis-mode input: the file named by the +// single optional positional argument, or stdin when it is absent or +// "-". The returned closer must be called when reading is done. +func openScanInput(args []string) (in io.Reader, name string, closer func()) { + if len(args) == 1 && args[0] != "-" { + f, err := os.Open(args[0]) + if err != nil { + fatalf("open %s: %v", args[0], err) + } + return f, args[0], func() { f.Close() } + } + return os.Stdin, "stdin", func() {} +} + +// parseScanStream reads every NUL-terminated record from in. A record +// that does not have exactly 5 fields or whose size is non-numeric is +// counted as malformed and skipped. Total records read is +// len(recs) + malformed. +func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) { + sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20)) + sc.Buffer(make([]byte, 0, 64<<10), 4<<20) + sc.Split(splitNUL) + for sc.Scan() { + // The path is the last field and may itself contain tabs, + // so split into at most 5 fields. + fields := strings.SplitN(sc.Text(), "\t", 5) + if len(fields) != 5 { + malformed++ + continue + } + size, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + malformed++ + continue + } + recs = append(recs, scanRec{ + size: size, + head: fields[2], + tail: fields[3], + path: fields[4], + }) + } + if err := sc.Err(); err != nil { + fatalf("read %s: %v", name, err) + } + return recs, malformed +} + +// dupeGroup is one set of candidate-duplicate files: identical size, +// head hash, and tail hash. paths is sorted lexicographically; the +// first entry is the group's "first", the rest are dupes. +type dupeGroup struct { + size int64 + paths []string +} + +// runReport implements the report subcommand: it reads a scan stream +// from the named file (or stdin when absent or "-") and prints the +// file-level duplicates report as TSV on stdout. It never touches the +// scanned filesystem; its only I/O is the scan input, stdout, and +// stderr. args holds the positional arguments already validated by +// cobra (at most one). +func runReport(args []string) { + in, name, closer := openScanInput(args) + defer closer() + recs, malformed := parseScanStream(in, name) + records := len(recs) + malformed + + type dupeKey struct { + size int64 + head string + tail string + } + groups := make(map[dupeKey][]string) + for _, r := range recs { + k := dupeKey{size: r.size, head: r.head, tail: r.tail} + groups[k] = append(groups[k], r.path) + } + + var dupes []dupeGroup + for k, paths := range groups { + if len(paths) < 2 { + continue + } + slices.Sort(paths) + dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) + } + // Biggest reclaimable space first; ties broken by first path. + slices.SortFunc(dupes, func(a, b dupeGroup) int { + if a.size != b.size { + if a.size > b.size { + return -1 + } + return 1 + } + return strings.Compare(a.paths[0], b.paths[0]) + }) + + out := bufio.NewWriterSize(os.Stdout, 1<<20) + if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil { + fatalf("write stdout: %v", err) + } + dupeFiles := 0 + var reclaimable int64 + for _, g := range dupes { + for _, p := range g.paths[1:] { + if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n", + g.paths[0], p, g.size); err != nil { + fatalf("write stdout: %v", err) + } + dupeFiles++ + reclaimable += g.size + } + } + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + + fmt.Fprintf(os.Stderr, + "report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n", + records, malformedNote(malformed), len(dupes), dupeFiles, + humanBytes(reclaimable)) +} + +// malformedNote formats the optional malformed-record note for the +// stderr summaries. +func malformedNote(malformed int) string { + if malformed == 0 { + return "" + } + return fmt.Sprintf(" (%d malformed, skipped)", malformed) +} + +// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing +// data without a terminator at EOF is returned as a final record. +func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) { + if i := bytes.IndexByte(data, 0); i >= 0 { + return i + 1, data[:i], nil + } + if atEOF && len(data) > 0 { + return len(data), data, nil + } + return 0, nil, nil +} + +// humanBytes formats a byte count in human units (binary prefixes). +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/scan.go b/scan.go new file mode 100644 index 0000000..818ee6c --- /dev/null +++ b/scan.go @@ -0,0 +1,231 @@ +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" +) + +// chunk is the number of bytes hashed from each end of a file. +const chunk = 1024 + +// fileRec carries one file between the stat and hash passes. +type fileRec struct { + path string + size int64 + mtime int64 +} + +// runScan implements the scan subcommand: three sequential passes (walk, +// stat, hash) over the tree under -root, emitting one NUL-terminated +// record per regular file on stdout. +func runScan(args []string) { + fl := flag.NewFlagSet("scan", flag.ExitOnError) + root := fl.String("root", "/srv", "directory tree to scan") + workers := fl.Int("workers", runtime.NumCPU(), + "concurrent workers for the stat and hash passes") + fl.Parse(args) + if fl.NArg() > 0 { + fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0)) + os.Exit(2) + } + if *workers < 1 { + *workers = 1 + } + + st, err := os.Stat(*root) + if err != nil { + fatalf("root %s: %v", *root, err) + } + if !st.IsDir() { + fatalf("root %s: not a directory", *root) + } + + paths, walkErrs := walkPass(*root) + recs, statErrs := statPass(paths, *workers) + emitted, hashErrs := hashPass(recs, *workers) + + skipped := walkErrs + statErrs + hashErrs + fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n", + emitted, skipped) +} + +// walkPass enumerates every regular file under root. It never follows +// symlinks, never descends into directories named .zfs, and warns and +// continues on any per-path error. +func walkPass(root string) (paths []string, errs int) { + prog := newProgress("walk", -1) + walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + errs++ + prog.warnf("walk %s: %v", p, err) + if d != nil && d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + // ZFS snapshot pseudo-dirs would list every file + // once per snapshot; never descend. + if d.Name() == ".zfs" { + return filepath.SkipDir + } + return nil + } + // Regular files only: skip symlinks, sockets, FIFOs, and + // device nodes. + if !d.Type().IsRegular() { + return nil + } + paths = append(paths, p) + prog.increment() + return nil + }) + prog.finish() + if walkErr != nil { + fatalf("walk %s: %v", root, walkErr) + } + return paths, errs +} + +// statPass lstats every collected path in a worker pool, recording size +// and mtime. Paths that fail to stat (or are no longer regular files) +// are warned about and dropped. +func statPass(paths []string, workers int) (recs []fileRec, errs int) { + type result struct { + rec fileRec + err error + } + jobs := make(chan string, 1024) + results := make(chan result, 1024) + for range workers { + go func() { + for p := range jobs { + fi, err := os.Lstat(p) + switch { + case err != nil: + results <- result{rec: fileRec{path: p}, err: err} + case !fi.Mode().IsRegular(): + results <- result{ + rec: fileRec{path: p}, + err: fmt.Errorf("no longer a regular file"), + } + default: + results <- result{rec: fileRec{ + path: p, + size: fi.Size(), + mtime: fi.ModTime().Unix(), + }} + } + } + }() + } + go func() { + for _, p := range paths { + jobs <- p + } + close(jobs) + }() + + prog := newProgress("stat", int64(len(paths))) + recs = make([]fileRec, 0, len(paths)) + for range paths { + r := <-results + if r.err != nil { + errs++ + prog.warnf("stat %s: %v", r.rec.path, r.err) + } else { + recs = append(recs, r.rec) + } + prog.increment() + } + prog.finish() + return recs, errs +} + +// hashPass hashes the first and last chunk bytes of every file in a +// worker pool and emits the output records on stdout from the main +// goroutine. Files that fail to open or read are warned about and +// dropped. +func hashPass(recs []fileRec, workers int) (emitted, errs int) { + type result struct { + rec fileRec + head string + tail string + err error + } + jobs := make(chan fileRec, 1024) + results := make(chan result, 1024) + for range workers { + go func() { + for rec := range jobs { + head, tail, err := hashHeadTail(rec.path, rec.size) + results <- result{rec: rec, head: head, tail: tail, err: err} + } + }() + } + go func() { + for _, rec := range recs { + jobs <- rec + } + close(jobs) + }() + + prog := newProgress("hash", int64(len(recs))) + out := bufio.NewWriterSize(os.Stdout, 1<<20) + for range recs { + r := <-results + if r.err != nil { + errs++ + prog.warnf("hash %s: %v", r.rec.path, r.err) + prog.increment() + continue + } + if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", + r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil { + fatalf("write stdout: %v", err) + } + emitted++ + prog.increment() + } + prog.finish() + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + return emitted, errs +} + +// hashHeadTail returns the lowercase-hex SHA-256 of the first +// min(chunk, size) bytes and of the last min(chunk, size) bytes of the +// file at path. The two reads overlap when size < 2*chunk; for +// size == 0 both hashes are of the empty input. size is the value +// recorded by the stat pass. +func hashHeadTail(path string, size int64) (head, tail string, err error) { + f, err := os.Open(path) + if err != nil { + return "", "", err + } + defer f.Close() + + n := min(int64(chunk), size) + buf := make([]byte, n) + if n > 0 { + if _, err := f.ReadAt(buf, 0); err != nil { + return "", "", err + } + } + h := sha256.Sum256(buf) + if n > 0 { + if _, err := f.ReadAt(buf, size-n); err != nil { + return "", "", err + } + } + t := sha256.Sum256(buf) + return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil +} diff --git a/trees.go b/trees.go new file mode 100644 index 0000000..86848fc --- /dev/null +++ b/trees.go @@ -0,0 +1,185 @@ +package main + +import ( + "bufio" + "crypto/sha256" + "fmt" + "os" + "slices" + "strconv" + "strings" +) + +// fileSig is a file's duplicate signature; mtime is excluded. +type fileSig struct { + size int64 + head string + tail string +} + +// treeNode is one directory reconstructed from the scan stream. +type treeNode struct { + path string + parent *treeNode + dirs map[string]*treeNode + files map[string]fileSig + digest [sha256.Size]byte + fileCount int64 + totalSize int64 +} + +// runTrees implements the trees subcommand: it reads a scan stream from +// the named file (or stdin when absent or "-"), reconstructs the +// directory hierarchy from the record paths, computes a Merkle-style +// digest per directory, and prints maximal duplicate-tree groups as TSV +// on stdout. It never touches the scanned filesystem; its only I/O is +// the scan input, stdout, and stderr. args holds the positional +// arguments already validated by cobra (at most one). +func runTrees(args []string) { + in, name, closer := openScanInput(args) + defer closer() + recs, malformed := parseScanStream(in, name) + records := len(recs) + malformed + + // Build the hierarchy under a synthetic super-root. Paths are + // split on "/"; for absolute paths the first component is empty, + // which simply becomes a top-level node representing "/". + super := &treeNode{} + var allDirs []*treeNode + for _, r := range recs { + comps := strings.Split(r.path, "/") + node := super + for _, c := range comps[:len(comps)-1] { + child := node.dirs[c] + if child == nil { + childPath := c + if node != super { + childPath = node.path + "/" + c + } + child = &treeNode{path: childPath, parent: node} + if node.dirs == nil { + node.dirs = make(map[string]*treeNode) + } + node.dirs[c] = child + allDirs = append(allDirs, child) + } + node = child + } + if node.files == nil { + node.files = make(map[string]fileSig) + } + node.files[comps[len(comps)-1]] = fileSig{ + size: r.size, head: r.head, tail: r.tail, + } + } + super.compute() + + // Group directories by digest; two or more dirs sharing a digest + // are candidate duplicate trees. + groups := make(map[[sha256.Size]byte][]*treeNode) + for _, d := range allDirs { + groups[d.digest] = append(groups[d.digest], d) + } + var dupes [][]*treeNode + for _, g := range groups { + if len(g) < 2 || suppressed(g, super) { + continue + } + slices.SortFunc(g, func(a, b *treeNode) int { + return strings.Compare(a.path, b.path) + }) + dupes = append(dupes, g) + } + // Biggest reclaimable space first; ties broken by first path. + slices.SortFunc(dupes, func(a, b []*treeNode) int { + if a[0].totalSize != b[0].totalSize { + if a[0].totalSize > b[0].totalSize { + return -1 + } + return 1 + } + return strings.Compare(a[0].path, b[0].path) + }) + + out := bufio.NewWriterSize(os.Stdout, 1<<20) + if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil { + fatalf("write stdout: %v", err) + } + dupeTrees := 0 + var reclaimable int64 + for _, g := range dupes { + first := g[0] + for _, n := range g[1:] { + if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", + first.path, n.path, first.fileCount, first.totalSize); err != nil { + fatalf("write stdout: %v", err) + } + dupeTrees++ + reclaimable += first.totalSize + } + } + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + + fmt.Fprintf(os.Stderr, + "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", + records, malformedNote(malformed), len(dupes), dupeTrees, + humanBytes(reclaimable)) +} + +// compute fills in digest, fileCount, and totalSize for n and all of +// its descendants. A directory's digest is the SHA-256 of its child +// entries — files serialized with name and signature, subdirectories +// with name and recursive digest — sorted byte-lexicographically. +// Filenames cannot contain NUL or "/", so NUL delimiters are +// unambiguous. +func (n *treeNode) compute() { + entries := make([]string, 0, len(n.dirs)+len(n.files)) + for name, sig := range n.files { + entries = append(entries, + "f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+ + "\x00"+sig.head+"\x00"+sig.tail) + n.fileCount++ + n.totalSize += sig.size + } + for name, child := range n.dirs { + child.compute() + entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) + n.fileCount += child.fileCount + n.totalSize += child.totalSize + } + slices.Sort(entries) + h := sha256.New() + for _, e := range entries { + h.Write([]byte(e)) + h.Write([]byte{0}) + } + copy(n.digest[:], h.Sum(nil)) +} + +// suppressed reports whether a duplicate-tree group is non-maximal: its +// members' parents are pairwise distinct real directories that all +// share a single digest, so the group is wholly implied by its parents' +// (or a further ancestor's) group. Groups containing siblings (shared +// parent) or members whose parents differ are always reported. +func suppressed(g []*treeNode, super *treeNode) bool { + seen := make(map[*treeNode]bool, len(g)) + var parentDigest [sha256.Size]byte + for i, n := range g { + p := n.parent + if p == nil || p == super { + return false + } + if seen[p] { + return false // siblings: not implied by any parent group + } + seen[p] = true + if i == 0 { + parentDigest = p.digest + } else if p.digest != parentDigest { + return false + } + } + return true +}