From 1fad4a7d1cb8e6af63be6b83d17eb21909e2c0d4 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 22 Jul 2026 22:06:22 +0700 Subject: [PATCH] Add sfdupes implementation: scan, report, and trees subcommands --- Makefile | 17 ++++ go.mod | 17 ++++ go.sum | 32 ++++++++ main.go | 81 ++++++++++++++++++ progress.go | 123 ++++++++++++++++++++++++++++ report.go | 181 ++++++++++++++++++++++++++++++++++++++++ scan.go | 231 ++++++++++++++++++++++++++++++++++++++++++++++++++++ trees.go | 185 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 867 insertions(+) create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 progress.go create mode 100644 report.go create mode 100644 scan.go create mode 100644 trees.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..80938d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +# sfdupes is pure Go; disable cgo so builds never touch the host C +# toolchain. +export CGO_ENABLED = 0 + +.PHONY: all build check clean + +all: build + +build: + go build + +check: build + test -z "$$(gofmt -l .)" + go vet ./... + +clean: + rm -f sfdupes files.dat diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..83bdfdb --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module sneak.berlin/go/sfdupes + +go 1.25.7 + +require ( + github.com/schollz/progressbar/v3 v3.19.1 + github.com/spf13/cobra v1.10.2 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cf50240 --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.19.1 h1:iv8BgwOvdML/S3p84uBpy/IMigv4U9594vPZYa2EdrU= +github.com/schollz/progressbar/v3 v3.19.1/go.mod h1:LFL7jqimKxfhero4K1eCkUr/6R39AgQeiPCJtlTWIW8= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..51e2900 --- /dev/null +++ b/main.go @@ -0,0 +1,81 @@ +// Command sfdupes quickly identifies candidate duplicate files across +// very large filesystems without reading full file contents. Files are +// considered duplicates when they have identical size, identical SHA-256 +// of their first 1024 bytes, and identical SHA-256 of their last 1024 +// bytes. +// +// Usage: +// +// sfdupes scan [-root /srv] [-workers N] > files.dat +// sfdupes report [files.dat|-] > dupes.tsv +// sfdupes trees [files.dat|-] > dupetrees.tsv +// +// See README.md for the complete specification. +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + root := &cobra.Command{ + Use: "sfdupes", + Short: "Find candidate duplicate files by size and head/tail SHA-256", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + // A missing subcommand prints usage and exits 2. + _ = cmd.Usage() + os.Exit(2) + }, + } + // Everything on stdout is machine-readable data; all human-facing + // output (help, usage, errors) goes to stderr. + root.SetOut(os.Stderr) + root.SetErr(os.Stderr) + root.CompletionOptions.DisableDefaultCmd = true + + scanCmd := &cobra.Command{ + Use: "scan [-root /srv] [-workers N]", + Short: "Walk a tree and emit one record per regular file on stdout", + // The README specifies single-dash flags (-root, -workers); + // parse them with the stdlib flag package inside runScan. + DisableFlagParsing: true, + Run: func(cmd *cobra.Command, args []string) { + runScan(args) + }, + } + + reportCmd := &cobra.Command{ + Use: "report [files.dat|-]", + Short: "Read a scan stream and print the file-level duplicates report", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + runReport(args) + }, + } + + treesCmd := &cobra.Command{ + Use: "trees [files.dat|-]", + Short: "Read a scan stream and print the duplicate-tree report", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + runTrees(args) + }, + } + + root.AddCommand(scanCmd, reportCmd, treesCmd) + if err := root.Execute(); err != nil { + // Cobra has already printed the error and usage to stderr; + // an invalid subcommand or bad arguments is a usage error. + os.Exit(2) + } +} + +// fatalf reports a fatal error and exits 1. +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...) + os.Exit(1) +} diff --git a/progress.go b/progress.go new file mode 100644 index 0000000..9aedbee --- /dev/null +++ b/progress.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/schollz/progressbar/v3" +) + +// plainInterval is the minimum time between progress lines when stderr +// is not a TTY. +const plainInterval = 5 * time.Second + +// stderrIsTTY reports whether stderr is attached to a terminal. +func stderrIsTTY() bool { + fi, err := os.Stderr.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// progress renders one scan pass's progress on stderr. On a TTY it +// delegates to the progressbar library (spinner style when the total is +// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When +// stderr is not a TTY it emits no ANSI redraws: it prints a plain +// one-line update no more often than every plainInterval. +// +// All methods must be called from the main goroutine only. +type progress struct { + label string + total int64 // -1 when unknown (walk pass) + bar *progressbar.ProgressBar + count int64 + start time.Time + last time.Time +} + +func newProgress(label string, total int64) *progress { + p := &progress{label: label, total: total, start: time.Now()} + if !stderrIsTTY() { + return p + } + opts := []progressbar.Option{ + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetDescription(label), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetItsString("files"), + progressbar.OptionSetElapsedTime(true), + progressbar.OptionThrottle(100 * time.Millisecond), + } + if total >= 0 { + opts = append(opts, + progressbar.OptionSetPredictTime(true), + progressbar.OptionShowElapsedTimeOnFinish(), + ) + } else { + opts = append(opts, + progressbar.OptionSetPredictTime(false), + progressbar.OptionSpinnerType(14), + ) + } + p.bar = progressbar.NewOptions64(total, opts...) + return p +} + +// increment records one completed item and refreshes the display. +func (p *progress) increment() { + p.count++ + if p.bar != nil { + _ = p.bar.Add(1) + return + } + if time.Since(p.last) >= plainInterval { + p.last = time.Now() + fmt.Fprintln(os.Stderr, p.plainLine()) + } +} + +// warnf prints a one-line warning to stderr without corrupting the bar. +func (p *progress) warnf(format string, args ...any) { + if p.bar != nil { + _ = p.bar.Clear() + } + fmt.Fprintf(os.Stderr, format+"\n", args...) +} + +// finish terminates the pass's display. +func (p *progress) finish() { + if p.bar != nil { + _ = p.bar.Finish() + fmt.Fprintln(os.Stderr) + return + } + fmt.Fprintln(os.Stderr, p.plainLine()) +} + +// plainLine formats a non-TTY progress line. +func (p *progress) plainLine() string { + elapsed := time.Since(p.start) + if p.total < 0 { + return fmt.Sprintf("%s: %d files, elapsed %s", + p.label, p.count, elapsed.Round(time.Second)) + } + pct := 100.0 + if p.total > 0 { + pct = 100 * float64(p.count) / float64(p.total) + } + rate := 0.0 + if s := elapsed.Seconds(); s > 0 { + rate = float64(p.count) / s + } + eta := "?" + if rate > 0 && p.count <= p.total { + remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second)) + eta = remaining.Round(time.Second).String() + } + return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s", + p.label, p.count, p.total, pct, rate, + elapsed.Round(time.Second), eta) +} diff --git a/report.go b/report.go new file mode 100644 index 0000000..096c4aa --- /dev/null +++ b/report.go @@ -0,0 +1,181 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" +) + +// scanRec is one well-formed record parsed from a scan stream. The +// signature (size, head, tail) is the duplicate key; mtime is +// informational and not retained. +type scanRec struct { + size int64 + head string + tail string + path string +} + +// openScanInput resolves the analysis-mode input: the file named by the +// single optional positional argument, or stdin when it is absent or +// "-". The returned closer must be called when reading is done. +func openScanInput(args []string) (in io.Reader, name string, closer func()) { + if len(args) == 1 && args[0] != "-" { + f, err := os.Open(args[0]) + if err != nil { + fatalf("open %s: %v", args[0], err) + } + return f, args[0], func() { f.Close() } + } + return os.Stdin, "stdin", func() {} +} + +// parseScanStream reads every NUL-terminated record from in. A record +// that does not have exactly 5 fields or whose size is non-numeric is +// counted as malformed and skipped. Total records read is +// len(recs) + malformed. +func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) { + sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20)) + sc.Buffer(make([]byte, 0, 64<<10), 4<<20) + sc.Split(splitNUL) + for sc.Scan() { + // The path is the last field and may itself contain tabs, + // so split into at most 5 fields. + fields := strings.SplitN(sc.Text(), "\t", 5) + if len(fields) != 5 { + malformed++ + continue + } + size, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + malformed++ + continue + } + recs = append(recs, scanRec{ + size: size, + head: fields[2], + tail: fields[3], + path: fields[4], + }) + } + if err := sc.Err(); err != nil { + fatalf("read %s: %v", name, err) + } + return recs, malformed +} + +// dupeGroup is one set of candidate-duplicate files: identical size, +// head hash, and tail hash. paths is sorted lexicographically; the +// first entry is the group's "first", the rest are dupes. +type dupeGroup struct { + size int64 + paths []string +} + +// runReport implements the report subcommand: it reads a scan stream +// from the named file (or stdin when absent or "-") and prints the +// file-level duplicates report as TSV on stdout. It never touches the +// scanned filesystem; its only I/O is the scan input, stdout, and +// stderr. args holds the positional arguments already validated by +// cobra (at most one). +func runReport(args []string) { + in, name, closer := openScanInput(args) + defer closer() + recs, malformed := parseScanStream(in, name) + records := len(recs) + malformed + + type dupeKey struct { + size int64 + head string + tail string + } + groups := make(map[dupeKey][]string) + for _, r := range recs { + k := dupeKey{size: r.size, head: r.head, tail: r.tail} + groups[k] = append(groups[k], r.path) + } + + var dupes []dupeGroup + for k, paths := range groups { + if len(paths) < 2 { + continue + } + slices.Sort(paths) + dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) + } + // Biggest reclaimable space first; ties broken by first path. + slices.SortFunc(dupes, func(a, b dupeGroup) int { + if a.size != b.size { + if a.size > b.size { + return -1 + } + return 1 + } + return strings.Compare(a.paths[0], b.paths[0]) + }) + + out := bufio.NewWriterSize(os.Stdout, 1<<20) + if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil { + fatalf("write stdout: %v", err) + } + dupeFiles := 0 + var reclaimable int64 + for _, g := range dupes { + for _, p := range g.paths[1:] { + if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n", + g.paths[0], p, g.size); err != nil { + fatalf("write stdout: %v", err) + } + dupeFiles++ + reclaimable += g.size + } + } + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + + fmt.Fprintf(os.Stderr, + "report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n", + records, malformedNote(malformed), len(dupes), dupeFiles, + humanBytes(reclaimable)) +} + +// malformedNote formats the optional malformed-record note for the +// stderr summaries. +func malformedNote(malformed int) string { + if malformed == 0 { + return "" + } + return fmt.Sprintf(" (%d malformed, skipped)", malformed) +} + +// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing +// data without a terminator at EOF is returned as a final record. +func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) { + if i := bytes.IndexByte(data, 0); i >= 0 { + return i + 1, data[:i], nil + } + if atEOF && len(data) > 0 { + return len(data), data, nil + } + return 0, nil, nil +} + +// humanBytes formats a byte count in human units (binary prefixes). +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/scan.go b/scan.go new file mode 100644 index 0000000..818ee6c --- /dev/null +++ b/scan.go @@ -0,0 +1,231 @@ +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" +) + +// chunk is the number of bytes hashed from each end of a file. +const chunk = 1024 + +// fileRec carries one file between the stat and hash passes. +type fileRec struct { + path string + size int64 + mtime int64 +} + +// runScan implements the scan subcommand: three sequential passes (walk, +// stat, hash) over the tree under -root, emitting one NUL-terminated +// record per regular file on stdout. +func runScan(args []string) { + fl := flag.NewFlagSet("scan", flag.ExitOnError) + root := fl.String("root", "/srv", "directory tree to scan") + workers := fl.Int("workers", runtime.NumCPU(), + "concurrent workers for the stat and hash passes") + fl.Parse(args) + if fl.NArg() > 0 { + fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0)) + os.Exit(2) + } + if *workers < 1 { + *workers = 1 + } + + st, err := os.Stat(*root) + if err != nil { + fatalf("root %s: %v", *root, err) + } + if !st.IsDir() { + fatalf("root %s: not a directory", *root) + } + + paths, walkErrs := walkPass(*root) + recs, statErrs := statPass(paths, *workers) + emitted, hashErrs := hashPass(recs, *workers) + + skipped := walkErrs + statErrs + hashErrs + fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n", + emitted, skipped) +} + +// walkPass enumerates every regular file under root. It never follows +// symlinks, never descends into directories named .zfs, and warns and +// continues on any per-path error. +func walkPass(root string) (paths []string, errs int) { + prog := newProgress("walk", -1) + walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + errs++ + prog.warnf("walk %s: %v", p, err) + if d != nil && d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + // ZFS snapshot pseudo-dirs would list every file + // once per snapshot; never descend. + if d.Name() == ".zfs" { + return filepath.SkipDir + } + return nil + } + // Regular files only: skip symlinks, sockets, FIFOs, and + // device nodes. + if !d.Type().IsRegular() { + return nil + } + paths = append(paths, p) + prog.increment() + return nil + }) + prog.finish() + if walkErr != nil { + fatalf("walk %s: %v", root, walkErr) + } + return paths, errs +} + +// statPass lstats every collected path in a worker pool, recording size +// and mtime. Paths that fail to stat (or are no longer regular files) +// are warned about and dropped. +func statPass(paths []string, workers int) (recs []fileRec, errs int) { + type result struct { + rec fileRec + err error + } + jobs := make(chan string, 1024) + results := make(chan result, 1024) + for range workers { + go func() { + for p := range jobs { + fi, err := os.Lstat(p) + switch { + case err != nil: + results <- result{rec: fileRec{path: p}, err: err} + case !fi.Mode().IsRegular(): + results <- result{ + rec: fileRec{path: p}, + err: fmt.Errorf("no longer a regular file"), + } + default: + results <- result{rec: fileRec{ + path: p, + size: fi.Size(), + mtime: fi.ModTime().Unix(), + }} + } + } + }() + } + go func() { + for _, p := range paths { + jobs <- p + } + close(jobs) + }() + + prog := newProgress("stat", int64(len(paths))) + recs = make([]fileRec, 0, len(paths)) + for range paths { + r := <-results + if r.err != nil { + errs++ + prog.warnf("stat %s: %v", r.rec.path, r.err) + } else { + recs = append(recs, r.rec) + } + prog.increment() + } + prog.finish() + return recs, errs +} + +// hashPass hashes the first and last chunk bytes of every file in a +// worker pool and emits the output records on stdout from the main +// goroutine. Files that fail to open or read are warned about and +// dropped. +func hashPass(recs []fileRec, workers int) (emitted, errs int) { + type result struct { + rec fileRec + head string + tail string + err error + } + jobs := make(chan fileRec, 1024) + results := make(chan result, 1024) + for range workers { + go func() { + for rec := range jobs { + head, tail, err := hashHeadTail(rec.path, rec.size) + results <- result{rec: rec, head: head, tail: tail, err: err} + } + }() + } + go func() { + for _, rec := range recs { + jobs <- rec + } + close(jobs) + }() + + prog := newProgress("hash", int64(len(recs))) + out := bufio.NewWriterSize(os.Stdout, 1<<20) + for range recs { + r := <-results + if r.err != nil { + errs++ + prog.warnf("hash %s: %v", r.rec.path, r.err) + prog.increment() + continue + } + if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", + r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil { + fatalf("write stdout: %v", err) + } + emitted++ + prog.increment() + } + prog.finish() + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + return emitted, errs +} + +// hashHeadTail returns the lowercase-hex SHA-256 of the first +// min(chunk, size) bytes and of the last min(chunk, size) bytes of the +// file at path. The two reads overlap when size < 2*chunk; for +// size == 0 both hashes are of the empty input. size is the value +// recorded by the stat pass. +func hashHeadTail(path string, size int64) (head, tail string, err error) { + f, err := os.Open(path) + if err != nil { + return "", "", err + } + defer f.Close() + + n := min(int64(chunk), size) + buf := make([]byte, n) + if n > 0 { + if _, err := f.ReadAt(buf, 0); err != nil { + return "", "", err + } + } + h := sha256.Sum256(buf) + if n > 0 { + if _, err := f.ReadAt(buf, size-n); err != nil { + return "", "", err + } + } + t := sha256.Sum256(buf) + return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil +} diff --git a/trees.go b/trees.go new file mode 100644 index 0000000..86848fc --- /dev/null +++ b/trees.go @@ -0,0 +1,185 @@ +package main + +import ( + "bufio" + "crypto/sha256" + "fmt" + "os" + "slices" + "strconv" + "strings" +) + +// fileSig is a file's duplicate signature; mtime is excluded. +type fileSig struct { + size int64 + head string + tail string +} + +// treeNode is one directory reconstructed from the scan stream. +type treeNode struct { + path string + parent *treeNode + dirs map[string]*treeNode + files map[string]fileSig + digest [sha256.Size]byte + fileCount int64 + totalSize int64 +} + +// runTrees implements the trees subcommand: it reads a scan stream from +// the named file (or stdin when absent or "-"), reconstructs the +// directory hierarchy from the record paths, computes a Merkle-style +// digest per directory, and prints maximal duplicate-tree groups as TSV +// on stdout. It never touches the scanned filesystem; its only I/O is +// the scan input, stdout, and stderr. args holds the positional +// arguments already validated by cobra (at most one). +func runTrees(args []string) { + in, name, closer := openScanInput(args) + defer closer() + recs, malformed := parseScanStream(in, name) + records := len(recs) + malformed + + // Build the hierarchy under a synthetic super-root. Paths are + // split on "/"; for absolute paths the first component is empty, + // which simply becomes a top-level node representing "/". + super := &treeNode{} + var allDirs []*treeNode + for _, r := range recs { + comps := strings.Split(r.path, "/") + node := super + for _, c := range comps[:len(comps)-1] { + child := node.dirs[c] + if child == nil { + childPath := c + if node != super { + childPath = node.path + "/" + c + } + child = &treeNode{path: childPath, parent: node} + if node.dirs == nil { + node.dirs = make(map[string]*treeNode) + } + node.dirs[c] = child + allDirs = append(allDirs, child) + } + node = child + } + if node.files == nil { + node.files = make(map[string]fileSig) + } + node.files[comps[len(comps)-1]] = fileSig{ + size: r.size, head: r.head, tail: r.tail, + } + } + super.compute() + + // Group directories by digest; two or more dirs sharing a digest + // are candidate duplicate trees. + groups := make(map[[sha256.Size]byte][]*treeNode) + for _, d := range allDirs { + groups[d.digest] = append(groups[d.digest], d) + } + var dupes [][]*treeNode + for _, g := range groups { + if len(g) < 2 || suppressed(g, super) { + continue + } + slices.SortFunc(g, func(a, b *treeNode) int { + return strings.Compare(a.path, b.path) + }) + dupes = append(dupes, g) + } + // Biggest reclaimable space first; ties broken by first path. + slices.SortFunc(dupes, func(a, b []*treeNode) int { + if a[0].totalSize != b[0].totalSize { + if a[0].totalSize > b[0].totalSize { + return -1 + } + return 1 + } + return strings.Compare(a[0].path, b[0].path) + }) + + out := bufio.NewWriterSize(os.Stdout, 1<<20) + if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil { + fatalf("write stdout: %v", err) + } + dupeTrees := 0 + var reclaimable int64 + for _, g := range dupes { + first := g[0] + for _, n := range g[1:] { + if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", + first.path, n.path, first.fileCount, first.totalSize); err != nil { + fatalf("write stdout: %v", err) + } + dupeTrees++ + reclaimable += first.totalSize + } + } + if err := out.Flush(); err != nil { + fatalf("write stdout: %v", err) + } + + fmt.Fprintf(os.Stderr, + "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", + records, malformedNote(malformed), len(dupes), dupeTrees, + humanBytes(reclaimable)) +} + +// compute fills in digest, fileCount, and totalSize for n and all of +// its descendants. A directory's digest is the SHA-256 of its child +// entries — files serialized with name and signature, subdirectories +// with name and recursive digest — sorted byte-lexicographically. +// Filenames cannot contain NUL or "/", so NUL delimiters are +// unambiguous. +func (n *treeNode) compute() { + entries := make([]string, 0, len(n.dirs)+len(n.files)) + for name, sig := range n.files { + entries = append(entries, + "f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+ + "\x00"+sig.head+"\x00"+sig.tail) + n.fileCount++ + n.totalSize += sig.size + } + for name, child := range n.dirs { + child.compute() + entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) + n.fileCount += child.fileCount + n.totalSize += child.totalSize + } + slices.Sort(entries) + h := sha256.New() + for _, e := range entries { + h.Write([]byte(e)) + h.Write([]byte{0}) + } + copy(n.digest[:], h.Sum(nil)) +} + +// suppressed reports whether a duplicate-tree group is non-maximal: its +// members' parents are pairwise distinct real directories that all +// share a single digest, so the group is wholly implied by its parents' +// (or a further ancestor's) group. Groups containing siblings (shared +// parent) or members whose parents differ are always reported. +func suppressed(g []*treeNode, super *treeNode) bool { + seen := make(map[*treeNode]bool, len(g)) + var parentDigest [sha256.Size]byte + for i, n := range g { + p := n.parent + if p == nil || p == super { + return false + } + if seen[p] { + return false // siblings: not implied by any parent group + } + seen[p] = true + if i == 0 { + parentDigest = p.digest + } else if p.digest != parentDigest { + return false + } + } + return true +}