Add sfdupes implementation: scan, report, and trees subcommands

trees.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// fileSig is a file's duplicate signature; mtime is excluded.
+type fileSig struct {
+	size int64
+	head string
+	tail string
+}
+
+// treeNode is one directory reconstructed from the scan stream.
+type treeNode struct {
+	path      string
+	parent    *treeNode
+	dirs      map[string]*treeNode
+	files     map[string]fileSig
+	digest    [sha256.Size]byte
+	fileCount int64
+	totalSize int64
+}
+
+// runTrees implements the trees subcommand: it reads a scan stream from
+// the named file (or stdin when absent or "-"), reconstructs the
+// directory hierarchy from the record paths, computes a Merkle-style
+// digest per directory, and prints maximal duplicate-tree groups as TSV
+// on stdout. It never touches the scanned filesystem; its only I/O is
+// the scan input, stdout, and stderr. args holds the positional
+// arguments already validated by cobra (at most one).
+func runTrees(args []string) {
+	in, name, closer := openScanInput(args)
+	defer closer()
+	recs, malformed := parseScanStream(in, name)
+	records := len(recs) + malformed
+
+	// Build the hierarchy under a synthetic super-root. Paths are
+	// split on "/"; for absolute paths the first component is empty,
+	// which simply becomes a top-level node representing "/".
+	super := &treeNode{}
+	var allDirs []*treeNode
+	for _, r := range recs {
+		comps := strings.Split(r.path, "/")
+		node := super
+		for _, c := range comps[:len(comps)-1] {
+			child := node.dirs[c]
+			if child == nil {
+				childPath := c
+				if node != super {
+					childPath = node.path + "/" + c
+				}
+				child = &treeNode{path: childPath, parent: node}
+				if node.dirs == nil {
+					node.dirs = make(map[string]*treeNode)
+				}
+				node.dirs[c] = child
+				allDirs = append(allDirs, child)
+			}
+			node = child
+		}
+		if node.files == nil {
+			node.files = make(map[string]fileSig)
+		}
+		node.files[comps[len(comps)-1]] = fileSig{
+			size: r.size, head: r.head, tail: r.tail,
+		}
+	}
+	super.compute()
+
+	// Group directories by digest; two or more dirs sharing a digest
+	// are candidate duplicate trees.
+	groups := make(map[[sha256.Size]byte][]*treeNode)
+	for _, d := range allDirs {
+		groups[d.digest] = append(groups[d.digest], d)
+	}
+	var dupes [][]*treeNode
+	for _, g := range groups {
+		if len(g) < 2 || suppressed(g, super) {
+			continue
+		}
+		slices.SortFunc(g, func(a, b *treeNode) int {
+			return strings.Compare(a.path, b.path)
+		})
+		dupes = append(dupes, g)
+	}
+	// Biggest reclaimable space first; ties broken by first path.
+	slices.SortFunc(dupes, func(a, b []*treeNode) int {
+		if a[0].totalSize != b[0].totalSize {
+			if a[0].totalSize > b[0].totalSize {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a[0].path, b[0].path)
+	})
+
+	out := bufio.NewWriterSize(os.Stdout, 1<<20)
+	if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
+		fatalf("write stdout: %v", err)
+	}
+	dupeTrees := 0
+	var reclaimable int64
+	for _, g := range dupes {
+		first := g[0]
+		for _, n := range g[1:] {
+			if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
+				first.path, n.path, first.fileCount, first.totalSize); err != nil {
+				fatalf("write stdout: %v", err)
+			}
+			dupeTrees++
+			reclaimable += first.totalSize
+		}
+	}
+	if err := out.Flush(); err != nil {
+		fatalf("write stdout: %v", err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
+		records, malformedNote(malformed), len(dupes), dupeTrees,
+		humanBytes(reclaimable))
+}
+
+// compute fills in digest, fileCount, and totalSize for n and all of
+// its descendants. A directory's digest is the SHA-256 of its child
+// entries — files serialized with name and signature, subdirectories
+// with name and recursive digest — sorted byte-lexicographically.
+// Filenames cannot contain NUL or "/", so NUL delimiters are
+// unambiguous.
+func (n *treeNode) compute() {
+	entries := make([]string, 0, len(n.dirs)+len(n.files))
+	for name, sig := range n.files {
+		entries = append(entries,
+			"f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+
+				"\x00"+sig.head+"\x00"+sig.tail)
+		n.fileCount++
+		n.totalSize += sig.size
+	}
+	for name, child := range n.dirs {
+		child.compute()
+		entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
+		n.fileCount += child.fileCount
+		n.totalSize += child.totalSize
+	}
+	slices.Sort(entries)
+	h := sha256.New()
+	for _, e := range entries {
+		h.Write([]byte(e))
+		h.Write([]byte{0})
+	}
+	copy(n.digest[:], h.Sum(nil))
+}
+
+// suppressed reports whether a duplicate-tree group is non-maximal: its
+// members' parents are pairwise distinct real directories that all
+// share a single digest, so the group is wholly implied by its parents'
+// (or a further ancestor's) group. Groups containing siblings (shared
+// parent) or members whose parents differ are always reported.
+func suppressed(g []*treeNode, super *treeNode) bool {
+	seen := make(map[*treeNode]bool, len(g))
+	var parentDigest [sha256.Size]byte
+	for i, n := range g {
+		p := n.parent
+		if p == nil || p == super {
+			return false
+		}
+		if seen[p] {
+			return false // siblings: not implied by any parent group
+		}
+		seen[p] = true
+		if i == 0 {
+			parentDigest = p.digest
+		} else if p.digest != parentDigest {
+			return false
+		}
+	}
+	return true
+}