package main import ( "bufio" "crypto/sha256" "fmt" "os" "slices" "strconv" "strings" ) // fileSig is a file's duplicate signature; mtime is excluded. type fileSig struct { size int64 head string tail string } // treeNode is one directory reconstructed from the scan stream. type treeNode struct { path string parent *treeNode dirs map[string]*treeNode files map[string]fileSig digest [sha256.Size]byte fileCount int64 totalSize int64 } // runTrees implements the trees subcommand: it reads a scan stream from // the named file (or stdin when absent or "-"), reconstructs the // directory hierarchy from the record paths, computes a Merkle-style // digest per directory, and prints maximal duplicate-tree groups as TSV // on stdout. It never touches the scanned filesystem; its only I/O is // the scan input, stdout, and stderr. args holds the positional // arguments already validated by cobra (at most one). func runTrees(args []string) { in, name, closer := openScanInput(args) defer closer() recs, malformed := parseScanStream(in, name) records := len(recs) + malformed // Build the hierarchy under a synthetic super-root. Paths are // split on "/"; for absolute paths the first component is empty, // which simply becomes a top-level node representing "/". super := &treeNode{} var allDirs []*treeNode for _, r := range recs { comps := strings.Split(r.path, "/") node := super for _, c := range comps[:len(comps)-1] { child := node.dirs[c] if child == nil { childPath := c if node != super { childPath = node.path + "/" + c } child = &treeNode{path: childPath, parent: node} if node.dirs == nil { node.dirs = make(map[string]*treeNode) } node.dirs[c] = child allDirs = append(allDirs, child) } node = child } if node.files == nil { node.files = make(map[string]fileSig) } node.files[comps[len(comps)-1]] = fileSig{ size: r.size, head: r.head, tail: r.tail, } } super.compute() // Group directories by digest; two or more dirs sharing a digest // are candidate duplicate trees. groups := make(map[[sha256.Size]byte][]*treeNode) for _, d := range allDirs { groups[d.digest] = append(groups[d.digest], d) } var dupes [][]*treeNode for _, g := range groups { if len(g) < 2 || suppressed(g, super) { continue } slices.SortFunc(g, func(a, b *treeNode) int { return strings.Compare(a.path, b.path) }) dupes = append(dupes, g) } // Biggest reclaimable space first; ties broken by first path. slices.SortFunc(dupes, func(a, b []*treeNode) int { if a[0].totalSize != b[0].totalSize { if a[0].totalSize > b[0].totalSize { return -1 } return 1 } return strings.Compare(a[0].path, b[0].path) }) out := bufio.NewWriterSize(os.Stdout, 1<<20) if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil { fatalf("write stdout: %v", err) } dupeTrees := 0 var reclaimable int64 for _, g := range dupes { first := g[0] for _, n := range g[1:] { if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", first.path, n.path, first.fileCount, first.totalSize); err != nil { fatalf("write stdout: %v", err) } dupeTrees++ reclaimable += first.totalSize } } if err := out.Flush(); err != nil { fatalf("write stdout: %v", err) } fmt.Fprintf(os.Stderr, "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", records, malformedNote(malformed), len(dupes), dupeTrees, humanBytes(reclaimable)) } // compute fills in digest, fileCount, and totalSize for n and all of // its descendants. A directory's digest is the SHA-256 of its child // entries — files serialized with name and signature, subdirectories // with name and recursive digest — sorted byte-lexicographically. // Filenames cannot contain NUL or "/", so NUL delimiters are // unambiguous. func (n *treeNode) compute() { entries := make([]string, 0, len(n.dirs)+len(n.files)) for name, sig := range n.files { entries = append(entries, "f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+ "\x00"+sig.head+"\x00"+sig.tail) n.fileCount++ n.totalSize += sig.size } for name, child := range n.dirs { child.compute() entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) n.fileCount += child.fileCount n.totalSize += child.totalSize } slices.Sort(entries) h := sha256.New() for _, e := range entries { h.Write([]byte(e)) h.Write([]byte{0}) } copy(n.digest[:], h.Sum(nil)) } // suppressed reports whether a duplicate-tree group is non-maximal: its // members' parents are pairwise distinct real directories that all // share a single digest, so the group is wholly implied by its parents' // (or a further ancestor's) group. Groups containing siblings (shared // parent) or members whose parents differ are always reported. func suppressed(g []*treeNode, super *treeNode) bool { seen := make(map[*treeNode]bool, len(g)) var parentDigest [sha256.Size]byte for i, n := range g { p := n.parent if p == nil || p == super { return false } if seen[p] { return false // siblings: not implied by any parent group } seen[p] = true if i == 0 { parentDigest = p.digest } else if p.digest != parentDigest { return false } } return true }