Add sfdupes implementation: scan, report, and trees subcommands

report.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// scanRec is one well-formed record parsed from a scan stream. The
+// signature (size, head, tail) is the duplicate key; mtime is
+// informational and not retained.
+type scanRec struct {
+	size int64
+	head string
+	tail string
+	path string
+}
+
+// openScanInput resolves the analysis-mode input: the file named by the
+// single optional positional argument, or stdin when it is absent or
+// "-". The returned closer must be called when reading is done.
+func openScanInput(args []string) (in io.Reader, name string, closer func()) {
+	if len(args) == 1 && args[0] != "-" {
+		f, err := os.Open(args[0])
+		if err != nil {
+			fatalf("open %s: %v", args[0], err)
+		}
+		return f, args[0], func() { f.Close() }
+	}
+	return os.Stdin, "stdin", func() {}
+}
+
+// parseScanStream reads every NUL-terminated record from in. A record
+// that does not have exactly 5 fields or whose size is non-numeric is
+// counted as malformed and skipped. Total records read is
+// len(recs) + malformed.
+func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
+	sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
+	sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
+	sc.Split(splitNUL)
+	for sc.Scan() {
+		// The path is the last field and may itself contain tabs,
+		// so split into at most 5 fields.
+		fields := strings.SplitN(sc.Text(), "\t", 5)
+		if len(fields) != 5 {
+			malformed++
+			continue
+		}
+		size, err := strconv.ParseInt(fields[0], 10, 64)
+		if err != nil {
+			malformed++
+			continue
+		}
+		recs = append(recs, scanRec{
+			size: size,
+			head: fields[2],
+			tail: fields[3],
+			path: fields[4],
+		})
+	}
+	if err := sc.Err(); err != nil {
+		fatalf("read %s: %v", name, err)
+	}
+	return recs, malformed
+}
+
+// dupeGroup is one set of candidate-duplicate files: identical size,
+// head hash, and tail hash. paths is sorted lexicographically; the
+// first entry is the group's "first", the rest are dupes.
+type dupeGroup struct {
+	size  int64
+	paths []string
+}
+
+// runReport implements the report subcommand: it reads a scan stream
+// from the named file (or stdin when absent or "-") and prints the
+// file-level duplicates report as TSV on stdout. It never touches the
+// scanned filesystem; its only I/O is the scan input, stdout, and
+// stderr. args holds the positional arguments already validated by
+// cobra (at most one).
+func runReport(args []string) {
+	in, name, closer := openScanInput(args)
+	defer closer()
+	recs, malformed := parseScanStream(in, name)
+	records := len(recs) + malformed
+
+	type dupeKey struct {
+		size int64
+		head string
+		tail string
+	}
+	groups := make(map[dupeKey][]string)
+	for _, r := range recs {
+		k := dupeKey{size: r.size, head: r.head, tail: r.tail}
+		groups[k] = append(groups[k], r.path)
+	}
+
+	var dupes []dupeGroup
+	for k, paths := range groups {
+		if len(paths) < 2 {
+			continue
+		}
+		slices.Sort(paths)
+		dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
+	}
+	// Biggest reclaimable space first; ties broken by first path.
+	slices.SortFunc(dupes, func(a, b dupeGroup) int {
+		if a.size != b.size {
+			if a.size > b.size {
+				return -1
+			}
+			return 1
+		}
+		return strings.Compare(a.paths[0], b.paths[0])
+	})
+
+	out := bufio.NewWriterSize(os.Stdout, 1<<20)
+	if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
+		fatalf("write stdout: %v", err)
+	}
+	dupeFiles := 0
+	var reclaimable int64
+	for _, g := range dupes {
+		for _, p := range g.paths[1:] {
+			if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
+				g.paths[0], p, g.size); err != nil {
+				fatalf("write stdout: %v", err)
+			}
+			dupeFiles++
+			reclaimable += g.size
+		}
+	}
+	if err := out.Flush(); err != nil {
+		fatalf("write stdout: %v", err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
+		records, malformedNote(malformed), len(dupes), dupeFiles,
+		humanBytes(reclaimable))
+}
+
+// malformedNote formats the optional malformed-record note for the
+// stderr summaries.
+func malformedNote(malformed int) string {
+	if malformed == 0 {
+		return ""
+	}
+	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
+}
+
+// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
+// data without a terminator at EOF is returned as a final record.
+func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if i := bytes.IndexByte(data, 0); i >= 0 {
+		return i + 1, data[:i], nil
+	}
+	if atEOF && len(data) > 0 {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
+// humanBytes formats a byte count in human units (binary prefixes).
+func humanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}