package main import ( "bufio" "bytes" "fmt" "io" "os" "slices" "strconv" "strings" ) // scanRec is one well-formed record parsed from a scan stream. The // signature (size, head, tail) is the duplicate key; mtime is // informational and not retained. type scanRec struct { size int64 head string tail string path string } // openScanInput resolves the analysis-mode input: the file named by the // single optional positional argument, or stdin when it is absent or // "-". The returned closer must be called when reading is done. func openScanInput(args []string) (in io.Reader, name string, closer func()) { if len(args) == 1 && args[0] != "-" { f, err := os.Open(args[0]) if err != nil { fatalf("open %s: %v", args[0], err) } return f, args[0], func() { f.Close() } } return os.Stdin, "stdin", func() {} } // parseScanStream reads every NUL-terminated record from in. A record // that does not have exactly 5 fields or whose size is non-numeric is // counted as malformed and skipped. Total records read is // len(recs) + malformed. func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) { sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20)) sc.Buffer(make([]byte, 0, 64<<10), 4<<20) sc.Split(splitNUL) for sc.Scan() { // The path is the last field and may itself contain tabs, // so split into at most 5 fields. fields := strings.SplitN(sc.Text(), "\t", 5) if len(fields) != 5 { malformed++ continue } size, err := strconv.ParseInt(fields[0], 10, 64) if err != nil { malformed++ continue } recs = append(recs, scanRec{ size: size, head: fields[2], tail: fields[3], path: fields[4], }) } if err := sc.Err(); err != nil { fatalf("read %s: %v", name, err) } return recs, malformed } // dupeGroup is one set of candidate-duplicate files: identical size, // head hash, and tail hash. paths is sorted lexicographically; the // first entry is the group's "first", the rest are dupes. type dupeGroup struct { size int64 paths []string } // runReport implements the report subcommand: it reads a scan stream // from the named file (or stdin when absent or "-") and prints the // file-level duplicates report as TSV on stdout. It never touches the // scanned filesystem; its only I/O is the scan input, stdout, and // stderr. args holds the positional arguments already validated by // cobra (at most one). func runReport(args []string) { in, name, closer := openScanInput(args) defer closer() recs, malformed := parseScanStream(in, name) records := len(recs) + malformed type dupeKey struct { size int64 head string tail string } groups := make(map[dupeKey][]string) for _, r := range recs { k := dupeKey{size: r.size, head: r.head, tail: r.tail} groups[k] = append(groups[k], r.path) } var dupes []dupeGroup for k, paths := range groups { if len(paths) < 2 { continue } slices.Sort(paths) dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) } // Biggest reclaimable space first; ties broken by first path. slices.SortFunc(dupes, func(a, b dupeGroup) int { if a.size != b.size { if a.size > b.size { return -1 } return 1 } return strings.Compare(a.paths[0], b.paths[0]) }) out := bufio.NewWriterSize(os.Stdout, 1<<20) if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil { fatalf("write stdout: %v", err) } dupeFiles := 0 var reclaimable int64 for _, g := range dupes { for _, p := range g.paths[1:] { if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n", g.paths[0], p, g.size); err != nil { fatalf("write stdout: %v", err) } dupeFiles++ reclaimable += g.size } } if err := out.Flush(); err != nil { fatalf("write stdout: %v", err) } fmt.Fprintf(os.Stderr, "report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n", records, malformedNote(malformed), len(dupes), dupeFiles, humanBytes(reclaimable)) } // malformedNote formats the optional malformed-record note for the // stderr summaries. func malformedNote(malformed int) string { if malformed == 0 { return "" } return fmt.Sprintf(" (%d malformed, skipped)", malformed) } // splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing // data without a terminator at EOF is returned as a final record. func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) { if i := bytes.IndexByte(data, 0); i >= 0 { return i + 1, data[:i], nil } if atEOF && len(data) > 0 { return len(data), data, nil } return 0, nil, nil } // humanBytes formats a byte count in human units (binary prefixes). func humanBytes(n int64) string { const unit = 1024 if n < unit { return fmt.Sprintf("%d B", n) } div, exp := int64(unit), 0 for m := n / unit; m >= unit; m /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) }