package main import ( "bufio" "bytes" "fmt" "io" "os" "slices" "strconv" "strings" ) // ioBufSize is the buffer size for the buffered scan-stream reader and // the buffered stdout writers. const ioBufSize = 1 << 20 // recordFieldCount is the number of tab-separated fields in a scan // record: size, mtime, head hash, tail hash, path. const recordFieldCount = 5 // scanInitBufSize and scanMaxRecordSize bound the scanner buffer used // to read scan records; a record longer than scanMaxRecordSize is a // fatal read error. const ( scanInitBufSize = 64 << 10 scanMaxRecordSize = 4 << 20 ) // minGroupSize is the smallest number of members that makes a // duplicate group. const minGroupSize = 2 // scanRec is one well-formed record parsed from a scan stream. The // signature (size, head, tail) is the duplicate key; mtime is // informational and not retained. type scanRec struct { size int64 head string tail string path string } // openScanInput resolves the analysis-mode input: the file named by the // single optional positional argument, or stdin when it is absent or // "-". The returned closer must be called when reading is done. func openScanInput(args []string) (io.Reader, string, func()) { if len(args) == 1 && args[0] != "-" { f, err := os.Open(args[0]) if err != nil { fatalf("open %s: %v", args[0], err) } return f, args[0], func() { _ = f.Close() } } return os.Stdin, "stdin", func() {} } // parseScanStream reads every NUL-terminated record from in. A record // that does not have exactly 5 fields or whose size is non-numeric is // counted as malformed and skipped. Total records read is // len(recs) + malformed. func parseScanStream(in io.Reader, name string) ([]scanRec, int) { sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize)) sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize) sc.Split(splitNUL) var ( recs []scanRec malformed int ) for sc.Scan() { // The path is the last field and may itself contain tabs, // so split into at most recordFieldCount fields. fields := strings.SplitN(sc.Text(), "\t", recordFieldCount) if len(fields) != recordFieldCount { malformed++ continue } size, err := strconv.ParseInt(fields[0], 10, 64) if err != nil { malformed++ continue } recs = append(recs, scanRec{ size: size, head: fields[2], tail: fields[3], path: fields[4], }) } err := sc.Err() if err != nil { fatalf("read %s: %v", name, err) } return recs, malformed } // dupeGroup is one set of candidate-duplicate files: identical size, // head hash, and tail hash. paths is sorted lexicographically; the // first entry is the group's "first", the rest are dupes. type dupeGroup struct { size int64 paths []string } // runReport implements the report subcommand: it reads a scan stream // from the named file (or stdin when absent or "-") and prints the // file-level duplicates report as TSV on stdout. It never touches the // scanned filesystem; its only I/O is the scan input, stdout, and // stderr. args holds the positional arguments already validated by // cobra (at most one). func runReport(args []string) { in, name, closer := openScanInput(args) defer closer() recs, malformed := parseScanStream(in, name) records := len(recs) + malformed dupes := collectDupeGroups(recs) out := bufio.NewWriterSize(os.Stdout, ioBufSize) _, err := fmt.Fprintln(out, "first\tdupe\tsize") if err != nil { fatalf("write stdout: %v", err) } dupeFiles := 0 var reclaimable int64 for _, g := range dupes { for _, p := range g.paths[1:] { _, err = fmt.Fprintf(out, "%s\t%s\t%d\n", g.paths[0], p, g.size) if err != nil { fatalf("write stdout: %v", err) } dupeFiles++ reclaimable += g.size } } err = out.Flush() if err != nil { fatalf("write stdout: %v", err) } fmt.Fprintf(os.Stderr, "report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n", records, malformedNote(malformed), len(dupes), dupeFiles, humanBytes(reclaimable)) } // collectDupeGroups groups records by signature and returns every group // with two or more paths, each group's paths sorted lexicographically, // groups ordered by size descending then by first path ascending. func collectDupeGroups(recs []scanRec) []dupeGroup { groups := make(map[fileSig][]string) for _, r := range recs { k := fileSig{size: r.size, head: r.head, tail: r.tail} groups[k] = append(groups[k], r.path) } var dupes []dupeGroup for k, paths := range groups { if len(paths) < minGroupSize { continue } slices.Sort(paths) dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) } // Biggest reclaimable space first; ties broken by first path. slices.SortFunc(dupes, func(a, b dupeGroup) int { if a.size != b.size { if a.size > b.size { return -1 } return 1 } return strings.Compare(a.paths[0], b.paths[0]) }) return dupes } // malformedNote formats the optional malformed-record note for the // stderr summaries. func malformedNote(malformed int) string { if malformed == 0 { return "" } return fmt.Sprintf(" (%d malformed, skipped)", malformed) } // splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing // data without a terminator at EOF is returned as a final record. func splitNUL(data []byte, atEOF bool) (int, []byte, error) { if i := bytes.IndexByte(data, 0); i >= 0 { return i + 1, data[:i], nil } if atEOF && len(data) > 0 { return len(data), data, nil } return 0, nil, nil } // humanBytes formats a byte count in human units (binary prefixes). func humanBytes(n int64) string { const unit = 1024 if n < unit { return fmt.Sprintf("%d B", n) } div, exp := int64(unit), 0 for m := n / unit; m >= unit; m /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) }