package main import ( "context" "database/sql" "fmt" "io" "os" "slices" "strings" "time" ) // ioBufSize is the buffer size for the buffered stdout writers. const ioBufSize = 1 << 20 // minGroupSize is the smallest number of members that makes a // duplicate group. const minGroupSize = 2 // scanRec is one file record from the database. The signature (size, // head, tail, content) is the duplicate key; mtime is informational // only and used by scan for change detection. type scanRec struct { size int64 mtime time.Time head string tail string content string path string } // reportPhase names one phase of report with its place among all of // them, as "[2/2] read", for the phase's progress display. The list // here is the one place the phases are counted. func reportPhase(name string) string { return phaseLabel([]string{"open", "read"}, name) } // runReport implements the report subcommand: it prints the file-level // duplicates report as TSV on stdout. It never touches the scanned // filesystem; its only I/O is the database, stdout, and stderr. It // names the database on stderr before it opens it. Any database // problem, including a missing database, is fatal. // // The work runs in two phases, each under a progress display of its // own on stderr: open (open the database) and read (count the records, // then read them in signature order, writing each size's duplicate // groups to stdout once that size is read; see readReport). Each // display ends before the next phase starts, and before an error is // returned to be printed. func runReport(ctx context.Context, stdout io.Writer) error { dbPath, source, err := databasePath() if err != nil { return err } fmt.Fprintf(os.Stderr, "report: database %s (%s)\n", escapePath(dbPath), source) // Opening can wait on a busy database for as long as the busy // timeout. prog := newProgress(progressParams{ label: reportPhase("open"), total: -1, waitingFor: "opening the database", }) db, err := openReportDatabase(ctx, dbPath) prog.finish() if err != nil { return err } defer func() { _ = db.Close() }() prog = newProgress(progressParams{ label: reportPhase("read"), unit: "records", total: -1, waitingFor: countingRecords, }) report, err := readReport(ctx, db, dbPath, stdout, prog) prog.finish() if err != nil { return err } fmt.Fprintf(os.Stderr, "report: %d records read, %d duplicate groups, %d dupe files, "+ "%s reclaimable\n", report.records, report.dupeGroups, report.dupeFiles, humanBytes(report.reclaimable)) return nil } // readReport gives prog the number of records in db, the database at // dbPath, as its total, then reads every record in signature order, // counting each on prog, and writes the report to stdout: the header // line, then the rows of each size's duplicate groups once that size is // read. stdout may be the terminal prog is drawn on, so the lines go // through prog's writer. It returns what it wrote, with the totals for // the summary line. func readReport(ctx context.Context, db *sql.DB, dbPath string, stdout io.Writer, prog *progress, ) (*reportWriter, error) { out := prog.writer(stdout) report := &reportWriter{out: out} _, err := fmt.Fprintln(out, "first\tdupe\tsize") if err != nil { return nil, fmt.Errorf("write stdout: %w", err) } var writeErr error err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error { prog.increment() writeErr = report.add(r) return writeErr }) if writeErr != nil { return nil, fmt.Errorf("write stdout: %w", writeErr) } if err != nil { return nil, fmt.Errorf("database %s: %w", dbPath, err) } err = report.finish() if err == nil { err = out.Flush() } if err != nil { return nil, fmt.Errorf("write stdout: %w", err) } return report, nil } // reportWriter writes the report's rows to out from the records, added // in signature order, largest size first, as loadSignatureRows reads // them. The records sharing a signature come together, and so do the // records of one size, so it holds only the duplicate groups of the // size being read, and writes them, ordered by first path, once a // record of a smaller size comes. It counts what the summary line // reports. type reportWriter struct { out io.Writer sig scanRec // the signature being read; its path is not used paths []string // the paths read so far with sig groups [][]string // the duplicate groups of sig.size, each sorted records, dupeGroups, dupeFiles int reclaimable int64 } // add takes the next record. One with the signature being read joins // its paths; any other ends them, and one of a smaller size first // writes the groups of the size before. func (w *reportWriter) add(r scanRec) error { w.records++ // A record without a content hash has unknown content (README // "Database"), so it is never a duplicate. if r.content == "" { return nil } if r.size == w.sig.size && r.head == w.sig.head && r.tail == w.sig.tail && r.content == w.sig.content { w.paths = append(w.paths, r.path) return nil } w.endGroup() if r.size != w.sig.size { err := w.writeGroups() if err != nil { return err } } w.sig = r w.paths = append(w.paths, r.path) return nil } // endGroup ends the paths read with w.sig: two or more are a duplicate // group, held with its paths sorted until writeGroups writes it. func (w *reportWriter) endGroup() { if len(w.paths) < minGroupSize { // A path with no duplicate is dropped, and its slice reused. w.paths = w.paths[:0] return } slices.Sort(w.paths) w.groups = append(w.groups, w.paths) w.paths = nil } // writeGroups writes the duplicate groups held, all of size w.sig.size, // ordered by first path: for each group, a row for every path but the // first. No two groups share a first path, since no two records share // a path. func (w *reportWriter) writeGroups() error { slices.SortFunc(w.groups, func(a, b []string) int { return strings.Compare(a[0], b[0]) }) for _, g := range w.groups { for _, path := range g[1:] { _, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n", escapePath(g[0]), escapePath(path), w.sig.size) if err != nil { return err } w.dupeFiles++ w.reclaimable += w.sig.size } w.dupeGroups++ } w.groups = nil return nil } // finish writes the groups of the last size read. func (w *reportWriter) finish() error { w.endGroup() return w.writeGroups() } // escapePath returns a path as it is written in a report column (README // "Report output format"): a backslash, tab, newline or carriage return // becomes \\, \t, \n or \r, and every other byte is kept as it is. // Grouping and sorting use the raw path, never this form. func escapePath(p string) string { // Most paths need no escaping; skip building a replacer for them. if !strings.ContainsAny(p, "\\\t\n\r") { return p } return strings.NewReplacer( `\`, `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`, ).Replace(p) } // humanBytes formats a byte count in human units (binary prefixes). func humanBytes(n int64) string { const unit = 1024 if n < unit { return fmt.Sprintf("%d B", n) } div, exp := int64(unit), 0 for m := n / unit; m >= unit; m /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) }