sfdupes/report.go

package main

import (
	"context"
	"database/sql"
	"fmt"
	"io"
	"os"
	"slices"
	"strings"
	"time"
)

// ioBufSize is the buffer size for the buffered stdout writers.
const ioBufSize = 1 << 20

// minGroupSize is the smallest number of members that makes a
// duplicate group.
const minGroupSize = 2

// scanRec is one file record from the database. The signature (size,
// head, tail, content) is the duplicate key; mtime is informational
// only and used by scan for change detection.
type scanRec struct {
	size    int64
	mtime   time.Time
	head    string
	tail    string
	content string
	path    string
}

// reportPhase names one phase of report with its place among all of
// them, as "[2/2] read", for the phase's progress display. The list
// here is the one place the phases are counted.
func reportPhase(name string) string {
	return phaseLabel([]string{openPhase, "read"}, name)
}

// runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. It never touches the scanned
// filesystem; its only I/O is the database, stdout, and stderr. It
// names the database on stderr before it opens it. Any database
// problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (count the records,
// then read them in signature order, writing each size's duplicate
// groups to stdout once that size is read; see readReport). Each
// display ends before the next phase starts, and before an error is
// returned to be printed.
func runReport(ctx context.Context, stdout io.Writer) error {
	dbPath, source, err := databasePath()
	if err != nil {
		return err
	}

	fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
		escapePath(dbPath), source)

	// Opening can wait on a busy database for as long as the busy
	// timeout.
	prog := newProgress(progressParams{
		label: reportPhase(openPhase), total: -1,
		waitingFor: openingDatabase,
	})
	db, err := openReportDatabase(ctx, dbPath)

	prog.finish()

	if err != nil {
		return err
	}

	defer func() { _ = db.Close() }()

	// The read phase writes the report while it reads the records, so a
	// stall may be in either.
	prog = newProgress(progressParams{
		label: reportPhase("read"), unit: "records", total: -1,
		waitingFor: countingRecords, doing: "read/write",
	})
	report, err := readReport(ctx, db, dbPath, stdout, prog)

	prog.finish()

	if err != nil {
		return err
	}

	fmt.Fprintf(os.Stderr,
		"report: %d records read, %d duplicate groups, %d dupe files, "+
			"%s reclaimable\n",
		report.records, report.dupeGroups, report.dupeFiles,
		humanBytes(report.reclaimable))

	return nil
}

// readReport gives prog the number of records in db, the database at
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
	stdout io.Writer, prog *progress,
) (*reportWriter, error) {
	out := prog.writer(stdout)
	report := &reportWriter{out: out}

	_, err := fmt.Fprintln(out, "first\tdupe\tsize")
	if err != nil {
		return nil, fmt.Errorf("write stdout: %w", err)
	}

	var writeErr error

	err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
		prog.increment()

		writeErr = report.add(r)

		return writeErr
	})

	if writeErr != nil {
		return nil, fmt.Errorf("write stdout: %w", writeErr)
	}

	if err != nil {
		return nil, fmt.Errorf("database %s: %w", dbPath, err)
	}

	err = report.finish()
	if err == nil {
		err = out.Flush()
	}

	if err != nil {
		return nil, fmt.Errorf("write stdout: %w", err)
	}

	return report, nil
}

// reportWriter writes the report's rows to out from the records, added
// in signature order, largest size first, as loadSignatureRows reads
// them. The records sharing a signature come together, and so do the
// records of one size, so it holds only the duplicate groups of the
// size being read, and writes them, ordered by first path, once a
// record of a smaller size comes. It counts what the summary line
// reports.
type reportWriter struct {
	out io.Writer

	sig    scanRec    // the signature being read; its path is not used
	paths  []string   // the paths read so far with sig
	groups [][]string // the duplicate groups of sig.size, each sorted

	records, dupeGroups, dupeFiles int
	reclaimable                    int64
}

// add takes the next record. One with the signature being read joins
// its paths; any other ends them, and one of a smaller size first
// writes the groups of the size before.
func (w *reportWriter) add(r scanRec) error {
	w.records++

	// A record without a content hash has unknown content (README
	// "Database"), so it is never a duplicate.
	if r.content == "" {
		return nil
	}

	if r.size == w.sig.size && r.head == w.sig.head &&
		r.tail == w.sig.tail && r.content == w.sig.content {
		w.paths = append(w.paths, r.path)

		return nil
	}

	w.endGroup()

	if r.size != w.sig.size {
		err := w.writeGroups()
		if err != nil {
			return err
		}
	}

	w.sig = r
	w.paths = append(w.paths, r.path)

	return nil
}

// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
func (w *reportWriter) endGroup() {
	if len(w.paths) < minGroupSize {
		// A path with no duplicate is dropped, and its slice reused.
		w.paths = w.paths[:0]

		return
	}

	slices.Sort(w.paths)
	w.groups = append(w.groups, w.paths)
	w.paths = nil
}

// writeGroups writes the duplicate groups held, all of size w.sig.size,
// ordered by first path: for each group, a row for every path but the
// first. No two groups share a first path, since no two records share
// a path.
func (w *reportWriter) writeGroups() error {
	slices.SortFunc(w.groups, func(a, b []string) int {
		return strings.Compare(a[0], b[0])
	})

	for _, g := range w.groups {
		for _, path := range g[1:] {
			_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
				escapePath(g[0]), escapePath(path), w.sig.size)
			if err != nil {
				return err
			}

			w.dupeFiles++
			w.reclaimable += w.sig.size
		}

		w.dupeGroups++
	}

	w.groups = nil

	return nil
}

// finish writes the groups of the last size read.
func (w *reportWriter) finish() error {
	w.endGroup()

	return w.writeGroups()
}

// escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
// Grouping and sorting use the raw path, never this form.
func escapePath(p string) string {
	// Most paths need no escaping; skip building a replacer for them.
	if !strings.ContainsAny(p, "\\\t\n\r") {
		return p
	}

	return strings.NewReplacer(
		`\`, `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`,
	).Replace(p)
}

// humanBytes formats a byte count in human units (binary prefixes).
func humanBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}

	div, exp := int64(unit), 0
	for m := n / unit; m >= unit; m /= unit {
		div *= unit
		exp++
	}

	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}