sfdupes/report.go

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"slices"
	"strconv"
	"strings"
)

// scanRec is one well-formed record parsed from a scan stream. The
// signature (size, head, tail) is the duplicate key; mtime is
// informational and not retained.
type scanRec struct {
	size int64
	head string
	tail string
	path string
}

// openScanInput resolves the analysis-mode input: the file named by the
// single optional positional argument, or stdin when it is absent or
// "-". The returned closer must be called when reading is done.
func openScanInput(args []string) (in io.Reader, name string, closer func()) {
	if len(args) == 1 && args[0] != "-" {
		f, err := os.Open(args[0])
		if err != nil {
			fatalf("open %s: %v", args[0], err)
		}
		return f, args[0], func() { f.Close() }
	}
	return os.Stdin, "stdin", func() {}
}

// parseScanStream reads every NUL-terminated record from in. A record
// that does not have exactly 5 fields or whose size is non-numeric is
// counted as malformed and skipped. Total records read is
// len(recs) + malformed.
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
	sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
	sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
	sc.Split(splitNUL)
	for sc.Scan() {
		// The path is the last field and may itself contain tabs,
		// so split into at most 5 fields.
		fields := strings.SplitN(sc.Text(), "\t", 5)
		if len(fields) != 5 {
			malformed++
			continue
		}
		size, err := strconv.ParseInt(fields[0], 10, 64)
		if err != nil {
			malformed++
			continue
		}
		recs = append(recs, scanRec{
			size: size,
			head: fields[2],
			tail: fields[3],
			path: fields[4],
		})
	}
	if err := sc.Err(); err != nil {
		fatalf("read %s: %v", name, err)
	}
	return recs, malformed
}

// dupeGroup is one set of candidate-duplicate files: identical size,
// head hash, and tail hash. paths is sorted lexicographically; the
// first entry is the group's "first", the rest are dupes.
type dupeGroup struct {
	size  int64
	paths []string
}

// runReport implements the report subcommand: it reads a scan stream
// from the named file (or stdin when absent or "-") and prints the
// file-level duplicates report as TSV on stdout. It never touches the
// scanned filesystem; its only I/O is the scan input, stdout, and
// stderr. args holds the positional arguments already validated by
// cobra (at most one).
func runReport(args []string) {
	in, name, closer := openScanInput(args)
	defer closer()
	recs, malformed := parseScanStream(in, name)
	records := len(recs) + malformed

	type dupeKey struct {
		size int64
		head string
		tail string
	}
	groups := make(map[dupeKey][]string)
	for _, r := range recs {
		k := dupeKey{size: r.size, head: r.head, tail: r.tail}
		groups[k] = append(groups[k], r.path)
	}

	var dupes []dupeGroup
	for k, paths := range groups {
		if len(paths) < 2 {
			continue
		}
		slices.Sort(paths)
		dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
	}
	// Biggest reclaimable space first; ties broken by first path.
	slices.SortFunc(dupes, func(a, b dupeGroup) int {
		if a.size != b.size {
			if a.size > b.size {
				return -1
			}
			return 1
		}
		return strings.Compare(a.paths[0], b.paths[0])
	})

	out := bufio.NewWriterSize(os.Stdout, 1<<20)
	if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
		fatalf("write stdout: %v", err)
	}
	dupeFiles := 0
	var reclaimable int64
	for _, g := range dupes {
		for _, p := range g.paths[1:] {
			if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
				g.paths[0], p, g.size); err != nil {
				fatalf("write stdout: %v", err)
			}
			dupeFiles++
			reclaimable += g.size
		}
	}
	if err := out.Flush(); err != nil {
		fatalf("write stdout: %v", err)
	}

	fmt.Fprintf(os.Stderr,
		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
		records, malformedNote(malformed), len(dupes), dupeFiles,
		humanBytes(reclaimable))
}

// malformedNote formats the optional malformed-record note for the
// stderr summaries.
func malformedNote(malformed int) string {
	if malformed == 0 {
		return ""
	}
	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
}

// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
// data without a terminator at EOF is returned as a final record.
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if i := bytes.IndexByte(data, 0); i >= 0 {
		return i + 1, data[:i], nil
	}
	if atEOF && len(data) > 0 {
		return len(data), data, nil
	}
	return 0, nil, nil
}

// humanBytes formats a byte count in human units (binary prefixes).
func humanBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}
	div, exp := int64(unit), 0
	for m := n / unit; m >= unit; m /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}