Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s
All checks were successful
check / check (push) Successful in 3s
scan now synchronizes a database that survives between runs (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of emitting a stream: operands are resolved to absolute paths, unchanged files (same size, mtime not newer than recorded) are never re-read, new and changed files are hashed, and records under the scanned operands that were not verified this run are deleted; records outside the operands are untouched. All changes commit in a single transaction, and WAL journaling with a busy timeout keeps a report run during a cron scan safe. report and trees read the database (no positional arguments); the NUL-terminated stream format, its parser, and the malformed-record handling are gone. The driver is modernc.org/sqlite (pure Go), so builds keep cgo disabled.
This commit is contained in:
153
report.go
153
report.go
@@ -2,106 +2,49 @@ package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ioBufSize is the buffer size for the buffered scan-stream reader and
|
||||
// the buffered stdout writers.
|
||||
// ioBufSize is the buffer size for the buffered stdout writers.
|
||||
const ioBufSize = 1 << 20
|
||||
|
||||
// recordFieldCount is the number of tab-separated fields in a scan
|
||||
// record: size, mtime, head hash, tail hash, path.
|
||||
const recordFieldCount = 5
|
||||
|
||||
// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
|
||||
// to read scan records; a record longer than scanMaxRecordSize is a
|
||||
// fatal read error.
|
||||
const (
|
||||
scanInitBufSize = 64 << 10
|
||||
scanMaxRecordSize = 4 << 20
|
||||
)
|
||||
|
||||
// minGroupSize is the smallest number of members that makes a
|
||||
// duplicate group.
|
||||
const minGroupSize = 2
|
||||
|
||||
// scanRec is one well-formed record parsed from a scan stream. The
|
||||
// signature (size, head, tail) is the duplicate key; mtime is
|
||||
// informational and not retained.
|
||||
// scanRec is one file record from the database. The signature (size,
|
||||
// head, tail) is the duplicate key; mtime is informational only and
|
||||
// used by scan for change detection.
|
||||
type scanRec struct {
|
||||
size int64
|
||||
head string
|
||||
tail string
|
||||
path string
|
||||
size int64
|
||||
mtime int64
|
||||
head string
|
||||
tail string
|
||||
path string
|
||||
}
|
||||
|
||||
// openScanInput resolves the analysis-mode input: the file named by the
|
||||
// single optional positional argument, or stdin when it is absent or
|
||||
// "-". The returned closer must be called when reading is done.
|
||||
func openScanInput(args []string) (io.Reader, string, func()) {
|
||||
if len(args) == 1 && args[0] != "-" {
|
||||
f, err := os.Open(args[0])
|
||||
if err != nil {
|
||||
fatalf("open %s: %v", args[0], err)
|
||||
}
|
||||
// loadRecords opens the database and reads every file record for the
|
||||
// report and trees subcommands. Any database problem — including a
|
||||
// missing database — is fatal.
|
||||
func loadRecords() []scanRec {
|
||||
dbPath := databasePath()
|
||||
|
||||
return f, args[0], func() { _ = f.Close() }
|
||||
}
|
||||
|
||||
return os.Stdin, "stdin", func() {}
|
||||
}
|
||||
|
||||
// parseScanStream reads every NUL-terminated record from in. A record
|
||||
// that does not have exactly 5 fields or whose size is non-numeric is
|
||||
// counted as malformed and skipped. Total records read is
|
||||
// len(recs) + malformed.
|
||||
func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
|
||||
sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
|
||||
sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
|
||||
sc.Split(splitNUL)
|
||||
|
||||
var (
|
||||
recs []scanRec
|
||||
malformed int
|
||||
)
|
||||
|
||||
for sc.Scan() {
|
||||
// The path is the last field and may itself contain tabs,
|
||||
// so split into at most recordFieldCount fields.
|
||||
fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
|
||||
if len(fields) != recordFieldCount {
|
||||
malformed++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
size, err := strconv.ParseInt(fields[0], 10, 64)
|
||||
if err != nil {
|
||||
malformed++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
recs = append(recs, scanRec{
|
||||
size: size,
|
||||
head: fields[2],
|
||||
tail: fields[3],
|
||||
path: fields[4],
|
||||
})
|
||||
}
|
||||
|
||||
err := sc.Err()
|
||||
db, err := openReportDatabase(dbPath)
|
||||
if err != nil {
|
||||
fatalf("read %s: %v", name, err)
|
||||
fatalf("%v", err)
|
||||
}
|
||||
|
||||
return recs, malformed
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
recs, err := loadFileRows(db)
|
||||
if err != nil {
|
||||
fatalf("database %s: %v", dbPath, err)
|
||||
}
|
||||
|
||||
return recs
|
||||
}
|
||||
|
||||
// dupeGroup is one set of candidate-duplicate files: identical size,
|
||||
@@ -112,18 +55,12 @@ type dupeGroup struct {
|
||||
paths []string
|
||||
}
|
||||
|
||||
// runReport implements the report subcommand: it reads a scan stream
|
||||
// from the named file (or stdin when absent or "-") and prints the
|
||||
// file-level duplicates report as TSV on stdout. It never touches the
|
||||
// scanned filesystem; its only I/O is the scan input, stdout, and
|
||||
// stderr. args holds the positional arguments already validated by
|
||||
// cobra (at most one).
|
||||
func runReport(args []string) {
|
||||
in, name, closer := openScanInput(args)
|
||||
defer closer()
|
||||
|
||||
recs, malformed := parseScanStream(in, name)
|
||||
records := len(recs) + malformed
|
||||
// runReport implements the report subcommand: it reads every record
|
||||
// from the database and prints the file-level duplicates report as TSV
|
||||
// on stdout. It never touches the scanned filesystem; its only I/O is
|
||||
// the database, stdout, and stderr.
|
||||
func runReport() {
|
||||
recs := loadRecords()
|
||||
dupes := collectDupeGroups(recs)
|
||||
|
||||
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||
@@ -156,9 +93,9 @@ func runReport(args []string) {
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
|
||||
records, malformedNote(malformed), len(dupes), dupeFiles,
|
||||
humanBytes(reclaimable))
|
||||
"report: %d records read, %d duplicate groups, %d dupe files, "+
|
||||
"%s reclaimable\n",
|
||||
len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
|
||||
}
|
||||
|
||||
// collectDupeGroups groups records by signature and returns every group
|
||||
@@ -199,30 +136,6 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
|
||||
return dupes
|
||||
}
|
||||
|
||||
// malformedNote formats the optional malformed-record note for the
|
||||
// stderr summaries.
|
||||
func malformedNote(malformed int) string {
|
||||
if malformed == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
|
||||
}
|
||||
|
||||
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
|
||||
// data without a terminator at EOF is returned as a final record.
|
||||
func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
|
||||
if i := bytes.IndexByte(data, 0); i >= 0 {
|
||||
return i + 1, data[:i], nil
|
||||
}
|
||||
|
||||
if atEOF && len(data) > 0 {
|
||||
return len(data), data, nil
|
||||
}
|
||||
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
// humanBytes formats a byte count in human units (binary prefixes).
|
||||
func humanBytes(n int64) string {
|
||||
const unit = 1024
|
||||
|
||||
Reference in New Issue
Block a user