Compare commits

..
Author SHA1 Message Date
sneak 16df355ef2 Rewrite README.md sentence by sentence for a newcomer
check / check (push) Canceled after 0s
Rewrites `README.md` from scratch for
#135. Every statement was checked
against the code on `next` and against what the built binary prints on
small trees under `/var/tmp`. The headings that the code comments cite
are kept. Five `main_test.go` comments cite "Settings", two cite what
"Error handling" says about a failed write to stdout, and one comment
each in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what
the README now says.

Where the old text and the repository disagreed, the new text states
what the code does: `@latest` installs the newest tagged release, ten
million files in both "Rationale" and "Design", no Windows excludes, and
an operand reached through a symlinked directory below another operand
is dropped (#144).

Model: fable-5-1
2026-10-26 18:05:31 +00:00
27 changed files with 1140 additions and 3263 deletions

No files matched your search

+2 -2
View File
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
BINARY := sfdupes
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
LDFLAGS := -X main.Version=$(VERSION)
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
@@ -14,7 +14,7 @@ LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
# Default target: build the binary. Phony so go build (which has its
# own build cache) always decides what to recompile.
sfdupes:
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes
go build -ldflags "$(LDFLAGS)" -o $(BINARY)
build: sfdupes
+220 -390
View File
File diff suppressed because it is too large. Load diff
+431 -655
View File
File diff suppressed because it is too large. Load diff
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker
package main
import (
"context"
@@ -359,7 +359,7 @@ func TestRunScanInterrupted(t *testing.T) {
dir := buildSmokeTree(t)
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
false, nil)
false)
if err != nil {
t.Fatal(err)
}
@@ -383,7 +383,7 @@ func TestRunScanInterrupted(t *testing.T) {
defer close(done)
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
false, false, nil)
false, false)
}()
awaitReturn(t, done, "runScan")
@@ -411,7 +411,7 @@ func TestRunScanInterrupted(t *testing.T) {
}
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
false, nil)
false)
if err != nil {
t.Fatal(err)
}
@@ -437,7 +437,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
dir := buildWalkCancelTree(t)
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
walkCancelWorkers, false, false, nil)
walkCancelWorkers, false, false)
if !errors.Is(err, errInterrupted) {
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
err, errInterrupted)
-29
View File
@@ -1,29 +0,0 @@
// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
// KiB, and of the content. The content hash covers the whole file under
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
// file is never read in full. scan maintains a persistent SQLite
// database of file signatures (SFDUPES_DATABASE, default
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes dwim
// sfdupes --version
//
// See README.md for the complete specification.
package main
import (
"os"
"sneak.berlin/go/sfdupes/internal/sfdupes"
)
func main() { os.Exit(sfdupes.Main()) }
+68 -179
View File
@@ -1,4 +1,4 @@
package sfdupes
package main
import (
"context"
@@ -47,15 +47,14 @@ const dbFilePerm = 0o644
const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
// disk_size holds the bytes the file takes on disk. mtime holds whole
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
// tail and content each hold the raw 32 bytes of a SHA-256.
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second. head, tail and content each hold the raw 32 bytes of a
// SHA-256.
const createTableSQL = `
CREATE TABLE files (
path BLOB PRIMARY KEY,
size INTEGER NOT NULL,
disk_size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL,
head BLOB NOT NULL,
@@ -70,59 +69,14 @@ const createIndexSQL = `
CREATE INDEX files_signature ON files (size, head, tail, content)
`
// createTotalsSQL creates the totals table, which holds one row: the
// number of records and the sizes of all records added up. Counting
// them over files would read every record. The three triggers below
// keep the row current, in the transaction of each change to files.
const createTotalsSQL = `
CREATE TABLE totals (
records INTEGER NOT NULL,
size INTEGER NOT NULL
)
`
// insertTotalsSQL adds the one row of totals, for an empty files table.
const insertTotalsSQL = `
INSERT INTO totals (records, size) VALUES (0, 0)
`
// totalsAfterInsertSQL counts each inserted record into totals.
const totalsAfterInsertSQL = `
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
UPDATE totals SET records = records + 1, size = size + NEW.size;
END
`
// totalsAfterDeleteSQL takes each deleted record out of totals.
const totalsAfterDeleteSQL = `
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
UPDATE totals SET records = records - 1, size = size - OLD.size;
END
`
// totalsAfterUpdateSQL replaces a record's old size in totals with its
// new one. scan's upsert of an existing record is such an update.
const totalsAfterUpdateSQL = `
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
UPDATE totals SET size = size + NEW.size - OLD.size;
END
`
// recordCountSQL reads the number of records from totals.
const recordCountSQL = `
SELECT records FROM totals
`
// upsertSQL inserts one file record, replacing any existing record for
// the same path.
const upsertSQL = `
INSERT INTO files (
path, size, disk_size, mtime, mtime_nsec, head, tail, content
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (path) DO UPDATE SET
size = excluded.size, disk_size = excluded.disk_size,
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
size = excluded.size, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
head = excluded.head, tail = excluded.tail,
content = excluded.content
`
@@ -146,44 +100,33 @@ var errHashSize = errors.New("hash is not 32 bytes")
// databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the
// path, made absolute by absolutePath, and where the path came from,
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
// prints first on stderr.
// path, with the working directory, its symlinks resolved, put in front
// of it when it is relative, and where the path came from, "default" or
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
// stderr.
func databasePath() (string, string, error) {
path, source := defaultDatabasePath, "default"
if p := os.Getenv(databaseEnv); p != "" {
path, source = p, "from "+databaseEnv
}
abs, err := absolutePath(path)
if filepath.IsAbs(path) {
return path, source, nil
}
// The working directory as the kernel reports it, with no symlink
// in it, as README "Database" says. os.Getwd may return PWD
// instead, which can name it through a symlink.
wd, err := unix.Getwd()
if err != nil {
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
}
return abs, source, nil
}
// absolutePath returns path unchanged when it is absolute, and
// otherwise the working directory, a "/" and path. The working
// directory is the one the kernel reports, with no symlink in it;
// os.Getwd may return PWD instead, which can name it through a symlink.
// The result is not cleaned, as filepath.Abs and filepath.Join would
// clean it. Cleaning drops each ".." together with the directory name
// before it. When that name is a symlink, the cleaned path names
// another file than the one the system reaches by following the symlink
// first.
func absolutePath(path string) (string, error) {
if filepath.IsAbs(path) {
return path, nil
}
wd, err := unix.Getwd()
if err != nil {
return "", err
}
// Not filepath.Abs or filepath.Join: they clean the path, which
// drops each ".." together with the directory name before it. When
// that directory is a symlink, the cleaned path names another file.
// The working directory ends in "/" only when it is the root.
return strings.TrimSuffix(wd, "/") + "/" + path, nil
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
}
// createDatabaseDirectory creates the directory the database at path
@@ -207,14 +150,9 @@ func createDatabaseDirectory(path string) error {
// scanParams are the connection parameters for scan: read-write, with
// WAL journaling and a busy timeout, so a report can run while a cron
// scan is in progress. closeScanDatabase leaves WAL mode again.
//
// Each write to files fires a trigger, so SQLite keeps a statement
// journal for it. temp_store(MEMORY) keeps that journal in memory,
// which costs far less than a temporary file.
const scanParams = "_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(WAL)" +
"&_pragma=synchronous(NORMAL)" +
"&_pragma=temp_store(MEMORY)"
"&_pragma=synchronous(NORMAL)"
// reportParams are the connection parameters for report, trees and
// info: read-only, with the same busy timeout. They set no journal
@@ -488,14 +426,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
defer func() { _ = tx.Rollback() }()
for _, stmt := range []string{
createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
} {
_, err = tx.ExecContext(ctx, stmt)
if err != nil {
return fmt.Errorf("create schema: %w", err)
}
_, err = tx.ExecContext(ctx, createTableSQL)
if err != nil {
return fmt.Errorf("create schema: %w", err)
}
_, err = tx.ExecContext(ctx, createIndexSQL)
if err != nil {
return fmt.Errorf("create schema: %w", err)
}
_, err = tx.ExecContext(ctx,
@@ -542,7 +480,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
var n int64
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
@@ -550,8 +488,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
total(n)
rows, err := tx.QueryContext(ctx,
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
"content FROM files ORDER BY path")
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
"FROM files ORDER BY path")
if err != nil {
return fmt.Errorf("read records: %w", err)
}
@@ -565,8 +503,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
r scanRec
)
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
&head, &tail, &content)
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
@@ -601,11 +538,10 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
`
// loadSignatureRows gives total the number of records, then streams
// every record's signature and path to fn, without its mtime or its
// disk_size, in the order of signatureRowsSQL. The count and the
// records are read in one transaction, so they agree while a scan is
// committing. An error from fn stops the reading and is returned as it
// is.
// every record's signature and path to fn, without its mtime, in the
// order of signatureRowsSQL. The count and the records are read in one
// transaction, so they agree while a scan is committing. An error from
// fn stops the reading and is returned as it is.
func loadSignatureRows(ctx context.Context, db *sql.DB,
total func(n int64), fn func(r scanRec) error,
) error {
@@ -620,7 +556,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
var n int64
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
@@ -686,7 +622,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return 0, fmt.Errorf("count records: %w", err)
}
@@ -694,10 +630,12 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
return n, nil
}
// databaseCountsSQL reads the number of records and the sizes of all
// records added up from totals.
// databaseCountsSQL counts the records and adds up their sizes, in one
// pass over the table. A SUM over no rows is NULL, so on an empty table
// the SUM falls back to 0.
const databaseCountsSQL = `
SELECT records, size FROM totals
SELECT COUNT(*), COALESCE(SUM(size), 0)
FROM files
`
// databaseCounts is what info reads from the database: its schema
@@ -740,15 +678,15 @@ func readDatabaseCounts(ctx context.Context,
return c, nil
}
// loadFileMeta streams every record's path, size, disk_size and mtime
// to fn. Scan change detection needs no hash values, and skipping the
// hash columns keeps the scan's in-memory index small on
// multi-million-file databases.
// loadFileMeta streams every record's path, size and mtime to fn. Scan
// change detection needs no hash values, and skipping the hash columns
// keeps the scan's in-memory index small on multi-million-file
// databases.
func loadFileMeta(ctx context.Context, db *sql.DB,
fn func(path string, m fileMeta),
fn func(path string, size int64, mtime time.Time),
) error {
rows, err := db.QueryContext(ctx,
"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
"SELECT path, size, mtime, mtime_nsec FROM files")
if err != nil {
return fmt.Errorf("read records: %w", err)
}
@@ -757,19 +695,16 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
for rows.Next() {
var (
path []byte
m fileMeta
sec, nsec int64
path []byte
size, sec, nsec int64
)
err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
err = rows.Scan(&path, &size, &sec, &nsec)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
m.mtime = time.Unix(sec, nsec)
fn(string(path), m)
fn(string(path), size, time.Unix(sec, nsec))
}
err = rows.Err()
@@ -787,21 +722,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
// WAL small and let concurrent reports observe progress.
const updateBatchSize = 10000
// applyChanges writes upserts for new and changed files and deletes for
// vanished ones, in batched transactions. Progress is rendered on prog
// (one increment per change).
// applyChanges writes one scan's database changes — upserts for new and
// changed files, deletes for vanished ones — in batched transactions.
// Progress is rendered on prog (one increment per change).
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
deletes []string, prog *progress,
) error {
for batch := range slices.Chunk(upserts, updateBatchSize) {
err := applyBatch(ctx, db, batch, nil, nil, prog)
err := applyBatch(ctx, db, batch, nil, prog)
if err != nil {
return err
}
}
for batch := range slices.Chunk(deletes, updateBatchSize) {
err := applyBatch(ctx, db, nil, nil, batch, prog)
err := applyBatch(ctx, db, nil, batch, prog)
if err != nil {
return err
}
@@ -810,26 +745,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
return nil
}
// applyDiskSizes writes the disk_size of each of recs into its record,
// in batched transactions, and leaves the rest of the record alone.
// Progress is rendered on prog (one increment per record).
func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
prog *progress,
) error {
for batch := range slices.Chunk(recs, updateBatchSize) {
err := applyBatch(ctx, db, nil, batch, nil, prog)
if err != nil {
return err
}
}
return nil
}
// applyBatch commits one batch of upserts, disk_size updates and
// deletes in a single transaction.
// applyBatch commits one batch of upserts and deletes in a single
// transaction.
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
diskSizes []fileRec, deletes []string, prog *progress,
deletes []string, prog *progress,
) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
@@ -843,11 +762,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
return err
}
err = execDiskSizes(ctx, tx, diskSizes, prog)
if err != nil {
return err
}
err = execDeletes(ctx, tx, deletes, prog)
if err != nil {
return err
@@ -873,7 +787,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
defer func() { _ = st.Close() }()
for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
r.mtime.Unix(), r.mtime.Nanosecond(),
r.head[:], r.tail[:], r.content[:])
if err != nil {
@@ -886,31 +800,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
return nil
}
// execDiskSizes sets the disk_size of the record of each of recs. It
// does not touch size, so the totals triggers do not fire.
func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
prog *progress,
) error {
st, err := tx.PrepareContext(ctx,
"UPDATE files SET disk_size = ? WHERE path = ?")
if err != nil {
return fmt.Errorf("prepare disk_size update: %w", err)
}
defer func() { _ = st.Close() }()
for _, r := range recs {
_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
if err != nil {
return fmt.Errorf("update disk_size of %s: %w", r.path, err)
}
prog.increment()
}
return nil
}
// execDeletes removes the records for paths no longer present.
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
prog *progress,
+11 -158
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // calls the unexported database openers
package main
import (
"context"
@@ -535,12 +535,17 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
}
}
// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
// PLAN for query.
func queryPlan(t *testing.T, db *sql.DB, query string) []string {
t.Helper()
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
t.Parallel()
rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
db := openTestDB(t)
rows, err := db.QueryContext(t.Context(),
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
if err != nil {
t.Fatal(err)
}
@@ -568,164 +573,12 @@ func queryPlan(t *testing.T, db *sql.DB, query string) []string {
t.Fatal(err)
}
return plan
}
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
t.Parallel()
db := openTestDB(t)
plan := queryPlan(t, db, signatureRowsSQL)
want := []string{"SCAN files USING COVERING INDEX files_signature"}
if !slices.Equal(plan, want) {
t.Errorf("query plan = %q, want %q", plan, want)
}
}
// TestCountsReadOnlyTotals checks that SQLite answers the queries for
// the number of records and their total size from the one row of
// totals. A plan that reads files or its index reads every record.
func TestCountsReadOnlyTotals(t *testing.T) {
t.Parallel()
db := openTestDB(t)
for _, query := range []string{recordCountSQL, databaseCountsSQL} {
plan := queryPlan(t, db, query)
want := []string{"SCAN totals"}
if !slices.Equal(plan, want) {
t.Errorf("query plan of %q = %q, want %q", query, plan, want)
}
}
}
// TestCountsComeFromTotals sets the row of totals to numbers that files
// does not hold, then checks that scan's load phase, the read phases of
// trees and report, and info each report the numbers from totals.
func TestCountsComeFromTotals(t *testing.T) {
t.Parallel()
// files is empty, so a count over files gives 0 records of 0 bytes.
const records, size = 7, 9000
db := openTestDB(t)
_, err := db.ExecContext(t.Context(),
"UPDATE totals SET records = ?, size = ?", records, size)
if err != nil {
t.Fatal(err)
}
scanTotal, err := countRecords(t.Context(), db)
if err != nil {
t.Fatal(err)
}
var treesTotal, reportTotal int64
err = loadFileRows(t.Context(), db,
func(n int64) { treesTotal = n }, func(scanRec) {})
if err != nil {
t.Fatal(err)
}
err = loadSignatureRows(t.Context(), db,
func(n int64) { reportTotal = n },
func(scanRec) error { return nil })
if err != nil {
t.Fatal(err)
}
info, err := readDatabaseCounts(t.Context(), db)
if err != nil {
t.Fatal(err)
}
for name, got := range map[string]int64{
"scan's load": scanTotal,
"trees' read": treesTotal,
"report's read": reportTotal,
"info": info.records,
} {
if got != records {
t.Errorf("%s: %d records, want %d from totals",
name, got, records)
}
}
if info.size != size {
t.Errorf("info: size %d, want %d from totals", info.size, size)
}
}
// assertTotalsMatchFiles fails unless the row of totals holds the
// number of records and the sizes of all records added up, as counted
// over files.
func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
t.Helper()
got, err := readDatabaseCounts(t.Context(), db)
if err != nil {
t.Fatal(err)
}
var records, size int64
err = db.QueryRowContext(t.Context(),
"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
Scan(&records, &size)
if err != nil {
t.Fatal(err)
}
if got.records != records || got.size != size {
t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
got.records, got.size, records, size)
}
}
// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
// change size, and rescans it after two files are removed. After each
// scan the row of totals must agree with files.
func TestTotalsMatchFiles(t *testing.T) {
t.Parallel()
dir := buildSmokeTree(t)
db := openTestDB(t)
syncTree(t, db, dir)
assertTotalsMatchFiles(t, db)
writeFile(t, dir, "a/one.bin", pattern(10, 2500))
writeFile(t, dir, "t1/f1", pattern(30, 1000))
if st := syncTree(t, db, dir); st.updated != 2 {
t.Fatalf("rescan stats = %+v, want 2 updated", st)
}
assertTotalsMatchFiles(t, db)
for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
err := os.Remove(filepath.Join(dir, name))
if err != nil {
t.Fatal(err)
}
}
if st := syncTree(t, db, dir); st.removed != 2 {
t.Fatalf("rescan stats = %+v, want 2 removed", st)
}
assertTotalsMatchFiles(t, db)
}
func TestApplyChangesRoundTrip(t *testing.T) {
t.Parallel()
+12 -104
View File
@@ -1,9 +1,8 @@
package sfdupes
package main
// This file holds three rules for the walk (README "Rules for the
// walk"): the paths given with --exclude, which the scan never enters;
// This file holds two rules for the walk (README "Rules for the walk"):
// the default excludes, the paths a scan's walk does not enter below an
// operand unless scan is given --no-default-excludes; and the
// operand unless scan is given --no-default-excludes, and the
// filesystems that are not persistent, on which the scan never records
// or reads anything.
@@ -13,7 +12,6 @@ import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
@@ -45,25 +43,17 @@ const typesPerHelpLine = 6
// readMountInfo reads.
var errMountInfoLine = errors.New("unexpected line")
// errExcludedOperand reports an operand at or below a path given with
// --exclude.
var errExcludedOperand = errors.New("operand excluded by --exclude")
// walkRules holds the rules for the walk that a scan sets up as it
// starts. excludePaths are the paths given with --exclude, each with
// the device ID and inode number of the file it names (see
// excludePathInodes), which the walk neither enters nor records,
// whatever path leads to them. defaultExcludes are the default
// excludes, or none with --no-default-excludes. nonPersistent maps the
// device ID of every mounted filesystem that is not persistent to its
// type; the scan records and reads nothing on one of those, whatever
// the flags. databaseFiles are the scan's own database files, which the
// walk passes over (see databaseFiles in db.go).
// starts. excludes are the default excludes, or none with
// --no-default-excludes. nonPersistent maps the device ID of every
// mounted filesystem that is not persistent to its type; the scan
// records and reads nothing on one of those, whatever the flags.
// databaseFiles are the scan's own database files, which the walk
// passes over (see databaseFiles in db.go).
type walkRules struct {
excludePaths []fileRec
defaultExcludes []string
nonPersistent map[uint64]string
databaseFiles []fileRec
excludes []string
nonPersistent map[uint64]string
databaseFiles []fileRec
}
// defaultExcludes returns the default excludes of the system goos
@@ -107,88 +97,6 @@ func isDefaultExclude(path string, excludes []string) bool {
return false
}
// excludePathInodes returns each of paths, the --exclude values, as its
// real path (see realPaths) with the device ID and inode number of the
// file there. The walk compares what it reaches with those, as it does
// with the database files, so it also leaves out an --exclude path
// spelled another way, such as in another letter case on macOS.
func excludePathInodes(paths []string) ([]fileRec, error) {
resolved, err := realPaths(paths)
if err != nil {
return nil, err
}
files := make([]fileRec, 0, len(resolved))
for _, path := range resolved {
fi, err := os.Lstat(path)
if err != nil {
return nil, err
}
dev, ino := inodeOfInfo(fi)
files = append(files, fileRec{path: path, dev: dev, ino: ino})
}
return files, nil
}
// matchingExcludePath returns the path of the one of excludePaths with
// the device ID and inode number of fi, an lstat result, or "" when
// there is none.
func matchingExcludePath(fi fs.FileInfo, excludePaths []fileRec) string {
dev, ino := inodeOfInfo(fi)
f := fileRec{dev: dev, ino: ino}
for _, exclude := range excludePaths {
if sameInode(exclude, f) {
return exclude.path
}
}
return ""
}
// checkOperandsNotExcluded returns an error when one of roots, the
// operands, or a directory above it is one of excludePaths, the
// --exclude paths, by device ID and inode number. The error names the
// operand and the --exclude path. The roots are absolute, so the
// directories above each one end at /. A path here that lstat cannot
// read is an error too.
func checkOperandsNotExcluded(roots []string, excludePaths []fileRec) error {
for _, root := range roots {
for path := root; ; path = filepath.Dir(path) {
fi, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("scan %s: %w", root, err)
}
exclude := matchingExcludePath(fi, excludePaths)
if exclude != "" {
return fmt.Errorf("scan %s: %w %s", root, errExcludedOperand,
exclude)
}
if path == "/" {
break
}
}
}
return nil
}
// excludePathLine returns the line the scan prints for path when fi,
// its lstat, makes it one of rules.excludePaths, and "" when it does
// not.
func (rules walkRules) excludePathLine(path string, fi fs.FileInfo) string {
if matchingExcludePath(fi, rules.excludePaths) == "" {
return ""
}
return fmt.Sprintf("scan: skipped %s (--exclude)", path)
}
// nonPersistentFilesystemTypes returns the filesystem types that are
// not persistent: filesystems that the kernel makes up or that are kept
// in memory. devfs is the macOS filesystem of /dev; the others are
@@ -1,6 +1,6 @@
//go:build darwin
package sfdupes
package main
import (
"slices"
@@ -1,6 +1,6 @@
//go:build !darwin
package sfdupes
package main
import (
"fmt"
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes
package main
import (
"bytes"
@@ -14,9 +14,6 @@ import (
"golang.org/x/sys/unix"
)
// excludeFlag is the scan flag that names a path the walk never enters.
const excludeFlag = "--exclude"
func TestDefaultExcludes(t *testing.T) {
t.Parallel()
@@ -191,7 +188,7 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
unseen := writeFile(t, excluded, "new", pattern(2, 10))
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
walkRules{defaultExcludes: []string{excluded}})
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
@@ -237,7 +234,7 @@ func TestScanWalksOperandInDefaultExclude(t *testing.T) {
db := openTestDB(t)
st, err := syncScan(t.Context(), db, roots, 1, false,
walkRules{defaultExcludes: []string{excluded}})
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
@@ -265,7 +262,7 @@ func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
f := writeFile(t, excluded, "f", pattern(1, 10))
st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
walkRules{defaultExcludes: []string{excluded}})
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
@@ -289,7 +286,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
dir := t.TempDir()
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
t.Skip(dir + " is not inside a default exclude")
}
@@ -359,217 +356,6 @@ func TestRunScanNoDefaultExcludes(t *testing.T) {
assertRecorded(t, path, f, true)
}
// TestRunScanExcludeNested scans a tree through the command with an
// --exclude path two levels below the operand. The walk records nothing
// at or below it and says so. The record an earlier scan left there is
// kept although its file is gone.
func TestRunScanExcludeNested(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
excluded := filepath.Join(dir, "a", "b")
kept := writeFile(t, dir, "a/f", pattern(1, 10))
gone := writeFile(t, excluded, "gone", pattern(2, 10))
scanOK(t, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
unseen := writeFile(t, excluded, "c/new", pattern(3, 10))
stderr := scanOK(t, excludeFlag, excluded, dir)
line := "scan: skipped " + excluded + " (--exclude)\n"
if !strings.Contains(stderr, line) {
t.Errorf("stderr = %q, want %q", stderr, line)
}
assertRecorded(t, path, kept, true)
assertRecorded(t, path, gone, true)
assertRecorded(t, path, unseen, false)
}
// TestRunScanExcludeRepeated gives --exclude twice, once for a
// directory and once for a file, the file through a symlink to it. The
// walk records neither, and records the file beside them.
func TestRunScanExcludeRepeated(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
inDir := writeFile(t, dir, "skip/f", pattern(1, 10))
file := writeFile(t, dir, "skip-file", pattern(2, 10))
kept := writeFile(t, dir, "keep", pattern(3, 10))
link := filepath.Join(t.TempDir(), "link")
symlink(t, file, link)
scanOK(t, excludeFlag, filepath.Dir(inDir), excludeFlag, link, dir)
assertRecorded(t, path, inDir, false)
assertRecorded(t, path, file, false)
assertRecorded(t, path, kept, true)
}
// TestRunScanExcludeReachedByAnotherPath gives --exclude a directory
// through a symlink and a file through "..", so the walk reaches
// neither by the path given. The walk also reaches the file through a
// hard link. It records none of these, prints a line naming each, and
// records the file beside them.
func TestRunScanExcludeReachedByAnotherPath(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
inDir := writeFile(t, dir, "a/cache/f", pattern(1, 10))
file := writeFile(t, dir, "a/big", pattern(2, 10))
kept := writeFile(t, dir, "keep", pattern(3, 10))
hardLink := filepath.Join(dir, "big-link")
err := os.Link(file, hardLink)
if err != nil {
t.Fatal(err)
}
link := filepath.Join(t.TempDir(), "link")
symlink(t, filepath.Join(dir, "a"), link)
// Joined as text, since filepath.Join would drop the "..".
dotDot := dir + "/a/cache/../big"
stderr := scanOK(t, excludeFlag, filepath.Join(link, "cache"),
excludeFlag, dotDot, dir)
for _, skipped := range []string{filepath.Dir(inDir), file, hardLink} {
line := "scan: skipped " + skipped + " (--exclude)\n"
if !strings.Contains(stderr, line) {
t.Errorf("stderr = %q, want %q", stderr, line)
}
}
assertRecorded(t, path, inDir, false)
assertRecorded(t, path, file, false)
assertRecorded(t, path, hardLink, false)
assertRecorded(t, path, kept, true)
}
// TestRunScanExcludedOperand checks that an operand at or below an
// --exclude path is a fatal error that names both, and so is an
// --exclude path that does not exist. An operand that is a hard link to
// an --exclude file is at it. Neither scan writes a database.
func TestRunScanExcludedOperand(t *testing.T) {
dir := realTempDir(t)
sub := filepath.Join(dir, "sub")
f := writeFile(t, sub, "f", pattern(1, 10))
hardLink := filepath.Join(dir, "f-link")
err := os.Link(f, hardLink)
if err != nil {
t.Fatal(err)
}
cases := map[string]struct {
args []string
want string
}{
"at it": {
[]string{excludeFlag, sub, sub},
"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
},
"below it": {
[]string{excludeFlag, dir, sub},
"scan " + sub + ": operand excluded by --exclude " + dir + "\n",
},
"after an operand it does not exclude": {
[]string{dir, excludeFlag, sub, sub},
"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
},
"a hard link to it": {
[]string{excludeFlag, f, hardLink},
"scan " + hardLink + ": operand excluded by --exclude " + f + "\n",
},
"exclude missing": {
[]string{excludeFlag, filepath.Join(dir, "nope"), dir},
"--exclude: resolve " + filepath.Join(dir, "nope") + ": ",
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
stdout := captureStdout(t)
stderr := captureStderr(t)
code := run(append([]string{cmdScan}, tc.args...), os.Stdout,
os.Stderr)
if code != exitFatal {
t.Errorf("run(scan %q) = %d, want %d", tc.args, code,
exitFatal)
}
assertFatalOutput(t, stderr(), stdout())
if !strings.Contains(stderr(), "sfdupes: "+tc.want) {
t.Errorf("stderr = %q, want %q", stderr(), tc.want)
}
_, err := os.Lstat(path)
if !errors.Is(err, os.ErrNotExist) {
t.Errorf("lstat %s: %v, want no database", path, err)
}
})
}
}
// TestRunScanRejectsEmptyExclude checks that an empty --exclude value,
// given as its own argument, after "=" or after another --exclude, is
// a usage error reported in one line before anything is scanned
// (README "Settings").
func TestRunScanRejectsEmptyExclude(t *testing.T) {
dir := t.TempDir()
cases := map[string][]string{
"separate": {excludeFlag, "", dir},
"joined by =": {excludeFlag + "=", dir},
"after a path": {excludeFlag, t.TempDir(), excludeFlag, "", dir},
}
for name, args := range cases {
t.Run(name, func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
var stdout, stderr bytes.Buffer
code := run(append([]string{cmdScan}, args...), &stdout,
&stderr)
if code != exitUsage {
t.Errorf("run(scan %q) = %d, want %d", args, code,
exitUsage)
}
want := "Error: --exclude must not be empty\n"
if stdout.String() != "" || stderr.String() != want {
t.Errorf("stdout, stderr = %q, %q, want \"\", %q",
stdout.String(), stderr.String(), want)
}
_, err := os.Lstat(path)
if !errors.Is(err, os.ErrNotExist) {
t.Errorf("lstat %s: %v, want no database", path, err)
}
})
}
}
// TestRunScanSkipsNonPersistentOperand scans, through the command, a
// directory on a filesystem that is not persistent, with and without
// --no-default-excludes: either way it is skipped, with a line saying
+7 -7
View File
@@ -1,4 +1,4 @@
package sfdupes
package main
import (
"context"
@@ -26,11 +26,11 @@ func infoPhase(name string) string {
// it. Any database problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and count (read the number of
// records and their total size from the totals table). Each display
// ends before the next phase starts, and before an error is returned
// to be printed. The details are written once the last display has
// ended, so no display is drawn while they are written.
// own on stderr: open (open the database) and count (count the records,
// with one query). Each display ends before the next phase starts, and
// before an error is returned to be printed. The details are written
// once the last display has ended, so no display is drawn while they
// are written.
func runInfo(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
defer func() { _ = db.Close() }()
// One row holds both counts, so the phase never has a total.
// One query counts everything, so the phase never has a total.
prog = newProgress(progressParams{
label: infoPhase("count"), total: -1,
waitingFor: countingRecords,
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec
package main
import (
"fmt"
-38
View File
@@ -1,38 +0,0 @@
package sfdupes
import (
"context"
"fmt"
"io"
"os"
"runtime"
)
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
// root" runs with no flags, then what "sfdupes trees" runs, and ends
// with a line on stderr saying how to print the report again. The dwim
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
// does, but only during the scan. An error from the scan, an interrupt
// included, ends it before the report. During the report, a signal
// ends it at once, as it ends trees.
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
scanCtx, stop := interruptContext(ctx)
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false,
nil)
stop()
if err != nil {
return err
}
err = runTrees(ctx, stdout)
if err != nil {
return err
}
fmt.Fprintln(os.Stderr,
`dwim: run "sfdupes trees" to print this report again`)
return nil
}
-83
View File
@@ -1,83 +0,0 @@
package sfdupes //nolint:testpackage // calls the unexported runDwim
import (
"bytes"
"path/filepath"
"strings"
"testing"
)
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
// dwim scans its root, then prints the trees report, the larger of
// two duplicate-tree pairs first, and ends by saying how to print
// the report again.
t.Setenv(databaseEnv, testDBPath(t))
root := realTempDir(t)
small := []string{
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
}
large := []string{
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
}
var stdout bytes.Buffer
stderr := captureStderr(t)
err := runDwim(t.Context(), root, &stdout)
if err != nil {
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
}
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
if len(lines) != 3 {
t.Fatalf("stdout = %q, want a header and two groups",
stdout.String())
}
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
if !strings.HasPrefix(lines[1], wantFirst) {
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
}
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
if !strings.HasPrefix(lines[2], wantSecond) {
t.Errorf("second group = %q, want it to start %q",
lines[2], wantSecond)
}
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
if got := stderr(); !strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to end %q", got, last)
}
}
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
// A fatal scan error ends dwim before the report: a root that does
// not exist fails the scan as it fails scan itself.
t.Setenv(databaseEnv, testDBPath(t))
missing := filepath.Join(realTempDir(t), "missing")
var stdout bytes.Buffer
stderr := captureStderr(t)
err := runDwim(t.Context(), missing, &stdout)
if err == nil {
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
}
if got := stdout.String(); got != "" {
t.Errorf("stdout = %q, want no report", got)
}
if got := stderr(); strings.Contains(got, "trees:") ||
strings.Contains(got, "dwim:") {
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
}
}
+35 -62
View File
@@ -1,6 +1,23 @@
// Package sfdupes implements the sfdupes command and the database its
// subcommands share.
package sfdupes
// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
// KiB, and of the content. The content hash covers the whole file under
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
// file is never read in full. scan maintains a persistent SQLite
// database of file signatures (SFDUPES_DATABASE, default
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes --version
//
// See README.md for the complete specification.
package main
import (
"context"
@@ -10,7 +27,6 @@ import (
"os"
"runtime"
"runtime/debug"
"slices"
"github.com/spf13/cobra"
)
@@ -29,7 +45,6 @@ const (
cmdReport = "report"
cmdTrees = "trees"
cmdInfo = "info"
cmdDwim = "dwim"
)
// errNoSubcommand is returned by the root command when it is invoked
@@ -41,11 +56,6 @@ var errNoSubcommand = errors.New("no subcommand")
// below 1.
var errWorkersBelowOne = errors.New("--workers must be at least 1")
// errEmptyExcludeValue is the usage error for an empty scan --exclude
// value, as from --exclude "$DIR" with DIR unset. Taken as a path, it
// would name the working directory.
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
// Version is the build version, injected at link time via -ldflags
// (see the Makefile); empty when the linker set none, as for a binary
// built by go install. buildVersion turns it into what --version prints.
@@ -74,22 +84,20 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
return info.Main.Version
}
// Main runs the command line in os.Args and returns the process exit
// code.
func Main() int {
func main() {
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
// head", the Go runtime ends the process with SIGPIPE on the next
// write instead of returning an error (README "Error handling").
// Registering for SIGPIPE with os/signal would change that.
return run(os.Args[1:], os.Stdout, os.Stderr)
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
}
// run executes args against the command tree and returns the process
// exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees, info and dwim
// subcommands write their data to stdout.
// runs before the process ends. The report, trees and info subcommands
// write their data to stdout.
func run(args []string, stdout, stderr io.Writer) int {
// A nil slice makes cobra fall back to os.Args, which would let a
// test binary's own flags reach the command tree.
@@ -193,57 +201,38 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
}),
}
dwimCmd := &cobra.Command{
Use: cmdDwim,
Short: "Scan / with the default excludes, then print the duplicate-tree report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runDwim(ctx, "/", stdout)
}),
}
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
return root
}
// newScanCommand builds the scan subcommand, with its --workers, -x,
// --no-default-excludes and --exclude flags. Its help lists the default
// excludes of the system it runs on, and the filesystem types it never
// records anything on.
// newScanCommand builds the scan subcommand, with its --workers, -x and
// --no-default-excludes flags. Its help lists the default excludes of
// the system it runs on, and the filesystem types it never records
// anything on.
func newScanCommand() *cobra.Command {
var (
scanWorkers int
scanOneFS bool
scanNoDefaultExcludes bool
scanExclude []string
)
short := "Walk trees and synchronize the scan database"
scanCmd := &cobra.Command{
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] " +
"[--exclude PATH]... PATH...",
// The Use line already names every flag, so cobra must not
// append its generic "[flags]" to it.
DisableFlagsInUseLine: true,
Short: short,
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
Args: cobra.MinimumNArgs(1),
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
Short: short,
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, _ []string) error {
err := checkScanWorkers(cmd, scanWorkers)
if err != nil {
return err
}
return rejectEmptyExcludeValue(cmd, scanExclude)
return checkScanWorkers(cmd, scanWorkers)
},
RunE: runE(func(ctx context.Context, args []string) error {
ctx, stop := interruptContext(ctx)
defer stop()
return runScan(ctx, args, scanWorkers, scanOneFS,
scanNoDefaultExcludes, scanExclude)
scanNoDefaultExcludes)
}),
}
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
@@ -252,10 +241,6 @@ func newScanCommand() *cobra.Command {
"do not cross filesystem boundaries")
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
false, "enter the default excludes, the paths listed above")
// A string array, not a slice, so a comma in a path is not a
// separator.
scanCmd.Flags().StringArrayVar(&scanExclude, "exclude", nil,
"never enter or record `PATH` or anything below it; repeatable")
return scanCmd
}
@@ -273,18 +258,6 @@ func checkScanWorkers(cmd *cobra.Command, workers int) error {
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
}
// rejectEmptyExcludeValue rejects an empty scan --exclude value. That
// is a usage error reported in one line, as for --workers below 1.
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
if !slices.Contains(excludePaths, "") {
return nil
}
cmd.SilenceUsage = true
return errEmptyExcludeValue
}
// runE adapts a subcommand implementation, or the version print, to
// cobra's RunE. Cobra prints the error and the command's usage text for
// every error RunE returns, but a subcommand that ran and failed has no
+68 -339
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported run
package main
import (
"bytes"
@@ -14,8 +14,6 @@ import (
"strconv"
"strings"
"testing"
"golang.org/x/sys/unix"
)
// usageMarker is the first line of cobra's usage text, which a usage
@@ -182,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
}
// brokenDatabase writes a database that opens cleanly and passes the
// schema-version check but has no tables, so the first query fails
// with the database already open: a fatal error on a path that owns an
// open database. It closes the database the way scan does.
// schema-version check but has no files table, so the first query
// fails with the database already open: a fatal error on a path that
// owns an open database. It closes the database the way scan does.
func brokenDatabase(t *testing.T) string {
t.Helper()
@@ -267,9 +265,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
assertFatalOutput(t, stderr.String(), stdout())
// Proof that the failure happened after the open: only a
// query against the opened database can report this. Each
// subcommand's first query reads totals.
if !strings.Contains(stderr.String(), "no such table: totals") {
// query against the opened database can report this.
if !strings.Contains(stderr.String(), "no such table: files") {
t.Errorf("stderr = %q, want the failure to come from a "+
"query on the open database", stderr.String())
}
@@ -570,8 +567,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
}{
cmdScan: {
run: func() error {
return runScan(t.Context(), []string{missing}, 1, false, false,
nil)
return runScan(t.Context(), []string{missing}, 1, false, false)
},
want: scanWant,
},
@@ -623,7 +619,6 @@ func TestRunUsageErrors(t *testing.T) {
"report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
"unknown subcommand": {[]string{"nope"}, "unknown command"},
}
@@ -713,45 +708,6 @@ func TestRunHelp(t *testing.T) {
args, got)
}
}
// The scan help lists --exclude with its argument.
var stdout, stderr bytes.Buffer
run([]string{cmdScan, "-h"}, &stdout, &stderr)
if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
t.Errorf("scan -h stderr = %q, want %s PATH", stderr.String(),
excludeFlag)
}
}
func TestRunScanUsageLine(t *testing.T) {
t.Parallel()
// README §Subcommands shows this usage line for scan, and the help
// and the usage error both print it.
const want = "\n sfdupes scan [--workers N] [-x] [--no-default-excludes] " +
"[--exclude PATH]... PATH...\n"
cases := map[string][]string{
"help": {cmdScan, "-h"},
"missing paths": {cmdScan},
}
for name, args := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
var stdout, stderr bytes.Buffer
run(args, &stdout, &stderr)
if !strings.Contains(stderr.String(), want) {
t.Errorf("run(%v) stderr = %q, want the usage line %q",
args, stderr.String(), want)
}
})
}
}
func TestRunVersion(t *testing.T) {
@@ -840,7 +796,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
func scanFixture(t *testing.T) []string {
t.Helper()
dir := realTempDir(t)
dir := t.TempDir()
dupes := []string{
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
@@ -861,23 +817,6 @@ func scanFixture(t *testing.T) []string {
return dupes
}
// storedDiskSize returns the disk_size of the record for path in the
// database at dbPath. A test of a trees run compares with this rather
// than with a new lstat, which on ZFS can differ from the scan's (see
// diskSizeOf).
func storedDiskSize(t *testing.T, dbPath, path string) int64 {
t.Helper()
db, err := openReportDatabase(t.Context(), dbPath)
if err != nil {
t.Fatal(err)
}
defer func() { _ = db.Close() }()
return recordByPath(t, dbRecords(t, db), path).diskSize
}
// scanOK runs scan over operands, fails the test unless it exits 0 with
// nothing on stdout, and returns everything it printed to stderr.
func scanOK(t *testing.T, operands ...string) string {
@@ -910,7 +849,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
dir := t.TempDir()
writeFile(t, dir, "a.bin", pattern(1, 300))
// Same size as a.bin, so the scan reads it, and the read fails.
@@ -938,278 +877,58 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
assertNoSidecars(t, path)
}
// realTempDir returns a new temporary directory by its real path, so
// tests hold where the temporary directory is reached through a
// symlink.
func realTempDir(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return dir
}
// symlink creates link pointing at target.
func symlink(t *testing.T, target, link string) {
t.Helper()
err := os.Symlink(target, link)
if err != nil {
t.Fatal(err)
}
}
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
// The operand is scanned at its real path, and its file is recorded
// there.
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
dir := t.TempDir()
writeFile(t, dir, "target/sub/f", pattern(1, 10))
link := filepath.Join(dir, "link")
symlink(t, filepath.Join(dir, "target"), link)
stderr := scanOK(t, link)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
err := os.Symlink(filepath.Join(dir, "target"), link)
if err != nil {
t.Fatal(err)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
// Scanning a directory through the symlink stores a record beneath
// the symlink's own path for a file beneath its target.
scanOK(t, filepath.Join(link, "sub"))
assertOperandSkipped(t, path, link, "symlink",
filepath.Join(link, "sub", "f"))
}
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
// with link a symlink to target. Both resolve under target, so
// target/sub is walked once, as part of target.
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
dir := t.TempDir()
writeFile(t, dir, "target/sub/f", pattern(1, 10))
link := filepath.Join(dir, "link")
symlink(t, filepath.Join(dir, "target"), link)
stderr := scanOK(t, link, filepath.Join(link, "sub"))
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
// with dir/link a symlink to a directory outside dir. The walk of dir
// does not follow link, so dir/link/sub is walked on its own at its
// real path, and the record an earlier scan stored there survives.
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
a := writeFile(t, dir, "a", pattern(1, 10))
outside := realTempDir(t)
f := writeFile(t, outside, "sub/f", pattern(2, 10))
symlink(t, outside, filepath.Join(dir, "link"))
scanOK(t, filepath.Join(outside, "sub"))
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
"1 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
want := []string{a, f}
slices.Sort(want)
if got := storedPaths(t, path); !slices.Equal(got, want) {
t.Errorf("records = %q, want %q", got, want)
}
}
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
// link, outside dir, a symlink to dir/real. link resolves inside dir,
// so it is dropped as covered by dir, and its file is recorded once.
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "real/f", pattern(1, 10))
link := filepath.Join(realTempDir(t), "link")
symlink(t, filepath.Join(dir, "real"), link)
stderr := scanOK(t, dir, link)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
// a symlink whose target does not exist. The scan fails with exit 1 and
// an error naming the symlink as given, before it walks anything, so no
// database is created.
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
writeFile(t, dir, "f", pattern(1, 10))
dangling := filepath.Join(realTempDir(t), "dangling")
symlink(t, filepath.Join(dir, "missing"), dangling)
stdout := captureStdout(t)
var stderr bytes.Buffer
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
if code != exitFatal {
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
exitFatal)
}
assertFatalOutput(t, stderr.String(), stdout())
want := "sfdupes: resolve " + dangling + ": "
if !strings.Contains(stderr.String(), want) {
t.Errorf("stderr = %q, want %q", stderr.String(), want)
}
_, err := os.Stat(path)
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("database: stat error %v, want it missing", err)
}
}
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
// after a cd through home/proj, a symlink to data/proj. The file is
// recorded under data/proj, its real path.
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
err := os.Symlink(filepath.Join(dir, "target"), link)
if err != nil {
t.Fatal(err)
}
symlink(t, filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
// link is dropped as a symlink, but link/sub must still be scanned,
// not dropped as lying under link.
scanOK(t, link, filepath.Join(link, "sub"))
// t.Chdir also sets PWD to the path through the symlink, as a
// shell does after cd.
t.Chdir(filepath.Join(dir, "home", "proj"))
scanOK(t, ".")
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
// a cd through home/proj. As with realpath, the ".." counts from
// data/proj, so data is scanned and home is not.
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
for _, tc := range []struct{ wd, operand string }{
{"", "home/proj/.."},
{"home/proj", ".."},
} {
t.Run(tc.operand, func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
g := writeFile(t, dir, "data/g", pattern(2, 10))
writeFile(t, dir, "home/h", pattern(3, 10))
symlink(t, filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
// t.Chdir also sets PWD to the path given, as a shell does
// after cd.
t.Chdir(filepath.Join(dir, tc.wd))
scanOK(t, tc.operand)
want := []string{f, g}
slices.Sort(want)
if got := storedPaths(t, path); !slices.Equal(got, want) {
t.Errorf("records = %q, want %q", got, want)
}
})
}
}
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
// The scan warns about the FIFO by its real path, counts it as skipped,
// and exits 0.
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
dir := realTempDir(t)
fifo := filepath.Join(dir, "fifo")
err := unix.Mkfifo(fifo, 0o600)
db, err := openDB(path, reportParams)
if err != nil {
t.Fatal(err)
}
link := filepath.Join(dir, "link")
symlink(t, fifo, link)
t.Cleanup(func() { _ = db.Close() })
stderr := scanOK(t, link)
warning := "walk " + fifo + ": skipping FIFO operand\n"
if !strings.Contains(stderr, warning) {
t.Errorf("stderr = %q, want %q", stderr, warning)
}
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
}
func TestRunScanSkipsZFSOperand(t *testing.T) {
zfs := filepath.Join(realTempDir(t), ".zfs")
zfs := filepath.Join(t.TempDir(), ".zfs")
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
// No scan records a file inside a .zfs directory, so the record
@@ -1260,7 +979,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
dir := t.TempDir()
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
writeFile(t, hourly, "f", pattern(1, 10))
f := writeFile(t, dir, "f", pattern(2, 10))
@@ -1304,19 +1023,23 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
// by its real path, counted as skipped, and not walked.
// by the path given, counted as skipped, and not walked.
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
dir := realTempDir(t)
dir := t.TempDir()
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
snaps := filepath.Join(dir, "snaps")
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
stderr := scanOK(t, filepath.Join(snaps, "daily"))
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(snaps, "daily")
stderr := scanOK(t, daily)
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n"
@@ -1328,19 +1051,24 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
}
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
// a symlink to pool/data. The operand is scanned as usual, and its file
// is recorded by its real path.
// a symlink to a directory outside any .zfs directory. The operand is
// scanned as usual, and its file is recorded by the path given.
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
dir := t.TempDir()
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
data := filepath.Join(dir, "data")
symlink(t, filepath.Join(dir, "pool", "data"), data)
stderr := scanOK(t, filepath.Join(data, "daily"))
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(data, "daily")
stderr := scanOK(t, daily)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
@@ -1348,9 +1076,14 @@ func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
db, err := openDB(path, reportParams)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
}
func TestRunReportSucceeds(t *testing.T) {
@@ -1407,9 +1140,8 @@ func TestRunTreesSucceeds(t *testing.T) {
// The two directories holding the duplicate pair are duplicate
// trees of each other.
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
want := "first\tdupe\tfiles\tsize\n" +
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
if got := stdout.String(); got != want {
t.Errorf("stdout = %q, want %q", got, want)
}
@@ -1443,10 +1175,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
cases := map[string]string{
cmdReport: "first\tdupe\tsize\n" +
dupes[0] + "\t" + dupes[1] + "\t300\n",
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
cmdTrees: "first\tdupe\tfiles\tsize\n" +
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
"\t1\t300\t" +
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
"\t1\t300\n",
}
for name, want := range cases {
@@ -1481,11 +1212,8 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
t.Errorf("report stdout = %q, want %q", got, want)
}
diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
strconv.FormatInt(diskSize, 10) + "\n"
want = "first\tdupe\tfiles\tsize\n" +
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
if got := runStdout(t, cmdTrees); got != want {
t.Errorf("trees stdout = %q, want %q", got, want)
}
@@ -1551,8 +1279,9 @@ func holdScanLock(t *testing.T, path string) {
}
func TestRunSecondScanFails(t *testing.T) {
// README §Database: a second scan that finds the lock held fails
// at once, naming the lock file, without opening the database.
// README §Database: while one scan holds the lock, a second scan
// fails at once, naming the lock file, without opening the
// database.
path := testDBPath(t)
t.Setenv(databaseEnv, path)
+2 -2
View File
@@ -1,4 +1,4 @@
package sfdupes
package main
import (
"fmt"
@@ -60,7 +60,7 @@ const fallbackWidth = 80
const spinnerMargin = 5
// countingRecords is what the display of a phase that reads the records
// says until it has the number of records, its total.
// says while it counts them, before it has its total.
const countingRecords = "counting the records"
// openPhase is the name of the phase in which report, trees and info
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported startProgress
package main
import (
"fmt"
@@ -642,9 +642,9 @@ func TestTerminalStalledLines(t *testing.T) {
{label: scanPhase("hash"), stalledOn: hashStalledOn},
{label: scanPhase("update"), stalledOn: "db"},
{label: treesPhase("read"), stalledOn: "db"},
{label: treesPhase("group"), stalledOn: "cpu/disk"},
{label: treesPhase("group"), stalledOn: "cpu"},
{label: treesPhase("write"), stalledOn: "out"},
{label: reportPhase("read"), stalledOn: "db/disk/out"},
{label: reportPhase("read"), stalledOn: "db/out"},
}
for _, params := range phases {
+30 -60
View File
@@ -1,17 +1,14 @@
package sfdupes
package main
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"io"
"io/fs"
"os"
"slices"
"strings"
"syscall"
"time"
)
@@ -24,16 +21,14 @@ const minGroupSize = 2
// scanRec is one file record from the database. The signature (size,
// head, tail, content) is the duplicate key; mtime is informational
// only and used by scan for change detection. diskSize is the bytes the
// file takes on disk, which trees adds up.
// only and used by scan for change detection.
type scanRec struct {
size int64
diskSize int64
mtime time.Time
head [sha256.Size]byte
tail [sha256.Size]byte
content [sha256.Size]byte
path string
size int64
mtime time.Time
head [sha256.Size]byte
tail [sha256.Size]byte
content [sha256.Size]byte
path string
}
// reportPhase names one phase of report with its place among all of
@@ -44,17 +39,17 @@ func reportPhase(name string) string {
}
// runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. Of the scanned filesystem it
// only checks, with pathExists, that each path in a duplicate group is
// still there. It names the database on stderr before it opens it. Any
// database problem, including a missing database, is fatal.
// duplicates report as TSV on stdout. It never touches the scanned
// filesystem; its only I/O is the database, stdout, and stderr. It
// names the database on stderr before it opens it. Any database
// problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (read the number of
// records from the totals table, then read the records in signature
// order, writing each size's duplicate groups to stdout once that size
// is read; see readReport). Each display ends before the next phase
// starts, and before an error is returned to be printed.
// own on stderr: open (open the database) and read (count the records,
// then read them in signature order, writing each size's duplicate
// groups to stdout once that size is read; see readReport). Each
// display ends before the next phase starts, and before an error is
// returned to be printed.
func runReport(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
@@ -80,14 +75,13 @@ func runReport(ctx context.Context, stdout io.Writer) error {
defer func() { _ = db.Close() }()
// The read phase checks the paths of each duplicate group on disk
// and writes the report to stdout while it reads the records, so it
// may wait on any of the three.
// The read phase writes the report to stdout while it reads the
// records, so it may wait on either.
prog = newProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db/disk/out",
waitingFor: countingRecords, stalledOn: "db/out",
})
report, err := readReport(ctx, db, dbPath, stdout, pathExists, prog)
report, err := readReport(ctx, db, dbPath, stdout, prog)
prog.finish()
@@ -108,15 +102,14 @@ func runReport(ctx context.Context, stdout io.Writer) error {
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. A path for which exists returns false is left out of its group.
// stdout may be the terminal prog is drawn on, so the lines go through
// prog's writer. It returns what it wrote, with the totals for the
// summary line.
// read. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
stdout io.Writer, exists func(path string) bool, prog *progress,
stdout io.Writer, prog *progress,
) (*reportWriter, error) {
out := prog.writer(stdout)
report := &reportWriter{out: out, exists: exists}
report := &reportWriter{out: out}
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
@@ -163,10 +156,6 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
type reportWriter struct {
out io.Writer
// exists reports whether a path is still on disk. A path that is
// gone is left out of its group.
exists func(path string) bool
sig scanRec // the signature being read; its path is not used
paths []string // the paths read so far with sig
groups [][]string // the duplicate groups of sig.size, each sorted
@@ -203,20 +192,11 @@ func (w *reportWriter) add(r scanRec) error {
return nil
}
// endGroup ends the paths read with w.sig: two or more that are still
// on disk are a duplicate group, held with its paths sorted until
// writeGroups writes it. Empty files form no group, since removing one
// frees nothing. Only the paths of a possible group are checked, never
// a path with no duplicate or an empty file.
// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
func (w *reportWriter) endGroup() {
if len(w.paths) >= minGroupSize && w.sig.size > 0 {
w.paths = slices.DeleteFunc(w.paths, func(p string) bool {
return !w.exists(p)
})
}
if len(w.paths) < minGroupSize || w.sig.size == 0 {
// The paths are dropped, and their slice reused.
if len(w.paths) < minGroupSize {
// A path with no duplicate is dropped, and its slice reused.
w.paths = w.paths[:0]
return
@@ -263,16 +243,6 @@ func (w *reportWriter) finish() error {
return w.writeGroups()
}
// pathExists reports whether path is still on disk, for report and
// trees. It is false only when lstat finds that the path, or a
// directory above it, is gone. Any other error counts as present, so
// a user who may not look inside a directory still sees its copies.
func pathExists(path string) bool {
_, err := os.Lstat(path)
return !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ENOTDIR)
}
// escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results
package main
import (
"bytes"
@@ -38,8 +38,8 @@ func awkwardPairRecs() []scanRec {
h, tl, c := testHash("h"), testHash("t"), testHash("c")
return []scanRec{
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
}
}
@@ -72,37 +72,15 @@ type dupeGroup struct {
paths []string
}
// everyPathExists stands in for pathExists where a test's records name
// made-up paths that are not on disk.
func everyPathExists(string) bool { return true }
// recsOnDisk returns a new directory and recs with each path moved
// under it, after creating an empty file at each moved path: report and
// trees then find every path still on disk.
func recsOnDisk(t *testing.T, recs []scanRec) (string, []scanRec) {
t.Helper()
dir := realTempDir(t)
moved := make([]scanRec, len(recs))
for i, r := range recs {
r.path = writeFile(t, dir, r.path, nil)
moved[i] = r
}
return dir, moved
}
// dupeGroups returns the duplicate groups report reads from db, in
// report order, from the rows it writes: a group's rows come together,
// each naming the group's first path. Every path counts as still on
// disk.
// each naming the group's first path.
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
t.Helper()
var out bytes.Buffer
_, err := readReport(t.Context(), db, "", &out, everyPathExists, nil)
_, err := readReport(t.Context(), db, "", &out, nil)
if err != nil {
t.Fatal(err)
}
@@ -149,8 +127,7 @@ func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup {
}
func TestRunReportEscapesPaths(t *testing.T) {
dir, recs := recsOnDisk(t, awkwardPairRecs())
t.Setenv(databaseEnv, seedDatabase(t, recs))
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
var stdout, stderr bytes.Buffer
@@ -161,7 +138,7 @@ func TestRunReportEscapesPaths(t *testing.T) {
}
want := "first\tdupe\tsize\n" +
dir + `/d/\tone\ntwo\rthree\\four/f` + "\t" + dir + "/d/A/f\t5\n"
`/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n"
if got := stdout.String(); got != want {
t.Errorf("stdout = %q, want %q", got, want)
}
@@ -195,8 +172,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
// Counts the records read, and draws nothing.
prog := &progress{now: time.Now}
_, err = readReport(t.Context(), db, "", failingWriter{},
everyPathExists, prog)
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
if !errors.Is(err, errWriteFailed) ||
!strings.HasPrefix(err.Error(), "write stdout: ") {
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
@@ -227,8 +203,7 @@ const smokeReport = "first\tdupe\tsize\n" +
// it ends; read counts the 6 records before it reads them. The summary
// comes last, and stdout holds the report alone.
func TestRunReportProgressNamesEachPhase(t *testing.T) {
dir, recs := recsOnDisk(t, smokeTreeRecs())
path := seedDatabase(t, recs)
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
@@ -241,9 +216,8 @@ func TestRunReportProgressNamesEachPhase(t *testing.T) {
code, exitOK, stderr())
}
report := strings.ReplaceAll(smokeReport, "/d/", dir+"/d/")
if got := stdout.String(); got != report {
t.Errorf("stdout = %q, want %q", got, report)
if got := stdout.String(); got != smokeReport {
t.Errorf("stdout = %q, want %q", got, smokeReport)
}
// How each line starts; the rate at the end of a phase varies.
@@ -292,7 +266,7 @@ func TestReadReportOnTerminal(t *testing.T) {
waitingFor: countingRecords,
}, terminalWidth, time.Hour, time.Now)
_, err = readReport(t.Context(), db, "", os.Stderr, everyPathExists, p)
_, err = readReport(t.Context(), db, "", os.Stderr, p)
p.finish()
@@ -327,8 +301,6 @@ func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
)
_, recs = recsOnDisk(t, recs)
reversed := slices.Clone(recs)
slices.Reverse(reversed)
@@ -462,140 +434,6 @@ func TestDupeGroups(t *testing.T) {
}
}
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
// duplicate group, while 1-byte files still do.
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
t.Parallel()
e, x := sha256.Sum256(nil), testHash("x")
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
}
groups := dupeGroupsOf(t, recs)
if len(groups) != 1 || groups[0].size != 1 ||
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
}
}
// TestReadReportChecksOnlyPossibleDuplicates checks that report asks
// whether a path is on disk only when its signature has two or more
// records, and never for an empty file.
func TestReadReportChecksOnlyPossibleDuplicates(t *testing.T) {
t.Parallel()
a, u, e := testHash("a"), testHash("u"), sha256.Sum256(nil)
recs := []scanRec{
{size: 9, head: a, tail: a, content: a, path: "/copy/1"},
{size: 9, head: a, tail: a, content: a, path: "/copy/2"},
{size: 9, head: a, tail: a, content: a, path: "/copy/3"},
{size: 9, head: u, tail: u, content: u, path: "/lonely"},
{size: 0, head: e, tail: e, content: e, path: "/empty1"},
{size: 0, head: e, tail: e, content: e, path: "/empty2"},
}
db := openTestDB(t)
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatal(err)
}
var checked []string
exists := func(path string) bool {
checked = append(checked, path)
return true
}
_, err = readReport(t.Context(), db, "", io.Discard, exists, nil)
if err != nil {
t.Fatal(err)
}
slices.Sort(checked)
want := []string{"/copy/1", "/copy/2", "/copy/3"}
if !slices.Equal(checked, want) {
t.Errorf("paths checked on disk = %q, want %q", checked, want)
}
}
// TestReadReportDropsGroupLeftWithOnePath checks that a group of two
// paths, one of them gone, is no duplicate group. If kept, it would
// write no row, but the summary line would count it.
func TestReadReportDropsGroupLeftWithOnePath(t *testing.T) {
t.Parallel()
a := testHash("a")
recs := []scanRec{
{size: 9, head: a, tail: a, content: a, path: "/a/kept"},
{size: 9, head: a, tail: a, content: a, path: "/a/gone"},
}
db := openTestDB(t)
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatal(err)
}
exists := func(path string) bool { return path != "/a/gone" }
report, err := readReport(t.Context(), db, "", io.Discard, exists, nil)
if err != nil {
t.Fatal(err)
}
if report.dupeGroups != 0 {
t.Errorf("duplicate groups = %d, want 0", report.dupeGroups)
}
}
// TestReadReportKeepsGroupLeftWithTwoPaths checks that a group of three
// paths, one of them gone, is a duplicate group of the other two.
func TestReadReportKeepsGroupLeftWithTwoPaths(t *testing.T) {
t.Parallel()
a := testHash("a")
recs := []scanRec{
{size: 9, head: a, tail: a, content: a, path: "/b/kept1"},
{size: 9, head: a, tail: a, content: a, path: "/b/gone"},
{size: 9, head: a, tail: a, content: a, path: "/b/kept2"},
}
db := openTestDB(t)
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatal(err)
}
exists := func(path string) bool { return path != "/b/gone" }
var out bytes.Buffer
report, err := readReport(t.Context(), db, "", &out, exists, nil)
if err != nil {
t.Fatal(err)
}
if report.dupeGroups != 1 {
t.Errorf("duplicate groups = %d, want 1", report.dupeGroups)
}
want := "first\tdupe\tsize\n/b/kept1\t/b/kept2\t9\n"
if got := out.String(); got != want {
t.Errorf("stdout = %q, want %q", got, want)
}
}
func TestDupeGroupsContentSeparates(t *testing.T) {
t.Parallel()
+126 -180
View File
@@ -1,4 +1,4 @@
package sfdupes
package main
import (
"cmp"
@@ -52,26 +52,23 @@ const workQueueDepth = 1024
// has already printed its line, so run prints nothing more.
var errInterrupted = errors.New("scan interrupted")
// fileRec carries one statted file between the scan phases. diskSize is
// the bytes the file takes on disk. dev and ino identify the underlying
// inode so hard-linked paths can share one read; both are zero when the
// platform exposes no inode.
// fileRec carries one statted file between the scan phases. dev and
// ino identify the underlying inode so hard-linked paths can share
// one read; both are zero when the platform exposes no inode.
type fileRec struct {
path string
size int64
diskSize int64
mtime time.Time
dev uint64
ino uint64
path string
size int64
mtime time.Time
dev uint64
ino uint64
}
// fileMeta is the in-memory index entry for one existing database
// record: what the walk compares the file with. Hashes stay on disk; at
// record: just enough for change detection. Hashes stay on disk; at
// tens of millions of records they would dominate the scan's memory.
type fileMeta struct {
size int64
diskSize int64
mtime time.Time
size int64
mtime time.Time
}
// scanUnit is what every scan phase counts, as its progress display
@@ -102,44 +99,40 @@ func scanPhase(name string) string {
// scan command rejects anything less. Before the lock is taken, the
// scan lists the filesystems that are not persistent, on which the walk
// records nothing; unless noDefaultExcludes is set, the walk does not
// enter the default excludes either. The walk neither enters nor
// records the paths in excludePaths, the --exclude values; an operand
// at or below one is a fatal error. Once the database is open, the scan
// notes its files, which the walk passes over.
// enter the default excludes either. Once the database is open, the
// scan notes its files, which the walk passes over.
func runScan(ctx context.Context, roots []string, workers int,
oneFS, noDefaultExcludes bool, excludePaths []string,
oneFS, noDefaultExcludes bool,
) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
printDatabasePath(dbPath, source)
// A database file missing now is created by lockScanDatabase below.
what := "database"
roots, err = realPaths(roots)
_, err = os.Stat(dbPath)
if errors.Is(err, fs.ErrNotExist) {
what = "new database"
}
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
source)
roots, err = resolveRoots(roots)
if err != nil {
return err
}
var rules walkRules
rules.excludePaths, err = excludePathInodes(excludePaths)
if err != nil {
return fmt.Errorf("--exclude: %w", err)
}
err = checkOperandsNotExcluded(roots, rules.excludePaths)
if err != nil {
return err
}
rules.nonPersistent, err = nonPersistentFilesystems()
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
return fmt.Errorf("list the mounted filesystems: %w", err)
}
rules := walkRules{nonPersistent: nonPersistent}
if !noDefaultExcludes {
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
rules.excludes = defaultExcludes(runtime.GOOS)
}
lock, err := lockScanDatabase(dbPath)
@@ -185,21 +178,6 @@ func runScan(ctx context.Context, roots []string, workers int,
return nil
}
// printDatabasePath prints on stderr the line that names the database
// at dbPath, with source, saying whether it is new. A database file
// missing now is new: lockScanDatabase creates it.
func printDatabasePath(dbPath, source string) {
what := "database"
_, err := os.Stat(dbPath)
if errors.Is(err, fs.ErrNotExist) {
what = "new database"
}
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
source)
}
// interruptContext returns a copy of ctx that the first SIGINT or
// SIGTERM cancels; the scan command runs the scan under it. stop
// releases the signals.
@@ -228,31 +206,29 @@ func interrupted(walked int) error {
return errInterrupted
}
// realPaths returns the real path of each of paths, the PATH operands
// or the --exclude values: absolute, with every symlink in it followed.
// Database records are keyed by that path, so scan results depend
// neither on the working directory nor on the symlinks an operand was
// given through. A path that cannot be resolved, because it or a
// symlink's target does not exist or cannot be read, is a fatal error
// before any scanning.
func realPaths(paths []string) ([]string, error) {
resolved := make([]string, 0, len(paths))
// resolveRoots converts each PATH operand to an absolute, lexically
// cleaned path (symlinks are not resolved) and verifies that it
// exists. Database records are keyed by absolute path, so scan results
// must not depend on the working directory.
func resolveRoots(roots []string) ([]string, error) {
abs := make([]string, 0, len(roots))
for _, path := range paths {
a, err := absolutePath(path)
for _, root := range roots {
a, err := filepath.Abs(root)
if err != nil {
return nil, fmt.Errorf("resolve %s: %w", path, err)
return nil, fmt.Errorf("resolve %s: %w", root, err)
}
r, err := filepath.EvalSymlinks(a)
// A nonexistent operand is a fatal error before any scanning.
_, err = os.Lstat(a)
if err != nil {
return nil, fmt.Errorf("resolve %s: %w", path, err)
return nil, err
}
resolved = append(resolved, r)
abs = append(abs, a)
}
return resolved, nil
return abs, nil
}
// pruneRoots drops operands already covered by another operand:
@@ -315,7 +291,7 @@ func walkReaches(above, r string, oneFS bool, rules walkRules,
}
for ; dir != above; dir = filepath.Dir(dir) {
if isDefaultExclude(dir, rules.defaultExcludes) {
if isDefaultExclude(dir, rules.excludes) {
return false
}
@@ -363,12 +339,11 @@ type scanStats struct {
// whatever remains after the walk and hash phases is deleted by the
// update phase.
type scanState struct {
db *sql.DB
existing map[string]fileMeta
toHash []fileRec // new and changed files: must be read
diskSizes []fileRec // unchanged files whose disk_size changed
batch []scanRec
st scanStats
db *sql.DB
existing map[string]fileMeta
toHash []fileRec // new and changed files: must be read
batch []scanRec
st scanStats
}
// syncScan synchronizes the database with the filesystem under roots;
@@ -438,16 +413,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
// walkableRoots returns the operands the walk can start from: regular
// files, and directories not named .zfs, none of them inside a
// directory named .zfs. The operands are real paths from realPaths,
// so these checks apply to what a symlink operand points at. Every
// other operand is warned about, counted as skipped, and dropped. A
// dropped operand is no longer a root, so the records stored beneath it
// are treated as if it had not been given: those under no other root
// count as outside the roots and are not deleted as unverified. An
// operand on a filesystem in rules that is not persistent is dropped
// the same way, with a line saying so that is not a warning, and is not
// counted as skipped. An operand that fails lstat here is kept, and the
// walk warns about it unless it no longer exists.
// directory named .zfs. Every other operand is warned about, counted as
// skipped, and dropped. A dropped operand is no longer a root, so the
// records stored beneath it are treated as if it had not been given:
// those under no other root count as outside the roots and are not
// deleted as unverified. An operand on a filesystem in rules that is not
// persistent is dropped the same way, with a line saying so that is not
// a warning, and is not counted as skipped. An operand that fails lstat
// here is kept, and the walk warns about it unless it no longer exists.
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
kept := make([]string, 0, len(roots))
@@ -482,8 +455,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
// included, so its display counts against the number of records.
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// Indexing tens of millions of records takes real time; without a
// display the scan looks hung before the walk begins. The number of
// records, read first, gives the display its total.
// display the scan looks hung before the walk begins. Counting them
// first gives the display its total.
prog := newProgress(progressParams{
label: scanPhase("load"), unit: scanUnit, total: -1,
waitingFor: countingRecords, stalledOn: "db",
@@ -499,25 +472,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
s.existing = make(map[string]fileMeta)
return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
prog.increment()
return loadFileMeta(ctx, s.db,
func(path string, size int64, mtime time.Time) {
prog.increment()
if underAnyRoot(path, roots) {
s.existing[path] = m
}
})
if underAnyRoot(path, roots) {
s.existing[path] = fileMeta{size: size, mtime: mtime}
}
})
}
// walkPhase drains the walk and queues every new or changed file for
// the hash phase. An unchanged file is not read, and its record counts
// as verified. When its disk_size differs from the record's, it is
// queued for the update phase, which writes the new disk_size. A record
// at or below a path the walk left out, because -x stopped at it, it is
// an --exclude path or a default exclude, or it is not on a persistent
// filesystem, counts as outside the roots: it is not deleted. That does
// not hold below a root at or inside that path, since that root's own
// walk verifies the records below it; nor is a line printed for such a
// root.
// the hash phase. An unchanged file needs nothing more, and its record
// counts as verified. A record at or below a path the walk left out,
// because -x stopped at it, it is a default exclude, or it is not on a
// persistent filesystem, counts as outside the roots: it is not
// deleted. That does not hold below a root at or inside that path,
// since that root's own walk verifies the records below it; nor is a
// line printed for such a root.
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
// The records the last scan left under the roots estimate how many
// files the walk will find. A first scan has none, and no estimate.
@@ -566,7 +538,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
prog.increment()
old, ok := s.existing[ev.rec.path]
if !ok || old.size != ev.rec.size || !ev.rec.mtime.Equal(old.mtime) {
if !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime) {
s.toHash = append(s.toHash, ev.rec)
continue
@@ -575,13 +547,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
delete(s.existing, ev.rec.path)
s.st.unchanged++
// The space a file takes on disk can change while its size and
// mtime stay the same. ZFS, for one, counts a new file's blocks
// up to seconds after the write.
if old.diskSize != ev.rec.diskSize {
s.diskSizes = append(s.diskSizes, ev.rec)
}
}
prog.finish()
@@ -730,13 +695,12 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
s.resolve(rec.path)
s.batch = append(s.batch, scanRec{
size: rec.size,
diskSize: rec.diskSize,
mtime: rec.mtime,
head: r.head,
tail: r.tail,
content: r.content,
path: rec.path,
size: rec.size,
mtime: rec.mtime,
head: r.head,
tail: r.tail,
content: r.content,
path: rec.path,
})
}
@@ -752,7 +716,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
return nil
}
err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
err := applyBatch(ctx, s.db, s.batch, nil, nil)
if err != nil {
return err
}
@@ -763,10 +727,9 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
}
// updatePhase writes the scan's tail under one progress display: the
// final partial batch of hashed records, the new disk_size of every
// unchanged file whose disk_size changed, and deletions for every
// record the scan did not verify (vanished files, plus paths that
// failed to stat or hash).
// final partial batch of hashed records, and deletions for every record
// the scan did not verify (vanished files, plus paths that failed to
// stat or hash).
func (s *scanState) updatePhase(ctx context.Context) error {
deletes := make([]string, 0, len(s.existing))
for path := range s.existing {
@@ -778,7 +741,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
s.st.removed = len(deletes)
total := len(s.batch) + len(s.diskSizes) + len(deletes)
total := len(s.batch) + len(deletes)
prog := newProgress(progressParams{
label: scanPhase("update"), unit: scanUnit, total: int64(total),
stalledOn: "db",
@@ -793,12 +756,17 @@ func (s *scanState) updatePhase(ctx context.Context) error {
s.batch = nil
err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
if err != nil {
return err
return applyChanges(ctx, s.db, nil, deletes, prog)
}
// mtimeAfter reports whether mtime a is later than mtime b.
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
func mtimeAfter(a, b time.Time) bool {
if a.Unix() != b.Unix() {
return a.Unix() > b.Unix()
}
return applyChanges(ctx, s.db, nil, deletes, prog)
return a.Nanosecond() > b.Nanosecond()
}
// underAnyRoot reports whether path is any of the roots or lies under
@@ -855,9 +823,9 @@ type dirJob struct {
// file's statted record, a warning when fail is set, or a path the walk
// left out: when otherFSDir is set, a directory -x did not enter
// because it is on another filesystem than its operand, and when
// leftOut is set, an --exclude path, a default exclude or a directory or
// file on a filesystem that is not persistent, with warn holding the
// line that says so.
// leftOut is set, a default exclude or a directory or file on a
// filesystem that is not persistent, with warn holding the line that
// says so.
type walkEvent struct {
rec fileRec
warn string
@@ -908,13 +876,19 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
// operandWarning returns the one-line warning for an operand the walk
// does not start from, naming the path and what it is, or "" for one it
// does: a regular file, or a directory not named .zfs, that is not
// inside a directory named .zfs. realPaths has already followed the
// operand's symlinks, so a symlink here is an operand replaced by one
// after it was resolved, and is not followed.
// inside a directory named .zfs, neither as given nor with its symlinks
// resolved. Symlinks are never followed, including as operands.
func operandWarning(root string, fi fs.FileInfo) string {
// The walk never enters a .zfs directory, so it starts from nothing
// inside one either.
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
// inside one either, not even through a symlink. An operand whose
// symlinks cannot be resolved is checked as given.
resolved, err := filepath.EvalSymlinks(root)
if err != nil {
resolved = root
}
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
root)
}
@@ -987,12 +961,11 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
dev, ino := inodeOfInfo(fi)
rec := fileRec{
path: root,
size: fi.Size(),
diskSize: diskSizeOfInfo(fi),
mtime: fi.ModTime(),
dev: dev,
ino: ino,
path: root,
size: fi.Size(),
mtime: fi.ModTime(),
dev: dev,
ino: ino,
}
if !isDatabaseFile(rec, rules) {
@@ -1139,9 +1112,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
// no longer exists, and one that has stopped being a regular file is
// skipped. A file on a filesystem in rules that is not persistent, as
// through a bind mount of one file, is not recorded: it gets an event
// naming it instead, and so does a file given with --exclude, reached
// by any path, a hard link to it included. One of the scan's own
// database files in rules gets no event at all.
// naming it instead. One of the scan's own database files in rules gets
// no event at all.
func emitFile(ctx context.Context, p string, e fs.DirEntry,
rules walkRules, events chan<- walkEvent,
) {
@@ -1165,10 +1137,6 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
}
line := rules.nonPersistentLine(p, info)
if line == "" {
line = rules.excludePathLine(p, info)
}
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
@@ -1177,12 +1145,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
dev, ino := inodeOfInfo(info)
rec := fileRec{
path: p,
size: info.Size(),
diskSize: diskSizeOfInfo(info),
mtime: info.ModTime(),
dev: dev,
ino: ino,
path: p,
size: info.Size(),
mtime: info.ModTime(),
dev: dev,
ino: ino,
}
if !isDatabaseFile(rec, rules) {
@@ -1193,11 +1160,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
// subdirJob applies the descent rules to directory p: never enter
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
// snapshot), never enter a directory on a filesystem in rules that is
// not persistent, nor an --exclude path or a default exclude in rules,
// and with -x never enter a directory on a different filesystem than
// its operand. For all but .zfs it sends an event that names the
// directory, so the scan keeps the records below it, except those below
// an operand at or inside it.
// not persistent, nor a default exclude in rules, and with -x never
// enter a directory on a different filesystem than its operand. For the
// last three it sends an event that names the directory, so the scan
// keeps the records below it, except those below an operand at or
// inside it.
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
parent dirJob, oneFS bool, rules walkRules,
events chan<- walkEvent,
@@ -1222,17 +1189,13 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
}
line := rules.nonPersistentLine(p, info)
if line == "" {
line = rules.excludePathLine(p, info)
}
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
return dirJob{}, false
}
if isDefaultExclude(p, rules.defaultExcludes) {
if isDefaultExclude(p, rules.excludes) {
sendEvent(ctx, events, walkEvent{
leftOut: p,
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
@@ -1278,23 +1241,6 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
return statDev(st), st.Ino
}
// statBlockSize is the unit of st_blocks in an lstat result on Linux and
// macOS, whatever the filesystem's own block size.
const statBlockSize = 512
// diskSizeOfInfo returns the bytes a file takes on disk: its allocated
// blocks, st_blocks × 512. On a compressed or sparse file it is less
// than the file's size. It is 0 when the platform exposes no block
// count.
func diskSizeOfInfo(fi fs.FileInfo) int64 {
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return 0
}
return st.Blocks * statBlockSize
}
// hashResult carries the hashes computed for one inode run (or the
// error that prevented computing them) from the pool's workers to the
// hash phase. Its run leaves out the paths found removed.
@@ -1,6 +1,6 @@
//go:build darwin
package sfdupes
package main
import "syscall"
@@ -1,6 +1,6 @@
//go:build !darwin
package sfdupes
package main
import "syscall"
+49 -338
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported syncTree
package main
import (
"bytes"
@@ -42,30 +42,6 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
return p
}
// diskSizeOf returns the bytes the file at path takes on disk, from its
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's
// blocks only when it commits the write, up to seconds later, and one
// commit can take the file through several growing sizes. So this value
// can change at any moment. Neither fsync nor unix.Sync hastens that,
// but a unix.Sync call often takes long enough that the blocks are
// counted when it returns. A test that needs the value to change waits
// for the change. A test that compares it with a scan's disk_size reads
// it before and after the scan. The rewrite test rescans until the two
// reads match, then compares exactly. The other two tests fail only on
// a disk_size outside the two reads.
func diskSizeOf(t *testing.T, path string) int64 {
t.Helper()
var st syscall.Stat_t
err := syscall.Lstat(path, &st)
if err != nil {
t.Fatal(err)
}
return st.Blocks * 512
}
// pattern returns n bytes of deterministic content seeded by tag.
func pattern(tag byte, n int) []byte {
data := make([]byte, n)
@@ -1094,7 +1070,7 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
func buildSmokeTree(t *testing.T) string {
t.Helper()
dir := realTempDir(t)
dir := t.TempDir()
one := pattern(10, 2000)
f1 := pattern(30, 3000)
f2 := pattern(40, 100)
@@ -1185,12 +1161,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
t.Helper()
groups := dupeGroups(t, db)
if len(groups) != 4 {
t.Fatalf("len(groups) = %d, want 4", len(groups))
if len(groups) != 5 {
t.Fatalf("len(groups) = %d, want 5", len(groups))
}
// The empty files empty1 and empty2 form no group.
wantSizes := []int64{3000, 2000, 100, 1}
wantSizes := []int64{3000, 2000, 100, 1, 0}
for i, g := range groups {
if g.size != wantSizes[i] {
t.Errorf("groups[%d].size = %d, want %d",
@@ -1215,7 +1190,7 @@ func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) {
super, dirs := dbTree(t, db)
tg := collectTreeGroups(dirs, super, everyPathExists, nil)
tg := collectTreeGroups(dirs, super, nil)
if len(tg) != 1 {
t.Fatalf("len(tree groups) = %d, want 1", len(tg))
}
@@ -1254,9 +1229,9 @@ func TestScanPipeline(t *testing.T) {
// TestScanProgressNamesEachPhase checks the plain progress lines of a
// first scan, and of a rescan of one operand holding two of the
// database's records. Each phase's first line names it with its place
// among the four: load gets the number of records before reading them,
// and the walk estimates its total from the records under its operand,
// or says its time remaining is unknown when there are none.
// among the four: load counts the records before reading them, and the
// walk estimates its total from the records under its operand, or says
// its time remaining is unknown when there are none.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanProgressNamesEachPhase(t *testing.T) {
@@ -1384,47 +1359,6 @@ func TestSyncScanMtimeBump(t *testing.T) {
}
}
// TestSyncScanMtimeMovedBack replaces a file with other content of the
// same size and an older mtime, as cp -p or a restore from backup does.
// The next scan must hash the file again.
func TestSyncScanMtimeMovedBack(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
recorded := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
err := os.Chtimes(a, recorded, recorded)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dir)
replaced := pattern(3, 500)
writeFile(t, dir, "a.bin", replaced)
older := recorded.Add(-time.Hour)
err = os.Chtimes(a, older, older)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 1, updated: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated", st)
}
assertWholeFileHashed(t, db, a, replaced)
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(older) {
t.Fatalf("mtime = %v, want %v", r.mtime, older)
}
}
// assertWholeFileHashed fails unless the record for path holds the
// whole-file hash of data as its head, tail, and content.
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
@@ -1483,159 +1417,6 @@ func TestScanStoresRawHashes(t *testing.T) {
}
}
// TestScanStoresDiskSize scans a sparse file, which takes less space on
// disk than its length, once through its directory and once as the
// operand itself. Each scan stores the disk_size of the file's lstat.
func TestScanStoresDiskSize(t *testing.T) {
t.Parallel()
const length = 1024 * 1024
p := sparseFile(t, t.TempDir(), "sparse", length)
pokeAt(t, p, 0, []byte("x"))
unix.Sync() // see diskSizeOf
size := diskSizeOf(t, p)
if size == 0 || size >= length {
t.Fatalf("the file takes %d bytes on disk, want more than none "+
"and fewer than its length %d", size, length)
}
for _, operand := range []string{filepath.Dir(p), p} {
db := openTestDB(t)
// ZFS may count the blocks during the scan; see diskSizeOf.
before := diskSizeOf(t, p)
syncTree(t, db, operand)
after := diskSizeOf(t, p)
got := recordByPath(t, dbRecords(t, db), p).diskSize
if got < before || got > after {
t.Errorf("scan of %s: disk_size = %d, want %d to %d",
operand, got, before, after)
}
}
}
// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
// sparse file and moves its mtime on. The rescan hashes the file again
// and rewrites its record with the new disk_size.
func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
p := sparseFile(t, dir, "sparse", 1024*1024)
pokeAt(t, p, 0, []byte("x"))
syncTree(t, db, dir)
recorded := recordByPath(t, dbRecords(t, db), p).diskSize
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
// The write may leave the mtime the scan recorded; a later one makes
// the rescan read the file again.
later := time.Now().Add(time.Hour)
err := os.Chtimes(p, later, later)
if err != nil {
t.Fatal(err)
}
// Wait for ZFS to count the write; see diskSizeOf.
deadline := time.Now().Add(30 * time.Second)
for diskSizeOf(t, p) == recorded {
if time.Now().After(deadline) {
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
"after the write into its hole", recorded)
}
unix.Sync()
}
// ZFS may still be counting, so the size may change during a rescan.
// Only a rescan with the same size on both sides of it can be compared
// exactly. A rescan that keeps the old size still fails, because the
// wait made before differ from it.
for {
before := diskSizeOf(t, p)
st := syncTree(t, db, dir)
after := diskSizeOf(t, p)
if st != (scanStats{walked: 1, updated: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated", st)
}
if before == after {
got := recordByPath(t, dbRecords(t, db), p).diskSize
if got != after {
t.Errorf("disk_size = %d, want %d", got, after)
}
return
}
if time.Now().After(deadline) {
t.Fatalf("the size on disk was still changing 30 seconds after "+
"the write into its hole: %d, then %d", before, after)
}
// A later mtime makes the next rescan read the file again.
later = later.Add(time.Second)
err = os.Chtimes(p, later, later)
if err != nil {
t.Fatal(err)
}
}
}
// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
// disk_size the file does not have, as when ZFS had not yet counted the
// file's blocks at the last scan. The rescan writes the file's own
// disk_size into the record without reading the file again.
func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
p := writeFile(t, dir, "a.bin", pattern(1, 500))
syncTree(t, db, dir)
// lstat counts 512-byte units, so no file takes 1 byte on disk.
_, err := db.ExecContext(t.Context(),
"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
if err != nil {
t.Fatal(err)
}
unix.Sync() // see diskSizeOf
// ZFS may count the blocks during the rescan; see diskSizeOf.
before := diskSizeOf(t, p)
// A file read again would count as updated.
st := syncTree(t, db, dir)
after := diskSizeOf(t, p)
if st != (scanStats{walked: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
}
got := recordByPath(t, dbRecords(t, db), p).diskSize
if got < before || got > after {
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
}
}
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
// size with an mtime later in the same second as the recorded one: the
// next scan must notice the change and re-hash the file. The operand is
@@ -2289,7 +2070,7 @@ func TestPruneRoots(t *testing.T) {
// The walk of /p does not enter /p/q when it is a default exclude, so
// /p/q is kept, without -x too; /p/q/r lies under it and is dropped.
got = pruneRoots(nested, false, walkRules{defaultExcludes: []string{pqRoot}},
got = pruneRoots(nested, false, walkRules{excludes: []string{pqRoot}},
device)
if !slices.Equal(got, want) {
t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want)
@@ -2307,86 +2088,23 @@ func TestPruneRoots(t *testing.T) {
}
}
// TestRunReportsLeaveOutRemovedCopies scans the smoke tree, then
// removes copies the reports name. report leaves out each removed file,
// and a group left with one path. trees checks only the root of each
// duplicate tree: with a file under t2 removed it still names t2, and
// with t2 removed it names nothing.
func TestRunReportsLeaveOutRemovedCopies(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
dir := buildSmokeTree(t)
p := func(name string) string { return filepath.Join(dir, name) }
scanOK(t, dir)
for _, name := range []string{"b/copy2.bin", "tiny2", "t2/sub/f2"} {
err := os.Remove(p(name))
if err != nil {
t.Fatal(err)
}
}
want := "first\tdupe\tsize\n" +
p("t1/f1") + "\t" + p("t2/f1") + "\t3000\n" +
p("t1/f1") + "\t" + p("t3/f1") + "\t3000\n" +
p("a/one.bin") + "\t" + p("b/copy.bin") + "\t2000\n" +
p("t1/sub/f2") + "\t" + p("t3/sub/f2renamed") + "\t100\n"
if got := runStdout(t, cmdReport); got != want {
t.Errorf("report stdout = %q, want %q", got, want)
}
// disk_size, the last column, depends on the filesystem.
header := "first\tdupe\tfiles\tsize\tdisk_size\n"
want = header + p("t1") + "\t" + p("t2") + "\t2\t3100\t"
if got := runStdout(t, cmdTrees); !strings.HasPrefix(got, want) ||
strings.Count(got, "\n") != 2 {
t.Errorf("trees stdout = %q, want one row starting %q", got, want)
}
err := os.RemoveAll(p("t2"))
if err != nil {
t.Fatal(err)
}
if got := runStdout(t, cmdTrees); got != header {
t.Errorf("trees stdout after removing t2 = %q, want %q", got, header)
}
}
func TestPathExists(t *testing.T) {
func TestReportsNeverTouchFilesystem(t *testing.T) {
t.Parallel()
dir := t.TempDir()
file := writeFile(t, dir, "file", nil)
locked := writeFile(t, dir, "locked/file", nil)
dir := buildSmokeTree(t)
db := openTestDB(t)
// Without search permission on its directory, a file cannot be
// looked at by anyone but root.
err := os.Chmod(filepath.Dir(locked), 0)
syncTree(t, db, dir)
// Remove the scanned tree entirely; the analysis must be
// unaffected because it reads the database alone.
err := os.RemoveAll(dir)
if err != nil {
t.Fatal(err)
}
// Runs before t.TempDir's own cleanup, which must delete the files.
t.Cleanup(func() {
//nolint:gosec // removing the directory needs its search bit back
_ = os.Chmod(filepath.Dir(locked), 0o700)
})
cases := map[string]bool{
dir: true,
file: true,
locked: true,
filepath.Join(dir, "gone"): false,
filepath.Join(file, "under-a-file"): false,
}
for path, want := range cases {
if got := pathExists(path); got != want {
t.Errorf("pathExists(%q) = %v, want %v", path, got, want)
}
}
assertSmokeDupeGroups(t, dir, db)
assertSmokeTreeGroups(t, dir, db)
}
func TestUnderRoot(t *testing.T) {
@@ -2415,24 +2133,33 @@ func TestUnderRoot(t *testing.T) {
}
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
// database, and in one case also a hard link to the database file. The
// walk knows the database's files by device ID and inode number, so it
// passes over the database file, its -wal and -shm files, its .lock
// file and the hard link with no line and no count. The record an
// earlier scan left for the .lock file is deleted. A rescan of the
// unchanged tree updates nothing, and report lists none of the
// database's files.
// database, reached directly and through a symlinked directory. The
// walk passes over the database file, its -wal and -shm files and its
// .lock file with no line and no count, and the record an earlier scan
// left for the .lock file is deleted. A rescan of the unchanged tree
// updates nothing, and report lists none of the database's files.
func TestRunScanPassesOverDatabase(t *testing.T) {
for _, viaHardLink := range []bool{false, true} {
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
tree := realTempDir(t)
hardLink := filepath.Join(tree, "hardlink")
for _, viaSymlink := range []bool{false, true} {
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
base := t.TempDir()
tree := filepath.Join(base, "real", "tree")
operand := tree
writeFile(t, tree, "a", pattern(1, 100))
writeFile(t, tree, "b", pattern(1, 100))
writeFile(t, tree, "empty1", nil)
writeFile(t, tree, "empty2", nil)
if viaSymlink {
err := os.Symlink(filepath.Join(base, "real"),
filepath.Join(base, "link"))
if err != nil {
t.Fatal(err)
}
operand = filepath.Join(base, "link", "tree")
}
dbPath := filepath.Join(tree, "db.sqlite")
t.Setenv(databaseEnv, dbPath)
@@ -2442,7 +2169,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
}
err = applyChanges(t.Context(), db, []scanRec{
{path: filepath.Join(tree, "db.sqlite.lock")},
{path: filepath.Join(operand, "db.sqlite.lock")},
}, nil, nil)
if err != nil {
t.Fatal(err)
@@ -2450,30 +2177,17 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
closeScanDatabase(t.Context(), db, dbPath)
if viaHardLink {
err = os.Link(dbPath, hardLink)
if err != nil {
t.Fatal(err)
}
}
stderr := assertScanSummary(t,
assertScanSummary(t,
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
tree)
stderr += assertScanSummary(t,
operand)
assertScanSummary(t,
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
tree)
if strings.Contains(stderr, hardLink) {
t.Errorf("scan stderr = %q, want no line naming %s",
stderr, hardLink)
}
operand)
report := runStdout(t, cmdReport)
if strings.Contains(report, "db.sqlite") ||
strings.Contains(report, hardLink) ||
!strings.Contains(report, filepath.Join(tree, "b")) {
t.Errorf("report = %q, want a and b and none of "+
!strings.Contains(report, filepath.Join(operand, "empty2")) {
t.Errorf("report = %q, want the empty files and none of "+
"the database's files", report)
}
})
@@ -2497,9 +2211,8 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
// assertScanSummary scans operands and fails the test unless the
// summary line gives counts with nothing skipped, and the line naming
// the database is the only one that names one of its files. It returns
// everything the scan printed to stderr.
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
// the database is the only one that names one of its files.
func assertScanSummary(t *testing.T, counts string, operands ...string) {
t.Helper()
stderr := scanOK(t, operands...)
@@ -2513,6 +2226,4 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) string {
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
"database to name one of its files", operands, stderr)
}
return stderr
}
+24 -48
View File
@@ -1,4 +1,4 @@
package sfdupes
package main
import (
"context"
@@ -22,10 +22,6 @@ type treeNode struct {
digest [sha256.Size]byte
fileCount int64
totalSize int64
// diskSize is the bytes the directory's files take on disk. Two
// copies of one tree have the same totalSize, but their diskSize
// can differ.
diskSize int64
}
// treesPhase names one phase of trees with its place among all of them,
@@ -38,18 +34,17 @@ func treesPhase(name string) string {
// runTrees implements the trees subcommand: it reads every record from
// the database in path order, reconstructs the directory hierarchy from
// the record paths, computes a Merkle-style digest per directory, and
// prints maximal duplicate-tree groups as TSV on stdout. Of the scanned
// filesystem it only checks, with pathExists, that the root of each
// duplicate tree is still there, never the files below it. It names the
// database on stderr before it opens it. Any database problem,
// including a missing database, is fatal.
// prints maximal duplicate-tree groups as TSV on stdout. It never
// touches the scanned filesystem; its only I/O is the database, stdout,
// and stderr. It names the database on stderr before it opens it. Any
// database problem, including a missing database, is fatal.
//
// The work runs in four phases, each under a progress display of its
// own on stderr: open (open the database), read (read the number of
// records from the totals table, then read the records into the
// hierarchy), group (group the directories by digest) and write (write
// the groups to stdout). Each display ends before the next phase
// starts, and before an error is returned to be printed.
// own on stderr: open (open the database), read (count the records,
// then read them into the hierarchy), group (group the directories by
// digest) and write (write the groups to stdout). Each display ends
// before the next phase starts, and before an error is returned to be
// printed.
func runTrees(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
@@ -87,13 +82,13 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
return fmt.Errorf("database %s: %w", dbPath, err)
}
// Grouping works in memory, then checks the root of each duplicate
// tree on disk, so it waits on the processor or the disk.
// Grouping works in memory, so it waits on nothing but the
// processor.
prog = newProgress(progressParams{
label: treesPhase("group"), unit: "directories",
total: int64(len(allDirs)), stalledOn: "cpu/disk",
total: int64(len(allDirs)), stalledOn: "cpu",
})
dupes := collectTreeGroups(allDirs, super, pathExists, prog)
dupes := collectTreeGroups(allDirs, super, prog)
prog.finish()
@@ -144,16 +139,15 @@ func readTree(ctx context.Context, db *sql.DB,
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
// a row for every member of each group but its first, counting each
// group on prog once its rows are written. The disk_size column is the
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
// go through prog's writer. It returns the number of rows (dupe trees)
// and the bytes they could reclaim.
// group on prog once its rows are written. stdout may be the terminal
// prog is drawn on, so the lines go through prog's writer. It returns
// the number of rows (dupe trees) and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress,
) (int, int64, error) {
out := prog.writer(stdout)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err)
}
@@ -165,9 +159,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
for _, g := range dupes {
first := g[0]
for _, n := range g[1:] {
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize, n.diskSize)
first.fileCount, first.totalSize)
if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err)
}
@@ -238,7 +232,6 @@ func (b *treeBuilder) add(r scanRec) {
dir.entries = append(dir.entries, fileEntry(name, r))
dir.fileCount++
dir.totalSize += r.size
dir.diskSize += r.diskSize
}
// openDir starts the directory called name inside the innermost open
@@ -277,7 +270,6 @@ func (b *treeBuilder) closeTo(n int) {
"d\x00"+name+"\x00"+string(dir.digest[:]))
dir.parent.fileCount += dir.fileCount
dir.parent.totalSize += dir.totalSize
dir.parent.diskSize += dir.diskSize
b.dirs = append(b.dirs, dir)
}
}
@@ -301,14 +293,10 @@ func fileEntry(name string, r scanRec) string {
// collectTreeGroups groups directories by digest, counting each one on
// prog as it is grouped, and returns every maximal group with two or
// more members for which exists returns true, each group's members
// sorted by path, groups ordered by tree size descending then by first
// path ascending. A group whose trees hold 0 bytes is left out, since
// removing one frees nothing. The groups it implies hold 0 bytes too,
// so leaving it out hides nothing. exists is called only on the members
// of maximal groups.
// more members, each group's members sorted by path, groups ordered by
// tree size descending then by first path ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
exists func(path string) bool, prog *progress,
prog *progress,
) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs {
@@ -320,19 +308,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
var dupes [][]*treeNode
for _, g := range groups {
// The members share a digest, so they share a total size.
if len(g) < minGroupSize || g[0].totalSize == 0 ||
suppressed(g, super) {
continue
}
// A member that is gone leaves the group. A suppressed group is
// still implied: its members still on disk have parents still
// on disk, and those parents' group is checked the same way.
g = slices.DeleteFunc(g, func(n *treeNode) bool {
return !exists(n.path)
})
if len(g) < minGroupSize {
if len(g) < minGroupSize || suppressed(g, super) {
continue
}
+25 -183
View File
@@ -1,12 +1,10 @@
package sfdupes //nolint:testpackage // checks the unexported treeNode results
package main
import (
"bytes"
"crypto/sha256"
"database/sql"
"os"
"slices"
"strconv"
"strings"
"testing"
"time"
@@ -19,24 +17,19 @@ const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
// smokeTreeRecs mirrors the tree of the test under README "Definition
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
// by one filename. Each file takes one 4 KiB block on disk.
// by one filename.
func smokeTreeRecs() []scanRec {
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
return []scanRec{
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
content: f1Content, path: "/d/t1/f1"},
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
content: f2Content, path: "/d/t1/sub/f2"},
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
content: f1Content, path: "/d/t2/f1"},
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
content: f2Content, path: "/d/t2/sub/f2"},
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
content: f1Content, path: "/d/t3/f1"},
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
content: f2Content, path: "/d/t3/sub/f2renamed"},
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
path: "/d/t3/sub/f2renamed"},
}
}
@@ -180,7 +173,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
t.Fatalf("directory paths = %q, want %q", got, want)
}
groups := collectTreeGroups(dirs, super, everyPathExists, nil)
groups := collectTreeGroups(dirs, super, nil)
gotGroups := groupPaths(groups)
wantGroups := [][]string{{"/a", "/c"}}
@@ -196,8 +189,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
}
func TestRunTreesEscapesPaths(t *testing.T) {
dir, recs := recsOnDisk(t, awkwardPairRecs())
t.Setenv(databaseEnv, seedDatabase(t, recs))
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
var stdout, stderr bytes.Buffer
@@ -207,45 +199,22 @@ func TestRunTreesEscapesPaths(t *testing.T) {
code, exitOK, stderr.String())
}
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
dir + `/d/\tone\ntwo\rthree\\four` + "\t" + dir + "/d/A\t1\t5\t4096\n"
want := "first\tdupe\tfiles\tsize\n" +
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
if got := stdout.String(); got != want {
t.Errorf("stdout = %q, want %q", got, want)
}
}
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
// take different space on disk. The row's disk_size is the sum over the
// dupe copy's files, those in its subdirectory included.
func TestRunTreesDiskSize(t *testing.T) {
f, g := testHash("f"), testHash("g")
recs := []scanRec{
{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
}
dir, recs := recsOnDisk(t, recs)
t.Setenv(databaseEnv, seedDatabase(t, recs))
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
dir + "/a\t" + dir + "/b\t2\t30\t1536\n"
if got := runStdout(t, cmdTrees); got != want {
t.Errorf("stdout = %q, want %q", got, want)
}
}
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the four as it starts and another as
// it ends. read counts the 6 records before it reads them, group counts
// the directories, and write the 1 group. The summary comes last, and
// the 8 directories, and write the 1 group. The summary comes last, and
// stdout holds the report alone.
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
dir, recs := recsOnDisk(t, smokeTreeRecs())
path := seedDatabase(t, recs)
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
@@ -258,16 +227,11 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
code, exitOK, stderr())
}
report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
dir + "/d/t1\t" + dir + "/d/t2\t2\t3100\t8192\n"
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
if got := stdout.String(); got != report {
t.Errorf("stdout = %q, want %q", got, report)
}
// The smoke-test records name 8 directories, / included. Moved under
// dir, they name one more for each slash in dir.
dirs := strconv.Itoa(8 + strings.Count(dir, "/"))
// How each line starts; the rate at the end of a phase varies.
want := []string{
"trees: database " + path + " (from " + databaseEnv + ")",
@@ -275,9 +239,8 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
treesOpenLine,
"[2/4] read: counting the records, elapsed 0s, remain unknown",
"[2/4] read: [6/6] 100%, ",
"[3/4] group: [0/" + dirs + "] 0%, 0 directories/s, elapsed 0s, " +
"remain unknown",
"[3/4] group: [" + dirs + "/" + dirs + "] 100%, ",
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
"[3/4] group: [8/8] 100%, ",
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
"[4/4] write: [1/1] 100%, ",
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
@@ -306,7 +269,7 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
stderr := captureStderr(t)
super, dirs := treeOf(t, smokeTreeRecs())
dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
dupes := collectTreeGroups(dirs, super, nil)
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
@@ -329,9 +292,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := []string{
"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
}
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
@@ -387,7 +348,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
super, dirs := treeOf(t, smokeTreeRecs())
groups := collectTreeGroups(dirs, super, everyPathExists, nil)
groups := collectTreeGroups(dirs, super, nil)
want := [][]string{{"/d/t1", "/d/t2"}}
if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -411,16 +372,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
super, dirs := treeOf(t, recs)
forward := groupPaths(collectTreeGroups(dirs, super, everyPathExists,
nil))
forward := groupPaths(collectTreeGroups(dirs, super, nil))
reversed := slices.Clone(recs)
slices.Reverse(reversed)
superR, dirsR := treeOf(t, reversed)
backward := groupPaths(collectTreeGroups(dirsR, superR, everyPathExists,
nil))
backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
if !slices.EqualFunc(forward, backward, slices.Equal) {
t.Fatalf("output depends on record order: %v vs %v",
forward, backward)
@@ -440,36 +399,7 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("groups = %v, want %v", got, want)
}
}
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
// holding only empty files form no group, and that an empty file still
// counts toward the digest of a tree that holds other files.
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
t.Parallel()
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
sha256.Sum256(nil)
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
// Without the empty file, /p/x3 is no copy of /p/x1.
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
}
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
got := groupPaths(collectTreeGroups(dirs, super, nil))
want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
@@ -492,98 +422,10 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
got := groupPaths(collectTreeGroups(dirs, super, nil))
want := [][]string{{"/p/a/x", "/q/b/x"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("groups = %v, want %v", got, want)
}
}
// TestCollectTreeGroupsChecksOnlyReportedCopies checks that trees asks
// whether a directory is on disk only for the members of the groups it
// returns. It never asks about /a/proj/sub, which lies inside a copy
// and whose group the group of /a/proj implies.
func TestCollectTreeGroupsChecksOnlyReportedCopies(t *testing.T) {
t.Parallel()
f, g, u := testHash("f"), testHash("g"), testHash("u")
recs := []scanRec{
{size: 10, head: f, tail: f, content: f, path: "/a/proj/f"},
{size: 20, head: g, tail: g, content: g, path: "/a/proj/sub/g"},
{size: 10, head: f, tail: f, content: f, path: "/b/proj/f"},
{size: 20, head: g, tail: g, content: g, path: "/b/proj/sub/g"},
// /a lacks this file, so /a and /b are no copies of each other.
{size: 30, head: u, tail: u, content: u, path: "/b/lonely"},
}
super, dirs := treeOf(t, recs)
var checked []string
exists := func(path string) bool {
checked = append(checked, path)
return true
}
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
want := [][]string{{"/a/proj", "/b/proj"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("groups = %v, want %v", got, want)
}
slices.Sort(checked)
if !slices.Equal(checked, want[0]) {
t.Errorf("directories checked on disk = %q, want %q",
checked, want[0])
}
}
// TestCollectTreeGroupsDropsGroupLeftWithOneCopy checks that a group of
// two copies, one of them gone, is not returned. If kept, it would
// write no row, but the summary line would count it.
func TestCollectTreeGroupsDropsGroupLeftWithOneCopy(t *testing.T) {
t.Parallel()
f := testHash("f")
recs := []scanRec{
{size: 10, head: f, tail: f, content: f, path: "/c/kept/f"},
{size: 10, head: f, tail: f, content: f, path: "/c/gone/f"},
}
super, dirs := treeOf(t, recs)
exists := func(path string) bool { return path != "/c/gone" }
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
if len(got) != 0 {
t.Errorf("groups = %v, want none", got)
}
}
// TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies checks that a group
// of three copies, one of them gone, is returned with the other two.
func TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies(t *testing.T) {
t.Parallel()
f := testHash("f")
recs := []scanRec{
{size: 10, head: f, tail: f, content: f, path: "/d/kept1/f"},
{size: 10, head: f, tail: f, content: f, path: "/d/gone/f"},
{size: 10, head: f, tail: f, content: f, path: "/d/kept2/f"},
}
super, dirs := treeOf(t, recs)
exists := func(path string) bool { return path != "/d/gone" }
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
want := [][]string{{"/d/kept1", "/d/kept2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Errorf("groups = %v, want %v", got, want)
}
}