Rewrite README.md sentence by sentence for a newcomer

Rewrites `README.md` from scratch for
#135. Every statement was checked
against the code on `next` and against what the built binary prints on
small trees under `/var/tmp`. The headings that the code comments cite
are kept.

Where the old text and the repository disagreed, the new text states
what the code does. #140 lists
every such point. Among them: `@latest` installs the newest tagged
release, ten million files in both "Rationale" and "Design", no Windows
excludes, a relative operand is joined to the working directory as `pwd`
prints it, and an operand reached through a symlinked directory below
another operand is dropped (#144).

Dropped from "Constraints": the permission to add `github.com/spf13/viper`
if configuration-file support is ever needed.

Model: fable-5-1

TODO.md

@@ -29,11 +29,13 @@
 
 # Completed Steps
 
-- the database keeps the number of records and the sum of their sizes in the one
+- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
-  row of a `totals` table, which three triggers on `files` keep current. `info`
+  it. It leads with what the program is and how to use it. A reference section
-  reads that row, and so do `scan`'s `load` phase and the `read` phases of
+  covers every setting and every subcommand. Development and testing come last.
-  `report` and `trees` for their totals; no count reads every record any more
+  Every statement was checked against the code on `next`. Five comments in
-  (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/146)
+  `main_test.go` cite "Settings", where their rules now live. One comment each
+  in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
+  now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
 
 - an operand at or inside a directory named `.zfs` is never scanned, whether
   given alone or under another operand. `scan` warns about it, counts it as

db.go

@@ -69,49 +69,6 @@ const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `
 
-// createTotalsSQL creates the totals table, which holds one row: the
-// number of records and the sizes of all records added up. Counting
-// them over files would read every record. The three triggers below
-// keep the row current, in the transaction of each change to files.
-const createTotalsSQL = `
-CREATE TABLE totals (
-    records INTEGER NOT NULL,
-    size    INTEGER NOT NULL
-)
-`
-
-// insertTotalsSQL adds the one row of totals, for an empty files table.
-const insertTotalsSQL = `
-INSERT INTO totals (records, size) VALUES (0, 0)
-`
-
-// totalsAfterInsertSQL counts each inserted record into totals.
-const totalsAfterInsertSQL = `
-CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
-    UPDATE totals SET records = records + 1, size = size + NEW.size;
-END
-`
-
-// totalsAfterDeleteSQL takes each deleted record out of totals.
-const totalsAfterDeleteSQL = `
-CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
-    UPDATE totals SET records = records - 1, size = size - OLD.size;
-END
-`
-
-// totalsAfterUpdateSQL replaces a record's old size in totals with its
-// new one. scan's upsert of an existing record is such an update.
-const totalsAfterUpdateSQL = `
-CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
-    UPDATE totals SET size = size + NEW.size - OLD.size;
-END
-`
-
-// recordCountSQL reads the number of records from totals.
-const recordCountSQL = `
-SELECT records FROM totals
-`
-
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
@@ -469,14 +426,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
 
 	defer func() { _ = tx.Rollback() }()
 
-	for _, stmt := range []string{
+	_, err = tx.ExecContext(ctx, createTableSQL)
-		createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
+	if err != nil {
-		totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
+		return fmt.Errorf("create schema: %w", err)
-	} {
+	}
-		_, err = tx.ExecContext(ctx, stmt)
+
-		if err != nil {
+	_, err = tx.ExecContext(ctx, createIndexSQL)
-			return fmt.Errorf("create schema: %w", err)
+	if err != nil {
-		}
+		return fmt.Errorf("create schema: %w", err)
 	}
 
 	_, err = tx.ExecContext(ctx,
@@ -523,7 +480,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
+	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -599,7 +556,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
+	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -665,7 +622,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
 func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64
 
-	err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
+	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
 	if err != nil {
 		return 0, fmt.Errorf("count records: %w", err)
 	}
@@ -673,10 +630,12 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	return n, nil
 }
 
-// databaseCountsSQL reads the number of records and the sizes of all
+// databaseCountsSQL counts the records and adds up their sizes, in one
-// records added up from totals.
+// pass over the table. A SUM over no rows is NULL, so on an empty table
+// the SUM falls back to 0.
 const databaseCountsSQL = `
-SELECT records, size FROM totals
+SELECT COUNT(*), COALESCE(SUM(size), 0)
+FROM files
 `
 
 // databaseCounts is what info reads from the database: its schema

db_test.go

@@ -535,12 +535,17 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
 	}
 }
 
-// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
+// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// PLAN for query.
+// from the files_signature index alone, with nothing to sort: the
-func queryPlan(t *testing.T, db *sql.DB, query string) []string {
+// first record comes at once, so report's read phase counts each
-	t.Helper()
+// record as it comes rather than waiting on SQLite for all of them.
+func TestSignatureRowsNeedNoSort(t *testing.T) {
+	t.Parallel()
 
-	rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
+	db := openTestDB(t)
+
+	rows, err := db.QueryContext(t.Context(),
+		"EXPLAIN QUERY PLAN "+signatureRowsSQL)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -568,164 +573,12 @@ func queryPlan(t *testing.T, db *sql.DB, query string) []string {
 		t.Fatal(err)
 	}
 
-	return plan
-}
-
-// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// from the files_signature index alone, with nothing to sort: the
-// first record comes at once, so report's read phase counts each
-// record as it comes rather than waiting on SQLite for all of them.
-func TestSignatureRowsNeedNoSort(t *testing.T) {
-	t.Parallel()
-
-	db := openTestDB(t)
-
-	plan := queryPlan(t, db, signatureRowsSQL)
-
 	want := []string{"SCAN files USING COVERING INDEX files_signature"}
 	if !slices.Equal(plan, want) {
 		t.Errorf("query plan = %q, want %q", plan, want)
 	}
 }
 
-// TestCountsReadOnlyTotals checks that SQLite answers each query that
-// counts the records from the one row of totals. A plan that reads
-// files or its index reads every record.
-func TestCountsReadOnlyTotals(t *testing.T) {
-	t.Parallel()
-
-	db := openTestDB(t)
-
-	for _, query := range []string{recordCountSQL, databaseCountsSQL} {
-		plan := queryPlan(t, db, query)
-
-		want := []string{"SCAN totals"}
-		if !slices.Equal(plan, want) {
-			t.Errorf("query plan of %q = %q, want %q", query, plan, want)
-		}
-	}
-}
-
-// TestCountsComeFromTotals sets the row of totals to numbers that files
-// does not hold, then checks that scan's load phase, the read phases of
-// trees and report, and info each report the numbers from totals.
-func TestCountsComeFromTotals(t *testing.T) {
-	t.Parallel()
-
-	// files is empty, so a count over files gives 0 records of 0 bytes.
-	const records, size = 7, 9000
-
-	db := openTestDB(t)
-
-	_, err := db.ExecContext(t.Context(),
-		"UPDATE totals SET records = ?, size = ?", records, size)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	scanTotal, err := countRecords(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var treesTotal, reportTotal int64
-
-	err = loadFileRows(t.Context(), db,
-		func(n int64) { treesTotal = n }, func(scanRec) {})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = loadSignatureRows(t.Context(), db,
-		func(n int64) { reportTotal = n },
-		func(scanRec) error { return nil })
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	info, err := readDatabaseCounts(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for name, got := range map[string]int64{
-		"scan's load":   scanTotal,
-		"trees' read":   treesTotal,
-		"report's read": reportTotal,
-		"info":          info.records,
-	} {
-		if got != records {
-			t.Errorf("%s: %d records, want %d from totals",
-				name, got, records)
-		}
-	}
-
-	if info.size != size {
-		t.Errorf("info: size %d, want %d from totals", info.size, size)
-	}
-}
-
-// assertTotalsMatchFiles fails unless the row of totals holds the
-// number of records and the sizes of all records added up, as counted
-// over files.
-func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
-	t.Helper()
-
-	got, err := readDatabaseCounts(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var records, size int64
-
-	err = db.QueryRowContext(t.Context(),
-		"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
-		Scan(&records, &size)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got.records != records || got.size != size {
-		t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
-			got.records, got.size, records, size)
-	}
-}
-
-// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
-// change size, and rescans it after two files are removed. After each
-// scan the row of totals must agree with files.
-func TestTotalsMatchFiles(t *testing.T) {
-	t.Parallel()
-
-	dir := buildSmokeTree(t)
-	db := openTestDB(t)
-
-	syncTree(t, db, dir)
-	assertTotalsMatchFiles(t, db)
-
-	writeFile(t, dir, "a/one.bin", pattern(10, 2500))
-	writeFile(t, dir, "t1/f1", pattern(30, 1000))
-
-	if st := syncTree(t, db, dir); st.updated != 2 {
-		t.Fatalf("rescan stats = %+v, want 2 updated", st)
-	}
-
-	assertTotalsMatchFiles(t, db)
-
-	for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
-		err := os.Remove(filepath.Join(dir, name))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if st := syncTree(t, db, dir); st.removed != 2 {
-		t.Fatalf("rescan stats = %+v, want 2 removed", st)
-	}
-
-	assertTotalsMatchFiles(t, db)
-}
-
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 

exclude_test.go

@@ -17,7 +17,9 @@ import (
 func TestDefaultExcludes(t *testing.T) {
 	t.Parallel()
 
-	// README "Rules for the walk": each system's list.
+	// README "Rules for the walk": the Linux and macOS lists, and none
+	// on any other system. The Windows list exists in the code only: the
+	// binary does not build for Windows, so the README leaves it out.
 	cases := map[string][]string{
 		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
 		goosDarwin: {

info.go

@@ -26,11 +26,11 @@ func infoPhase(name string) string {
 // it. Any database problem, including a missing database, is fatal.
 //
 // The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and count (read the number of
+// own on stderr: open (open the database) and count (count the records,
-// records and their total size from the totals table). Each display
+// with one query). Each display ends before the next phase starts, and
-// ends before the next phase starts, and before an error is returned
+// before an error is returned to be printed. The details are written
-// to be printed. The details are written once the last display has
+// once the last display has ended, so no display is drawn while they
-// ended, so no display is drawn while they are written.
+// are written.
 func runInfo(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
 
 	defer func() { _ = db.Close() }()
 
-	// One row holds both counts, so the phase never has a total.
+	// One query counts everything, so the phase never has a total.
 	prog = newProgress(progressParams{
 		label: infoPhase("count"), total: -1,
 		waitingFor: countingRecords,

main_test.go

@@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
 }
 
 // brokenDatabase writes a database that opens cleanly and passes the
-// schema-version check but has no tables, so the first query fails
+// schema-version check but has no files table, so the first query
-// with the database already open: a fatal error on a path that owns an
+// fails with the database already open: a fatal error on a path that
-// open database. It closes the database the way scan does.
+// owns an open database. It closes the database the way scan does.
 func brokenDatabase(t *testing.T) string {
 	t.Helper()
 
@@ -265,9 +265,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
 			assertFatalOutput(t, stderr.String(), stdout())
 
 			// Proof that the failure happened after the open: only a
-			// query against the opened database can report this. Each
+			// query against the opened database can report this.
-			// subcommand's first query reads totals.
+			if !strings.Contains(stderr.String(), "no such table: files") {
-			if !strings.Contains(stderr.String(), "no such table: totals") {
 				t.Errorf("stderr = %q, want the failure to come from a "+
 					"query on the open database", stderr.String())
 			}
@@ -394,7 +393,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
 }
 
 func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
-	// README §Database: the path names the file exactly. With link a
+	// README §Settings: the path names the file exactly. With link a
 	// symlink to the directory real/sub, link/../db.sqlite names
 	// real/db.sqlite, not db.sqlite beside link. Every subcommand must
 	// use that file, relative or absolute, and its first line must name
@@ -649,7 +648,7 @@ func TestRunUsageErrors(t *testing.T) {
 }
 
 func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
-	// README §scan mode: --workers below 1 is a usage error reported in
+	// README §Settings: --workers below 1 is a usage error reported in
 	// one line on stderr, before the scan opens the database.
 	for _, workers := range []string{"0", "-1"} {
 		t.Run(workers, func(t *testing.T) {
@@ -687,8 +686,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 func TestRunHelp(t *testing.T) {
 	t.Parallel()
 
-	// README §Subcommands: help goes to stderr, exits 0, and leaves
+	// README §Settings: help goes to stderr, exits 0, and leaves stdout
-	// stdout empty.
+	// empty.
 	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
 
 	for _, args := range cases {
@@ -714,8 +713,8 @@ func TestRunHelp(t *testing.T) {
 func TestRunVersion(t *testing.T) {
 	t.Parallel()
 
-	// README §Subcommands: the version is one line on stdout, with
+	// README §Settings: the version is one line on stdout, with nothing
-	// nothing on stderr, and exits 0.
+	// on stderr, and exits 0.
 	info, _ := debug.ReadBuildInfo()
 
 	for _, arg := range []string{"--version", "-v"} {
@@ -774,8 +773,8 @@ func TestBuildVersion(t *testing.T) {
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()
 
-	// README §Error handling: a stdout write failure exits 1, reported
+	// README §Error handling: a failed write to stdout is a fatal error,
-	// in one line on stderr.
+	// exit 1, reported on stderr starting with "sfdupes:".
 	var stderr bytes.Buffer
 
 	code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -1235,7 +1234,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 }
 
 func TestRunDatabasePathUsedAsGiven(t *testing.T) {
-	// README §Database: the path names the database file exactly. In
+	// README §Settings: the path names the database file exactly. In
 	// SQLite's connection string an unescaped ? or # would end the file
 	// name and % would start an escape, and a path starting with //
 	// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1280,9 +1279,8 @@ func holdScanLock(t *testing.T, path string) {
 }
 
 func TestRunSecondScanFails(t *testing.T) {
-	// README §Database: while one scan holds the lock, a second scan
+	// README §Database: a second scan that finds the lock held fails
-	// fails at once, naming the lock file, without opening the
+	// at once, naming the lock file, without opening the database.
-	// database.
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
 
@@ -1341,7 +1339,7 @@ func assertSecondScanFails(t *testing.T, path string) {
 }
 
 func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
-	// README §Database: every path to one database names the same lock
+	// README §Database: a path through a symlink names the same lock
 	// file, so while a scan holds it through one path, a scan through
 	// another fails, naming the lock file beside the database's real
 	// path.
@@ -1446,8 +1444,8 @@ func TestRunReportsDuringScan(t *testing.T) {
 }
 
 func TestRunStdoutClosedIsFatal(t *testing.T) {
-	// README §Error handling: a stdout write failure exits 1, reported
+	// README §Error handling: a failed write to stdout is a fatal error,
-	// in one line on stderr.
+	// exit 1, reported on stderr starting with "sfdupes:".
 	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))

scan.go

@@ -455,8 +455,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 // included, so its display counts against the number of records.
 func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 	// Indexing tens of millions of records takes real time; without a
-	// display the scan looks hung before the walk begins. The number of
+	// display the scan looks hung before the walk begins. Counting them
-	// records, read first, gives the display its total.
+	// first gives the display its total.
 	prog := newProgress(progressParams{
 		label: scanPhase("load"), unit: scanUnit, total: -1,
 		waitingFor: countingRecords, stalledOn: "db",

scan_test.go

@@ -1064,8 +1064,9 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
 	}
 }
 
-// buildSmokeTree recreates the README smoke-test filesystem layout
+// buildSmokeTree recreates the tree of the test under README
-// with deterministic content and returns the tree root.
+// "Definition of done" with deterministic content and returns the tree
+// root.
 func buildSmokeTree(t *testing.T) string {
 	t.Helper()
 

trees_test.go

@@ -15,8 +15,9 @@ import (
 const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
 	"remain unknown"
 
-// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
+// smokeTreeRecs mirrors the tree of the test under README "Definition
-// /d/t2 are identical, /d/t3 differs from them only by one filename.
+// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
+// by one filename.
 func smokeTreeRecs() []scanRec {
 	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
 	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")