From ded0ac11ae348533ef3603a81fc89d71630cc757 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 19:59:51 +0000 Subject: [PATCH] Add sfdupes info: show the database's details (closes #117) info prints the database's path and source, the sizes of the database file and its -wal file, when the database file last changed, the schema version and the record counts, one per line as a name, a tab and the value. It opens the database as report does and shows an open and a count phase on stderr. The schema version and the counts come from one read-only transaction, the counts from one query. seedDatabase now closes test databases out of WAL mode, as scan does; otherwise info's own connection creates an empty -wal file. The open phase's name and text became constants for the linter, and the scan command moved to newScanCommand to keep newRootCommand short enough. Judgement call: the lines are written after the last display ends, so they skip the display's stdout writer. Model: opus-5-5 --- README.md | 123 ++++++++++++++++++++++++++---------- TODO.md | 10 +++ db.go | 69 +++++++++++++++++--- info.go | 128 +++++++++++++++++++++++++++++++++++++ info_test.go | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 65 ++++++++++++------- main_test.go | 38 ++++++----- progress.go | 8 +++ report.go | 6 +- report_test.go | 8 +-- trees.go | 6 +- 11 files changed, 540 insertions(+), 89 deletions(-) create mode 100644 info.go create mode 100644 info_test.go diff --git a/README.md b/README.md index ac3d4cc..f709ab8 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,9 @@ database persists between runs; a rescan only hashes files that are new or changed, or that may have gained a duplicate since the last scan, and removes records for files that no longer exist. `report` reads the database and prints the file-level duplicates report. `trees` reads the same database and prints the -duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation -with no `PATH` operand — prints a usage message and exits 2. +duplicate-tree report. `info` prints the database's details: its path, the sizes +of its files and the number of its records. A missing/invalid subcommand — or a +`scan` invocation with no `PATH` operand — prints a usage message and exits 2. The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` @@ -109,9 +110,9 @@ As a file `/etc/cron.d/sfdupes`: - A scan still running when the next one starts carries on. The new one fails at once, and the lines cron mails include `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. -- `report` and `trees` need only read access to the database (see "Database"). - Under the usual umask of `022` the first scan creates it readable by every - user, so an unprivileged user can run them against root's database. +- `report`, `trees` and `info` need only read access to the database (see + "Database"). Under the usual umask of `022` the first scan creates it readable + by every user, so an unprivileged user can run them against root's database. ### Reading the reports @@ -235,7 +236,7 @@ Goals, in order: ### Subcommands -Three subcommands, all implemented: +Four subcommands, all implemented: 1. `scan` — walk the filesystem and synchronize the database: one signature record per regular file. @@ -243,11 +244,14 @@ Three subcommands, all implemented: 3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy from the database records, compute a Merkle-style digest per directory, and report maximal groups of identical trees. +4. `info` — the database's details: its path, the sizes of its files, its + schema version and the counts of its records. ``` sfdupes scan [--workers N] [-x] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv +sfdupes info sfdupes --version sfdupes [command] --help ``` @@ -262,7 +266,7 @@ stdout. ### Database -All three subcommands operate on a single SQLite database file: +All four subcommands operate on a single SQLite database file: - Location: the value of the `SFDUPES_DATABASE` environment variable when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no @@ -285,13 +289,13 @@ All three subcommands operate on a single SQLite database file: symlinks resolved. After a `cd` through a symlink to a directory, a path starting `../` therefore names a file beside the directory the symlink leads to, as it does for any other program. `scan` says `new database` when no - file is at that path as it starts. `report` and `trees` print the line - before they open the database, so the error for a missing database comes - after it. + file is at that path as it starts. `report`, `trees` and `info` print the + line before they open the database, so the error for a missing database + comes after it. -- `scan` creates the database (and its parent directory) on first use. `report` - and `trees` require an existing database; a missing database file is a fatal - error (exit 1) telling the user to run `scan` first. +- `scan` creates the database (and its parent directory) on first use. `report`, + `trees` and `info` require an existing database; a missing database file is a + fatal error (exit 1) telling the user to run `scan` first. - Only one `scan` runs against a database at a time. For its whole run, `scan` holds an exclusive `flock(2)` lock on a lock file beside the database, named by appending `.lock` to the database's path with symlinks resolved @@ -307,8 +311,8 @@ All three subcommands operate on a single SQLite database file: on. The lock file is created on first use, open to its owner only, and left in place: a leftover file blocks nothing, because the lock ends with the process holding it however it ends, a fatal error or an interrupt included, and - deleting the file while a scan runs would let a second scan start. `report` - and `trees` never take the lock, so they run during a scan. + deleting the file while a scan runs would let a second scan start. `report`, + `trees` and `info` never take the lock, so they run during a scan. - While `scan` runs, the database is in WAL journal mode with a busy timeout, so running a report while a cron `scan` is in progress is safe. The filesystem is authoritative; the database is an eventually-consistent reflection of it. @@ -341,10 +345,10 @@ All three subcommands operate on a single SQLite database file: - In the content phase: content hashes, again committed 10,000 at a time. - At the end, as it closes the database: the `-wal` and `-shm` files go away, unless a report still has the database open (see above). -- `report` and `trees` open the database read-only and need only read access to - the database file, and no write access to its directory. While the database is - in WAL mode they also read the `-wal` and `-shm` files beside it, which SQLite - creates with the database file's permissions. +- `report`, `trees` and `info` open the database read-only and need only read + access to the database file, and no write access to its directory. While the + database is in WAL mode they also read the `-wal` and `-shm` files beside it, + which SQLite creates with the database file's permissions. - Schema (`PRAGMA user_version` is the schema version, currently 1; a database with any other version is a fatal error. `scan` creates the schema and sets the version in one transaction, so a first scan stopped while doing so leaves @@ -727,13 +731,55 @@ Summary to stderr: records read, number of duplicate-tree groups, number of dupe trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human units. +### `info` mode + +`info` takes no positional arguments and prints the database's details on +stdout, one per line: a name, a tab, and the value. It opens the database the +way `report` does, read-only and without the scan lock, so it runs during a scan +and needs only read access (see "Database"). Like `report`, it never touches the +scanned filesystem. + +``` +database /var/lib/sfdupes/db.sqlite (default) +database file size 2147483648 (2.0 GiB) +-wal file size 4124152 (3.9 MiB) +modified 2026-10-16T12:02:50Z +schema version 1 +records 12345678 +records with head and tail hashes 1234567 +records with content hash 123456 +size of recorded files 150000000000000 (136.4 TiB) +``` + +- `database` is the path and where it came from, as on the line every subcommand + prints first on stderr (see "Database"). +- `database file size` and `-wal file size` are in bytes, then in human units. + The `-wal` line is there only when a `-wal` file is beside the database, as + while a scan runs. +- `modified` is the time the database file was last changed, in UTC. While the + database is in WAL mode, the newest changes may be in the `-wal` file alone. +- `schema version` is the database's `PRAGMA user_version`. A database with + another version than this build's is refused, as by `report`. +- `records` is the number of records. `records with head and tail hashes` and + `records with content hash` count the records that have those hashes (see + "Database"), and `size of recorded files` adds up the sizes of all records, in + bytes, then in human units. + +`info` runs in two phases, each with its own progress display on stderr (see +"Progress"): `open` opens the database; `count` reads the schema version and +counts the records with one query, a single pass over the table, both in one +read-only transaction, so the numbers agree with each other while a scan is +committing. `info` writes its lines once the `count` display has ended, so on a +terminal they come after the displays. + ### Progress -Use the progress-bar library for all progress of `scan`, `report` and `trees`; -rendering in the style of `pv` is the model. All progress goes to stderr. +Use the progress-bar library for all progress of `scan`, `report`, `trees` and +`info`; rendering in the style of `pv` is the model. All progress goes to +stderr. -`scan`, `report` and `trees` must always show that they are working and not -hung. Each phase gets its own display from the moment it starts, naming the +`scan`, `report`, `trees` and `info` must always show that they are working and +not hung. Each phase gets its own display from the moment it starts, naming the phase and its place among the command's phases and showing its progress, its elapsed time and its estimated time remaining (ETA). @@ -773,6 +819,14 @@ A scan has five phases (`[1/5] load`, `[2/5] walk`, `[3/5] hash`, the rows to stdout as it goes. While the count runs, the display says it is counting the records. +`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"): + +- `open` has no total. While it opens the database, waiting on a busy one + included, the display says it is opening the database. +- `count` has no total either, because one query counts everything: the display + says it is counting the records until the query returns, and its ETA is + unknown. + Each display is one line holding: - the phase and its place among the phases @@ -792,13 +846,14 @@ Example shape (exact layout is flexible, content is not): Additional requirements: -- On a terminal, a phase with an exact total (every phase but `scan`'s `walk` - and the `open` phase of `report` and `trees`, once it has counted what it will - do) is drawn as its line followed by the progress-bar library's bar, 10 - columns wide, which shows the percent in place of the line's own. Any other - phase, and a phase whose bar has no room, is drawn as the library's spinner - followed by its line. Each display is drawn again several times a second, so - its elapsed time keeps moving while a phase waits for its next item. +- On a terminal, a phase with an exact total (every phase but `scan`'s `walk`, + the `open` phase of `report` and `trees`, and both phases of `info`, once it + has counted what it will do) is drawn as its line followed by the progress-bar + library's bar, 10 columns wide, which shows the percent in place of the line's + own. Any other phase, and a phase whose bar has no room, is drawn as the + library's spinner followed by its line. Each display is drawn again several + times a second, so its elapsed time keeps moving while a phase waits for its + next item. - On a terminal no line reaches the terminal's last column, however narrow the terminal is. The phase and its place, the progress, the elapsed time and the ETA are kept whole: a line too wide for the terminal leaves out its rate first @@ -831,10 +886,12 @@ Additional requirements: - `0`: success, even if individual files were skipped with warnings. - `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is already running against the same database, the database cannot be - created/opened/read/written, a missing database for `report`/`trees`, stdout - write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below). + created/opened/read/written, a missing database for `report`/`trees`/`info`, + stdout write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see + below). - `2`: usage error (including `scan` with no `PATH` operand, `scan` with - `--workers` below 1, and `report`/`trees` with any positional argument). + `--workers` below 1, and `report`/`trees`/`info` with any positional + argument). A stdout write failure, such as a full disk, is reported in one line on stderr and exits 1. Two cases never reach sfdupes as a failed write: diff --git a/TODO.md b/TODO.md index 1690a99..62e8918 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,16 @@ # Completed Steps +- new subcommand `info` prints the database's details on stdout, one per line as + a name, a tab and the value: its path and where it came from, the sizes of the + database file and of its `-wal` file when there is one, when the database file + was last changed, the schema version, the number of records, how many have + head and tail hashes and how many a content hash, and the sizes of all records + added up. It opens the database as `report` does, and shows two phases on + stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come + from one read-only transaction, the counts from one query (2026-10-16, + https://git.eeqj.de/sneak/sfdupes/issues/117) + - `report` shows its work on stderr in two phases, `[1/2] open` and `[2/2] read`, with the progress display `scan` and `trees` use, from the moment it starts until the summary line: `open` says it is opening the diff --git a/db.go b/db.go index 40cad40..9404dc1 100644 --- a/db.go +++ b/db.go @@ -79,7 +79,8 @@ ON CONFLICT (path) DO UPDATE SET content = excluded.content ` -// errNoDatabase reports a missing database file for report/trees. +// errNoDatabase reports a missing database file for report, trees and +// info. var errNoDatabase = errors.New( "no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")") @@ -131,9 +132,9 @@ const scanParams = "_pragma=busy_timeout(10000)" + "&_pragma=journal_mode(WAL)" + "&_pragma=synchronous(NORMAL)" -// reportParams are the connection parameters for report and trees: -// read-only, with the same busy timeout. They set no journal mode, -// because setting one is a write. +// reportParams are the connection parameters for report, trees and +// info: read-only, with the same busy timeout. They set no journal +// mode, because setting one is a write. const reportParams = "mode=ro" + "&_pragma=busy_timeout(10000)" + "&_pragma=query_only(1)" @@ -262,9 +263,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) { _ = db.Close() } -// openReportDatabase opens an existing database for the report and -// trees subcommands. A missing database file is an error directing the -// user to run scan first; the schema version must match exactly. +// openReportDatabase opens an existing database for the report, trees +// and info subcommands. A missing database file is an error directing +// the user to run scan first; the schema version must match exactly. func openReportDatabase(ctx context.Context, path string, ) (*sql.DB, error) { @@ -545,6 +546,60 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) { return n, nil } +// databaseCountsSQL counts the records, those with head and tail +// hashes and those with a content hash, and adds up their sizes, in one +// pass over the table. A SUM over no rows is NULL, so on an empty table +// each SUM falls back to 0. +const databaseCountsSQL = ` +SELECT COUNT(*), + COALESCE(SUM(head <> '' AND tail <> ''), 0), + COALESCE(SUM(content <> ''), 0), + COALESCE(SUM(size), 0) +FROM files +` + +// databaseCounts is what info reads from the database: its schema +// version and the counts of databaseCountsSQL. +type databaseCounts struct { + schemaVersion int + records int64 // every record + headTail int64 // records with head and tail hashes + content int64 // records with a content hash + size int64 // the sizes of all records added up +} + +// readDatabaseCounts reads the schema version, then the counts of +// databaseCountsSQL, in one read-only transaction, so both describe the +// database at one moment while a scan is committing. +func readDatabaseCounts(ctx context.Context, + db *sql.DB, +) (databaseCounts, error) { + var c databaseCounts + + // Everything goes through tx: the report connection is the only + // one, so a query on db would wait for tx forever. + tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return c, fmt.Errorf("count records: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + err = tx.QueryRowContext(ctx, "PRAGMA user_version"). + Scan(&c.schemaVersion) + if err != nil { + return c, fmt.Errorf("read schema version: %w", err) + } + + err = tx.QueryRowContext(ctx, databaseCountsSQL). + Scan(&c.records, &c.headTail, &c.content, &c.size) + if err != nil { + return c, fmt.Errorf("count records: %w", err) + } + + return c, nil +} + // loadFileMeta streams every record's path, size, mtime, and whether // it carries hashes to fn. Scan change detection needs no hash // values, and skipping the hash columns keeps the scan's in-memory diff --git a/info.go b/info.go new file mode 100644 index 0000000..4f705c4 --- /dev/null +++ b/info.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// infoPhase names one phase of info with its place among all of them, +// as "[2/2] count", for the phase's progress display. The list here is +// the one place the phases are counted. +func infoPhase(name string) string { + return phaseLabel([]string{openPhase, "count"}, name) +} + +// runInfo implements the info subcommand: it prints the database's +// details on stdout (see writeInfo). It opens the database as report +// does, read-only and without the scan lock, and never touches the +// scanned filesystem. It names the database on stderr before it opens +// it. Any database problem, including a missing database, is fatal. +// +// The work runs in two phases, each under a progress display of its +// own on stderr: open (open the database) and count (count the records, +// with one query). Each display ends before the next phase starts, and +// before an error is returned to be printed. The details are written +// once the last display has ended, so no display is drawn while they +// are written. +func runInfo(ctx context.Context, stdout io.Writer) error { + dbPath, source, err := databasePath() + if err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "info: database %s (%s)\n", + escapePath(dbPath), source) + + // Opening can wait on a busy database for as long as the busy + // timeout. + prog := newProgress(progressParams{ + label: infoPhase(openPhase), total: -1, + waitingFor: openingDatabase, + }) + db, err := openReportDatabase(ctx, dbPath) + + prog.finish() + + if err != nil { + return err + } + + defer func() { _ = db.Close() }() + + // One query counts everything, so the phase never has a total. + prog = newProgress(progressParams{ + label: infoPhase("count"), total: -1, + waitingFor: countingRecords, + }) + counts, err := readDatabaseCounts(ctx, db) + + prog.finish() + + if err != nil { + return fmt.Errorf("database %s: %w", dbPath, err) + } + + return writeInfo(stdout, dbPath, source, counts) +} + +// writeInfo writes the details of the database at dbPath to stdout, +// one per line, as a name, a tab and the value (README "`info` mode"): +// the path and source as databasePath returns them, the sizes of the +// database file and of its -wal file when there is one, when the +// database file was last changed, and the schema version and record +// counts that readDatabaseCounts read. +func writeInfo(stdout io.Writer, dbPath, source string, + counts databaseCounts, +) error { + // SQLite keeps the -wal file beside the database file, named from + // its path with symlinks resolved. + file, err := filepath.EvalSymlinks(dbPath) + if err != nil { + return fmt.Errorf("database: %w", err) + } + + dbFile, err := os.Stat(file) + if err != nil { + return fmt.Errorf("database: %w", err) + } + + walFile, err := os.Stat(file + "-wal") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("database: %w", err) + } + + var b strings.Builder + + fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source) + fmt.Fprintf(&b, "database file size\t%d (%s)\n", + dbFile.Size(), humanBytes(dbFile.Size())) + + if walFile != nil { + fmt.Fprintf(&b, "-wal file size\t%d (%s)\n", + walFile.Size(), humanBytes(walFile.Size())) + } + + fmt.Fprintf(&b, "modified\t%s\n", + dbFile.ModTime().UTC().Format(time.RFC3339)) + fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion) + fmt.Fprintf(&b, "records\t%d\n", counts.records) + fmt.Fprintf(&b, "records with head and tail hashes\t%d\n", + counts.headTail) + fmt.Fprintf(&b, "records with content hash\t%d\n", counts.content) + fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n", + counts.size, humanBytes(counts.size)) + + _, err = io.WriteString(stdout, b.String()) + if err != nil { + return fmt.Errorf("write stdout: %w", err) + } + + return nil +} diff --git a/info_test.go b/info_test.go new file mode 100644 index 0000000..1d308ed --- /dev/null +++ b/info_test.go @@ -0,0 +1,168 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// infoOpenLine is the plain line of info's open phase when it opens the +// database at once; it prints it as the phase starts and as it ends. +const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " + + "eta unknown" + +// infoCountLine is the plain line of info's count phase when the count +// takes under a second; it prints it as the phase starts and as it +// ends. +const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " + + "eta unknown" + +// infoRecs are four records: a duplicate pair with every hash, a file +// of 20 MiB with head and tail hashes but no content hash yet, and a +// file never hashed. +func infoRecs() []scanRec { + return []scanRec{ + {size: 5, head: "h", tail: "t", content: "c", path: "/d/a"}, + {size: 5, head: "h", tail: "t", content: "c", path: "/d/b"}, + {size: 20 << 20, head: "bh", tail: "bt", path: "/d/big"}, + {size: 7, path: "/d/unique"}, + } +} + +// TestRunInfo runs info on an empty and on a filled database, with +// stdout and stderr the same file that is not a terminal, and checks +// everything written there: the line naming the database, each phase's +// line as it starts and as it ends, and then the details, each on a +// line of its own. No -wal file is beside a database scan has closed. +func TestRunInfo(t *testing.T) { + cases := map[string]struct { + recs []scanRec + want string // the details after the database file's time + }{ + "empty": { + want: "schema version\t1\n" + + "records\t0\n" + + "records with head and tail hashes\t0\n" + + "records with content hash\t0\n" + + "size of recorded files\t0 (0 B)\n", + }, + "filled": { + recs: infoRecs(), + want: "schema version\t1\n" + + "records\t4\n" + + "records with head and tail hashes\t3\n" + + "records with content hash\t2\n" + + "size of recorded files\t20971537 (20.0 MiB)\n", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + path := seedDatabase(t, tc.recs) + t.Setenv(databaseEnv, path) + + file, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + stderr := captureStderr(t) + + code := run([]string{cmdInfo}, os.Stderr, os.Stderr) + if code != exitOK { + t.Fatalf("run(info) = %d, want %d; stderr: %s", + code, exitOK, stderr()) + } + + named := path + " (from " + databaseEnv + ")\n" + want := "info: database " + named + + infoOpenLine + "\n" + infoOpenLine + "\n" + + infoCountLine + "\n" + infoCountLine + "\n" + + "database\t" + named + + fmt.Sprintf("database file size\t%d (%s)\n", + file.Size(), humanBytes(file.Size())) + + "modified\t" + file.ModTime().UTC().Format(time.RFC3339) + + "\n" + tc.want + + if got := stderr(); got != want { + t.Errorf("stdout and stderr = %q, want %q", got, want) + } + }) + } +} + +// TestRunInfoDuringScan runs info while a scan holds the lock and has +// the database open in WAL mode, with records committed to the -wal +// file only. info must count those records and show the -wal file's +// size, also when the database path is a symlink: SQLite keeps the +// -wal file beside the symlink's target. +func TestRunInfoDuringScan(t *testing.T) { + path := testDBPath(t) + link := filepath.Join(t.TempDir(), "link.sqlite") + + err := os.Symlink(path, link) + if err != nil { + t.Fatal(err) + } + + db, err := openScanDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + err = applyChanges(t.Context(), db, infoRecs(), nil, nil) + if err != nil { + t.Fatal(err) + } + + holdScanLock(t, path) + + wal, err := os.Stat(path + "-wal") + if err != nil { + t.Fatal(err) + } + + for _, named := range []string{path, link} { + t.Run(filepath.Base(named), func(t *testing.T) { + t.Setenv(databaseEnv, named) + + stdout := runStdout(t, cmdInfo) + + for _, want := range []string{ + "database\t" + named + " (from " + databaseEnv + ")\n", + fmt.Sprintf("\n-wal file size\t%d (%s)\n", + wal.Size(), humanBytes(wal.Size())), + "\nrecords\t4\n", + } { + if !strings.Contains(stdout, want) { + t.Errorf("stdout = %q, want it to hold %q", stdout, want) + } + } + }) + } +} + +func TestRunInfoNeedsOnlyReadAccess(t *testing.T) { + // README "`info` mode": like report, info needs only read access to + // the database file and none to its directory. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + makeReadOnly(t, path) + + stdout := runStdout(t, cmdInfo) + + if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) { + t.Errorf("stdout = %q, want it to hold %q", stdout, want) + } + + if strings.Contains(stdout, "-wal") { + t.Errorf("stdout = %q, want no -wal file between scans", stdout) + } +} diff --git a/main.go b/main.go index 6133b50..8f33973 100644 --- a/main.go +++ b/main.go @@ -8,13 +8,14 @@ // under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB // or larger. scan maintains a persistent SQLite database of file // signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) -// that the reporting subcommands read. +// that the other subcommands read. // // Usage: // // sfdupes scan [--workers N] [-x] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv +// sfdupes info // sfdupes --version // // See README.md for the complete specification. @@ -44,6 +45,7 @@ const ( cmdScan = "scan" cmdReport = "report" cmdTrees = "trees" + cmdInfo = "info" ) // errNoSubcommand is returned by the root command when it is invoked @@ -73,8 +75,8 @@ func main() { // exit code. It is the program's single exit point: the subcommands // return their errors instead of exiting, so every deferred cleanup — // above all closing the database, which checkpoints the SQLite WAL — -// runs before the process ends. The report and trees subcommands write -// their data to stdout. +// runs before the process ends. The report, trees and info subcommands +// write their data to stdout. func run(args []string, stdout, stderr io.Writer) int { // A nil slice makes cobra fall back to os.Args, which would let a // test binary's own flags reach the command tree. @@ -148,6 +150,41 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { root.Flags().BoolVarP(&showVersion, "version", "v", false, "print the version to stdout") + reportCmd := &cobra.Command{ + Use: cmdReport, + Short: "Read the scan database and print the file-level duplicates report", + Args: cobra.NoArgs, + RunE: runE(func(ctx context.Context, _ []string) error { + return runReport(ctx, stdout) + }), + } + + treesCmd := &cobra.Command{ + Use: cmdTrees, + Short: "Read the scan database and print the duplicate-tree report", + Args: cobra.NoArgs, + RunE: runE(func(ctx context.Context, _ []string) error { + return runTrees(ctx, stdout) + }), + } + + infoCmd := &cobra.Command{ + Use: cmdInfo, + Short: "Read the scan database and print its details", + Args: cobra.NoArgs, + RunE: runE(func(ctx context.Context, _ []string) error { + return runInfo(ctx, stdout) + }), + } + + root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd) + + return root +} + +// newScanCommand builds the scan subcommand, with its --workers and -x +// flags. +func newScanCommand() *cobra.Command { var ( scanWorkers int scanOneFS bool @@ -172,27 +209,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, "do not cross filesystem boundaries") - reportCmd := &cobra.Command{ - Use: cmdReport, - Short: "Read the scan database and print the file-level duplicates report", - Args: cobra.NoArgs, - RunE: runE(func(ctx context.Context, _ []string) error { - return runReport(ctx, stdout) - }), - } - - treesCmd := &cobra.Command{ - Use: cmdTrees, - Short: "Read the scan database and print the duplicate-tree report", - Args: cobra.NoArgs, - RunE: runE(func(ctx context.Context, _ []string) error { - return runTrees(ctx, stdout) - }), - } - - root.AddCommand(scanCmd, reportCmd, treesCmd) - - return root + return scanCmd } // checkScanWorkers rejects a scan --workers value below 1. That is a diff --git a/main_test.go b/main_test.go index 9e4db36..f93fca2 100644 --- a/main_test.go +++ b/main_test.go @@ -284,10 +284,16 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) { } func TestRunMissingDatabaseIsFatal(t *testing.T) { - // README §Database: report and trees need an existing database; a - // missing one exits 1 with a message telling the user to run scan, - // after the line naming the database. - for _, name := range []string{cmdReport, cmdTrees} { + // README §Database: report, trees and info need an existing + // database; a missing one exits 1 with a message telling the user to + // run scan, after the line naming the database. + openLines := map[string]string{ + cmdReport: reportOpenLine, + cmdTrees: treesOpenLine, + cmdInfo: infoOpenLine, + } + + for name, openLine := range openLines { t.Run(name, func(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) @@ -305,13 +311,8 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) { // Each shows its open phase, as it starts and as it ends, // before the error. - opening := reportOpenLine + "\n" + reportOpenLine + "\n" - if name == cmdTrees { - opening = treesOpenLine + "\n" + treesOpenLine + "\n" - } - want := name + ": database " + path + " (from " + databaseEnv + - ")\n" + opening + + ")\n" + openLine + "\n" + openLine + "\n" + "sfdupes: " + path + ": no database (run \"sfdupes " + "scan\" first, or set " + databaseEnv + ")\n" if got := stderr(); got != want { @@ -512,9 +513,9 @@ func TestRunNamesDefaultDatabase(t *testing.T) { // exist at the default path on the machine running the test, so // nothing here may create or read one: scan gets an operand that // does not exist, which stops it before it takes the lock, and - // report and trees a cancelled context, which stops them before - // they read anything. report and trees show their open phase after - // the line. + // report, trees and info a cancelled context, which stops them + // before they read anything. They show their open phase after the + // line. t.Setenv(databaseEnv, "") // Whether scan calls the database new depends on that machine too. @@ -552,6 +553,13 @@ func TestRunNamesDefaultDatabase(t *testing.T) { want: "trees: database " + defaultDatabasePath + " (default)\n" + treesOpenLine + "\n" + treesOpenLine + "\n", }, + cmdInfo: { + run: func() error { + return runInfo(cancelledContext(t), io.Discard) + }, + want: "info: database " + defaultDatabasePath + " (default)\n" + + infoOpenLine + "\n" + infoOpenLine + "\n", + }, } for name, tc := range cases { @@ -578,6 +586,7 @@ func TestRunUsageErrors(t *testing.T) { "scan without paths": {[]string{cmdScan}, usageMarker}, "report with args": {[]string{cmdReport, "x"}, usageMarker}, "trees with args": {[]string{cmdTrees, "x"}, usageMarker}, + "info with args": {[]string{cmdInfo, "x"}, usageMarker}, "unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker}, "unknown subcommand": {[]string{"nope"}, "unknown command"}, } @@ -1258,7 +1267,7 @@ func TestRunReportsDuringScan(t *testing.T) { func TestRunStdoutClosedIsFatal(t *testing.T) { // README §Error handling: a stdout write failure exits 1, reported // in one line on stderr. - for _, name := range []string{cmdReport, cmdTrees} { + for _, name := range []string{cmdReport, cmdTrees, cmdInfo} { t.Run(name, func(t *testing.T) { t.Setenv(databaseEnv, testDBPath(t)) @@ -1308,6 +1317,7 @@ func TestStdoutWriteErrorPropagates(t *testing.T) { cases := map[string]func(context.Context, io.Writer) error{ cmdReport: runReport, cmdTrees: runTrees, + cmdInfo: runInfo, } for name, fn := range cases { diff --git a/progress.go b/progress.go index 7b0f236..321b226 100644 --- a/progress.go +++ b/progress.go @@ -51,6 +51,14 @@ const barMargin = barWidth + 11 // says while it counts them, before it has its total. const countingRecords = "counting the records" +// openPhase is the name of the phase in which report, trees and info +// open the database, and openingDatabase what its display says while +// it does. +const ( + openPhase = "open" + openingDatabase = "opening the database" +) + // stderrIsTTY reports whether stderr is attached to a terminal. func stderrIsTTY() bool { return term.IsTerminal(int(os.Stderr.Fd())) diff --git a/report.go b/report.go index 2555c2a..0767e4d 100644 --- a/report.go +++ b/report.go @@ -34,7 +34,7 @@ type scanRec struct { // them, as "[2/2] read", for the phase's progress display. The list // here is the one place the phases are counted. func reportPhase(name string) string { - return phaseLabel([]string{"open", "read"}, name) + return phaseLabel([]string{openPhase, "read"}, name) } // runReport implements the report subcommand: it prints the file-level @@ -61,8 +61,8 @@ func runReport(ctx context.Context, stdout io.Writer) error { // Opening can wait on a busy database for as long as the busy // timeout. prog := newProgress(progressParams{ - label: reportPhase("open"), total: -1, - waitingFor: "opening the database", + label: reportPhase(openPhase), total: -1, + waitingFor: openingDatabase, }) db, err := openReportDatabase(ctx, dbPath) diff --git a/report_test.go b/report_test.go index 994564a..c60066e 100644 --- a/report_test.go +++ b/report_test.go @@ -29,7 +29,8 @@ func awkwardPairRecs() []scanRec { } } -// seedDatabase writes recs into a fresh database and returns its path. +// seedDatabase writes recs into a fresh database, closes it as scan +// does, out of WAL mode, and returns its path. func seedDatabase(t *testing.T, recs []scanRec) string { t.Helper() @@ -45,10 +46,7 @@ func seedDatabase(t *testing.T, recs []scanRec) string { t.Fatal(err) } - err = db.Close() - if err != nil { - t.Fatal(err) - } + closeScanDatabase(t.Context(), db, path) return path } diff --git a/trees.go b/trees.go index 0944b87..4e4028f 100644 --- a/trees.go +++ b/trees.go @@ -28,7 +28,7 @@ type treeNode struct { // as "[2/4] read", for the phase's progress display. The list here is // the one place the phases are counted. func treesPhase(name string) string { - return phaseLabel([]string{"open", "read", "group", "write"}, name) + return phaseLabel([]string{openPhase, "read", "group", "write"}, name) } // runTrees implements the trees subcommand: it reads every record from @@ -57,8 +57,8 @@ func runTrees(ctx context.Context, stdout io.Writer) error { // Opening can wait on a busy database for as long as the busy // timeout. prog := newProgress(progressParams{ - label: treesPhase("open"), total: -1, - waitingFor: "opening the database", + label: treesPhase(openPhase), total: -1, + waitingFor: openingDatabase, }) db, err := openReportDatabase(ctx, dbPath) -- 2.54.0