From e0d578a707c5c9fd0a9f231e1c7e34f83be613d8 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 24 Jul 2026 02:54:08 +0700 Subject: [PATCH 1/2] Specify persistent SQLite scan database in README; plan in TODO.md scan will maintain a persistent database of file signatures (default /var/lib/sfdupes/db.sqlite, overridable via SFDUPES_DATABASE) that survives between runs; rescans hash only new or changed files (mtime/size) and remove records for vanished files, so scan can be cronned daily. report and trees will read the database instead of a scan stream. --- README.md | 291 +++++++++++++++++++++++++++++++++++------------------- TODO.md | 39 +++++++- 2 files changed, 228 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index fcaed1a..02b1ee0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,12 @@ SHA-256 of their first 1024 bytes, and identical SHA-256 of their last content (the middle of the file is never read); the intended use is finding duplicate downloads and duplicated directory trees on multi-terabyte ZFS servers where reading every byte is prohibitively -expensive. +expensive. `scan` maintains a persistent SQLite database of file +signatures that survives between runs, so it can be run from cron and +the reports can be generated at any time from the most recent scan. + +This tool was created by [@sneak](https://sneak.berlin) to scratch an itch, +using Claude Code/Fable. This README is the complete and authoritative specification. @@ -20,29 +25,39 @@ This README is the complete and authoritative specification. ```sh make build -./sfdupes scan /srv > files.dat -./sfdupes report files.dat > dupes.tsv -./sfdupes trees files.dat > dupetrees.tsv +export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite" +./sfdupes scan /srv +./sfdupes report > dupes.tsv +./sfdupes trees > dupetrees.tsv ``` -`scan` walks one or more filesystem trees and emits one record per -regular file (path, size, mtime, head hash, tail hash). `report` -ingests that stream and prints the file-level duplicates report. -`trees` ingests the same stream and prints the duplicate-tree report. A -missing/invalid subcommand — or a `scan` invocation with no `PATH` -operand — prints a usage message and exits 2. +`scan` walks one or more filesystem trees and maintains one database +record per regular file (path, size, mtime, head hash, tail hash). The +database persists between runs; a rescan only hashes files that are new +or changed, and removes records for files that no longer exist. +`report` reads the database and prints the file-level duplicates +report. `trees` reads the same database and prints the duplicate-tree +report. A missing/invalid subcommand — or a `scan` invocation with no +`PATH` operand — prints a usage message and exits 2. + +The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed +anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a +daily `sfdupes scan` cron job, with the reporting commands run +interactively whenever needed; their results are as fresh as the last +completed scan. ## Rationale Duplicate finders that hash entire files do not scale to the target environment: ~10 million files and ~150 TB on possibly slow or busy disks (a ZFS pool under resilver). Reading at most 2 KiB per file makes -a full-filesystem sweep tractable, and the resulting scan stream is -self-contained, so the expensive filesystem pass runs exactly once and -all analysis happens offline. The end goal is not individual files but -whole duplicated trees — duplicate extractions, duplicate downloads, -copied project trees — which an operator can consider removing as a -unit. +a full-filesystem sweep tractable, and the signatures are kept in a +persistent database, so the expensive filesystem pass is incremental: a +rescan re-hashes only files whose recorded mtime or size changed, and +all analysis happens offline from the database alone. The end goal is +not individual files but whole duplicated trees — duplicate +extractions, duplicate downloads, copied project trees — which an +operator can consider removing as a unit. ## Design @@ -59,10 +74,12 @@ Goals, in order: filesystem, possibly slow or busy disks (ZFS pool under resilver). Holding the full file list in memory is acceptable; reading file contents beyond 2 KiB per file is not. -3. **Scan once, analyze offline.** The expensive filesystem scan - produces a self-contained stream; all analysis (`report`, `trees`) - works from that stream alone and must never touch the scanned - filesystem again. +3. **Scan incrementally, analyze offline.** The expensive filesystem + scan maintains a persistent database; an unchanged file is never + read again on a rescan. All analysis (`report`, `trees`) works from + the database alone and must never touch the scanned filesystem + again. `scan` is designed to be cronned; the reports run at any + time against the last completed scan. 4. **Clean stream separation.** Everything on stdout is machine-readable data. All progress, warnings, and summaries go to stderr. Never mix them. @@ -71,54 +88,116 @@ Goals, in order: - Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: `sfdupes`. -- Dependencies: standard library, `github.com/spf13/cobra` for the CLI, - and **one progress-bar library** - (`github.com/schollz/progressbar/v3`). `github.com/spf13/viper` is - permitted if configuration-file support is ever needed, but is not - currently used. No other third-party deps. +- Dependencies: standard library, `github.com/spf13/cobra` for the + CLI, **one progress-bar library** + (`github.com/schollz/progressbar/v3`), and **one SQLite driver** + (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled). + `github.com/spf13/viper` is permitted if configuration-file support + is ever needed, but is not currently used. No other third-party + deps. - Cross-compilation is not a concern. Builds run with cgo disabled (the `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. - Analysis modes (`report`, `trees`) must be deterministic: identical - input stream, identical output, regardless of record order. + database contents, identical output, regardless of the order in + which records were inserted. ### Subcommands Three subcommands, all implemented: -1. `scan` — walk the filesystem and emit one signature record per - regular file. -2. `report` — file-level duplicate report from the scan stream. +1. `scan` — walk the filesystem and synchronize the database: one + signature record per regular file. +2. `report` — file-level duplicate report from the database. 3. `trees` — tree-level duplicate report: reconstruct the directory - hierarchy from the scan stream, compute a Merkle-style digest per - directory, and report maximal groups of identical trees. + hierarchy from the database records, compute a Merkle-style digest + per directory, and report maximal groups of identical trees. ``` -sfdupes scan [--workers N] [-x] PATH... > files.dat -sfdupes report [files.dat|-] > dupes.tsv -sfdupes trees [files.dat|-] > dupetrees.tsv +sfdupes scan [--workers N] [-x] PATH... +sfdupes report > dupes.tsv +sfdupes trees > dupetrees.tsv ``` +### Database + +All three subcommands operate on a single SQLite database file: + +- Location: the value of the `SFDUPES_DATABASE` environment variable + when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. + There is no command-line flag. +- `scan` creates the database (and its parent directory) on first + use. `report` and `trees` require an existing database; a missing + database file is a fatal error (exit 1) telling the user to run + `scan` first. +- The database uses WAL journal mode and a busy timeout, so running a + report while a cron `scan` is in progress is safe; the reports see + the last committed state. Each scan commits its changes in a single + transaction, so a report never observes a half-finished scan and a + scan that dies partway leaves the previous state intact. +- Schema (`PRAGMA user_version` is the schema version, currently 1; a + database with any other version is a fatal error): + + ```sql + CREATE TABLE files ( + path BLOB PRIMARY KEY, -- absolute path, raw bytes + size INTEGER NOT NULL, -- bytes, from lstat + mtime INTEGER NOT NULL, -- Unix seconds, from lstat + head TEXT NOT NULL, -- lowercase-hex SHA-256, first 1 KiB + tail TEXT NOT NULL -- lowercase-hex SHA-256, last 1 KiB + ) WITHOUT ROWID; + ``` + + Paths are stored as BLOBs because Unix paths are raw bytes, not + guaranteed UTF-8. `mtime` is used only for change detection; it is + not part of the duplicate key. + ### `scan` mode `scan` requires one or more `PATH` operands naming the trees to scan. There is no default path; invoking `scan` with no operand is a usage error (usage message on stderr, exit 2). An operand may be a directory or a regular file; an operand that does not exist is a fatal error -(exit 1). Operands are walked in the order given; overlapping operands -(one containing another) emit their common files once per operand, so -callers should pass disjoint paths. +(exit 1). Because database records persist between runs and are keyed +by absolute path, each operand is resolved to an absolute, lexically +cleaned path (symlinks are not resolved) before walking, so results do +not depend on the working directory. Operands are walked in the order +given; overlapping operands (one containing another) are harmless — a +file reached via multiple operands produces one database record. -`scan` runs **three sequential passes**, in this order, so that every +`scan` synchronizes the database with the filesystem state under the +scanned operands: + +- A file not yet in the database is hashed and inserted. +- A file already in the database is **skipped without reading its + contents** when its lstat size equals the recorded size and its + lstat mtime is not newer than the recorded mtime. This is what + makes a daily rescan cheap. +- A file whose mtime is newer than recorded, or whose size differs, + is re-hashed and its record updated. +- A database record whose path lies under one of the scanned operands + but was not successfully processed this run is deleted. This + removes records for deleted files. It also removes records for + paths that failed to stat or hash this run: the database only ever + contains signatures verified by the most recent scan that covered + them (a subsequent successful scan re-adds such files). +- Database records outside the scanned operands are untouched, so + disjoint trees can be scanned on different schedules into the same + database. + +`scan` runs **four sequential passes**, in this order, so that every expensive pass has an exact total for meaningful progress and ETA: 1. **walk** — recursively enumerate the tree under each `PATH` in turn, collecting the list of regular-file paths. Total unknown while running: show a live count, not a percentage. 2. **stat** — `lstat` every collected path, recording size and mtime. -3. **hash** — for each file, read the first `min(1024, size)` bytes and - the last `min(1024, size)` bytes (the two reads overlap when - `size < 2048`; for `size == 0` hash the empty input) and compute the - SHA-256 of each. Emit the output record. +3. **hash** — for each new or changed file (per the rules above), read + the first `min(1024, size)` bytes and the last `min(1024, size)` + bytes (the two reads overlap when `size < 2048`; for `size == 0` + hash the empty input) and compute the SHA-256 of each. Unchanged + files are not read and do not appear in this pass's total. +4. **update** — apply all insertions, updates, and deletions to the + database in a single transaction. Rules for the walk: @@ -134,52 +213,46 @@ Rules for the walk: - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final - summary reports how many were skipped. + summary reports how many were skipped. As specified above, a + skipped path that has a database record from an earlier scan loses + that record; an unreadable directory subtree likewise loses its + records (accepted: the database mirrors what the latest scan could + actually verify). Concurrency: the stat and hash passes use a worker pool (`--workers`, -default `runtime.NumCPU()`). The main goroutine owns stdout writing and -progress rendering; progress display must never block the workers. +default `runtime.NumCPU()`). The main goroutine owns database writes +and progress rendering; progress display must never block the workers. -#### Output record format - -One record per file on stdout, NUL-terminated (`\x00`), with -tab-separated fields, **path last** so tabs or newlines embedded in -paths cannot corrupt the record structure: +`scan` writes nothing to stdout. The summary line on stderr reports the +files seen this run broken down by disposition, plus skips: ``` -\t\t\t\t\x00 +scan: 123456 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped ``` -- `size`: decimal bytes, from the stat pass. -- `mtime_unix`: decimal Unix seconds. Informational only; not part of - the duplicate key. -- Hashes: lowercase hex, 64 chars each. -- Record order is unspecified (workers complete out of order); the - analysis modes must not depend on ordering. +(`removed` counts deleted database records, which are not part of the +files-seen total.) ### `report` mode -`report` reads the scan stream from the file named in its first -positional argument, or from stdin if the argument is absent or `-`. +`report` reads every record from the database and takes no positional +arguments. **`report` must never touch the filesystem being analyzed.** It does not stat, open, or otherwise access any path that appears in the records; its -only I/O is reading the scan file/stdin and writing stdout/stderr. It must +only I/O is reading the database and writing stdout/stderr. It must produce identical output whether or not the scanned filesystem is still mounted. Processing: -- Parse records; a record that does not have exactly 5 fields or whose - size is non-numeric is counted as malformed and skipped (warn once - with the total malformed count in the summary, not per record). - Group records by the key `(size, head_hash, tail_hash)`. - Every group with two or more paths is a duplicate group. - Within each group, sort paths lexicographically (byte order). The first path is the group's `first`; every other path is a `dupe`. - Order groups by size descending (biggest reclaimable space first), tie-broken by `first` path ascending. Output must be fully - deterministic for a given input. + deterministic for a given database state. #### Report output format @@ -192,16 +265,16 @@ first dupe size /srv/a/big.iso /srv/c/big-copy2.iso 4294967296 ``` -Summary to stderr: records read, malformed count (if any), number of -duplicate groups, number of dupe files, and total reclaimable bytes -(sum of `size` over all dupe rows) in human units. +Summary to stderr: records read, number of duplicate groups, number of +dupe files, and total reclaimable bytes (sum of `size` over all dupe +rows) in human units. ### `trees` mode -`trees` reads the same scan stream as `report` (same argument handling, -same parsing and malformed-record rules) and reports **entire duplicate -directory trees**: directories under which the exact same set of relative -paths exists with the exact same file signatures. +`trees` reads the same database as `report` (no positional arguments) +and reports **entire duplicate directory trees**: directories under +which the exact same set of relative paths exists with the exact same +file signatures. **`trees` must never touch the filesystem being analyzed** — the same rule as `report`. The directory hierarchy is reconstructed purely from @@ -222,10 +295,10 @@ Definitions: equal. Equal digests imply equal recursive file count and equal total byte size. -Known limitation (accepted): only regular files that appear in the scan -stream define a tree. Empty directories are invisible, and a file skipped -during the scan (e.g. permission error) in one copy but not the other -will make otherwise-identical trees compare as different. +Known limitation (accepted): only regular files that appear in the +database define a tree. Empty directories are invisible, and a file +skipped during the scan (e.g. permission error) in one copy but not the +other will make otherwise-identical trees compare as different. Processing: @@ -254,22 +327,22 @@ first dupe files size /srv/a/project /srv/backup/project 3417 104857600 ``` -Summary to stderr: records read, malformed count (if any), number of -duplicate-tree groups, number of dupe trees, and total reclaimable bytes -(sum of `size` over all dupe rows) in human units. +Summary to stderr: records read, number of duplicate-tree groups, +number of dupe trees, and total reclaimable bytes (sum of `size` over +all dupe rows) in human units. ### Progress Use the progress-bar library for all scan-pass progress; rendering in the style of `pv` is the model. All progress goes to stderr. -Each scan pass gets its own bar. Required elements for the stat and hash -passes (known totals): +Each scan pass gets its own bar. Required elements for the stat, hash, +and update passes (known totals): - elapsed time - estimated time remaining -- a `[m/n] x%` display (files processed / total files, percent) -- current rate (files/s) +- a `[m/n] x%` display (items processed / total items, percent) +- current rate (items/s) Example shape (exact layout is flexible, content is not): @@ -292,9 +365,11 @@ Additional requirements: ### Error handling and exit codes - `0`: success, even if individual files were skipped with warnings. -- `1`: fatal error (e.g., a `PATH` operand does not exist, cannot - read the scan input, stdout write failure). -- `2`: usage error (including `scan` with no `PATH` operand). +- `1`: fatal error (e.g., a `PATH` operand does not exist, the + database cannot be created/opened/read/written, a missing database + for `report`/`trees`, stdout write failure). +- `2`: usage error (including `scan` with no `PATH` operand and + `report`/`trees` with any positional argument). ## Build @@ -310,7 +385,7 @@ The `Makefile` is the single source of truth for all operations: - `make docker` — build the Docker image, which runs `make check` as a build stage. - `make hooks` — install the pre-commit hook. -- `make clean` — remove the binary and any local `files.dat`. +- `make clean` — remove the binary and any legacy local `files.dat`. ### Definition of done @@ -323,6 +398,7 @@ All of the following, run in this directory, must pass: ```sh d=$(mktemp -d) + export SFDUPES_DATABASE="$d/db.sqlite" mkdir -p "$d/a" "$d/b" head -c 2000 /dev/urandom > "$d/a/one.bin" cp "$d/a/one.bin" "$d/b/copy.bin" @@ -339,28 +415,41 @@ All of the following, run in this directory, must pass: cp "$d/t1/sub/f2" "$d/t2/sub/f2" cp "$d/t1/f1" "$d/t3/f1" cp "$d/t1/sub/f2" "$d/t3/sub/f2renamed" - ./sfdupes scan "$d" > files.dat - ./sfdupes report files.dat - ./sfdupes trees files.dat + ./sfdupes scan "$d" + ./sfdupes report + ./sfdupes trees + # incremental behavior (scan a subtree; records elsewhere persist): + ./sfdupes scan "$d/a" # everything unchanged, nothing hashed + printf 'z' >> "$d/a/one.bin" # modify: next scan re-hashes it + rm "$d/a/unique.bin" # delete: next scan removes its record + ./sfdupes scan "$d/a" # 1 updated, 1 removed + ./sfdupes report ``` - Expected from `report`: `one.bin`/`copy.bin`/`copy2.bin` form one - group (two dupe rows, `first` is the lexicographically smallest - path); `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/ - `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair; - `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere; - groups ordered by size descending; piping scan directly into report - (`./sfdupes scan "$d" | ./sfdupes report`) gives the same - rows. + (The scan database lives inside `$d` here purely for test hygiene; + scanning `$d` therefore also records the SQLite file itself, which + is harmless.) + + Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` + form one group (two dupe rows, `first` is the lexicographically + smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group; + `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group; + `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and + `tiny3` appear nowhere; groups ordered by size descending. Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set differs by name). + Expected from the second `report` (after the modify/delete rescan): + `one.bin` has left its group (its content changed), so + `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is + gone from the database. + The test suite automates this scenario (see `scan_test.go`), plus a - negative check: `report` and `trees` operate on the captured stream - alone and never touch the scanned filesystem. + negative check: `report` and `trees` operate on the database alone + and never touch the scanned filesystem. ## TODO @@ -371,7 +460,9 @@ Tracked in [TODO.md](TODO.md). - No full-content verification, no byte-for-byte compare, no deletion or linking of duplicates. The reports are advisory; acting on them is the user's job. -- No persistence formats beyond the scan stream described above. +- No persistence beyond the SQLite database described above; no + export/import formats. +- No daemon or filesystem watcher; scheduling rescans is cron's job. ## License diff --git a/TODO.md b/TODO.md index f9b1099..407e861 100644 --- a/TODO.md +++ b/TODO.md @@ -14,8 +14,41 @@ # Next Step -- convert Makefile targets to scripts-to-rule-them-all `script/` - entrypoints like the other managed repos +- persistent scan database (branch `persistent-database`): `scan` + maintains a SQLite database that survives between runs so it can be + cronned daily; `report` and `trees` read the database instead of a + scan stream. Database path defaults to `/var/lib/sfdupes/db.sqlite`, + overridable via `SFDUPES_DATABASE`. Plan: + - [x] update `README.md` (the authoritative spec) for the database: + new Database section, revised `scan`/`report`/`trees` modes, + dependency list (`modernc.org/sqlite`, pure Go, cgo stays + disabled), exit codes, smoke test with incremental steps + - [ ] add `modernc.org/sqlite` (hash-pinned via `go.sum`) + - [ ] `db.go`: path resolution (`SFDUPES_DATABASE` env, default + `/var/lib/sfdupes/db.sqlite`), open/create with WAL + + busy-timeout pragmas and `PRAGMA user_version` schema check, + `files` table (path BLOB primary key, size, mtime, head, + tail), load-all-rows, and single-transaction apply of + upserts/deletes + - [ ] `scan.go`: resolve operands to absolute paths; diff stat + results against loaded rows (reuse hashes when size matches + and mtime is not newer); hash only new/changed files; delete + rows under the scanned operands not successfully processed; + leave rows outside the operands untouched; new summary line + (added/updated/removed/unchanged/skipped); "update" progress + pass for the database write + - [ ] `report.go`/`trees.go`: read records from the database (no + positional args, no stream parsing); drop the NUL-stream + parser and malformed-record handling + - [ ] `main.go`: updated usage strings; `report`/`trees` take no + args (usage error otherwise) + - [ ] tests: db open/env-override/schema tests; incremental scan + tests (add, mtime update, delete, unchanged-hash-reuse, + out-of-scope rows untouched, error paths); rework pipeline + test to go through the database; keep walk/stat/hash/grouping + unit tests + - [ ] `.gitignore`: local `*.sqlite` artifacts + - [ ] `make check`, `make docker`, and the README smoke test pass # Completed Steps @@ -33,6 +66,8 @@ # Future Steps +- convert Makefile targets to scripts-to-rule-them-all `script/` + entrypoints like the other managed repos - possible later features (explicitly out of scope per README): full-content verification of candidates, removal-script helpers From d8fbcb32c24d0097785ff266c2b2f0a1ea41b3b4 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 24 Jul 2026 03:08:49 +0700 Subject: [PATCH 2/2] Implement persistent SQLite scan database scan now synchronizes a database that survives between runs (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of emitting a stream: operands are resolved to absolute paths, unchanged files (same size, mtime not newer than recorded) are never re-read, new and changed files are hashed, and records under the scanned operands that were not verified this run are deleted; records outside the operands are untouched. All changes commit in a single transaction, and WAL journaling with a busy timeout keeps a report run during a cron scan safe. report and trees read the database (no positional arguments); the NUL-terminated stream format, its parser, and the malformed-record handling are gone. The driver is modernc.org/sqlite (pure Go), so builds keep cgo disabled. --- .gitignore | 3 + TODO.md | 50 ++---- db.go | 319 ++++++++++++++++++++++++++++++++++++++ db_test.go | 180 ++++++++++++++++++++++ go.mod | 9 ++ go.sum | 48 ++++++ main.go | 32 ++-- report.go | 153 ++++--------------- report_test.go | 110 ++----------- scan.go | 269 ++++++++++++++++++++++++++++---- scan_test.go | 408 +++++++++++++++++++++++++++++++++++++++++-------- trees.go | 27 ++-- 12 files changed, 1229 insertions(+), 379 deletions(-) create mode 100644 db.go create mode 100644 db_test.go diff --git a/.gitignore b/.gitignore index 5f4b081..fe37e88 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,9 @@ node_modules/ # Local scan data files.dat +*.sqlite +*.sqlite-shm +*.sqlite-wal # Agent worktrees .claude/worktrees/ diff --git a/TODO.md b/TODO.md index 407e861..ecf49ef 100644 --- a/TODO.md +++ b/TODO.md @@ -14,44 +14,22 @@ # Next Step -- persistent scan database (branch `persistent-database`): `scan` - maintains a SQLite database that survives between runs so it can be - cronned daily; `report` and `trees` read the database instead of a - scan stream. Database path defaults to `/var/lib/sfdupes/db.sqlite`, - overridable via `SFDUPES_DATABASE`. Plan: - - [x] update `README.md` (the authoritative spec) for the database: - new Database section, revised `scan`/`report`/`trees` modes, - dependency list (`modernc.org/sqlite`, pure Go, cgo stays - disabled), exit codes, smoke test with incremental steps - - [ ] add `modernc.org/sqlite` (hash-pinned via `go.sum`) - - [ ] `db.go`: path resolution (`SFDUPES_DATABASE` env, default - `/var/lib/sfdupes/db.sqlite`), open/create with WAL + - busy-timeout pragmas and `PRAGMA user_version` schema check, - `files` table (path BLOB primary key, size, mtime, head, - tail), load-all-rows, and single-transaction apply of - upserts/deletes - - [ ] `scan.go`: resolve operands to absolute paths; diff stat - results against loaded rows (reuse hashes when size matches - and mtime is not newer); hash only new/changed files; delete - rows under the scanned operands not successfully processed; - leave rows outside the operands untouched; new summary line - (added/updated/removed/unchanged/skipped); "update" progress - pass for the database write - - [ ] `report.go`/`trees.go`: read records from the database (no - positional args, no stream parsing); drop the NUL-stream - parser and malformed-record handling - - [ ] `main.go`: updated usage strings; `report`/`trees` take no - args (usage error otherwise) - - [ ] tests: db open/env-override/schema tests; incremental scan - tests (add, mtime update, delete, unchanged-hash-reuse, - out-of-scope rows untouched, error paths); rework pipeline - test to go through the database; keep walk/stat/hash/grouping - unit tests - - [ ] `.gitignore`: local `*.sqlite` artifacts - - [ ] `make check`, `make docker`, and the README smoke test pass +- convert Makefile targets to scripts-to-rule-them-all `script/` + entrypoints like the other managed repos # Completed Steps +- persistent scan database (2026-07-24, branch `persistent-database`): + `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure + Go, cgo stays disabled) keyed by absolute path that survives between + runs — a rescan hashes only new or changed files (by mtime/size), + deletes records for files vanished from under the scanned operands, + and leaves records outside them untouched, so `scan` can be cronned + daily; `report` and `trees` read the database (no positional + arguments) instead of a scan stream. Database at + `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; + WAL journaling plus a single-transaction update keep a report run + during a scan safe - add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`, and push `main` plus tags (2026-07-23) - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required @@ -66,8 +44,6 @@ # Future Steps -- convert Makefile targets to scripts-to-rule-them-all `script/` - entrypoints like the other managed repos - possible later features (explicitly out of scope per README): full-content verification of candidates, removal-script helpers diff --git a/db.go b/db.go new file mode 100644 index 0000000..1b54c67 --- /dev/null +++ b/db.go @@ -0,0 +1,319 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + + // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo + // disabled. + _ "modernc.org/sqlite" +) + +// defaultDatabasePath is where the persistent scan database lives when +// SFDUPES_DATABASE is not set. +const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite" + +// databaseEnv is the environment variable that overrides the database +// path. +const databaseEnv = "SFDUPES_DATABASE" + +// schemaVersion is the database schema version this build reads and +// writes, stored in PRAGMA user_version. +const schemaVersion = 1 + +// dbDirPerm is the mode for a database parent directory created by +// scan. +const dbDirPerm = 0o755 + +// createTableSQL is the schema applied to a fresh database. Paths are +// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. +const createTableSQL = ` +CREATE TABLE files ( + path BLOB PRIMARY KEY, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL, + head TEXT NOT NULL, + tail TEXT NOT NULL +) WITHOUT ROWID +` + +// upsertSQL inserts one file record, replacing any existing record for +// the same path. +const upsertSQL = ` +INSERT INTO files (path, size, mtime, head, tail) +VALUES (?, ?, ?, ?, ?) +ON CONFLICT (path) DO UPDATE SET + size = excluded.size, mtime = excluded.mtime, + head = excluded.head, tail = excluded.tail +` + +// errNoDatabase reports a missing database file for report/trees. +var errNoDatabase = errors.New( + "no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")") + +// errSchemaVersion reports a database whose schema version this build +// does not understand. +var errSchemaVersion = errors.New("unsupported database schema version") + +// databasePath resolves the database location: SFDUPES_DATABASE when +// set and non-empty, the compiled-in default otherwise. +func databasePath() string { + if p := os.Getenv(databaseEnv); p != "" { + return p + } + + return defaultDatabasePath +} + +// openDB opens the SQLite database at path with WAL journaling and a +// busy timeout, so a report can run while a cron scan is in progress. +// It does not create or verify the schema. +func openDB(path string) (*sql.DB, error) { + dsn := "file:" + path + + "?_pragma=busy_timeout(10000)" + + "&_pragma=journal_mode(WAL)" + + "&_pragma=synchronous(NORMAL)" + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("open database %s: %w", path, err) + } + + // A single connection avoids SQLITE_BUSY between this process's + // own connections; concurrency lives in the worker pools, not in + // parallel database access. + db.SetMaxOpenConns(1) + + return db, nil +} + +// openScanDatabase opens the database for the scan subcommand, creating +// the file, its parent directory, and the schema as needed. +func openScanDatabase(path string) (*sql.DB, error) { + err := os.MkdirAll(filepath.Dir(path), dbDirPerm) + if err != nil { + return nil, fmt.Errorf("create database directory: %w", err) + } + + db, err := openDB(path) + if err != nil { + return nil, err + } + + err = initSchema(db) + if err != nil { + _ = db.Close() + + return nil, fmt.Errorf("database %s: %w", path, err) + } + + return db, nil +} + +// openReportDatabase opens an existing database for the report and +// trees subcommands. A missing database file is an error directing the +// user to run scan first; the schema version must match exactly. +func openReportDatabase(path string) (*sql.DB, error) { + _, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%s: %w", path, errNoDatabase) + } + + if err != nil { + return nil, fmt.Errorf("database: %w", err) + } + + db, err := openDB(path) + if err != nil { + return nil, err + } + + v, err := userVersion(db) + if err != nil { + _ = db.Close() + + return nil, fmt.Errorf("database %s: %w", path, err) + } + + if v != schemaVersion { + _ = db.Close() + + return nil, fmt.Errorf("database %s: version %d, want %d: %w", + path, v, schemaVersion, errSchemaVersion) + } + + return db, nil +} + +// initSchema creates the schema on a fresh database and verifies the +// schema version on an existing one. +func initSchema(db *sql.DB) error { + v, err := userVersion(db) + if err != nil { + return err + } + + switch v { + case 0: + return createSchema(db) + case schemaVersion: + return nil + default: + return fmt.Errorf("version %d, want %d: %w", + v, schemaVersion, errSchemaVersion) + } +} + +// createSchema applies the schema to a fresh database and stamps the +// schema version. +func createSchema(db *sql.DB) error { + ctx := context.Background() + + _, err := db.ExecContext(ctx, createTableSQL) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + + _, err = db.ExecContext(ctx, + "PRAGMA user_version = "+strconv.Itoa(schemaVersion)) + if err != nil { + return fmt.Errorf("set schema version: %w", err) + } + + return nil +} + +// userVersion reads the database's PRAGMA user_version. +func userVersion(db *sql.DB) (int, error) { + var v int + + err := db.QueryRowContext(context.Background(), + "PRAGMA user_version").Scan(&v) + if err != nil { + return 0, fmt.Errorf("read schema version: %w", err) + } + + return v, nil +} + +// loadFileRows reads every record from the files table. +func loadFileRows(db *sql.DB) ([]scanRec, error) { + rows, err := db.QueryContext(context.Background(), + "SELECT path, size, mtime, head, tail FROM files") + if err != nil { + return nil, fmt.Errorf("read records: %w", err) + } + + defer func() { _ = rows.Close() }() + + var recs []scanRec + + for rows.Next() { + var ( + path []byte + r scanRec + ) + + err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail) + if err != nil { + return nil, fmt.Errorf("read record: %w", err) + } + + r.path = string(path) + recs = append(recs, r) + } + + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("read records: %w", err) + } + + return recs, nil +} + +// applyChanges writes one scan's database changes — upserts for new and +// changed files, deletes for vanished ones — in a single transaction, +// so a concurrent report never observes a half-finished scan. Progress +// is rendered on prog (one increment per change). +func applyChanges(db *sql.DB, upserts []scanRec, deletes []string, + prog *progress, +) error { + ctx := context.Background() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + err = execUpserts(ctx, tx, upserts, prog) + if err != nil { + return err + } + + err = execDeletes(ctx, tx, deletes, prog) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("commit: %w", err) + } + + return nil +} + +// execUpserts inserts or updates one record per new or changed file. +func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, + prog *progress, +) error { + st, err := tx.PrepareContext(ctx, upsertSQL) + if err != nil { + return fmt.Errorf("prepare upsert: %w", err) + } + + defer func() { _ = st.Close() }() + + for _, r := range upserts { + _, err = st.ExecContext(ctx, + []byte(r.path), r.size, r.mtime, r.head, r.tail) + if err != nil { + return fmt.Errorf("upsert %s: %w", r.path, err) + } + + prog.increment() + } + + return nil +} + +// execDeletes removes the records for paths no longer present. +func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string, + prog *progress, +) error { + st, err := tx.PrepareContext(ctx, "DELETE FROM files WHERE path = ?") + if err != nil { + return fmt.Errorf("prepare delete: %w", err) + } + + defer func() { _ = st.Close() }() + + for _, p := range deletes { + _, err = st.ExecContext(ctx, []byte(p)) + if err != nil { + return fmt.Errorf("delete %s: %w", p, err) + } + + prog.increment() + } + + return nil +} diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..27eae1e --- /dev/null +++ b/db_test.go @@ -0,0 +1,180 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "slices" + "strings" + "testing" +) + +// testDBPath returns a database path inside a fresh temp dir. +func testDBPath(t *testing.T) string { + t.Helper() + + return filepath.Join(t.TempDir(), "db.sqlite") +} + +// openTestDB creates a fresh scan database in a temp dir. +func openTestDB(t *testing.T) *sql.DB { + t.Helper() + + db, err := openScanDatabase(testDBPath(t)) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + return db +} + +func TestDatabasePath(t *testing.T) { + t.Setenv(databaseEnv, "") + + if got := databasePath(); got != defaultDatabasePath { + t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath) + } + + t.Setenv(databaseEnv, "/custom/place.sqlite") + + if got := databasePath(); got != "/custom/place.sqlite" { + t.Errorf("databasePath() = %q, want the env override", got) + } +} + +func TestOpenScanDatabaseCreates(t *testing.T) { + t.Parallel() + + // The parent directory does not exist yet; scan must create it. + path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite") + + db, err := openScanDatabase(path) + if err != nil { + t.Fatalf("openScanDatabase: %v", err) + } + + v, err := userVersion(db) + if err != nil || v != schemaVersion { + t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion) + } + + _ = db.Close() + + // Reopening an existing database must succeed and find the schema. + db, err = openScanDatabase(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + + defer func() { _ = db.Close() }() + + recs, err := loadFileRows(db) + if err != nil || len(recs) != 0 { + t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err) + } +} + +func TestOpenReportDatabaseMissing(t *testing.T) { + t.Parallel() + + _, err := openReportDatabase(testDBPath(t)) + if !errors.Is(err, errNoDatabase) { + t.Fatalf("err = %v, want errNoDatabase", err) + } +} + +func TestOpenReportDatabaseVersionMismatch(t *testing.T) { + t.Parallel() + + path := testDBPath(t) + + db, err := openScanDatabase(path) + if err != nil { + t.Fatal(err) + } + + _, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99") + if err != nil { + t.Fatal(err) + } + + _ = db.Close() + + _, err = openReportDatabase(path) + if !errors.Is(err, errSchemaVersion) { + t.Fatalf("err = %v, want errSchemaVersion", err) + } +} + +func TestOpenReportDatabaseOK(t *testing.T) { + t.Parallel() + + path := testDBPath(t) + + db, err := openScanDatabase(path) + if err != nil { + t.Fatal(err) + } + + _ = db.Close() + + db, err = openReportDatabase(path) + if err != nil { + t.Fatalf("openReportDatabase: %v", err) + } + + _ = db.Close() +} + +func TestApplyChangesRoundTrip(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + // Paths may contain tabs and newlines; the database must store + // them byte-exactly. + recs := []scanRec{ + {size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"}, + {size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"}, + } + + err := applyChanges(db, recs, nil, newProgress("update", 2)) + if err != nil { + t.Fatalf("applyChanges: %v", err) + } + + got, err := loadFileRows(db) + if err != nil { + t.Fatal(err) + } + + slices.SortFunc(got, func(a, b scanRec) int { + return strings.Compare(a.path, b.path) + }) + + if !slices.Equal(got, recs) { + t.Fatalf("rows = %+v, want %+v", got, recs) + } + + // An upsert for an existing path updates in place; a delete + // removes exactly its path. + upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"} + + err = applyChanges(db, []scanRec{upd}, + []string{"/a/tab\tnew\nline"}, newProgress("update", 2)) + if err != nil { + t.Fatalf("applyChanges: %v", err) + } + + got, err = loadFileRows(db) + if err != nil { + t.Fatal(err) + } + + if len(got) != 1 || got[0] != upd { + t.Fatalf("rows = %+v, want just %+v", got, upd) + } +} diff --git a/go.mod b/go.mod index 83bdfdb..9b0faaa 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,22 @@ go 1.25.7 require ( github.com/schollz/progressbar/v3 v3.19.1 github.com/spf13/cobra v1.10.2 + modernc.org/sqlite v1.54.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index cf50240..a247cbd 100644 --- a/go.sum +++ b/go.sum @@ -3,14 +3,28 @@ github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -23,10 +37,44 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/main.go b/main.go index 1e0ca25..5d35d24 100644 --- a/main.go +++ b/main.go @@ -2,13 +2,15 @@ // very large filesystems without reading full file contents. Files are // considered duplicates when they have identical size, identical SHA-256 // of their first 1024 bytes, and identical SHA-256 of their last 1024 -// bytes. +// bytes. scan maintains a persistent SQLite database of file signatures +// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the +// reporting subcommands read. // // Usage: // -// sfdupes scan [--workers N] [-x] PATH... > files.dat -// sfdupes report [files.dat|-] > dupes.tsv -// sfdupes trees [files.dat|-] > dupetrees.tsv +// sfdupes scan [--workers N] [-x] PATH... +// sfdupes report > dupes.tsv +// sfdupes trees > dupetrees.tsv // // See README.md for the complete specification. package main @@ -60,7 +62,7 @@ func main() { scanCmd := &cobra.Command{ Use: "scan [--workers N] [-x] PATH...", - Short: "Walk trees and emit one record per regular file on stdout", + Short: "Walk trees and synchronize the scan database", Args: cobra.MinimumNArgs(1), Run: func(_ *cobra.Command, args []string) { runScan(args, scanWorkers, scanOneFS) @@ -72,20 +74,20 @@ func main() { "do not cross filesystem boundaries") reportCmd := &cobra.Command{ - Use: "report [files.dat|-]", - Short: "Read a scan stream and print the file-level duplicates report", - Args: cobra.MaximumNArgs(1), - Run: func(_ *cobra.Command, args []string) { - runReport(args) + Use: "report", + Short: "Read the scan database and print the file-level duplicates report", + Args: cobra.NoArgs, + Run: func(_ *cobra.Command, _ []string) { + runReport() }, } treesCmd := &cobra.Command{ - Use: "trees [files.dat|-]", - Short: "Read a scan stream and print the duplicate-tree report", - Args: cobra.MaximumNArgs(1), - Run: func(_ *cobra.Command, args []string) { - runTrees(args) + Use: "trees", + Short: "Read the scan database and print the duplicate-tree report", + Args: cobra.NoArgs, + Run: func(_ *cobra.Command, _ []string) { + runTrees() }, } diff --git a/report.go b/report.go index a66104e..e0141ae 100644 --- a/report.go +++ b/report.go @@ -2,106 +2,49 @@ package main import ( "bufio" - "bytes" "fmt" - "io" "os" "slices" - "strconv" "strings" ) -// ioBufSize is the buffer size for the buffered scan-stream reader and -// the buffered stdout writers. +// ioBufSize is the buffer size for the buffered stdout writers. const ioBufSize = 1 << 20 -// recordFieldCount is the number of tab-separated fields in a scan -// record: size, mtime, head hash, tail hash, path. -const recordFieldCount = 5 - -// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used -// to read scan records; a record longer than scanMaxRecordSize is a -// fatal read error. -const ( - scanInitBufSize = 64 << 10 - scanMaxRecordSize = 4 << 20 -) - // minGroupSize is the smallest number of members that makes a // duplicate group. const minGroupSize = 2 -// scanRec is one well-formed record parsed from a scan stream. The -// signature (size, head, tail) is the duplicate key; mtime is -// informational and not retained. +// scanRec is one file record from the database. The signature (size, +// head, tail) is the duplicate key; mtime is informational only and +// used by scan for change detection. type scanRec struct { - size int64 - head string - tail string - path string + size int64 + mtime int64 + head string + tail string + path string } -// openScanInput resolves the analysis-mode input: the file named by the -// single optional positional argument, or stdin when it is absent or -// "-". The returned closer must be called when reading is done. -func openScanInput(args []string) (io.Reader, string, func()) { - if len(args) == 1 && args[0] != "-" { - f, err := os.Open(args[0]) - if err != nil { - fatalf("open %s: %v", args[0], err) - } +// loadRecords opens the database and reads every file record for the +// report and trees subcommands. Any database problem — including a +// missing database — is fatal. +func loadRecords() []scanRec { + dbPath := databasePath() - return f, args[0], func() { _ = f.Close() } - } - - return os.Stdin, "stdin", func() {} -} - -// parseScanStream reads every NUL-terminated record from in. A record -// that does not have exactly 5 fields or whose size is non-numeric is -// counted as malformed and skipped. Total records read is -// len(recs) + malformed. -func parseScanStream(in io.Reader, name string) ([]scanRec, int) { - sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize)) - sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize) - sc.Split(splitNUL) - - var ( - recs []scanRec - malformed int - ) - - for sc.Scan() { - // The path is the last field and may itself contain tabs, - // so split into at most recordFieldCount fields. - fields := strings.SplitN(sc.Text(), "\t", recordFieldCount) - if len(fields) != recordFieldCount { - malformed++ - - continue - } - - size, err := strconv.ParseInt(fields[0], 10, 64) - if err != nil { - malformed++ - - continue - } - - recs = append(recs, scanRec{ - size: size, - head: fields[2], - tail: fields[3], - path: fields[4], - }) - } - - err := sc.Err() + db, err := openReportDatabase(dbPath) if err != nil { - fatalf("read %s: %v", name, err) + fatalf("%v", err) } - return recs, malformed + defer func() { _ = db.Close() }() + + recs, err := loadFileRows(db) + if err != nil { + fatalf("database %s: %v", dbPath, err) + } + + return recs } // dupeGroup is one set of candidate-duplicate files: identical size, @@ -112,18 +55,12 @@ type dupeGroup struct { paths []string } -// runReport implements the report subcommand: it reads a scan stream -// from the named file (or stdin when absent or "-") and prints the -// file-level duplicates report as TSV on stdout. It never touches the -// scanned filesystem; its only I/O is the scan input, stdout, and -// stderr. args holds the positional arguments already validated by -// cobra (at most one). -func runReport(args []string) { - in, name, closer := openScanInput(args) - defer closer() - - recs, malformed := parseScanStream(in, name) - records := len(recs) + malformed +// runReport implements the report subcommand: it reads every record +// from the database and prints the file-level duplicates report as TSV +// on stdout. It never touches the scanned filesystem; its only I/O is +// the database, stdout, and stderr. +func runReport() { + recs := loadRecords() dupes := collectDupeGroups(recs) out := bufio.NewWriterSize(os.Stdout, ioBufSize) @@ -156,9 +93,9 @@ func runReport(args []string) { } fmt.Fprintf(os.Stderr, - "report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n", - records, malformedNote(malformed), len(dupes), dupeFiles, - humanBytes(reclaimable)) + "report: %d records read, %d duplicate groups, %d dupe files, "+ + "%s reclaimable\n", + len(recs), len(dupes), dupeFiles, humanBytes(reclaimable)) } // collectDupeGroups groups records by signature and returns every group @@ -199,30 +136,6 @@ func collectDupeGroups(recs []scanRec) []dupeGroup { return dupes } -// malformedNote formats the optional malformed-record note for the -// stderr summaries. -func malformedNote(malformed int) string { - if malformed == 0 { - return "" - } - - return fmt.Sprintf(" (%d malformed, skipped)", malformed) -} - -// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing -// data without a terminator at EOF is returned as a final record. -func splitNUL(data []byte, atEOF bool) (int, []byte, error) { - if i := bytes.IndexByte(data, 0); i >= 0 { - return i + 1, data[:i], nil - } - - if atEOF && len(data) > 0 { - return len(data), data, nil - } - - return 0, nil, nil -} - // humanBytes formats a byte count in human units (binary prefixes). func humanBytes(n int64) string { const unit = 1024 diff --git a/report_test.go b/report_test.go index f75d228..e540985 100644 --- a/report_test.go +++ b/report_test.go @@ -1,92 +1,10 @@ package main import ( - "bufio" - "fmt" "slices" - "strings" "testing" ) -// mkRecord serializes one scan record in the on-the-wire format. -func mkRecord(size, mtime int64, head, tail, path string) string { - return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00", - size, mtime, head, tail, path) -} - -func TestSplitNULScanner(t *testing.T) { - t.Parallel() - - sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail")) - sc.Split(splitNUL) - - var got []string - for sc.Scan() { - got = append(got, sc.Text()) - } - - err := sc.Err() - if err != nil { - t.Fatalf("scanner error: %v", err) - } - - want := []string{"a", "bb", "", "tail"} - if !slices.Equal(got, want) { - t.Fatalf("tokens = %q, want %q", got, want) - } -} - -func TestParseScanStream(t *testing.T) { - t.Parallel() - - stream := mkRecord(10, 1, "h1", "t1", "/a/x") + - "garbage-without-tabs\x00" + - "notanumber\t1\th\tt\t/a/bad\x00" + - mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") + - "30\t3\th3\tt3\t/trailing/no-nul" - - recs, malformed := parseScanStream(strings.NewReader(stream), "test") - - if malformed != 2 { - t.Errorf("malformed = %d, want 2", malformed) - } - - want := []scanRec{ - {size: 10, head: "h1", tail: "t1", path: "/a/x"}, - {size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"}, - {size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"}, - } - if !slices.Equal(recs, want) { - t.Fatalf("recs = %+v, want %+v", recs, want) - } -} - -func TestParseScanStreamPathWithNewline(t *testing.T) { - t.Parallel() - - in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline")) - - recs, malformed := parseScanStream(in, "test") - if malformed != 0 || len(recs) != 1 { - t.Fatalf("got %d recs, %d malformed, want 1, 0", - len(recs), malformed) - } - - if recs[0].path != "/a/new\nline" { - t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline") - } -} - -func TestParseScanStreamEmpty(t *testing.T) { - t.Parallel() - - recs, malformed := parseScanStream(strings.NewReader(""), "test") - if len(recs) != 0 || malformed != 0 { - t.Fatalf("got %d recs, %d malformed, want 0, 0", - len(recs), malformed) - } -} - func TestCollectDupeGroups(t *testing.T) { t.Parallel() @@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) { } } +func TestCollectDupeGroupsMtimeExcluded(t *testing.T) { + t.Parallel() + + // mtime is informational only; records differing only in mtime + // still group together. + recs := []scanRec{ + {size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"}, + {size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"}, + } + + groups := collectDupeGroups(recs) + if len(groups) != 1 { + t.Fatalf("len(groups) = %d, want 1", len(groups)) + } +} + func TestCollectDupeGroupsTieBreak(t *testing.T) { t.Parallel() @@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) { } } } - -func TestMalformedNote(t *testing.T) { - t.Parallel() - - if got := malformedNote(0); got != "" { - t.Errorf("malformedNote(0) = %q, want empty", got) - } - - if got := malformedNote(3); got != " (3 malformed, skipped)" { - t.Errorf("malformedNote(3) = %q", got) - } -} diff --git a/scan.go b/scan.go index 42d17c2..b70faf3 100644 --- a/scan.go +++ b/scan.go @@ -1,14 +1,16 @@ package main import ( - "bufio" "crypto/sha256" + "database/sql" "encoding/hex" "errors" "fmt" "io/fs" "os" "path/filepath" + "slices" + "strings" "syscall" ) @@ -30,30 +32,242 @@ type fileRec struct { mtime int64 } -// runScan implements the scan subcommand: three sequential passes -// (walk, stat, hash) over the trees named by the PATH operands, -// emitting one NUL-terminated record per regular file on stdout. Flag -// parsing and the at-least-one-operand check are done by cobra. +// runScan implements the scan subcommand: four sequential passes +// (walk, stat, hash, update) that synchronize the persistent database +// with the filesystem state under the PATH operands. Flag parsing and +// the at-least-one-operand check are done by cobra. func runScan(roots []string, workers int, oneFS bool) { if workers < 1 { workers = 1 } - // A nonexistent operand is a fatal error before any scanning. + roots = resolveRoots(roots) + dbPath := databasePath() + + db, err := openScanDatabase(dbPath) + if err != nil { + fatalf("%v", err) + } + + defer func() { _ = db.Close() }() + + st, err := syncScan(db, roots, workers, oneFS) + if err != nil { + fatalf("update database %s: %v", dbPath, err) + } + + fmt.Fprintf(os.Stderr, + "scan: %d files seen (%d added, %d updated, %d removed, "+ + "%d unchanged), %d skipped\n", + st.added+st.updated+st.unchanged, st.added, st.updated, + st.removed, st.unchanged, st.skipped) +} + +// resolveRoots converts each PATH operand to an absolute, lexically +// cleaned path (symlinks are not resolved) and verifies that it +// exists. Database records are keyed by absolute path, so scan results +// must not depend on the working directory. +func resolveRoots(roots []string) []string { + abs := make([]string, 0, len(roots)) + for _, root := range roots { - _, err := os.Lstat(root) + a, err := filepath.Abs(root) + if err != nil { + fatalf("resolve %s: %v", root, err) + } + + // A nonexistent operand is a fatal error before any scanning. + _, err = os.Lstat(a) if err != nil { fatalf("%v", err) } + + abs = append(abs, a) + } + + return abs +} + +// scanStats summarizes one scan's database synchronization for the +// final stderr summary. +type scanStats struct { + added int + updated int + removed int + unchanged int + skipped int +} + +// syncScan synchronizes the database with the filesystem under roots: +// walk and stat everything, hash only new or changed files, and apply +// the resulting record insertions, updates, and deletions in a single +// transaction. Records outside the roots are never touched. +func syncScan(db *sql.DB, roots []string, workers int, + oneFS bool, +) (scanStats, error) { + var st scanStats + + existing, err := loadScopedRows(db, roots) + if err != nil { + return st, err } paths, walkErrs := walkPass(roots, oneFS) - recs, statErrs := statPass(paths, workers) - emitted, hashErrs := hashPass(recs, workers) + recs, statErrs := statPass(uniquePaths(paths), workers) + toHash, unchanged := partitionChanged(recs, existing) + hashed, hashErrs := hashPass(toHash, workers) - skipped := walkErrs + statErrs + hashErrs - fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n", - emitted, skipped) + st.unchanged = len(unchanged) + st.skipped = walkErrs + statErrs + hashErrs + + for _, r := range hashed { + if _, ok := existing[r.path]; ok { + st.updated++ + } else { + st.added++ + } + } + + deletes := collectDeletes(existing, unchanged, hashed) + st.removed = len(deletes) + + return st, applyPass(db, hashed, deletes) +} + +// loadScopedRows loads the database records whose paths lie under any +// of the scan roots, keyed by path. Records outside the roots belong +// to other trees and are left untouched by this scan. +func loadScopedRows(db *sql.DB, roots []string) (map[string]scanRec, error) { + all, err := loadFileRows(db) + if err != nil { + return nil, err + } + + scoped := make(map[string]scanRec) + + for _, r := range all { + if underAnyRoot(r.path, roots) { + scoped[r.path] = r + } + } + + return scoped, nil +} + +// underAnyRoot reports whether path is any of the roots or lies under +// one of them. +func underAnyRoot(path string, roots []string) bool { + for _, root := range roots { + if underRoot(path, root) { + return true + } + } + + return false +} + +// underRoot reports whether path is root itself or lies under it. Both +// must be absolute and lexically clean. +func underRoot(path, root string) bool { + if path == root { + return true + } + + prefix := root + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + return strings.HasPrefix(path, prefix) +} + +// uniquePaths deduplicates the walked paths, preserving order. +// Overlapping operands can reach the same file more than once, but the +// database keys records by path, so each file is processed once. +func uniquePaths(paths []string) []string { + seen := make(map[string]bool, len(paths)) + out := make([]string, 0, len(paths)) + + for _, p := range paths { + if seen[p] { + continue + } + + seen[p] = true + + out = append(out, p) + } + + return out +} + +// partitionChanged splits the stat results into files that must be +// hashed (new, or changed) and files whose existing records are reused +// without reading them: a file is unchanged when its statted size +// equals the recorded size and its statted mtime is not newer than the +// recorded mtime. +func partitionChanged(recs []fileRec, + existing map[string]scanRec, +) ([]fileRec, []scanRec) { + var ( + toHash []fileRec + unchanged []scanRec + ) + + for _, rec := range recs { + old, ok := existing[rec.path] + if ok && old.size == rec.size && old.mtime >= rec.mtime { + unchanged = append(unchanged, old) + + continue + } + + toHash = append(toHash, rec) + } + + return toHash, unchanged +} + +// collectDeletes returns the existing record paths that were not +// successfully processed this run: vanished files, plus paths that +// failed to stat or hash. The database keeps only records verified by +// the latest scan covering them. The result is sorted so the update +// pass is deterministic. +func collectDeletes(existing map[string]scanRec, + unchanged, hashed []scanRec, +) []string { + kept := make(map[string]bool, len(unchanged)+len(hashed)) + + for _, r := range unchanged { + kept[r.path] = true + } + + for _, r := range hashed { + kept[r.path] = true + } + + var deletes []string + + for p := range existing { + if !kept[p] { + deletes = append(deletes, p) + } + } + + slices.Sort(deletes) + + return deletes +} + +// applyPass writes the scan's changes to the database under an update +// progress display (one item per insertion, update, or deletion). +func applyPass(db *sql.DB, upserts []scanRec, deletes []string) error { + prog := newProgress("update", int64(len(upserts)+len(deletes))) + err := applyChanges(db, upserts, deletes, prog) + + prog.finish() + + return err } // treeWalker carries the walk-pass state shared by all PATH operands. @@ -271,15 +485,15 @@ func startHashWorkers(recs []fileRec, workers int) <-chan hashResult { } // hashPass hashes the first and last chunk bytes of every file in a -// worker pool and emits the output records on stdout from the main +// worker pool and collects the resulting records on the main // goroutine. Files that fail to open or read are warned about and -// dropped. -func hashPass(recs []fileRec, workers int) (int, int) { +// dropped. Only new or changed files reach this pass. +func hashPass(recs []fileRec, workers int) ([]scanRec, int) { results := startHashWorkers(recs, workers) prog := newProgress("hash", int64(len(recs))) - out := bufio.NewWriterSize(os.Stdout, ioBufSize) + out := make([]scanRec, 0, len(recs)) - var emitted, errs int + var errs int for range recs { r := <-results @@ -292,25 +506,20 @@ func hashPass(recs []fileRec, workers int) (int, int) { continue } - _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00", - r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path) - if err != nil { - fatalf("write stdout: %v", err) - } - - emitted++ + out = append(out, scanRec{ + size: r.rec.size, + mtime: r.rec.mtime, + head: r.head, + tail: r.tail, + path: r.rec.path, + }) prog.increment() } prog.finish() - err := out.Flush() - if err != nil { - fatalf("write stdout: %v", err) - } - - return emitted, errs + return out, errs } // hashHeadTail returns the lowercase-hex SHA-256 of the first diff --git a/scan_test.go b/scan_test.go index 5eba987..2a3130d 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1,15 +1,16 @@ package main import ( - "bytes" + "context" "crypto/sha256" + "database/sql" "encoding/hex" - "io" "os" "path/filepath" "slices" "strings" "testing" + "time" ) // writeFile creates a file with the given content and returns its path. @@ -254,38 +255,6 @@ func TestStatPass(t *testing.T) { } } -// captureStdout runs fn with os.Stdout redirected to a temp file and -// returns everything fn wrote to it. -func captureStdout(t *testing.T, fn func()) []byte { - t.Helper() - - f, err := os.CreateTemp(t.TempDir(), "stdout") - if err != nil { - t.Fatal(err) - } - - orig := os.Stdout - os.Stdout = f - - defer func() { os.Stdout = orig }() - - fn() - - _, err = f.Seek(0, io.SeekStart) - if err != nil { - t.Fatal(err) - } - - data, err := io.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - _ = f.Close() - - return data -} - // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { @@ -316,45 +285,65 @@ func buildSmokeTree(t *testing.T) string { return dir } -// scanToRecords runs the walk, stat, and hash passes over dir and -// parses the emitted stream back into records. -func scanToRecords(t *testing.T, dir string) []scanRec { +// smokeTreeFiles is the number of regular files buildSmokeTree creates. +const smokeTreeFiles = 15 + +// syncTree synchronizes the database with the given roots and returns +// the scan stats. +func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { t.Helper() - paths, walkErrs := walkPass([]string{dir}, false) - if walkErrs != 0 { - t.Fatalf("walk errors: %d", walkErrs) + st, err := syncScan(db, roots, 4, false) + if err != nil { + t.Fatalf("syncScan: %v", err) } - recs, statErrs := statPass(paths, 4) - if statErrs != 0 { - t.Fatalf("stat errors: %d", statErrs) - } - - var emitted, hashErrs int - - out := captureStdout(t, func() { - emitted, hashErrs = hashPass(recs, 4) - }) - - if emitted != len(paths) || hashErrs != 0 { - t.Fatalf("emitted %d of %d, %d hash errors", - emitted, len(paths), hashErrs) - } - - parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe") - if malformed != 0 || len(parsed) != len(paths) { - t.Fatalf("parsed %d records, %d malformed, want %d, 0", - len(parsed), malformed, len(paths)) - } - - return parsed + return st } -//nolint:paralleltest // redirects the process-wide os.Stdout -func TestScanPipeline(t *testing.T) { - dir := buildSmokeTree(t) - parsed := scanToRecords(t, dir) +// dbRecords returns every record currently in the database. +func dbRecords(t *testing.T, db *sql.DB) []scanRec { + t.Helper() + + recs, err := loadFileRows(db) + if err != nil { + t.Fatal(err) + } + + return recs +} + +// recordByPath finds the record with the given path. +func recordByPath(t *testing.T, recs []scanRec, path string) scanRec { + t.Helper() + + for _, r := range recs { + if r.path == path { + return r + } + } + + t.Fatalf("no record for %q", path) + + return scanRec{} +} + +// recordPaths returns the sorted paths of recs. +func recordPaths(recs []scanRec) []string { + paths := make([]string, 0, len(recs)) + for _, r := range recs { + paths = append(paths, r.path) + } + + slices.Sort(paths) + + return paths +} + +// assertSmokeDupeGroups checks the file-level duplicate groups for the +// smoke tree rooted at dir. +func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { + t.Helper() groups := collectDupeGroups(parsed) if len(groups) != 5 { @@ -377,6 +366,12 @@ func TestScanPipeline(t *testing.T) { if !slices.Equal(groups[0].paths, wantF1) { t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1) } +} + +// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke +// tree rooted at dir. +func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) { + t.Helper() super, dirs := buildHierarchy(parsed) super.compute() @@ -396,3 +391,282 @@ func TestScanPipeline(t *testing.T) { tg[0][0].fileCount, tg[0][0].totalSize) } } + +func TestScanPipeline(t *testing.T) { + t.Parallel() + + dir := buildSmokeTree(t) + db := openTestDB(t) + + st := syncTree(t, db, dir) + if st != (scanStats{added: smokeTreeFiles}) { + t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles) + } + + parsed := dbRecords(t, db) + if len(parsed) != smokeTreeFiles { + t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles) + } + + assertSmokeDupeGroups(t, dir, parsed) + assertSmokeTreeGroups(t, dir, parsed) +} + +func TestSyncScanUnchangedReuse(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + writeFile(t, dir, "b.bin", pattern(2, 600)) + + st := syncTree(t, db, dir) + if st != (scanStats{added: 2}) { + t.Fatalf("first scan stats = %+v, want 2 added", st) + } + + // An immediate rescan reuses every record without reading file + // contents. Prove the files are not re-read by corrupting a stored + // hash and observing that it survives the rescan. + _, err := db.ExecContext(context.Background(), + "UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a)) + if err != nil { + t.Fatal(err) + } + + st = syncTree(t, db, dir) + if st != (scanStats{unchanged: 2}) { + t.Fatalf("rescan stats = %+v, want 2 unchanged", st) + } + + if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" { + t.Fatalf("head = %q, want sentinel (file must not be re-read)", + r.head) + } +} + +func TestSyncScanMtimeBump(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + syncTree(t, db, dir) + + // Bump the mtime forward: the file must be re-hashed even though + // its size is unchanged. + future := time.Now().Add(time.Hour) + + err := os.Chtimes(a, future, future) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{updated: 1}) { + t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) + } + + if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() { + t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix()) + } +} + +func TestSyncScanAddRemove(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + writeFile(t, dir, "b.bin", pattern(2, 600)) + syncTree(t, db, dir) + + // Add one file, remove another. + c := writeFile(t, dir, "c.bin", pattern(3, 700)) + + err := os.Remove(a) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{added: 1, removed: 1, unchanged: 1}) { + t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged", + st) + } + + want := []string{filepath.Join(dir, "b.bin"), c} + if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) { + t.Fatalf("paths = %q, want %q", got, want) + } +} + +func TestSyncScanSizeChange(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + p := writeFile(t, dir, "f", pattern(1, 100)) + + syncTree(t, db, dir) + + // Rewrite with a different size but force the mtime back to the + // recorded value: the size mismatch alone must trigger a re-hash. + old := recordByPath(t, dbRecords(t, db), p) + + writeFile(t, dir, "f", pattern(1, 200)) + + mt := time.Unix(old.mtime, 0) + + err := os.Chtimes(p, mt, mt) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st.updated != 1 { + t.Fatalf("stats = %+v, want 1 updated", st) + } + + if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 { + t.Fatalf("size = %d, want 200", got.size) + } +} + +func TestSyncScanScope(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + writeFile(t, dir, "a/keep", pattern(1, 10)) + + gone := writeFile(t, dir, "b/gone", pattern(2, 10)) + + syncTree(t, db, dir) + + // Deleting a file outside the rescanned root must not remove its + // record: records outside the scanned operands are untouched. + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, filepath.Join(dir, "a")) + if st.removed != 0 || st.unchanged != 1 { + t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st) + } + + if got := recordPaths(dbRecords(t, db)); len(got) != 2 { + t.Fatalf("records = %q, want both retained", got) + } + + // Rescanning the parent now removes the vanished file's record. + st = syncTree(t, db, dir) + if st.removed != 1 { + t.Fatalf("parent stats = %+v, want 1 removed", st) + } +} + +func TestSyncScanRemovesNonRegular(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + p := writeFile(t, dir, "f", pattern(1, 10)) + keep := writeFile(t, dir, "g", pattern(2, 10)) + + syncTree(t, db, dir) + + // Replace the file with a symlink: it is no longer walked, so its + // record must be deleted. + err := os.Remove(p) + if err != nil { + t.Fatal(err) + } + + err = os.Symlink(keep, p) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st.removed != 1 || st.unchanged != 1 { + t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st) + } +} + +func TestSyncScanOverlappingRoots(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + writeFile(t, dir, "sub/f", pattern(1, 10)) + + // A file reachable via two overlapping operands yields one record. + st := syncTree(t, db, dir, filepath.Join(dir, "sub")) + if st.added != 1 { + t.Fatalf("stats = %+v, want 1 added", st) + } +} + +func TestReportsNeverTouchFilesystem(t *testing.T) { + t.Parallel() + + dir := buildSmokeTree(t) + db := openTestDB(t) + + syncTree(t, db, dir) + + // Remove the scanned tree entirely; the analysis must be + // unaffected because it reads the database alone. + err := os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + + recs := dbRecords(t, db) + + assertSmokeDupeGroups(t, dir, recs) + assertSmokeTreeGroups(t, dir, recs) +} + +func TestUnderRoot(t *testing.T) { + t.Parallel() + + const abRoot = "/a/b" + + cases := []struct { + path string + root string + want bool + }{ + {"/a/b/c", abRoot, true}, + {abRoot, abRoot, true}, + {"/a/bc", abRoot, false}, + {"/a", abRoot, false}, + {"/x/y", "/", true}, + {"/", "/", true}, + } + for _, c := range cases { + if got := underRoot(c.path, c.root); got != c.want { + t.Errorf("underRoot(%q, %q) = %v, want %v", + c.path, c.root, got, c.want) + } + } +} + +func TestUniquePaths(t *testing.T) { + t.Parallel() + + got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"}) + + want := []string{"/a", "/b", "/c"} + if !slices.Equal(got, want) { + t.Fatalf("uniquePaths = %q, want %q", got, want) + } +} diff --git a/trees.go b/trees.go index 5f068e6..7e43de1 100644 --- a/trees.go +++ b/trees.go @@ -28,19 +28,14 @@ type treeNode struct { totalSize int64 } -// runTrees implements the trees subcommand: it reads a scan stream from -// the named file (or stdin when absent or "-"), reconstructs the -// directory hierarchy from the record paths, computes a Merkle-style -// digest per directory, and prints maximal duplicate-tree groups as TSV -// on stdout. It never touches the scanned filesystem; its only I/O is -// the scan input, stdout, and stderr. args holds the positional -// arguments already validated by cobra (at most one). -func runTrees(args []string) { - in, name, closer := openScanInput(args) - defer closer() - - recs, malformed := parseScanStream(in, name) - records := len(recs) + malformed +// runTrees implements the trees subcommand: it reads every record from +// the database, reconstructs the directory hierarchy from the record +// paths, computes a Merkle-style digest per directory, and prints +// maximal duplicate-tree groups as TSV on stdout. It never touches the +// scanned filesystem; its only I/O is the database, stdout, and +// stderr. +func runTrees() { + recs := loadRecords() super, allDirs := buildHierarchy(recs) super.compute() @@ -78,9 +73,9 @@ func runTrees(args []string) { } fmt.Fprintf(os.Stderr, - "trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n", - records, malformedNote(malformed), len(dupes), dupeTrees, - humanBytes(reclaimable)) + "trees: %d records read, %d duplicate tree groups, %d dupe trees, "+ + "%s reclaimable\n", + len(recs), len(dupes), dupeTrees, humanBytes(reclaimable)) } // buildHierarchy reconstructs the directory hierarchy from the record