From 2c9eaf9c49348f539d628235903bf8218e320080 Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 26 Oct 2026 06:13:34 +0000 Subject: [PATCH] Rewrite README.md sentence by sentence for a newcomer The old README was one dense specification: long sentences, lists chained with commas, exceptions ahead of the rules, and release history in the user text. This rewrite leads with what the program is and how to use it, then gives a reference for the settings, each subcommand, the database, the output formats, the progress display and the exit codes. Non-goals come before the reference; the constraints, the entrypoints and the build come last. Every statement was checked against the code on next and against the built binary on small trees. The section names the code comments cite are kept. Five comments in main_test.go cite Settings, and one each in scan_test.go, trees_test.go and exclude_test.go cites what the README now says. TODO.md records the step. Model: fable-5-1 --- README.md | 1866 +++++++++++++++++++++++++---------------------- TODO.md | 8 + exclude_test.go | 4 +- main_test.go | 14 +- scan_test.go | 5 +- trees_test.go | 5 +- 6 files changed, 1018 insertions(+), 884 deletions(-) diff --git a/README.md b/README.md index 6a4b5c3..04123bc 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,27 @@ ## Description -`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that -quickly finds _candidate_ duplicate files, and ultimately whole duplicate -directory trees, on very large filesystems. It does this without reading every -byte of every file. +`sfdupes` is an MIT-licensed Go command-line tool by +[@sneak](https://sneak.berlin) that finds candidate duplicate files, and whole +duplicate directory trees, on large filesystems. It does this without reading +every byte of every file. -Two files are duplicates when they have the same size and the same three SHA-256 -hashes. Two of the hashes cover the first and the last 64 KiB of the file. The -third, the content hash, covers the whole file when it is under 100 MiB, so a -match proves identical content. At 100 MiB and above it covers only 1 MiB at -each gigabyte, so such a file is never read in full. A match there is a strong -candidate rather than proof. +Two files count as duplicates when they have the same size and the same three +SHA-256 hashes. A hash is a short fingerprint of a stretch of bytes. Two +stretches with the same fingerprint are, in practice, identical. Two of the +hashes cover the first and the last 64 KiB of the file. The third covers the +whole file when the file is under 100 MiB. A match then proves identical +content. At 100 MiB and above, the third hash covers only 1 MiB at each +gigabyte. A large file is never read in full. A match between two large files is +a strong candidate, not proof. -The intended use is finding duplicate downloads and duplicated directory trees -on multi-terabyte ZFS servers, where reading every byte of every file is -prohibitively expensive. `scan` keeps a persistent SQLite database of file -signatures between runs, so it can run from cron. The reports can be generated -at any time from the most recent scan. - -This README is the complete and authoritative specification. +`sfdupes` is meant for multi-terabyte servers, where reading every byte of every +file takes too long. It has four subcommands. `scan` walks the filesystem and +records the files it finds in an SQLite database. SQLite keeps a whole database +in one file. The database is kept between runs. A rescan reads only new and +changed files. That makes `scan` cheap enough to run every day from `cron`, the +system's job scheduler. `report`, `trees` and `info` read the database alone and +can run at any time. ## Getting Started @@ -30,60 +32,51 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite" ./sfdupes scan /srv ./sfdupes report > dupes.tsv ./sfdupes trees > dupetrees.tsv +./sfdupes info ``` -`scan` walks one or more filesystem trees and maintains one database record per -regular file (path, size, mtime, head hash, tail hash, content hash). The -database persists between runs. A rescan hashes only the files that are new or -changed, and removes the records of files that no longer exist. `report` reads -the database and prints the file-level duplicates report. `trees` reads the same -database and prints the duplicate-tree report. `info` prints the database's -details: its path, the sizes of its files and the number of its records. A -missing/invalid subcommand — or a `scan` invocation with no `PATH` operand — -prints a usage message and exits 2. +`scan` walks the trees named by its operands, the paths given after `scan` on +the command line. The walk is the listing of every directory under them. `scan` +keeps one database record per regular file. A regular file is an ordinary file, +as opposed to a directory, a symlink or a device. `report` prints the file-level +duplicates. `trees` prints the duplicate directory trees. `info` prints the +database's path, size and record count. -The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere -by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` -cron job, with the reporting commands run interactively whenever needed; their -results are as fresh as the last completed scan. +The database is `/var/lib/sfdupes/db.sqlite` unless `SFDUPES_DATABASE` names +another file. The first `scan` creates it. + +`report`, `trees` and `info` write their data to stdout. Progress, warnings and +errors go to stderr. stdout and stderr are a program's two output streams. The +shell shows both on the terminal, and `>` redirects stdout alone. ### Install -With Go installed, this builds and installs the newest release: +With Go installed, this builds and installs the newest tagged release: ```sh go install sneak.berlin/go/sfdupes@latest ``` -Until 1.0, releases are `v0.0.N` tags on the tip of the `main` branch: each time -`main` moves and its tip passes `script/cibuild`, that tip is tagged with the -next `v0.0.N`. So `@latest` installs `main`'s tip once it has passed the build. -`@main` installs the tip before it is tagged: +`@main` installs the tip of the `main` branch instead: ```sh go install sneak.berlin/go/sfdupes@main ``` -`v0.0.1` is an early build without a database. It does not work as this README -describes, and `go.mod` retracts it. +The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. -The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. To -see which version an installed binary is: +`sfdupes --version` prints `sfdupes VERSION` and exits. For a binary from +`go install`, `VERSION` is the version Go recorded in it. For `@latest` that is +a release tag such as `v0.0.3`. For `@main` it is a pseudo-version, Go's version +string for an untagged commit. That string ends in the commit's hash. A binary +built from a clone carries the version `git describe --tags --always --dirty` +prints in that clone, such as `v0.0.3` on a tagged commit. `VERSION` is `dev` +when no version is known. To see everything Go recorded in an installed binary: ```sh -sfdupes --version go version -m "$(command -v sfdupes)" ``` -`sfdupes --version` prints `sfdupes VERSION`. For a binary from `go install`, -`VERSION` is the version Go recorded in it: the release tag, such as `v0.0.3`, -or for `@main` a pseudo-version ending in the first 12 characters of the -commit's hash. A binary built from a clone or into the Docker image carries the -git tag or commit it was built from instead, and `VERSION` is `dev` only when -the binary carries no version at all. `go version -m` prints what Go recorded in -the binary; its `mod` line names the module's version. Binaries from `v0.0.2` -and earlier print `dev`, so check those with `go version -m`. - From a clone, `make build` writes the binary to `./sfdupes`: ```sh @@ -92,14 +85,16 @@ cd sfdupes make build ``` -Copy the binary to `/usr/local/bin` for the cron job below. +Copy the binary to `/usr/local/bin` for the `cron` job below. -`make docker` builds the Docker image, tagged `sfdupes`, after running the tests -and the linter (see "Build"). The image runs `sfdupes` as root with the database -at its default path, so a bind mount of `/var/lib/sfdupes` keeps the database -between runs. Mount the scanned tree at the same path inside the container as on -the host; read-only is enough. The database records paths as the container sees -them, so the reports then name the host's paths. +`make docker` builds a Docker image tagged `sfdupes`. The build first runs the +tests and the linter, a checker for common mistakes in Go code (see "Build"). +Inside the image `sfdupes` runs as root, with the database at its default path. +Each `-v` flag below makes a bind mount, which shows a host directory at a path +inside the container. Bind-mount `/var/lib/sfdupes` to keep the database between +runs. Mount the tree to scan at the same path inside the container as on the +host. Read-only access is enough. The database then records the host's paths, +and so do the reports. ```sh make docker @@ -108,60 +103,62 @@ docker run --rm -v /srv:/srv:ro -v /var/lib/sfdupes:/var/lib/sfdupes \ docker run --rm -v /var/lib/sfdupes:/var/lib/sfdupes sfdupes report > dupes.tsv ``` -### Daily scan from cron +### Daily scan from `cron` -Run `scan` as root, so that it can read every file: a path it cannot read is -skipped with a warning and loses its database record (see "Rules for the walk"). -As a file `/etc/cron.d/sfdupes`: +Run `scan` as root, so that it can read every file. A path it cannot read is +skipped with a warning and loses its database record. As the file +`/etc/cron.d/sfdupes`: ``` 30 3 * * * root /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log ``` -- The database is `/var/lib/sfdupes/db.sqlite`, created with its directory by - the first scan. To keep it elsewhere, set - `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line. +- The first scan creates `/var/lib/sfdupes/db.sqlite` and its directory. To keep + the database elsewhere, put `SFDUPES_DATABASE=/path/to/db.sqlite` before the + command on the same line. - `scan` writes nothing to stdout. Its stderr, appended here to - `/var/log/sfdupes.log`, holds these lines: - - a first line naming the database (see "Database"); - - plain progress lines naming each phase and its place among the four, as - `[2/4] walk`: one as the phase starts, one every 5 seconds while it runs, - and one as it ends; + `/var/log/sfdupes.log`, holds: + - one line naming the database; + - progress lines, as described under "Progress"; - a warning for each path it skips; - - a line for each default exclude its walk does not enter, and for each path - it leaves out because it is not on a persistent filesystem (see "Rules for - the walk"); - - the summary line (see "Progress" and "`scan` mode"). - - The log grows with every scan; rotate it like any other. - -- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A - scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the - reason among the last lines of the log; `tail` prints them, and cron mails - them to root if the host can send mail. + - a line for each system or temporary directory the walk leaves out (see + "Rules for the walk"); + - a line for each path left out because its filesystem is one the kernel + makes up or keeps in memory, such as `tmpfs`; + - the summary line; + - a warning when a report still has the database open as the scan ends (see + "Concurrent reads"). +- The log grows with every scan. Rotate it like any other log. +- Skipped paths do not fail a scan. The scan exits 0 all the same, and `cron` + sends no mail. +- A scan that fails exits 1, with the reason among the last lines of the log. + `tail` prints those lines, and `cron` mails them to root if the host can send + mail. - A scan still running when the next one starts carries on. The new one fails at - once, and the lines cron mails include + once, and the lines `cron` mails include `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. -- `report`, `trees` and `info` need only read access to the database (see - "Database"). Under the usual umask of `022` the first scan creates it readable - by every user, so an unprivileged user can run them against root's database. +- `report`, `trees` and `info` need only read access to the database. The umask + is the setting that removes permissions from the files a process creates. + Under the usual umask of `022`, the first scan creates the database readable + by every user. An unprivileged user can then run those three commands against + root's database. ### Reading the reports -Each row of `report` names two copies of one file, and each row of `trees` two -copies of one directory tree (see "Report output format" and "Trees output -format"). In a group of copies, the path that sorts first byte by byte is -`first` and every other path is a `dupe` of it. `first` says nothing about which -copy is the original or the oldest; which copy to keep is your choice. +Each row of `report` names two copies of one file. Each row of `trees` names two +copies of one directory tree. Within a group of copies, the path that sorts +first byte by byte is `first`. Every other path in the group is a `dupe` of it. +`first` says nothing about which copy is the original or the oldest. Which copy +to keep is your choice. A row is a candidate, not proof: -- The reports read only the database, so they show the files as of the last - scan; a file may have changed or gone since. +- The reports show the files as they were at the last scan. A file may have + changed or gone since. - A file of 100 MiB or more is compared only on samples of its content (see "Duplicate detection"). -- Paths that are hard links to one file are listed as duplicates, but they share - their data, so removing one frees nothing. +- Two paths that are hard links, two names for one file, are listed as + duplicates. They share their data, so removing one frees nothing. Compare a pair byte for byte before removing either copy. For the row `/srv/a/big.iso`, `/srv/b/big-copy.iso`, `4294967296`: @@ -171,108 +168,81 @@ cmp /srv/a/big.iso /srv/b/big-copy.iso && echo identical [ /srv/a/big.iso -ef /srv/b/big-copy.iso ] && echo "hard links" ``` -`cmp` prints nothing and exits 0 only when every byte matches, and otherwise -reports where the files differ. The second line prints `hard links` when the two -paths are the same file, so removing either frees nothing. +`cmp` prints nothing and exits 0 when every byte matches. Otherwise it reports +where the files differ. The second line prints `hard links` when both paths are +the same file. -A path holding a backslash, tab, newline or carriage return is escaped in the -reports (see "Report output format"). Undo the escapes before using it. -`printf '%b'` does exactly that, because every backslash in an escaped path -starts one of the four escapes. Command substitution drops trailing newlines, so -print an `x` after the path and remove it afterwards, or a path that ends in a -newline names a different file: +A path holding a special character, such as a tab, is escaped in the reports. +The character is written as a backslash and one more character, `\t` for a tab +(see "Report output format"). Undo the escapes before using such a path. +`printf '%b'` does that, because every backslash in an escaped path starts an +escape. The shell drops trailing newlines from the output of `$(...)`. To keep a +newline at the end of a path, print an `x` after the path and remove it +afterwards: ```sh p="$(printf '%bx' '/srv/a/tab\tname.txt')"; p="${p%x}" cmp "$p" /srv/b/tab-copy.txt ``` -Check a `trees` row with `diff -r`, which compares the two trees file by file -and also names anything present in only one of them, such as an empty directory -or a symlink, which `trees` does not see. +Check a `trees` row with `diff -r`. It compares the two trees file by file. It +also names anything present in only one of them, such as an empty directory or a +symlink. `trees` does not see those. ## Rationale Duplicate finders that hash every file in full do not scale to the target -environment. It holds about 10 million files and 150 TB, on disks that may be -slow or busy, such as a ZFS pool under resilver. +environment. That is about ten million files and 150 TB. The disks may be slow +or busy. A ZFS storage pool, a set of disks managed as one filesystem, is busy +while it rebuilds a disk. -sfdupes reads a file under 100 MiB in full, and a larger one only at its two -ends and at 1 MiB per gigabyte. A full read of a small file is cheap, and it -makes the comparison exact. A full read of every large file would take -unreasonably long. Sampling cuts a 150 GiB file to about 150 MiB of reads, at -the price of a comparison that is not exact. +`sfdupes` reads a file under 100 MiB in full. It reads a larger file only at its +two ends and at 1 MiB per gigabyte. A full read of a small file is cheap, and it +makes the comparison exact. Reading samples cuts a 150 GiB file to about 150 MiB +of reads, at the price of a comparison that is not exact. -The hashes are kept in a persistent database, so the expensive filesystem pass -is incremental. A rescan reads only the files that are new or whose size or -mtime changed. All analysis then works from the database alone. +The hashes live in a database that is kept between runs. A rescan reads only the +files that are new or whose size or modification time (mtime) changed. All +analysis then works from the database alone. -The end goal is not individual files but whole duplicated trees, such as a -download extracted twice. An operator can consider removing such a tree as a -unit. +The goal is whole duplicated trees, not only single files. A download extracted +twice, or a project copied once too often, is such a tree. It can be removed as +a unit. ## Design -Goals, in order: +### Goals -1. **Find whole duplicate trees, not just files.** The end goal is to identify - places where the exact same set of files and directories exists at two or - more paths (duplicate extractions, duplicate downloads, copied project - trees), so the operator can consider removing an entire subtree at once. - File-level duplicate detection is the foundation; tree-level detection is - built on top of it. -2. **Bound the I/O per file.** A file is read only when it is new or has - changed. A file under 100 MiB is then read in full, and a larger one only at - its ends and at 1 MiB per gigabyte (see "Duplicate detection" below). - Therefore no file of 100 MiB or more is read in full. Scale target: tens of - millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool - under resilver). Holding one small record (path, size, mtime) per file in - memory during a scan is acceptable; holding every file's hashes is not (they - stay in the database). The reporting commands do not hold every file's - hashes either: `report` reads the records in signature order and holds the - duplicate paths of one size at a time, so its memory grows with the - duplicates of the size that has the most of them, and `trees` reads the - records in path order and keeps each directory's path, digest and totals, - plus the hashes of only the files in the directories holding the record - being read, so its memory grows with the number of directories and with the - size of the largest directory. -3. **Scan incrementally, analyze offline.** The expensive filesystem scan - maintains a persistent database, and an unchanged file is never read again - on a rescan. All analysis (`report`, `trees`) works from the database alone - and must never touch the scanned filesystem again. `scan` is designed to be - cronned; the reports run at any time against the last completed scan. -4. **Clean stream separation.** Everything on stdout is machine-readable data. - All progress, warnings, summaries, and help and usage text go to stderr. - Never mix them. +In order: -### Constraints - -- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: `sfdupes`. -- Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one - progress-bar library** (`github.com/schollz/progressbar/v3`), - `golang.org/x/term` to tell whether stderr is a terminal, **one SQLite - driver** (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and - `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). - `github.com/spf13/viper` is permitted if configuration-file support is ever - needed, but is not currently used. No other third-party deps. -- Cross-compilation is not a concern. Builds run with cgo disabled (the - `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. -- Analysis modes (`report`, `trees`) must be deterministic: identical database - contents, identical output, regardless of the order in which records were - inserted. +1. **Find whole duplicate trees, not just files.** The aim is to find places + where the same set of files and directories exists at two or more paths. A + whole subtree can then be removed at once. Tree-level detection is built on + file-level detection. +2. **Bound the reads per file.** A file is read only when it is new or has + changed. A file under 100 MiB is then read in full, and a larger one only at + its ends and at 1 MiB per gigabyte. No file of 100 MiB or more is ever read + in full. The target is a filesystem of about ten million files and 150 TB, on + disks that may be slow or busy. During a scan, one small record per file is + held in memory. It holds the path and the two values the walk compares, the + size and the mtime. The hashes stay in the database. `report` holds the + duplicate paths of one file size at a time. `trees` holds one entry per + directory, plus the files of the directories it is reading. +3. **Rescan cheaply, analyze from the database.** The scan keeps its database + between runs, and an unchanged file is never read again. `report`, `trees` + and `info` work from the database alone and never touch the scanned + filesystem. `scan` is built to run from `cron`. The reports run at any time, + against what the database holds. +4. **Keep the streams apart.** Everything on stdout is machine-readable data. + Everything for a human reader, such as progress and warnings, goes to stderr. ### Subcommands -Four subcommands, all implemented: - -1. `scan` — walk the filesystem and synchronize the database: one signature - record per regular file. -2. `report` — file-level duplicate report from the database. -3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy - from the database records, compute a Merkle-style digest per directory, and - report maximal groups of identical trees. -4. `info` — the database's details: its path, the sizes of its files, its - schema version, the number of its records and their total size. +- `scan` walks the filesystem and updates the database to match it, one record + per regular file. +- `report` prints the file-level duplicate report from the database. +- `trees` prints the duplicate-tree report from the database. +- `info` prints the database's details. ``` sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... @@ -283,424 +253,402 @@ sfdupes --version sfdupes [command] --help ``` -`--workers N` sets the size of each `scan` worker pool (default: the number of -CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that -operand's filesystem, and `--no-default-excludes` lets the walk enter the -default excludes, a list of paths; all three are described under "`scan` mode". -`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION` (see -"Install"), to stdout and exits 0, writing nothing to stderr. `-h` or `--help`, -alone or after a subcommand, prints the help text to stderr and exits 0, writing -nothing to stdout. +A missing or unknown subcommand is a usage error, a mistake in the command line. +`sfdupes` then prints the usage text, a summary of the commands and their flags, +on stderr and exits 2. -### Database +The reference below is also the program's specification. -All four subcommands operate on a single SQLite database file: +## Non-goals -- Location: the value of the `SFDUPES_DATABASE` environment variable when set - and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no - command-line flag. The path names the file exactly, whatever characters it - holds (`?`, `#` and `%` included); a relative path is relative to the working - directory. -- Each subcommand first prints one line on stderr naming the database it uses, - before anything else it prints (progress displays included), whether or not - stderr is a terminal. The line gives the path with the working directory put - in front of it when it is relative, and otherwise as set (a `..` in it stays, - so the line names the same file), escaped like the paths in a report (see - "Report output format"), and says where the path came from: +- No byte-for-byte compare, and no deleting or linking of duplicates. The + reports are advisory. Acting on them is your job. +- The SQLite database is the only stored data. There are no export or import + formats. +- No daemon or filesystem watcher. Scheduling rescans is `cron`'s job. - ``` - scan: new database /var/lib/sfdupes/db.sqlite (default) - trees: database /srv/tmp/sfdupes/db.db (from SFDUPES_DATABASE) - ``` +## Reference - The working directory put in front is the one `pwd -P` prints, with its - symlinks resolved. After a `cd` through a symlink to a directory, a path - starting `../` therefore names a file beside the directory the symlink leads - to, as it does for any other program. `scan` says `new database` when no - file is at that path as it starts. `report`, `trees` and `info` print the - line before they open the database, so the error for a missing database - comes after it. +### Settings -- `scan` creates the database (and its parent directory) on first use. `report`, - `trees` and `info` require an existing database; a missing database file is a - fatal error (exit 1) telling the user to run `scan` first. -- Only one `scan` runs against a database at a time. For its whole run, `scan` - holds an exclusive `flock(2)` lock on a lock file beside the database, named - by appending `.lock` to the database's path with symlinks resolved - (`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the - filesystem or opens the database. Every path to one database, through a - symlink to the file or to a directory above it, names the same lock file. Just - before it takes the lock, `scan` creates the database file, empty, if it is - missing (through a symlink at the database path, it creates the symlink's - target), so the lock file is named the same way before the first scan sets up - the database as after. A second `scan` against the same database does not - wait: it fails at once with a one-line error naming the lock file and exits 1, - without walking anything or opening the database, and the running scan carries - on. The lock file is created on first use, open to its owner only, and left in - place: a leftover file blocks nothing, because the lock ends with the process - holding it however it ends, a fatal error or an interrupt included, and - deleting the file while a scan runs would let a second scan start. `report`, - `trees` and `info` never take the lock, so they run during a scan. -- While `scan` runs, the database is in WAL journal mode with a busy timeout, so - running a report while a cron `scan` is in progress is safe. The filesystem is - authoritative; the database is an eventually-consistent reflection of it. - Hashed records are committed in batched transactions while the scan is still - running (keeping the WAL small and letting concurrent reports observe - progress), so a report may see a scan's changes partially applied, and a scan - that dies partway leaves a valid database holding every batch committed so far - (an interrupted scan also commits the batch in progress, see "Error handling - and exit codes"); the next scan skips those records and converges toward the - filesystem. -- `scan` switches the database back to rollback-journal mode when it closes it, - so between scans the database file alone holds the whole database. Each switch - needs the database to itself: a `scan` that starts while a report is still - reading waits for it up to the 10-second busy timeout, then fails; a `scan` - that ends while a report has the database open warns and leaves the database - in WAL mode until the next scan. `report` writes its rows while it reads the - records, so it is still reading while its output is paused (a pager, a stalled - pipe), and a `scan` started then fails after the busy timeout. -- A `scan` writes to the database and the files beside it at these points (its - phases are described under "`scan` mode"): - - As it starts, before the walk: the directory, an empty database file and - the `.lock` file, each only if missing. - - Then, still before the walk: the schema, in a new database, and the `-wal` - and `-shm` files, which stay beside the database while the scan runs. - - While it loads the existing records (`load`) and walks: no records. A - first scan's database holds none, however long its walk has run. - - During the hash phase: the first records, committed 10,000 at a time. - - In the update phase: the rest of the hashed records, and the deletions. - - At the end, as it closes the database: the `-wal` and `-shm` files go - away, unless a report still has the database open (see above). -- `report`, `trees` and `info` open the database read-only and need only read - access to the database file, and no write access to its directory. While the - database is in WAL mode they also read the `-wal` and `-shm` files beside it, - which SQLite creates with the database file's permissions. -- Schema (`PRAGMA user_version` is the schema version, currently 1; a database - with any other version is a fatal error. `scan` creates the schema and sets - the version in one transaction, so a first scan stopped while doing so leaves - an empty database the next scan sets up. A database at version 0 that already - has a `files` table was therefore not made by sfdupes; every subcommand - refuses it with an error telling the user to remove the file and rescan): - - ```sql - CREATE TABLE files ( - path BLOB PRIMARY KEY, -- absolute path, raw bytes - size INTEGER NOT NULL, -- bytes, from lstat - mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat - mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 - head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB - tail BLOB NOT NULL, -- raw 32-byte SHA-256 of the last 64 KiB - content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples - ) WITHOUT ROWID; - CREATE INDEX files_signature ON files (size, head, tail, content); - ``` - - Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed - UTF-8. `mtime` and `mtime_nsec` hold the file's mtime to the nanosecond: - `mtime` the whole Unix seconds, rounded down, and `mtime_nsec` the - nanoseconds past that second. Split this way they hold any mtime a - filesystem can record, one before 1678 or after 2262 included, which a - single 64-bit count of nanoseconds cannot. They are used only for change - detection and are not part of the duplicate key. Every record holds the - three hashes that "Duplicate detection" below describes. The - `files_signature` index lets `report` read the records in signature order - without sorting the table. +- `SFDUPES_DATABASE` names the database file. When it is unset or empty, the + database is `/var/lib/sfdupes/db.sqlite`. There is no command-line flag for + it. The path names the file exactly, whatever characters it holds. A relative + path is relative to the working directory. +- `--workers N` (`scan`) sets how many directories `scan` reads at once while it + lists the trees. It also sets how many files `scan` reads at once while it + hashes them. The default is the number of CPUs. `N` must be at least 1. A + smaller value is a usage error, reported in one line on stderr with exit 2 + before anything is scanned. +- `-x`, `--one-file-system` (`scan`) keeps the walk of each operand on that + operand's filesystem, as the same flag does for `du` and `rsync`. By default + the walk also enters directories on other filesystems. +- `--no-default-excludes` (`scan`) lets the walk enter the default excludes. + Those are the system and temporary directories the walk otherwise leaves out, + such as `/var/tmp` on Linux (see "Rules for the walk"). +- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and + exits 0. It writes nothing to stderr. +- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr + and exits 0. It writes nothing to stdout. `sfdupes scan --help` also lists the + default excludes of the system it runs on, and the filesystem types the walk + never records anything on. ### Duplicate detection Two files are duplicates when they have the same size and the same three hashes. -`scan` computes the hashes and stores them in the database. `report` and `trees` -group the files whose size and hashes all match, whichever scan recorded them. +A file's size and its three hashes together are its signature. `scan` computes +the hashes and stores them in the database. `report` and `trees` group the files +with the same signature, whichever scan recorded them. -`scan` gives every file it records three SHA-256 hashes: +Every record holds three SHA-256 hashes: -- `head`, the hash of the file's first 64 KiB. -- `tail`, the hash of the file's last 64 KiB. +- `head`, the hash of the file's first 64 KiB; +- `tail`, the hash of the file's last 64 KiB; - `content`, the content hash described below. -In a file of 64 KiB or less, the first and the last 64 KiB are both the whole -file, so all three hashes are the hash of the whole file. - The content hash depends on the file's size: -- Under 100 MiB, it is the SHA-256 of the whole file. Agreement on it is proof - of identical content, barring a SHA-256 collision. -- At 100 MiB and above, it is the SHA-256 of samples, so the file is never read - in full. A sample is the 1 MiB at each gigabyte-aligned offset: 0, 1 GiB, 2 - GiB and so on. The samples are fed into one hash in order. A sample that would - run past the end of the file stops there. +- Under 100 MiB, it is the SHA-256 of the whole file. A match proves identical + content. Two different files with the same SHA-256 are not a practical + concern. +- At 100 MiB and above, it is the SHA-256 of samples. A sample is the 1 MiB that + starts at each gigabyte boundary, the first at offset 0 and the next at 1 GiB. + The samples go into one hash in order. A sample that would run past the end of + the file stops there. -The sampled hash is **deliberately probabilistic**. Bytes outside the samples -and outside the first and last 64 KiB are never read. Two large files that -differ only in such bytes are reported as duplicates. That is the price of never -reading a 150 GB file end to end. +In a file of 64 KiB or less, each 64 KiB stretch is the whole file. All three +hashes are then the hash of the whole file. An empty file has fixed hashes and +is never opened. -Only files of the same size are compared, so their samples always lie at the -same offsets. A file under 100 MiB and one of 100 MiB or more never share a -size, so a whole-file hash is never compared with a sampled one. +The sampled hash is not exact. Bytes outside the samples and outside the first +and last 64 KiB are never read. Two large files that differ only in such bytes +are reported as duplicates. That is the price of never reading a 150 GiB file +end to end. + +Only files of the same size are compared, so their samples lie at the same +offsets. A file under 100 MiB and one of 100 MiB or more never share a size. So +a whole-file hash is never compared with a sampled one. `scan` reads a file only to hash it, and only when the file is new or has -changed since its record was written (see "`scan` mode"). +changed since its record was written. ### `scan` mode -`scan` requires one or more `PATH` operands naming the trees to scan. There is -no default path; invoking `scan` with no operand is a usage error (usage message -on stderr, exit 2). An operand may be a directory or a regular file; an operand -that does not exist is a fatal error (exit 1). Because database records persist -between runs and are keyed by absolute path, each operand is resolved to an -absolute, lexically cleaned path (symlinks are not resolved) before walking, so -results do not depend on the working directory. All operands belong to a single -scan and are enumerated concurrently: every operand seeds the shared walk worker -pool. Overlapping operands are harmless — an operand that duplicates another or -lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. An operand under another one is kept, -though, and walked on its own when any directory from it (from its directory, -for a regular file) up to but not including the operand covering it is a default -exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with -`-x`, is on a different filesystem than that operand, because that operand's -walk does not enter such a directory and so does not reach it. +`scan` takes one or more `PATH` operands naming the trees to scan. There is no +default. An operand may be a directory or a regular file. -An operand is not scanned when it is: +#### Operands -- a symlink, which `scan` never follows, not even as an operand; -- a socket, FIFO or device node; -- at or inside a directory named `.zfs`. +- `scan` with no operand is a usage error. The usage text goes to stderr, and + the exit code is 2. +- An operand that does not exist is a fatal error. `scan` then stops before it + walks anything, and exits 1. +- Each operand is made absolute before the walk. A `.` or `..` in it is resolved + as text, and symlinks are not followed. Each record is stored under that + absolute path, so results do not depend on the working directory. +- All operands belong to one scan, and one walk covers them all at once (see + "Concurrency"). +- An operand that repeats another is dropped before the walk, without a line. +- An operand under another operand is dropped the same way, unless one of the + cases below applies. No file is then reached twice. +- An operand under another is kept, and walked on its own, when the inner + operand or a directory between the two is one of these: + - a default exclude (see "Rules for the walk"); + - with `-x`, a directory on another filesystem than the outer operand. -An operand reached through a symlink into a `.zfs` directory is not scanned -either. + It is also kept when a directory between the two is on a filesystem that is + not persistent. Such a filesystem is one the kernel makes up or keeps in + memory (see "Rules for the walk"). The outer operand's walk does not enter + such a directory, so the inner operand is walked on its own. For an inner + operand that is a regular file, the check starts at its directory. -This holds even when the operand lies under another operand. `scan` prints a -one-line warning naming the path and what it is, counts it as skipped, and drops -it from the scanned operands before reading the database. An operand beneath a -symlink operand is still scanned. The records stored beneath the dropped operand -are not deleted: they are treated like any other record outside the scanned -operands. If it lies under another operand, they are under that operand instead, -and are treated like any other record there. This is not an error: a scan whose -every operand is dropped walks nothing and exits 0. An operand that is not on a -persistent filesystem (see "Rules for the walk") is dropped the same way. The -line `scan` prints for it is not a warning, though, and it is not counted as -skipped. +- No other case keeps an operand under another. An operand reached through a + symlinked directory below the outer operand is dropped too, without a line. + The outer operand's walk does not follow the symlink, so no file under the + inner operand is scanned. The records an earlier scan left under it are + deleted. +- An operand that is neither a regular file nor a directory is not scanned. + `scan` prints a warning naming the path and what it is. It counts the operand + as skipped and drops it. An operand beneath a dropped one is scanned all the + same. The scan keeps the records beneath a dropped operand, unless another + operand covers them. +- An operand at or inside a directory named `.zfs` is never scanned, whether + given alone or under another operand. The check looks at the path as given and + at the path with its symlinks resolved. `scan` prints a warning naming the + path, counts the operand as skipped and drops it. The records beneath it are + kept, unless it lies under another operand. Then they are deleted, because + that operand's walk never enters `.zfs` (see "Rules for the walk"). +- An operand on a filesystem that is not persistent is dropped too. `scan` + prints a line naming it and its filesystem type, and does not count it as + skipped. +- A scan whose every operand is dropped walks nothing and exits 0. -`scan` synchronizes the database with the filesystem state under the scanned +#### What a scan does to the database + +`scan` brings the database in line with the filesystem under the scanned operands: - A file not yet in the database is hashed and inserted. -- A file already in the database is **skipped without reading its contents** - when its lstat size equals the recorded size and its lstat mtime is not newer - than the recorded mtime. This is what makes a daily rescan cheap. -- A file whose mtime is newer than recorded, or whose size differs, is hashed - again. Change detection compares the mtime to the nanosecond, as finely as the - filesystem records it, so a same-size rewrite counts as a change whenever the - filesystem gives it a later mtime than recorded, even within the same second. -- A database record whose path lies under one of the scanned operands but was - not successfully processed this run is deleted. This removes records for - deleted files. It also removes records for paths that failed to stat or hash - this run: the database only ever contains signatures verified by the most - recent scan that covered them (a subsequent successful scan re-adds such - files). -- Database records outside the scanned operands are untouched, so disjoint trees - can be scanned on different schedules into the same database. -- A record at or below a path the walk left out counts as outside the scanned - operands, so it is not deleted. The walk leaves out a path that: - - is a default exclude; - - is not on a persistent filesystem; - - with `-x`, is a directory on another filesystem than its operand. +- A file already in the database is not read again when its size and mtime match + its record. The mtime matches when it is not newer than the recorded one. This + is what makes a daily rescan cheap. +- A file whose size differs, or whose mtime is newer, is hashed again. The mtime + is compared to the nanosecond, as finely as the filesystem records it. A + same-size rewrite counts as a change whenever the filesystem gives it a later + mtime, even within the same second. +- The scan deletes each record under a scanned operand whose file it could not + verify. That covers deleted files, and paths the scan could not stat or read + this run. To stat a file is to read its details, such as its size, without + opening it. So the database holds only what the latest scan over each path + could verify. A later successful scan adds such files again. +- Records outside the scanned operands are left alone, so separate trees can be + scanned on different schedules into one database. +- The scan keeps every record at or below a path that the walk leaves out for + one of these reasons: + - it is a default exclude; + - it is not on a persistent filesystem; + - with `-x`, it is a directory on another filesystem than its operand. - A filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. Records below an operand at or inside such - a path are not kept this way, since that operand's own walk verifies them. + Such records count as outside the scanned operands. So with `-x`, a + filesystem mounted under an operand can be scanned on its own schedule into + the same database. An operand may lie at or inside a default exclude, or, + with `-x`, at or inside a directory on another filesystem. Such an operand + is walked on its own. Its walk verifies the records below it, so they are + deleted when their files are gone. -`scan` runs **four sequential phases over the whole scan**. Parallelism lives -inside the walk and hash phases, and batched database writes begin during the -hash phase: +#### Phases -1. **load** — read the existing records. Each record under the operands is kept - in memory, by path, with its size and its mtime, for the walk to compare - each file with. -2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently - with the walk worker pool: every operand seeds the shared queue, and each - worker reads one directory at a time, handing discovered subdirectories back - to the queue and running `lstat` on each regular file as it is discovered - (while the directory's metadata is still hot). Sequential directory - enumeration is metadata-latency-bound and takes hours at tens of millions of - files; per-directory parallelism is what makes the walk tractable on large - or busy pools. The walk compares each file with its record as it goes, and - an unchanged file needs nothing more. Every new or changed file is carried - to the hash phase as a (path, size, mtime) record. -3. **hash** — the worker pool hashes every carried file as described in - "Duplicate detection" above, which gives the file its `head`, `tail` and - `content`. Zero-length files have constant hashes and are never opened. - Files are hashed in **inode order**, which minimizes seeks on spinning - disks. Paths that are hard links to the same inode are **read once**, and - all share the one result. A hard-link backup farm therefore costs one read - per inode, not per path. The phase total counts actual reads, so its - progress and its time remaining are meaningful. Completed records are - committed in batched transactions **while hashing runs**. A scan interrupted - after hours therefore keeps everything hashed so far, and the next scan - resumes cheaply, skipping records already written. -4. **update** — commit the final partial batch of hashed records. Then delete - the records the scan did not verify: those of vanished files, and of paths - that failed to stat or hash. +`scan` runs four phases in order. Parallel work happens inside the walk and hash +phases. -Rules for the walk: +1. `load` reads the existing records. Each record under the operands is kept in + memory by path, with its size and mtime, for the walk to compare against. +2. `walk` lists the trees under all operands. Several workers share one queue of + directories to read. A worker is one thread of the program, and "Concurrency" + says how many there are. Each worker reads one directory at a time. It hands + the subdirectories it finds back to the queue, and it stats each regular file + as it finds it. The walk compares each file with its record. An unchanged + file needs nothing more. Every new or changed file is passed to the hash + phase. +3. `hash` reads every new or changed file and computes its three hashes. Files + are read in inode order. An inode is the filesystem's own record of a file. + Files with nearby inode numbers usually lie near each other on disk. Reading + them in that order spares a spinning disk from seeking back and forth. Paths + that are hard links to one file are read once and share the result. A backup + tree made of hard links therefore costs one read per file. Completed records + are written to the database 10,000 at a time while hashing runs. So a scan + interrupted after hours keeps everything hashed so far. +4. `update` writes the hashed records not yet written, then deletes the records + the scan did not verify. -- Only regular files. Skip directories, symlinks (do not follow, including - symlink operands), sockets, FIFOs, and device nodes. An operand that is a - symlink, socket, FIFO, or device node is dropped as described in "`scan` mode" - above. -- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking - them would list every file once per snapshot), not even when it is an operand; - such an operand is dropped the same way. -- Never record the scan's own database. Once the database and its `.lock` file - are open, `scan` notes the device and inode of the database file, its `-wal` - and `-shm` files and its `.lock` file, and the walk passes over a file with - one of them, however it reaches it: as an operand, below one, or through a - symlinked directory. It prints no line for such a file and does not count it - as skipped. A record that an earlier scan left for one of them is deleted, - like the record of any file the scan no longer verifies. There is no - `-journal` file during the walk, because the database is in WAL mode then (see - "Database"). -- Filesystem boundaries are crossed by default. With `-x` (long form - `--one-file-system`, following the GNU `du`/`rsync` convention), never descend - into a directory on a different filesystem than its `PATH` operand; each - operand is bounded by its own filesystem. The records below such a directory - count as outside the scanned operands, as described in "`scan` mode" above: - they are not deleted. That does not hold below an operand at or inside that - directory. -- Never record anything that is not on a persistent filesystem. A filesystem is - not persistent when the kernel makes it up or keeps it in memory: one of type - `proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`, - `debugfs`, `tracefs`, `securityfs`, `pstore`, `bpf`, `configfs`, `fusectl`, - `mqueue`, `hugetlbfs`, `efivarfs`, `binfmt_misc`, `autofs`, `nsfs`, - `rpc_pipefs`, `selinuxfs` or `devfs` (the macOS filesystem of `/dev`). After - it names the database and before it takes the lock, `scan` lists the mounted - filesystems, from `/proc/self/mountinfo` on Linux and from the system's list - of mounted filesystems on macOS (on any other system it lists none), and notes - the device ID of each one of those types. On Linux that is the device ID the - list gives, so a filesystem that a later mount hides still gives its own. On - macOS it is read with `lstat` on the mount point, and where one filesystem is - mounted over another on the same mount point, the one mounted last counts. If - `scan` cannot list them, it fails (exit 1). The walk then never enters a - directory, and never records a file, on one of those devices, however it - reaches it: as an operand, below one, or through a bind mount, which keeps the - device ID of what it binds. No flag turns this off. For each operand, - directory or file it leaves out this way, `scan` prints a line on stderr - naming it and the type of its filesystem. It prints that line as it prints a - warning, so the line never lands inside a progress display: +Reading a tree one directory at a time waits on the disk at every step and takes +hours at millions of files. Reading many directories at once is what makes the +walk practical on large or busy storage pools. + +#### Rules for the walk + +- Only regular files are recorded. Every other kind of file, such as a symlink + or a device node, is passed over. Symlinks are never followed. +- The walk never enters a directory named `.zfs`. Such a directory holds the + filesystem's ZFS snapshots, which are its saved earlier states. Walking it + would list every file once per snapshot. `scan` never starts a walk at or + inside one either (see "Operands"). +- The walk never records the scan's own database files: + - the database file; + - its `.lock` file (see "The scan lock"); + - the `-wal` and `-shm` files SQLite keeps beside it while the scan runs + (see "Concurrent reads"). + + The walk knows them by device ID and inode number. The device ID is the + number by which the kernel tells filesystems apart. The inode number tells + the files of one filesystem apart. So the walk passes over them whatever + path leads to them. The walk follows no symlink it finds. It does open each + operand at the path given, even when that path runs through a symlinked + directory. Such an operand can lead to the database files, and the walk + passes over them there as well. The walk prints no line for the database + files and does not count them as skipped. A record an earlier scan left for + one of them is deleted. + +- By default the walk enters directories on other filesystems than its + operand's. With `-x`, it never enters a directory on a different filesystem + than its operand. `scan` prints no line for that directory and does not count + it as skipped. The records below it are kept, as described under "What a scan + does to the database". +- The walk never records anything on a filesystem that is not persistent. A + filesystem is not persistent when the kernel makes it up or keeps it in + memory. These are the types, as `sfdupes scan --help` lists them on Linux and + macOS: + + ``` + proc, sysfs, devtmpfs, devpts, tmpfs, ramfs + cgroup, cgroup2, debugfs, tracefs, securityfs, pstore + bpf, configfs, fusectl, mqueue, hugetlbfs, efivarfs + binfmt_misc, autofs, nsfs, rpc_pipefs, selinuxfs, devfs + ``` + + `devfs` is the macOS filesystem of `/dev`. The others are Linux names. As it + starts, `scan` lists the mounted filesystems and notes the device ID of each + one of those types. On Linux the list comes from `/proc/self/mountinfo`, + which gives each filesystem's own device ID. On macOS it comes from the + system's mount list. There the device ID is read from the mount point, the + directory the filesystem is mounted on. Where two filesystems share a mount + point on macOS, the one mounted last counts. On any other system the list is + empty. A list that `scan` cannot read is a fatal error, and nothing is + scanned. No flag turns this rule off. + + The walk never enters a directory on one of those devices, and never records + a file on one. That holds however the walk reaches the path, through a bind + mount included. A bind mount shows a directory at a second path, and the + files under it keep their device ID. For each path it leaves out this way, + `scan` prints a line on stderr naming the path and its filesystem type: ``` scan: skipped /dev/shm (tmpfs, not a persistent filesystem) ``` - That line is not a warning, and the path is not counted as skipped. An - earlier scan's records at or below it count as outside the scanned operands, - as described in "`scan` mode" above: they are not deleted. That does not - hold below an operand at or inside it. + The path is not counted as skipped. The records below it are kept, as + described under "What a scan does to the database". -- Below an operand, never descend into a default exclude: a path that holds the - kernel's view of the running system, or temporary files, so that - `sfdupes scan /` walks only the files worth comparing. The default excludes - depend on the system, and `sfdupes scan --help` lists those of the system it - runs on: - - On Linux: `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`. - - On macOS: `/dev`, `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, - `/System/Volumes/VM` and `/System/Volumes/Data`. A walk of `/` reaches - everything in `/System/Volumes/Data` through the firmlinks, so walking it - as well would record each of those files twice. - - On Windows, relative to the root of the drive an operand lies on: the - page, swap and hibernation files `pagefile.sys`, `swapfile.sys` and - `hiberfil.sys`, and the directories `System Volume Information`, - `$Recycle.Bin`, `Windows\Temp` and `Users\*\AppData\Local\Temp`, where `*` - matches one path component, as it does in any default exclude. sfdupes - does not build for Windows yet, so this list is not used. - - On any other system there are none. +- Below an operand, the walk does not enter a default exclude. The default + excludes hold the kernel's view of the running system and temporary files, so + that `sfdupes scan /` walks only the files worth comparing. They depend on the + system, and `sfdupes scan --help` lists those of the system it runs on. On + Linux they are: - `--no-default-excludes` turns them all off; what is not on a persistent - filesystem stays left out. An operand at or inside a default exclude is - still walked, so `sfdupes scan /tmp/x` works, unless it is not on a - persistent filesystem. For each default exclude its walk does not enter, - `scan` prints a line on stderr naming it, as it prints a warning: + ``` + /proc, /sys, /dev, /run, /tmp, /var/tmp + ``` + + On macOS they are: + + ``` + /dev, /private/tmp, /private/var/tmp, /private/var/vm + /System/Volumes/VM, /System/Volumes/Data + ``` + + A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already, + through firmlinks. Firmlinks are the links macOS uses to show the data + volume inside the system volume. Walking `/System/Volumes/Data` as well + would record each of those files twice. Any other system has no default + excludes. + + An operand at or inside a default exclude is walked, so + `sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem. + `--no-default-excludes` turns the default excludes off. What is not on a + persistent filesystem stays left out. For each default exclude the walk does + not enter, `scan` prints a line on stderr naming it: ``` scan: skipped /var/tmp (default exclude) ``` - That line is not a warning, and the directory is not counted as skipped. A - default exclude that is not on a persistent filesystem, as `/proc` is, gets - the line for its filesystem instead. One that is itself an operand gets no - line, since it is walked as that operand. When the walk does not enter a - default exclude, the records below it count as outside the scanned operands, - as described in "`scan` mode" above: they are not deleted. That does not - hold below an operand at or inside it. + The directory is not counted as skipped. A default exclude on a filesystem + that is not persistent, such as `/proc`, gets the line for its filesystem + instead. One that is itself an operand gets no line, because it is walked as + that operand. The scan keeps the records below a default exclude that the + walk did not enter, as described under "What a scan does to the database". -- On a per-path error, such as permission denied or an unreadable file: print a - one-line warning to stderr, skip the path, and continue. Per-file errors never - abort the run; the final summary reports how many were skipped. As specified - above, a skipped path that has a database record from an earlier scan loses - that record. An unreadable directory subtree likewise loses its records. This - is accepted: the database mirrors what the latest scan could actually verify. - The exception is an operand dropped before the database was read, whose record - counts as outside the scanned operands. -- A file or directory removed while the scan runs is not an error. The scan - prints no line for it and does not count it as skipped. A path gone when the - walk reaches it gets no record. A file gone when the hash phase opens it gets - none either. The records an earlier scan left at or below such a path are - deleted. +- A path that cannot be read, for lack of permission or any other error, gets a + one-line warning on stderr and is skipped. The scan continues, and the summary + line says how many paths were skipped. A skipped path that has a record from + an earlier scan loses that record, and so does every path under an unreadable + directory. The database mirrors what the latest scan could verify. +- A file or directory that disappears while the scan runs is not an error. The + scan prints no line for it and does not count it as skipped. A path gone when + the walk reaches it gets no record, and so does a file gone when the hash + phase opens it. The records an earlier scan left at or below it are deleted. -Concurrency: the walk phase, which also stats files, and the hash phase each use -a worker pool of `--workers` workers (default `runtime.NumCPU()`). The walk -parallelizes across directories, and hashing across files. `--workers` must be -at least 1: a smaller value is a usage error, reported in one line on stderr -with exit 2 before anything is scanned. Both phases are seek-bound on spinning -disks, so raising `--workers` well past the core count can help on pools with -many spindles. The main goroutine owns the database writes and the progress -counts. Each progress display draws the counts from a goroutine of its own, and -must never block the workers. +#### Concurrency -`scan` writes nothing to stdout. The summary line on stderr reports the files -seen this run broken down by disposition, plus skips: +The walk phase and the hash phase each have `--workers` workers of their own. +The walk works on several directories at once, and the hash phase on several +files at once. On spinning disks both phases wait mostly on seeks, the time the +disk head takes to move. A `--workers` value well above the CPU count can +therefore help on storage pools with many disks. The main thread of the program +owns the database writes and the progress counts. Each progress display draws +from a thread of its own and never blocks the workers. + +#### What `scan` prints + +`scan` writes nothing to stdout. On stderr it prints, in this order: + +- one line naming the database (see "Database"); +- for each dropped operand, in operand order, the warning or the line described + under "Operands". An operand dropped as a repeat of another, or as lying under + another, gets neither; +- the progress display of each phase (see "Progress"). `scan` prints each + warning for a skipped path as it happens. It prints a line the same way for + each path left out as a default exclude or as not on a persistent filesystem; +- the summary line; +- a warning when a report still has the database open as the scan ends (see + "Concurrent reads"). + +The summary line reports the files seen this run, by what happened to each, plus +the skips: ``` scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped ``` -(`removed` counts deleted database records, which are not part of the files-seen -total.) +`removed` counts deleted records, which are not part of the files seen. + +#### Stopping a scan + +`scan` stops cleanly on `SIGINT`, the signal Ctrl-C sends, or `SIGTERM`, the +signal `kill` sends by default: + +- The workers stop taking work. Each finishes no more than the directory listing + or the file it is reading. The progress display is finished. +- The records hashed but not yet written are written, so the next scan does not + hash them again. +- Nothing else is written or deleted. Records are deleted only after a complete + walk. So the records under paths an interrupted walk never reached are kept. +- The database is closed and the lock released, as on any other exit. +- The line `scan: interrupted after N files` goes to stderr. `N` is the number + of files the walk reached. The exit code is 1. + +The next scan reads none of the files already recorded and brings the database +up to date as usual. + +After the first signal `scan` stops catching signals. A second one ends it at +once, as any uncaught signal does. The records not yet written are lost, and the +database stays valid. `scan` catches `SIGTERM` even when it inherits it as +ignored. A background job started from a script inherits `SIGINT` as ignored, +and `scan` leaves it ignored. ### `report` mode -`report` reads every record from the database and takes no positional arguments. - -**`report` must never touch the filesystem being analyzed.** It does not stat, -open, or otherwise access any path that appears in the records; its only I/O is -reading the database and writing stdout/stderr. `report` must produce identical -output whether or not the scanned filesystem is still mounted. +`report` reads every record from the database. It takes no positional arguments. +A positional argument is anything on the command line that is not a flag, such +as the paths `scan` takes. `report` never touches the scanned filesystem. It +neither stats nor opens any path in the records. It reads the database and +writes to stdout and stderr, and nothing else. Its output is the same whether or +not the scanned filesystem is still mounted. Processing: -- Group the records by the key `(size, head, tail, content)`. +- Records are grouped by signature, the four values + `(size, head, tail, content)`. - Every group with two or more paths is a duplicate group. -- Within each group, sort paths lexicographically (byte order). The first path - is the group's `first`; every other path is a `dupe`. -- Order groups by size descending (biggest reclaimable space first), tie-broken - by `first` path ascending. Output must be fully deterministic for a given - database state. +- Within a group, paths are sorted in byte order. The first is `first`, and + every other path is a `dupe`. +- Groups are ordered by size, largest first, then by `first` path. The output is + the same for the same database contents. -`report` runs in two phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `read` counts the records, then reads -them in signature order, largest size first. That is the order of the -`files_signature` index read backwards, so SQLite sorts nothing and returns the -first record at once. The records sharing a signature come together, and so do -the records of one size, so `report` holds only the duplicate groups of the size -it is reading, and writes them, ordered by `first` path, once it reaches a -smaller size. Its memory holds at most the duplicate paths of one size at a -time, never every record. +`report` has two phases, each with its own progress display on stderr (see +"Progress"). `open` opens the database. `read` counts the records, then reads +them in signature order, largest size first. The database keeps an index named +`files_signature`, a sorted list of the records by signature (see "Schema"). +Reading it backwards gives that order, so SQLite sorts nothing and returns the +first record at once. Records with one signature come together, and so do the +records of one size. So `report` holds only the duplicate groups of the size it +is reading. Once it reaches a smaller size, it writes them, ordered by `first` +path. #### Report output format -TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a -group of N): +The report is TSV (tab-separated values) on stdout. It holds a header line, then +one row per duplicate file. A group of N paths gives N-1 rows. ``` first dupe size @@ -708,97 +656,112 @@ first dupe size /srv/a/big.iso /srv/c/big-copy2.iso 4294967296 ``` -Paths are raw bytes and may hold any byte except NUL, so the path columns -(`first` and `dupe`) are escaped to keep every row one line of tab-separated -fields: a backslash is written as `\\`, a tab as `\t`, a newline as `\n`, and a -carriage return as `\r`. Every other byte is written unchanged, including bytes -that are not valid UTF-8. Undoing those four escapes gives back the stored path. -Grouping and ordering use the stored path, not the escaped one. The warnings -`scan` prints on stderr are escaped the same way, so each warning is one line. +Paths are raw bytes and may hold any byte except NUL, the zero byte. The path +columns are escaped, so that every row stays one line of tab-separated fields. +Four bytes are written as escapes: -Summary to stderr: records read, number of duplicate groups, number of dupe -files, and total reclaimable bytes (sum of `size` over all dupe rows) in human -units. +- a backslash as `\\`; +- a tab as `\t`; +- a newline as `\n`; +- a carriage return as `\r`. + +Every other byte is written unchanged, including bytes that are not valid UTF-8, +the usual text encoding. Undoing the four escapes gives back the stored path. +Grouping and ordering use the stored path. The warnings `scan` prints are +escaped the same way, so each warning stays one line. + +The summary line on stderr gives: + +- the records read; +- the duplicate groups; +- the dupe files, which is the number of rows; +- the reclaimable bytes, the sum of `size` over all rows. + +``` +report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable +``` ### `trees` mode -`trees` reads the same database as `report` (no positional arguments) and -reports **entire duplicate directory trees**: directories under which the exact -same set of relative paths exists with the exact same file signatures. - -**`trees` must never touch the filesystem being analyzed** — the same rule as -`report`. The directory hierarchy is reconstructed purely from the paths in the -records, split on `/`. +`trees` reads the same database as `report` and takes no positional arguments. +It reports whole duplicate directory trees. Two directories are duplicate trees +when they hold the same relative paths, and each path holds a file with the same +signature in both. Like `report`, it never touches the scanned filesystem. It +rebuilds the directory tree from the paths in the records, split on `/`. Definitions: -- A file's **signature** is `(size, head, tail, content)` — mtime is - informational and excluded. -- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up: - serialize the directory's child entries — for a file child, its name and - signature; for a subdirectory child, its name and that subdirectory's digest — - sort the serialized entries byte-lexicographically, and hash the - concatenation. Names are part of the digest: two trees whose files differ only - in name are _not_ duplicates. +- A file's **signature** is `(size, head, tail, content)`, as for `report`. The + mtime is not part of it. +- A directory's **digest** is a SHA-256 hash computed from the bottom up. Each + child contributes one entry. A file's entry is its name and signature. A + subdirectory's entry is its name and digest. The entries are sorted in byte + order and hashed together. Names are part of the digest, so two trees whose + files differ only in name are not duplicates. - Two directories are **duplicate trees** when their digests are equal. Equal - digests imply equal recursive file count and equal total byte size. - -Known limitation (accepted): hard-linked paths are reported as duplicates by -`report` and count toward duplicate trees — their content is genuinely identical -— even though they share storage, so removing one reclaims no space. Inode -identity is used during the scan to avoid redundant reads but is not persisted -in the database. - -Known limitation (accepted): only regular files that appear in the database -define a tree. Empty directories are invisible, and a file skipped during the -scan (e.g. permission error) in one copy but not the other will make -otherwise-identical trees compare as different. + digests imply the same number of files and the same total size. Processing: -- Build the hierarchy, compute every directory's digest, and group directories - by digest. Every group with two or more directories is a duplicate-tree group. -- **Report only maximal trees.** A group is suppressed when its members' parents - are pairwise distinct directories that all share a single digest — such a - group is wholly implied by its parents' (or a further ancestor's) group. - Groups containing sibling directories, or members whose parents differ, are - always reported. -- Within each group, sort paths lexicographically (byte order); the first path - is `first`, every other path is a `dupe`. -- Order groups by total tree size descending, tie-broken by `first` path - ascending. Output must be fully deterministic for a given input. +- `trees` builds the directory tree and computes every directory's digest. It + then groups the directories by digest. Every group with two or more + directories is a duplicate-tree group. +- Only the outermost duplicate trees are reported. A group is left out when its + members' parents are distinct directories that all have the same digest. The + parents' group, or an ancestor's, already implies it. A group holding sibling + directories, or members whose parents have different digests, is always + reported. +- Within a group, paths are sorted in byte order. The first is `first`, and + every other path is a `dupe`. +- Groups are ordered by total tree size, largest first, then by `first` path. + The output is the same for the same database contents. -`trees` runs in four phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `read` counts the records, then reads -them in path order, building the hierarchy and computing the digests as it goes; -`group` groups the directories by digest and picks the maximal groups; `write` -writes the groups to stdout. +Two limitations are accepted: + +- Hard-linked paths count toward duplicate trees. Their content is identical. + They share storage, so removing one frees nothing. The scan uses the inode to + avoid reading a file twice, and does not store it. +- Only the regular files in the database define a tree. Empty directories are + invisible. A file skipped during the scan in one copy but not in the other + makes two otherwise identical trees differ. + +`trees` has four phases, each with its own progress display on stderr (see +"Progress"). `open` opens the database. `read` counts the records, then reads +them in path order. It builds the directory tree and computes the digests as it +goes. `group` groups the directories by digest and picks the groups to report. +`write` writes them to stdout. #### Trees output format -TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a -group of N). `files` is the recursive regular-file count of one copy of the -tree; `size` is the recursive total byte size of one copy: +The report is TSV on stdout, with a header line and then one row per duplicate +tree. A group of N trees gives N-1 rows. `files` is the number of regular files +in one copy of the tree, and `size` their total size in bytes: ``` first dupe files size /srv/a/project /srv/backup/project 3417 104857600 ``` -The `first` and `dupe` paths are escaped as described under "Report output -format". The root directory's path is `/`. +The paths are escaped as described under "Report output format". The root +directory's path is `/`. -Summary to stderr: records read, number of duplicate-tree groups, number of dupe -trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human -units. +The summary line on stderr gives: + +- the records read; +- the duplicate-tree groups; +- the dupe trees, which is the number of rows; +- the reclaimable bytes, the sum of `size` over all rows. + +``` +trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable +``` ### `info` mode `info` takes no positional arguments and prints the database's details on -stdout, one per line: a name, a tab, and the value. It opens the database the -way `report` does, read-only and without the scan lock, so it runs during a scan -and needs only read access (see "Database"). Like `report`, it never touches the -scanned filesystem. +stdout, one per line. Each line holds a name and its value, separated by a tab. +`info` opens the database read-only, as `report` does, so it runs during a scan +and needs only read access. It never touches the scanned filesystem. ``` database /var/lib/sfdupes/db.sqlite (default) @@ -812,87 +775,215 @@ size of recorded files 150000000000000 (136.4 TiB) - `database` is the path and where it came from, as on the line every subcommand prints first on stderr (see "Database"). -- `database file size` and `-wal file size` are in bytes, then in human units. - The `-wal` line is there only when a `-wal` file is beside the database, as - while a scan runs. -- `modified` is the time the database file was last changed, in UTC. While the - database is in WAL mode, the newest changes may be in the `-wal` file alone. -- `schema version` is the database's `PRAGMA user_version`. A database with - another version than this build's is refused, as by `report`. +- `database file size` and `-wal file size` are in bytes, then rounded to a unit + such as MiB. The `-wal` line appears only while a `-wal` file is beside the + database, as during a scan. +- `modified` is when the database file was last changed, in UTC. While a scan + runs, the newest changes may be in the `-wal` file alone (see "Concurrent + reads"). +- `schema version` is the version of the database's layout, read from + `PRAGMA user_version` (see "Schema"). A database with another version than + this build's is refused, as by `report`. - `records` is the number of records. `size of recorded files` adds up the sizes - of all records, in bytes, then in human units. + of all recorded files, in bytes, then rounded to a unit such as TiB. -`info` runs in two phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `count` reads the schema version and -counts the records with one query, a single pass over the table, both in one -read-only transaction, so the numbers agree with each other while a scan is -committing. `info` writes its lines once the `count` display has ended, so on a -terminal they come after the displays. +`info` has two phases, each with its own progress display on stderr (see +"Progress"). `open` opens the database. `count` reads the schema version and +counts the records in one transaction. A transaction is a group of reads or +writes that SQLite carries out as one step, against the database as it stands at +one moment. So both describe the database at one moment, even while a scan is +writing. `info` writes its lines after the `count` display has ended. + +### Database + +All four subcommands work on one SQLite database file. + +#### Location + +The database is the file `SFDUPES_DATABASE` names, when that is set and not +empty, and otherwise `/var/lib/sfdupes/db.sqlite`. + +Each subcommand prints one line on stderr naming the database it uses, before +anything else and whether or not stderr is a terminal. The line gives the path +and where it came from: + +``` +scan: new database /var/lib/sfdupes/db.sqlite (default) +trees: database /srv/tmp/sfdupes/db.db (from SFDUPES_DATABASE) +``` + +A relative path is printed with the working directory in front of it. The +working directory is printed as `pwd -P` prints it, with its symlinks resolved. +The rest of the path is printed as set, so a `..` in it stays. The line names +the file the setting names. After a `cd` through a symlink, a `..` counts from +the directory the symlink leads to, as it does for any other program. The +printed path shows that directory. The path is escaped like the paths in a +report. `scan` says `new database` when no file is at that path as it starts. +`report`, `trees` and `info` print the line before they open the database, so +the error for a missing database comes after it. + +#### Creation and access + +- `scan` creates the database and its parent directory on first use. +- `report`, `trees` and `info` need an existing database. A missing database is + a fatal error, exit 1. The error tells you to run `scan` first. +- `report`, `trees` and `info` open the database read-only. They need read + access to the database file and no write access to its directory. While the + database has `-wal` and `-shm` files beside it (see "Concurrent reads"), they + read those too. SQLite creates them with the database file's permissions. + +#### The scan lock + +Only one `scan` runs against a database at a time. + +- For its whole run, `scan` holds a lock on a lock file beside the database. The + lock is taken with the `flock(2)` system call, and only one process can hold + it. The lock file's name is the database's path, with symlinks resolved, plus + `.lock`. By default that is `/var/lib/sfdupes/db.sqlite.lock`. Every path to + one database therefore names the same lock file. +- `scan` takes the lock before it walks the filesystem or opens the database. + Just before that, it creates the database file, empty, if it is missing. So + the lock file is named the same way before the first scan as after. A symlink + at the database path is followed, and its target is created. +- A second `scan` against the same database does not wait. It fails at once with + a one-line error naming the lock file, and exits 1. It has walked nothing and + opened nothing. The running scan carries on. +- The lock file is created on first use and never deleted. Only its owner may + open it. A leftover file blocks nothing. The lock ends with the process that + holds it, however that process ends. Deleting the file while a scan runs would + let a second scan start. +- `report`, `trees` and `info` never take the lock, so they run during a scan. + +#### Concurrent reads + +- While `scan` runs, the database is in write-ahead log (WAL) mode. In WAL mode + SQLite appends changes to a `-wal` file beside the database, and keeps a + lookup table for that file in a `-shm` file. Readers do not block the writer, + so running a report during a scan is safe. +- Every connection has a busy timeout of 10 seconds. That is how long it waits + for a database another connection has locked before it gives up. +- The filesystem is authoritative, and the database follows it. Hashed records + are written 10,000 at a time while the scan runs, so a report may see a scan's + changes partly applied. A scan that dies partway leaves a valid database + holding everything written so far, and the next scan carries on from there. +- When `scan` closes the database, it switches it back to SQLite's default mode, + called rollback-journal mode. In that mode the database file alone holds + everything. Between scans a reader therefore needs no `-wal` or `-shm` file. + Each switch needs the database to itself. A `scan` that starts while a report + is still reading waits for it up to the busy timeout, then fails. A `scan` + that ends while a report has the database open prints a warning and leaves the + database in WAL mode until the next scan. `report` writes its rows while it + reads. So it is still reading while a pager or a stalled pipe holds up its + output. A pager, such as `less`, shows output one screen at a time and waits + for a key press. + +#### What `scan` writes, and when + +- As it starts, before the walk, `scan` creates whichever of these are missing: + - the database's directory; + - an empty database file; + - the `.lock` file. +- Then, before the walk as well, it writes: + - the `-wal` and `-shm` files, which stay beside the database while the scan + runs; + - in a new database, the table, the index and the schema version described + under "Schema". +- During the `load` and `walk` phases, no records are written. A first scan's + database holds no records until the hash phase, however long its walk runs. +- During the `hash` phase, records are written 10,000 at a time. +- During the `update` phase, the rest of the hashed records are written, and the + deletions are made. +- As `scan` closes the database, the `-wal` and `-shm` files go away, unless a + report still has the database open. + +#### Schema + +The schema is the database's layout, its table and its index. +`PRAGMA user_version` holds the schema version, currently 1. A database with any +other version is a fatal error. `scan` creates the table and the index and sets +the version in one transaction (see "`info` mode"). So the three land together +or not at all. A first scan stopped while doing so leaves an empty database, +which the next scan sets up. A database at version 0 that already has a `files` +table was not made by `sfdupes`. Every subcommand refuses it, with an error +telling you to remove the file and rescan. + +```sql +CREATE TABLE files ( + path BLOB PRIMARY KEY, -- absolute path, raw bytes + size INTEGER NOT NULL, -- bytes + mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime + mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second + head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB + tail BLOB NOT NULL, -- raw 32-byte SHA-256 of the last 64 KiB + content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples +) WITHOUT ROWID; +CREATE INDEX files_signature ON files (size, head, tail, content); +``` + +- `path` is a BLOB, a column of raw bytes, because Unix paths are bytes and not + necessarily UTF-8 text. +- `mtime` holds the whole seconds of the file's mtime, rounded down, and + `mtime_nsec` the nanoseconds past that second. Split this way, they hold any + mtime a filesystem can record, even one before 1678 or after 2262. One 64-bit + count of nanoseconds cannot hold those. They serve change detection only and + are not part of the signature. +- `head`, `tail` and `content` each hold the raw 32 bytes of a SHA-256 (see + "Duplicate detection"). `report` and `trees` fail on a record whose hash is + not 32 bytes. +- `files_signature` lets `report` read the records in signature order without + sorting the table. ### Progress -Use the progress-bar library for all progress of `scan`, `report`, `trees` and -`info`; rendering in the style of `pv` is the model. All progress goes to -stderr. +Every subcommand always shows that it is working. Each phase gets a display of +its own on stderr from the moment it starts. The display names the phase and +shows its progress. The parts of its line are listed below. -`scan`, `report`, `trees` and `info` must always show that they are working and -not hung. Each phase gets its own display from the moment it starts. The display -names the phase and its place among the command's phases. It shows the phase's -progress, its elapsed time and its time remaining. +The phases, and what each display counts: -A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`, -`[4/4] update`): - -- `load` counts the records first, then reads them against that count. While the - count runs, the display says it is counting the records. -- `walk` has no exact total. Its estimate is the number of records the last scan - left under the operands, known once `load` is done and shown with a `~`. The - time remaining is unknown once the walk passes the estimate, and when there - are no such records, as on a first scan. -- `hash` and `update` have exact totals. Only files that actually need hashing - appear in the hash total, so its time remaining is meaningful. - -`trees` has four phases (`[1/4] open`, `[2/4] read`, `[3/4] group`, -`[4/4] write`, see "`trees` mode"): - -- `open` has no total. While it opens the database, waiting on a busy one - included, the display says it is opening the database. -- `read` counts the records first, then reads them against that count. While the - count runs, the display says it is counting the records. -- `group` has the directories `read` found as its total, and `write` the - duplicate-tree groups. - -`report` has two phases (`[1/2] open`, `[2/2] read`, see "`report` mode"): - -- `open` has no total. While it opens the database, waiting on a busy one - included, the display says it is opening the database. -- `read` counts the records first, then reads them against that count, writing - the rows to stdout as it goes. While the count runs, the display says it is - counting the records. - -`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"): - -- `open` has no total. While it opens the database, waiting on a busy one - included, the display says it is opening the database. -- `count` has no total either, because one query counts everything: the display - says it is counting the records until the query returns, and its time - remaining is unknown. +- `scan` has four phases: + - `[1/4] load` counts the records first, then reads them against that count. + While it counts, the display says `counting the records`. + - `[2/4] walk` has no exact total. Its estimate is the number of records + under the operands, shown with a `~`. A scan with no records under its + operands, such as a first scan, has no estimate. The time remaining is + `unknown` when there is no estimate, and once the walk passes the + estimate. + - `[3/4] hash` and `[4/4] update` have exact totals. Only the files that + need hashing count toward the hash total, so its time remaining is + meaningful. +- `report` has two phases: + - `[1/2] open` has no total. The display says `opening the database`, also + while it waits on a busy one. + - `[2/2] read` counts the records first, then reads them against that count. + It writes rows to stdout as it goes. +- `trees` has four phases: + - `[1/4] open` and `[2/4] read` are as for `report`, except that `read` + writes nothing. + - `[3/4] group` has the directories `read` found as its total. + - `[4/4] write` has the duplicate-tree groups as its total. +- `info` has two phases: + - `[1/2] open` is as for `report`. + - `[2/2] count` has no total either, because one request to the database + counts everything. The display says `counting the records` until that + request returns. Each display is one line holding: -- the phase and its place among the phases -- a `[m/n] x%` display (items processed / total items, percent); before a phase - has a total, the items processed, or what the phase is waiting for -- current rate (items/s), naming the items the phase counts: files, records, - directories or groups. It covers the items completed over the last 5 seconds, - or, when stderr is not a terminal, since the line before, so it falls to 0 - while nothing completes, and a burst of items after a wait shows as a burst - rather than as a steady rate. Once no item has completed for 10 seconds, the - word `stalled` and where the phase waits take the rate's place (see below). -- elapsed time, labelled `elapsed` -- time remaining, labelled `remain`: a duration such as `12m9s`, or `unknown`. - It comes from the phase's average rate since it started counting, so that it - does not swing with each burst. +- the phase and its place among the phases; +- the progress, written `[m/n] x%` for `m` items done of a total `n`, which is + `x` percent of it. A phase that has no total shows the items done instead, as + `1234 files`. A phase waiting for its total shows what it waits for, as + `counting the records`, and no rate; +- the current rate, in the items the phase counts, such as files or records. It + covers the items completed over the last 5 seconds, or, when stderr is not a + terminal, since the line before. It falls to 0 while nothing completes. A + burst of items after a wait shows as a high rate for the next 5 seconds, then + the rate falls again; +- the elapsed time, labelled `elapsed`; +- the time remaining, labelled `remain`, as a duration such as `12m9s` or as + `unknown`. It comes from the phase's average rate since it started counting, + so it does not swing with each burst. Example lines, as printed when stderr is not a terminal: @@ -902,28 +993,29 @@ Example lines, as printed when stderr is not a terminal: [3/4] hash: [12345/45678] 27%, stalled disk/db, elapsed 4m30s, remain 12m9s ``` -When no item has completed for 10 seconds, as while one large file is read, a -disk is slow or the database is busy, the line says `stalled` and where the -phase waits in place of the rate, which would read 0: `disk` for files, `db` for -the database, `out` for stdout, and two of them, as `disk/db`, for a phase that -may wait on either. `walk` says `disk`. `hash` says `disk/db`, as it reads files -and writes their records to the database between them. `load`, `update` and the -`read` phase of `trees` say `db`. `trees`' `write` says `out`, and its `group`, -which works in memory, says `cpu`. The `read` phase of `report` writes the -report while it reads the records, so it says `db/out`. A phase still waiting -for its total already says what it waits for, and says nothing more. +When no item has completed for 10 seconds, the line says `stalled` and where the +phase waits, in place of the rate. That happens while one large file is read, or +while a disk or the database is slow. Each phase names the places it may wait +in, and a phase that may wait in two places names both, as `disk/db`: -Additional requirements: +- `walk` says `disk`, for files. +- `hash` says `disk/db`, because it reads files and writes their records to the + database between them. +- `load`, `update` and the `read` phase of `trees` say `db`. +- The `read` phase of `report` says `db/out`, because it writes the report to + stdout while it reads. +- `group` of `trees` says `cpu`, for work in memory, and its `write` says `out`. +- A phase still waiting for its total says what it waits for, and nothing more. -- On a terminal, every display starts with the progress-bar library's spinner in - its first column, followed by its line. Each display is drawn again several - times a second, so its spinner turns and its elapsed time keeps moving while a - phase waits for its next item. -- On a terminal, a phase with an exact total ends its line with its percent and - a bar 10 columns wide. This percent takes the place of the line's own. The bar - stays at the terminal's right edge, so it does not move as the line grows or - shrinks. A phase with nothing to do shows no bar. Every phase has an exact - total once it has counted what it will do, except these: +On a terminal: + +- Every display starts with a spinner, a turning character, followed by its + line. The display is drawn again several times a second. So the spinner turns + and the elapsed time keeps moving while a phase waits for its next item. +- A phase with an exact total ends its line with its percent and a bar 10 + columns wide, at the terminal's right edge. The percent after the count is + left out then. A phase with nothing to do shows no bar. Every phase has an + exact total once it has counted its work, except these: - `scan`'s `walk`; - the `open` phase of `report` and `trees`; - both phases of `info`. @@ -934,200 +1026,240 @@ Additional requirements: ⠹ [3/4] hash: [12345/98765], 92 files/s, elapsed 2m32s, remain 17m44s 12% |█ | ``` -- On a terminal no line reaches the terminal's last column, however narrow the - terminal is. A line too wide for the terminal first loses whichever of these - it shows: its rate, the words of a stalled phase, or what the phase is waiting - for. The other four parts are kept whole: +- No line reaches the terminal's last column, however narrow the terminal is. A + line too wide for the terminal first leaves out its rate, the words of a + stalled phase, or what the phase is waiting for. It leaves out its bar next. + It is cut short at the edge only when the rest does not fit on its own. The + rest is: - the phase and its place; - - the progress; + - the progress, when the phase shows one; - the elapsed time; - the time remaining. - The line loses its bar only when those four and the bar do not fit. It is - cut short at the terminal's edge only when those four alone are too wide, - and then the end of the time remaining is lost first. That happens hours - into a scan of millions of files, on a terminal narrower than about 75 - columns. On a terminal 100 columns wide, the stalled line of the example - above keeps its words and its bar. The next draw fits the line to a terminal - made narrower while a phase runs. It erases the old line within the new - width, and the display goes on redrawing in place. + That happens hours into a scan of millions of files, on a terminal narrower + than about 75 columns. A terminal made narrower while a phase runs is fitted + at the next draw. -- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI - redraws: print the line the moment each phase starts, every 5 seconds while it - runs (whether or not an item has completed, and never more often), and once - more as the phase ends. -- Progress must be non-blocking with respect to the worker pool: the main - goroutine counts the items, and each display draws them from a goroutine of - its own. -- A warning printed during a phase always lands on a line of its own, never - inside the progress display. So does every line written to stdout when stdout - and stderr are the same file (the same device and inode): one terminal, as in - a plain `sfdupes report` or `sfdupes trees`, or one pipe or file that both are - sent to. When they are not, stdout is written without waiting on the display, - and the display keeps being drawn while a slow reader of stdout holds up a - write. -- A phase that stops short of its total, as an interrupted one does, is left - showing the count it stopped at rather than filled up. +- A warning printed during a phase lands on a line of its own, never inside the + display. So does every line written to stdout when stdout and stderr are the + same file. That is one terminal, or one pipe or file that both are sent to. + When they are not the same file, stdout is written without waiting on the + display. The display then keeps being drawn while a slow reader holds up + stdout. + +When stderr is not a terminal, such as a pipe or a file, there are no redraws. +Each phase prints its line: + +- as it starts; +- every 5 seconds while it runs, whether or not an item has completed; +- once more as it ends. + +A phase that stops short of its total, as an interrupted one does, keeps showing +the count it stopped at. ### Error handling and exit codes -- `0`: success, even if individual files were skipped with warnings. -- `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is - already running against the same database, the database cannot be - created/opened/read/written, a missing database for `report`/`trees`/`info`, - stdout write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see - below). -- `2`: usage error (including `scan` with no `PATH` operand, `scan` with - `--workers` below 1, and `report`/`trees`/`info` with any positional - argument). +- `0`: success, even when files were skipped with warnings. +- `1`: a fatal error, or a `scan` stopped by `SIGINT` or `SIGTERM`. A fatal + error is one of these: + - an operand that does not exist, or whose details `scan` cannot read; + - a list of mounted filesystems that `scan` cannot read; + - another `scan` already running against the same database; + - a database that cannot be created or opened, or whose reads or writes + fail, its directory and its lock file included; + - a relative database path when the working directory cannot be read; + - a database with another schema version, or one at version 0 that already + has a `files` table; + - a record with a hash that is not 32 bytes, read by `report` or `trees`; + - a missing database for `report`, `trees` or `info`; + - a failed write to stdout. +- `2`: a usage error, which is one of these: + - a missing or unknown subcommand; + - an unknown flag; + - a flag without its value, or with a value that cannot be parsed, such as + `--workers abc`; + - `scan` with no operand; + - `--workers` below 1; + - `report`, `trees` or `info` with a positional argument. -A stdout write failure, such as a full disk, is reported in one line on stderr -and exits 1. Two cases never reach sfdupes as a failed write: +A fatal error is reported in one line on stderr that starts with `sfdupes:`. A +failed write to stdout, such as a full disk, is reported the same way. Two cases +never reach `sfdupes` as a failed write: -- When the reader of a stdout pipe exits early, as in `sfdupes report | head`, - the next write ends sfdupes with `SIGPIPE`, quietly and without a summary, the - way `cat` or `sort` end. The shell reports the signal (status 141 in most - shells), not exit 1. -- When stdout is closed outright (`sfdupes report >&-`), the Go runtime opens - `/dev/null` in its place before sfdupes starts, so the output is discarded and - the run succeeds, as with `> /dev/null`. - -`scan` stops cleanly on `SIGINT` (Ctrl-C) or `SIGTERM`. Its workers stop taking -work, each finishing at most the directory listing or file it is reading; the -progress display is finished; and the records it has hashed but not yet -committed are committed, so the next scan does not hash them again. Apart from -that commit it starts no further writes or deletions: records are deleted only -after a complete walk, so those under paths an interrupted walk never reached -are kept. The database is closed and the lock released as on any other exit, the -line `scan: interrupted after N files` goes to stderr, N being the number of -files the walk reached, and the exit code is 1. The next scan skips the records -already written and converges as usual. - -After the first signal `scan` stops catching them, so a second one ends it at -once, as an uncaught signal does: the records not yet committed are lost, and -the database is left valid, as when any scan dies (see "Database"). A `SIGINT` -that `scan` inherits as ignored, as a script's background job does, stays -ignored. +- The reader of a stdout pipe may exit early, as `head` does in + `sfdupes report | head`. The next write then ends `sfdupes` with `SIGPIPE`, + the signal for a write to a closed pipe. `sfdupes` ends quietly and without a + summary, as `cat` or `sort` would. The shell reports the signal, status 141 in + most shells, not exit 1. +- When stdout is closed outright, as in `sfdupes report >&-`, the Go runtime + opens `/dev/null` in its place before `sfdupes` starts. The output is + discarded, and the run succeeds, as with `> /dev/null`. ## Entrypoints -This repository adheres to the +This repository follows the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) -standard: the normalized executables in `script/` are the entrypoints for the -development workflow, and the `Makefile` targets are thin shims that call them. -Every script is POSIX `sh`, resolves the repository root itself so it can be run -from any working directory, and may be invoked directly. The provided -entrypoints are: +standard. The executables in `script/` are the entrypoints of the development +workflow, and the `Makefile` targets call them, except `build` and `clean` (see +"Build"). Every script runs under plain `sh`, the standard Unix shell. It finds +the repository root itself, so it can be run from any directory. -- `script/bootstrap` — install everything needed to build and develop this - repository, idempotently, assuming nothing is present. `git`, `make`, and `go` - come from the first of nix, apt, brew, or apk found on the host, and are - presence-checked only. An installed node is used when its major version is 22; - otherwise node 22.17.0 is installed through nvm, which comes from a release - archive whose sha256 the script checks, and yarn 1.22.22 is activated under it - through corepack. With an installed node 22, a yarn already on `PATH` is used - as it is; otherwise yarn 1.22.22 is activated through corepack, or installed - with `npm` when there is no corepack. `yarn install --frozen-lockfile` then - installs the prettier that `package.json` and `yarn.lock` pin. `golangci-lint` - is never installed: it runs in Docker (see `script/lint`). `docker` is not - installed either; testing, linting and the image build need it. Ends with - `go mod download`. -- `script/setup` — make a fresh clone ready for development: runs +- `script/bootstrap` installs what building and developing need. It can be run + again at any time, and installed tools are kept. `git`, `make` and `go` are + checked for presence only. A missing one is installed with the first of these + package managers found on the host: + - `nix`; + - `apt`; + - `brew`; + - `apk`. + + Then come `node` and `yarn`. `prettier`, the Markdown formatter, is a `node` + program. `yarn` is the package manager that installs it: + - An installed `node` is used when its major version is 22. A `yarn` already + on `PATH` is then used as it is. When `yarn` is missing, `yarn` 1.22.22 is + activated through `corepack`, the `yarn` installer that ships with `node`. + When there is no `corepack` either, `npm`, the package manager that ships + with `node`, installs it. + - Otherwise the script installs `node` 22.17.0 through `nvm`, a `node` + version manager, and activates `yarn` 1.22.22 under that `node` through + `corepack`. When `nvm` is missing, the script first downloads it and + checks the archive's SHA-256. + + `yarn install --frozen-lockfile` then installs the `prettier` that + `package.json` and `yarn.lock` pin. The linter is never installed. It runs + in Docker (see `script/lint`). `docker` is not installed either. + `make check` and `make docker` both need it. The script ends with + `go mod download`. + +- `script/setup` makes a fresh clone ready for development. It runs `script/bootstrap`, then `script/install-precommit`. -- `script/projectname` — print this project's name (`sfdupes`). Scripts that - need the name call it, so they stay identical across repositories. -- `script/test` — run the test suite under the race detector, with a 90-second - timeout and coverage enabled, rerunning verbosely on failure so the logs show - which test failed. It builds the `Dockerfile`'s `test` phase alone and writes - no image. The race detector needs cgo and a C compiler, which the build never - uses, so the phase starts from a digest-pinned Debian `golang` image, which - has `gcc`. The tests run as `nobody`, because several of them make a file - unreadable and root reads it anyway. -- `script/lint` — run the linter. It builds the `Dockerfile`'s `lint` phase - alone and writes no image: in the digest-pinned `golangci/golangci-lint` - image, the gofmt check, `golangci-lint config verify` and `golangci-lint run` - run as build steps, so a successful build is a clean lint. The linter is never - run on the host, which makes a working `docker` the one prerequisite for - linting. -- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every - Markdown file with prettier, at the settings in `.prettierrc` (4-space - indents, prose wrapped at 80 columns). Both run on the host, prettier through - yarn at the version `yarn.lock` pins. When the node on `PATH` is not node 22, - this loads the node 22.17.0 that `script/bootstrap` installed through nvm. -- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any - unformatted file and exits non-zero instead of writing. gofmt and prettier +- `script/projectname` prints `sfdupes`. Scripts that need the name call it, so + they stay identical across repositories. +- `script/test` runs the test suite under the race detector, Go's check for + unsynchronized access to memory from several threads. The suite runs with a + 90-second timeout, and its coverage of the code is measured. On failure it + reruns the suite verbosely, so that the log names the failing test. It builds + only the `test` stage of the `Dockerfile` and writes no image. A stage is one + named part of the `Dockerfile`, which starts from an image of its own and can + be built alone. The race detector needs cgo, Go's bridge to C code, and a C + compiler. The stage therefore starts from a Debian `golang` image, which has + the C compiler `gcc`. The image is pinned by its digest, its content hash. The + tests run as the user `nobody`, because several of them make a file + unreadable, and root reads it anyway. +- `script/lint` runs the linter, `golangci-lint`, which checks the Go sources + for common mistakes. It builds only the `lint` stage of the `Dockerfile` and + writes no image. The stage starts from the `golangci/golangci-lint` image, + pinned by its digest, and runs three build steps: + - the `gofmt` check, which fails on a Go source that `gofmt`, Go's own + formatter, would change; + - `golangci-lint config verify`, which checks the linter's settings file + `.golangci.yml`; + - `golangci-lint run`, which lints the Go sources. + + A successful build is a clean lint. The linter never runs on the host, so a + working `docker` is the one thing linting needs. + +- `script/fmt` formats the Go sources with `gofmt -s -w` and every Markdown file + with `prettier`, in place. `prettier` indents by 4 spaces and wraps prose, as + `.prettierrc` sets and as the script also passes on the command line. It wraps + at 80 columns, its default width. Both run on the host. `prettier` runs + through `yarn` at the version `yarn.lock` pins. When the `node` on `PATH` is + not `node` 22, the script loads the `node` 22.17.0 that `script/bootstrap` + installed. +- `script/fmt-check` is the read-only form of `script/fmt`. It names any + unformatted file and exits non-zero instead of writing. `gofmt` and `prettier` both run every time, and each names itself when it fails. The `Dockerfile`'s - `lint` phase runs the same gofmt check. -- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in - that order. Modifies nothing. The first two need `docker`, the third what - `script/bootstrap` installs. -- `script/docker` — build the Docker image, tagged with the name from - `script/projectname`, passing the output of - `git describe --tags --always --dirty` (or `unknown` when that is empty) as - the `VERSION` build argument. The `Dockerfile`'s build stage depends on its - `lint` and `test` phases, so this also runs the tests and the linter. -- `script/cibuild` — run `script/bootstrap`, then `script/check`, then build the - image as `script/docker` does. This is what the Gitea workflow runs on push; a - successful run means every check passed. The tests and the linter run twice: - once in `script/check`, and again as stages of the image build. -- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a - no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then - `script/check`. -- `script/install-precommit` — install the git pre-commit hook, as - `.git/hooks/pre-commit`, that runs `script/precommit`. + `lint` stage runs the same `gofmt` check. +- `script/check` runs three scripts in order, and changes nothing: + - `script/test` and `script/lint`, which need `docker`; + - `script/fmt-check`, which needs what `script/bootstrap` installs. +- `script/docker` builds the Docker image, tagged with the name from + `script/projectname`. It passes the output of + `git describe --tags --always --dirty`, or `unknown` when that is empty, to + the `Dockerfile` as the build argument `VERSION`. A build argument is a value + that a `docker build` hands to the `Dockerfile`. The build stage depends on + the `lint` and `test` stages, so this also runs the tests and the linter. +- `script/cibuild` runs `script/bootstrap` and then `script/check`. Then it + builds the image as `script/docker` does. The Gitea workflow in + `.gitea/workflows/check.yml` runs it on every push. A successful run means + every check passed. The tests and the linter run twice, once in `script/check` + and again as stages of the image build. +- `script/precommit` runs from the `git` pre-commit hook, a script that `git` + runs before each commit. It fails the commit when `go mod tidy` would change + anything. Then it runs `script/check`. +- `script/install-precommit` installs `.git/hooks/pre-commit`, which runs + `script/precommit`. Every `docker build` in `script/` passes a new `CACHEBUST` build argument. In -every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps -that install system packages and download the Go modules, or right after `FROM` -in a stage with neither, and every step that lints, tests or compiles comes -after it, so every `RUN` step after it runs again each time. On an unchanged -tree Docker would otherwise serve the gate steps from its cache, and the build -would pass having run no test and no linter. The layers above that line stay in -Docker's build cache until the base image, their own lines, `go.mod` or `go.sum` -change, or until that cache is cleared; only then are the packages installed and -the modules downloaded again. Go's build cache is kept between builds on the -steps that compile, so only what changed is compiled again; the tests run with -`-count=1`, so none reports a stored result. `golangci-lint run` also keeps -golangci-lint's cache, and takes a package's findings from it only when the -package's files and those of the packages it imports, `go.mod`, the linters and -their settings in `.golangci.yml`, and the golangci-lint version are all -unchanged. +every stage of the `Dockerfile`, `ARG CACHEBUST` comes after the steps that +install system packages and download the Go modules. Every step that checks or +compiles the code comes after it. Every `RUN` step after it runs again on each +build. Without that, Docker would serve the test and lint steps from its cache +on an unchanged tree, and the build would pass without running them. The steps +above `ARG CACHEBUST` stay cached until something they depend on changes, such +as `go.mod`. Go's build cache is kept between builds, so only what changed is +compiled again. The tests run with `-count=1`, so no test reports a stored +result. `golangci-lint run` keeps the linter's own cache as well, so an +unchanged package is not linted again. ## Build -The `script/` entrypoints above are where the implementations live; the -`Makefile` targets are shims onto them, except `build`, which carries the -compile recipe: +The `script/` entrypoints above hold the implementations. Each `Makefile` target +does nothing but call its script, except `build` and `clean`, which have +commands of their own: -- `make` / `make build` — build the `sfdupes` binary (cgo disabled); building is - the default target. -- `make bootstrap` — install the build and development dependencies. -- `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook. -- `make test` — run the test suite under the race detector, in Docker (90-second - timeout; reruns with `-v` on failure; see `script/test`); requires `docker`. -- `make lint` — run `golangci-lint` with the repo config and the gofmt check, in - Docker (see `script/lint`); requires `docker`. -- `make fmt` / `make fmt-check` — format the Go sources and the Markdown / - verify formatting without writing, on the host; requires `go` and what - `make bootstrap` installs (see `script/fmt`). -- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires - `docker` and what `make bootstrap` installs. -- `make docker` — build the Docker image, which runs the gates as build stages. -- `make hooks` — install the pre-commit hook. -- `make clean` — remove the binary. +- `make` or `make build` builds the `sfdupes` binary with cgo disabled. Building + is the default target. +- `make bootstrap` installs the build and development dependencies. +- `make setup` prepares a fresh clone: `bootstrap`, then the pre-commit hook. +- `make test` runs the test suite under the race detector, in Docker (see + `script/test`). It needs `docker`. +- `make lint` runs the linter and the `gofmt` check, in Docker (see + `script/lint`). It needs `docker`. +- `make fmt` formats the Go sources and the Markdown on the host, and + `make fmt-check` checks them without writing. Both need `go` and what + `make bootstrap` installs. +- `make check` runs `script/check`, described above, and changes nothing. It + needs `docker` and what `make bootstrap` installs. +- `make docker` builds the Docker image, which runs the tests and the linter as + build stages. +- `make hooks` installs the pre-commit hook. +- `make clean` removes the binary. + +### Constraints + +- The language is Go. The module is `sneak.berlin/go/sfdupes`, and the binary is + `sfdupes`. +- The code imports these modules beyond the standard library: + - `github.com/spf13/cobra`, for the command line; + - `github.com/schollz/progressbar/v3`, for the progress display; + - `golang.org/x/term`, to tell whether stderr is a terminal; + - `modernc.org/sqlite`, the SQLite driver; + - `golang.org/x/sys`, for the scan lock and the list of mounted filesystems. + + No other dependency outside the standard library is added. + +- The code stays pure Go, with no C code. Building on one system for another is + not a goal. `modernc.org/sqlite` is pure Go too, so the binary builds without + cgo. The `Makefile` exports `CGO_ENABLED=0`. The test build is the one + exception, because the race detector needs cgo (see `script/test`). +- `report` and `trees` give the same output for the same database contents, + whatever order the records were inserted in. ### Definition of done All of the following, run in this directory, must pass: -1. `make check` passes (tests, lint, `gofmt`, prettier). -2. `make docker` succeeds. -3. Smoke test — create a throwaway tree in a temp dir (never test against real - data): +1. `make check` passes. +2. `make docker` succeeds. +3. The test below behaves as described. It builds a throwaway tree in a new + directory under `/var/tmp`, which is on a persistent filesystem. On many + systems `/tmp` is a `tmpfs`, which `scan` leaves out. Never test against real + data. ```sh - d=$(mktemp -d) - export SFDUPES_DATABASE="$(mktemp -d)/db.sqlite" + d=$(mktemp -d /var/tmp/sfdupes.XXXXXX) + export SFDUPES_DATABASE="$(mktemp -d /var/tmp/sfdupes.XXXXXX)/db.sqlite" mkdir -p "$d/a" "$d/b" head -c 2000 /dev/urandom > "$d/a/one.bin" cp "$d/a/one.bin" "$d/b/copy.bin" @@ -1155,44 +1287,34 @@ All of the following, run in this directory, must pass: ./sfdupes report ``` - Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one - group (two dupe rows, `first` is the lexicographically smallest path); - `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/ - `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair; - `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere; groups - ordered by size descending. + Expected from the first `report`: + - `one.bin`, `copy.bin` and `copy2.bin` form one group of two rows. `first` + is the path that sorts first. + - `t1/f1`, `t2/f1` and `t3/f1` form one group. + - `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group. + - `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`. + - `unique.bin` and `tiny3` appear nowhere. + - The groups come in order of size, largest first. - Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2 - files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal - (implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set - differs by name). + `trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`. + The row shows 2 files and 3100 bytes. The two `sub` directories are left + out, because the `t1` and `t2` group implies them. `t3` appears nowhere, + since one of its file names differs. - Expected from the second `report` (after the modify/delete rescan): - `one.bin` has left its group (its content changed), so - `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is gone from the - database. + After the rescan, the second `report` shows `one.bin` out of its group, + because its content changed. `copy.bin` and `copy2.bin` remain as one pair. + `unique.bin` is gone from the database. - The test suite automates this scenario (see `scan_test.go`), plus a negative - check: `report` and `trees` operate on the database alone and never touch - the scanned filesystem. + The test suite automates this scenario (see `scan_test.go`). It also checks + that `report` and `trees` never touch the scanned filesystem. ## TODO -Tracked in [TODO.md](TODO.md). - -## Non-goals - -- No byte-for-byte compare, and no deletion or linking of duplicates. Files that - match are compared by a SHA-256 of the whole file below 100 MiB, and only by - samples at 100 MiB and over. The reports are advisory; acting on them is the - user's job. -- No persistence beyond the SQLite database described above; no export/import - formats. -- No daemon or filesystem watcher; scheduling rescans is cron's job. +Tracked in [`TODO.md`](TODO.md). ## License -MIT. See [LICENSE](LICENSE). +MIT. See [`LICENSE`](LICENSE). ## Author diff --git a/TODO.md b/TODO.md index def2438..4b301b6 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,14 @@ # Completed Steps +- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads + it. It leads with what the program is and how to use it. A reference section + covers every setting and every subcommand. Development and testing come last. + Every statement was checked against the code on `next`. Five comments in + `main_test.go` cite "Settings", where their rules now live. One comment each + in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README + now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135) + - an operand at or inside a directory named `.zfs` is never scanned, whether given alone or under another operand. `scan` warns about it, counts it as skipped and does not walk it (2026-10-26, diff --git a/exclude_test.go b/exclude_test.go index b6b48c7..fb8aa67 100644 --- a/exclude_test.go +++ b/exclude_test.go @@ -17,7 +17,9 @@ import ( func TestDefaultExcludes(t *testing.T) { t.Parallel() - // README "Rules for the walk": each system's list. + // README "Rules for the walk": the Linux and macOS lists, and none + // on any other system. The Windows list exists in the code only: the + // binary does not build for Windows, so the README leaves it out. cases := map[string][]string{ goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, goosDarwin: { diff --git a/main_test.go b/main_test.go index cca359a..53d7875 100644 --- a/main_test.go +++ b/main_test.go @@ -393,7 +393,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) { } func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) { - // README §Database: the path names the file exactly. With link a + // README §Settings: the path names the file exactly. With link a // symlink to the directory real/sub, link/../db.sqlite names // real/db.sqlite, not db.sqlite beside link. Every subcommand must // use that file, relative or absolute, and its first line must name @@ -648,7 +648,7 @@ func TestRunUsageErrors(t *testing.T) { } func TestRunScanRejectsWorkersBelowOne(t *testing.T) { - // README §scan mode: --workers below 1 is a usage error reported in + // README §Settings: --workers below 1 is a usage error reported in // one line on stderr, before the scan opens the database. for _, workers := range []string{"0", "-1"} { t.Run(workers, func(t *testing.T) { @@ -686,8 +686,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) { func TestRunHelp(t *testing.T) { t.Parallel() - // README §Subcommands: help goes to stderr, exits 0, and leaves - // stdout empty. + // README §Settings: help goes to stderr, exits 0, and leaves stdout + // empty. cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}} for _, args := range cases { @@ -713,8 +713,8 @@ func TestRunHelp(t *testing.T) { func TestRunVersion(t *testing.T) { t.Parallel() - // README §Subcommands: the version is one line on stdout, with - // nothing on stderr, and exits 0. + // README §Settings: the version is one line on stdout, with nothing + // on stderr, and exits 0. info, _ := debug.ReadBuildInfo() for _, arg := range []string{"--version", "-v"} { @@ -1234,7 +1234,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) { } func TestRunDatabasePathUsedAsGiven(t *testing.T) { - // README §Database: the path names the database file exactly. In + // README §Settings: the path names the database file exactly. In // SQLite's connection string an unescaped ? or # would end the file // name and % would start an escape, and a path starting with // // could be read as a host name. %25 is a valid escape, so unescaped diff --git a/scan_test.go b/scan_test.go index 58ab1db..0ff7538 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1064,8 +1064,9 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) { } } -// buildSmokeTree recreates the README smoke-test filesystem layout -// with deterministic content and returns the tree root. +// buildSmokeTree recreates the tree of the test under README +// "Definition of done" with deterministic content and returns the tree +// root. func buildSmokeTree(t *testing.T) string { t.Helper() diff --git a/trees_test.go b/trees_test.go index 540c117..bc2610c 100644 --- a/trees_test.go +++ b/trees_test.go @@ -15,8 +15,9 @@ import ( const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " + "remain unknown" -// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and -// /d/t2 are identical, /d/t3 differs from them only by one filename. +// smokeTreeRecs mirrors the tree of the test under README "Definition +// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only +// by one filename. func smokeTreeRecs() []scanRec { f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c") f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")