Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be47255598 | ||
|
|
3732ecef1f | ||
|
|
a9ab6124ae | ||
|
|
db56fd0e4c |
No files matched your search
@@ -22,14 +22,14 @@ records the files it finds in an SQLite database. SQLite keeps a whole database
|
|||||||
in one file. The database is kept between runs. A rescan reads only new and
|
in one file. The database is kept between runs. A rescan reads only new and
|
||||||
changed files. That makes `scan` cheap enough to run every day from `cron`, the
|
changed files. That makes `scan` cheap enough to run every day from `cron`, the
|
||||||
system's job scheduler. `report`, `trees` and `info` work from the database and
|
system's job scheduler. `report`, `trees` and `info` work from the database and
|
||||||
can run at any time. `report` and `trees` leave out a copy that was deleted
|
can run at any time. `dwim` scans `/` and then prints the `trees` report, for a
|
||||||
after the scan. `dwim` scans `/` and then prints the `trees` report, for a first
|
first look at a new machine with one command.
|
||||||
look at a new machine with one command.
|
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
make build
|
make build
|
||||||
|
export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
|
||||||
./sfdupes scan /srv
|
./sfdupes scan /srv
|
||||||
./sfdupes report > dupes.tsv
|
./sfdupes report > dupes.tsv
|
||||||
./sfdupes trees > dupetrees.tsv
|
./sfdupes trees > dupetrees.tsv
|
||||||
@@ -45,10 +45,8 @@ that stands for a piece of hardware, such as a disk or a terminal. `report`
|
|||||||
prints the file-level duplicates. `trees` prints the duplicate directory trees.
|
prints the file-level duplicates. `trees` prints the duplicate directory trees.
|
||||||
`info` prints the database's path, size and record count.
|
`info` prints the database's path, size and record count.
|
||||||
|
|
||||||
Root keeps the database in `/var/lib/sfdupes/db.sqlite`. Any other user keeps it
|
The database is `/var/lib/sfdupes/db.sqlite` unless `SFDUPES_DATABASE` names
|
||||||
in `sfdupes/db.sqlite` in the XDG state directory, `~/.local/state` by default
|
another file. The first `scan` creates it.
|
||||||
on Linux. `SFDUPES_DATABASE` names another file (see "Location"). The first
|
|
||||||
`scan` creates the database.
|
|
||||||
|
|
||||||
`report`, `trees` and `info` write their data to stdout. Progress, warnings and
|
`report`, `trees` and `info` write their data to stdout. Progress, warnings and
|
||||||
errors go to stderr. stdout and stderr are a program's two output streams. The
|
errors go to stderr. stdout and stderr are a program's two output streams. The
|
||||||
@@ -150,9 +148,8 @@ walk"). As the file `/etc/cron.d/sfdupes`:
|
|||||||
- `report`, `trees` and `info` need only read access to the database. The umask
|
- `report`, `trees` and `info` need only read access to the database. The umask
|
||||||
is the setting that removes permissions from the files a process creates.
|
is the setting that removes permissions from the files a process creates.
|
||||||
Under the usual umask of `022`, the first scan creates the database readable
|
Under the usual umask of `022`, the first scan creates the database readable
|
||||||
by every user. Another user can then run those three commands against root's
|
by every user. An unprivileged user can then run those three commands against
|
||||||
database, with `SFDUPES_DATABASE=/var/lib/sfdupes/db.sqlite` before the
|
root's database.
|
||||||
command.
|
|
||||||
|
|
||||||
### Reading the reports
|
### Reading the reports
|
||||||
|
|
||||||
@@ -257,15 +254,14 @@ In order:
|
|||||||
- `report` prints the file-level duplicate report from the database.
|
- `report` prints the file-level duplicate report from the database.
|
||||||
- `trees` prints the duplicate-tree report from the database.
|
- `trees` prints the duplicate-tree report from the database.
|
||||||
- `info` prints the database's details.
|
- `info` prints the database's details.
|
||||||
- `dwim` scans `/` with the default excludes, then prints the duplicate-tree
|
- `dwim` scans `/`, then prints the duplicate-tree report.
|
||||||
report.
|
|
||||||
|
|
||||||
```
|
```
|
||||||
sfdupes scan [--workers N] [-x] [--no-default-excludes] [--exclude PATH]... PATH...
|
sfdupes scan [--workers N] [-x] [--no-default-excludes] [--exclude PATH]... PATH...
|
||||||
sfdupes report > dupes.tsv
|
sfdupes report > dupes.tsv
|
||||||
sfdupes trees > dupetrees.tsv
|
sfdupes trees > dupetrees.tsv
|
||||||
sfdupes info
|
sfdupes info
|
||||||
sfdupes dwim
|
sfdupes dwim [--workers N] [-x] [--no-default-excludes] [--exclude PATH]...
|
||||||
sfdupes --version
|
sfdupes --version
|
||||||
sfdupes [command] --help
|
sfdupes [command] --help
|
||||||
```
|
```
|
||||||
@@ -292,23 +288,23 @@ The reference below is also the program's specification.
|
|||||||
### Settings
|
### Settings
|
||||||
|
|
||||||
- `SFDUPES_DATABASE` names the database file. When it is unset or empty, the
|
- `SFDUPES_DATABASE` names the database file. When it is unset or empty, the
|
||||||
database is at its default path (see "Location"). There is no command-line
|
database is `/var/lib/sfdupes/db.sqlite`. There is no command-line flag for
|
||||||
flag for it. The path names the file exactly, whatever characters it holds. A
|
it. The path names the file exactly, whatever characters it holds. A relative
|
||||||
relative path is relative to the working directory.
|
path is relative to the working directory.
|
||||||
- `--workers N` (`scan`) sets how many directories `scan` reads at once while it
|
- `--workers N` (`scan`, `dwim`) sets how many directories `scan` reads at once
|
||||||
lists the trees. It also sets how many files `scan` reads at once while it
|
while it lists the trees. It also sets how many files `scan` reads at once
|
||||||
hashes them. The default is the number of CPUs. `N` must be at least 1. A
|
while it hashes them. The default is the number of CPUs. `N` must be at
|
||||||
smaller value is a usage error, reported in one line on stderr with exit 2
|
least 1. A smaller value is a usage error, reported in one line on stderr with
|
||||||
before anything is scanned.
|
exit 2 before anything is scanned.
|
||||||
- `-x`, `--one-file-system` (`scan`) keeps the walk of each operand out of
|
- `-x`, `--one-file-system` (`scan`, `dwim`) keeps the walk of each operand out
|
||||||
directories on other filesystems, as the same flag does for `du` and `rsync`.
|
of directories on other filesystems, as the same flag does for `du` and
|
||||||
By default the walk also enters directories on other filesystems.
|
`rsync`. By default the walk also enters directories on other filesystems.
|
||||||
- `--no-default-excludes` (`scan`) lets the walk enter the default excludes.
|
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
||||||
Those are the system and temporary directories the walk otherwise leaves out,
|
excludes. Those are the system and temporary directories the walk otherwise
|
||||||
such as `/var/tmp` on Linux (see "Rules for the walk").
|
leaves out, such as `/var/tmp` on Linux (see "Rules for the walk").
|
||||||
- `--exclude PATH` (`scan`) keeps the walk out of `PATH` and everything below
|
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
||||||
it. `PATH` may be a directory or a file. The flag may be given more than once.
|
below it. `PATH` may be a directory or a file. The flag may be given more than
|
||||||
The walk knows `PATH` by its device ID and inode number, as it knows the
|
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
||||||
database files (see "Rules for the walk"). So the walk leaves `PATH` out
|
database files (see "Rules for the walk"). So the walk leaves `PATH` out
|
||||||
whatever path leads to it. That includes a hard link to a file given as
|
whatever path leads to it. That includes a hard link to a file given as
|
||||||
`PATH`. A symlink given as `PATH` excludes its target. An empty `PATH`, as in
|
`PATH`. A symlink given as `PATH` excludes its target. An empty `PATH`, as in
|
||||||
@@ -319,8 +315,10 @@ The reference below is also the program's specification.
|
|||||||
- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and
|
- `--version`, `-v` prints `sfdupes VERSION` (see "Install") to stdout and
|
||||||
exits 0. It writes nothing to stderr.
|
exits 0. It writes nothing to stderr.
|
||||||
- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr
|
- `--help`, `-h`, alone or after a subcommand, prints the help text to stderr
|
||||||
and exits 0. It writes nothing to stdout. `sfdupes scan --help` also lists the
|
and exits 0. It writes nothing to stdout. The help of `sfdupes` and of `scan`
|
||||||
default excludes of the system it runs on, and the filesystem types the walk
|
names the database. The help of `report`, `trees`, `info` and `dwim` says what
|
||||||
|
each prints. `sfdupes scan --help` and `sfdupes dwim --help` also list the
|
||||||
|
default excludes of the system they run on, and the filesystem types the walk
|
||||||
never records anything on.
|
never records anything on.
|
||||||
|
|
||||||
### Duplicate detection
|
### Duplicate detection
|
||||||
@@ -859,14 +857,25 @@ trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimab
|
|||||||
### `dwim` mode
|
### `dwim` mode
|
||||||
|
|
||||||
`dwim`, short for "do what I mean", scans a new machine and reports its
|
`dwim`, short for "do what I mean", scans a new machine and reports its
|
||||||
duplicate trees in one command. It takes no positional arguments and no flags of
|
duplicate trees in one command. It takes no positional arguments. Like `scan`,
|
||||||
its own. Like `scan`, it should run as root, so that it can read every file.
|
it should run as root, so that it can read every file.
|
||||||
|
|
||||||
`dwim` runs what `sfdupes scan /` runs with no flags. The walk skips the default
|
`dwim` takes these flags of `scan`, with the same meaning (see "Settings"):
|
||||||
excludes and crosses into other filesystems. `--workers` keeps its default. The
|
|
||||||
|
- `--workers N`;
|
||||||
|
- `-x`, `--one-file-system`;
|
||||||
|
- `--no-default-excludes`;
|
||||||
|
- `--exclude PATH`, which may be given more than once.
|
||||||
|
|
||||||
|
`dwim` runs what `sfdupes scan /` runs with the same flags. With no flags, the
|
||||||
|
walk skips the default excludes and crosses into other filesystems. The
|
||||||
database, its creation and the scan lock are those of `scan`. Then `dwim` runs
|
database, its creation and the scan lock are those of `scan`. Then `dwim` runs
|
||||||
what `sfdupes trees` runs. The largest trees come first.
|
what `sfdupes trees` runs. The largest trees come first.
|
||||||
|
|
||||||
|
```
|
||||||
|
sfdupes dwim -x --exclude /srv/cache
|
||||||
|
```
|
||||||
|
|
||||||
The report goes to stdout, exactly as `trees` prints it. On stderr comes what
|
The report goes to stdout, exactly as `trees` prints it. On stderr comes what
|
||||||
`scan` prints, then what `trees` prints, then one last line:
|
`scan` prints, then what `trees` prints, then one last line:
|
||||||
|
|
||||||
@@ -928,22 +937,7 @@ All five subcommands work on one SQLite database file.
|
|||||||
#### Location
|
#### Location
|
||||||
|
|
||||||
The database is the file named by `SFDUPES_DATABASE`, when that is set and not
|
The database is the file named by `SFDUPES_DATABASE`, when that is set and not
|
||||||
empty. Otherwise it is at its default path, which depends on the user:
|
empty, and otherwise `/var/lib/sfdupes/db.sqlite`.
|
||||||
|
|
||||||
- For root, it is `/var/lib/sfdupes/db.sqlite`.
|
|
||||||
- For any other user, it is `sfdupes/db.sqlite` in the XDG state directory. That
|
|
||||||
directory is `$XDG_STATE_HOME` when that is set and not empty. Otherwise it is
|
|
||||||
`~/.local/state` on Linux and `~/Library/Application Support` on macOS. An
|
|
||||||
`XDG_STATE_HOME` that is not an absolute path is a fatal error, exit 1, for
|
|
||||||
every subcommand. The error names `XDG_STATE_HOME`.
|
|
||||||
|
|
||||||
`scan` creates each missing directory on the way to the database with mode 755.
|
|
||||||
At the default path of a user other than root, it uses mode 700 instead, as the
|
|
||||||
XDG Base Directory specification asks. Other users cannot enter a directory with
|
|
||||||
mode 700, so they cannot read a database in it.
|
|
||||||
|
|
||||||
All five subcommands use the same default, so a user's `report` reads the
|
|
||||||
database that user's `scan` wrote.
|
|
||||||
|
|
||||||
Each subcommand prints one line on stderr naming the database it uses, before
|
Each subcommand prints one line on stderr naming the database it uses, before
|
||||||
anything else and whether or not stderr is a terminal. The line gives the path
|
anything else and whether or not stderr is a terminal. The line gives the path
|
||||||
@@ -983,9 +977,9 @@ Only one `scan` at a time can hold the lock.
|
|||||||
- For its whole run, `scan` holds a lock on a lock file beside the database. The
|
- For its whole run, `scan` holds a lock on a lock file beside the database. The
|
||||||
lock is taken with the `flock(2)` system call, a request a program makes to
|
lock is taken with the `flock(2)` system call, a request a program makes to
|
||||||
the kernel, and only one process can hold it. The lock file's name is the
|
the kernel, and only one process can hold it. The lock file's name is the
|
||||||
database's path, with symlinks resolved, plus `.lock`. For root, by default,
|
database's path, with symlinks resolved, plus `.lock`. By default that is
|
||||||
that is `/var/lib/sfdupes/db.sqlite.lock`. A path through a symlink names the
|
`/var/lib/sfdupes/db.sqlite.lock`. A path through a symlink names the same
|
||||||
same lock file.
|
lock file.
|
||||||
- `scan` takes the lock before it walks the filesystem or opens the database.
|
- `scan` takes the lock before it walks the filesystem or opens the database.
|
||||||
Just before that, it creates the database file, empty, if it is missing. So
|
Just before that, it creates the database file, empty, if it is missing. So
|
||||||
the lock file is named the same way before the first scan as after. A symlink
|
the lock file is named the same way before the first scan as after. A symlink
|
||||||
@@ -1207,6 +1201,14 @@ On a terminal:
|
|||||||
⠹ [3/4] hash: [12345/98765], 92 files/s, elapsed 2m32s, remain 17m44s 12% |█ |
|
⠹ [3/4] hash: [12345/98765], 92 files/s, elapsed 2m32s, remain 17m44s 12% |█ |
|
||||||
```
|
```
|
||||||
|
|
||||||
|
- During `scan`'s `walk`, the line below the display shows the path of the last
|
||||||
|
file the walk found. Each draw shows the path of that moment, cut short before
|
||||||
|
the terminal's last column. The path is escaped as in the reports (see "Report
|
||||||
|
output format"). Each control character left after that is written as `\x` and
|
||||||
|
its code in two hexadecimal digits, such as `\x1b` for the escape character. A
|
||||||
|
control character is one that a terminal acts on instead of showing. The line
|
||||||
|
is erased when the walk ends, and the next phase is drawn on it.
|
||||||
|
|
||||||
- No line reaches the terminal's last column. A line too wide for the terminal
|
- No line reaches the terminal's last column. A line too wide for the terminal
|
||||||
first leaves out its rate, the words of a stalled phase, or what the phase is
|
first leaves out its rate, the words of a stalled phase, or what the phase is
|
||||||
waiting for. It leaves out its bar next. It is cut short at the edge only when
|
waiting for. It leaves out its bar next. It is cut short at the edge only when
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
the tracker yet is filed as an issue first.
|
the tracker yet is filed as an issue first.
|
||||||
- Branch from `next`.
|
- Branch from `next`.
|
||||||
- Do the work, with tests, in small focused commits.
|
- Do the work, with tests, in small focused commits.
|
||||||
- Record the step at the top of Completed Steps, in the same commit as the work.
|
- Delete Next Step from `TODO.md`, in the same commit as the work.
|
||||||
- Push the branch and open a PR against `next`. Its title ends with
|
- Push the branch and open a PR against `next`. Its title ends with
|
||||||
` (closes #N)`.
|
` (closes #N)`.
|
||||||
- An independent review gates each merge to `next`. Every finding is addressed
|
- An independent review gates each merge to `next`. Every finding is addressed
|
||||||
@@ -15,8 +15,7 @@
|
|||||||
|
|
||||||
- Pre-1.0.
|
- Pre-1.0.
|
||||||
- The tracker is authoritative for the pre-1.0 backlog. The open issues under
|
- The tracker is authoritative for the pre-1.0 backlog. The open issues under
|
||||||
the `1.0.0` milestone are what remains before the tag. This file records
|
the `1.0.0` milestone are what remains before the tag.
|
||||||
history and process, not the queue.
|
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
@@ -24,767 +23,6 @@
|
|||||||
https://git.eeqj.de/sneak/sfdupes/milestone/17. The issues are not restated
|
https://git.eeqj.de/sneak/sfdupes/milestone/17. The issues are not restated
|
||||||
here, because a copy drifts as soon as the tracker moves.
|
here, because a copy drifts as soon as the tracker moves.
|
||||||
|
|
||||||
# Completed Steps
|
|
||||||
|
|
||||||
Newest first. An entry ends with its date and its issue or branch, where those
|
|
||||||
are known.
|
|
||||||
|
|
||||||
- Without `SFDUPES_DATABASE`, a user other than root keeps the database in
|
|
||||||
`sfdupes/db.sqlite` under the XDG state directory, `~/.local/state` by
|
|
||||||
default. Root keeps `/var/lib/sfdupes/db.sqlite`. `sfdupes --help` says where
|
|
||||||
the database is (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/171)
|
|
||||||
|
|
||||||
- New `scan` flag `--exclude PATH`, which may be repeated. The walk neither
|
|
||||||
enters nor records an excluded path, and keeps the records below it. An
|
|
||||||
operand at or below one is a fatal error (2026-11-11,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/163)
|
|
||||||
|
|
||||||
- `report` and `trees` leave out a copy that was deleted after the scan. Each
|
|
||||||
checks only the copy itself with `lstat`: the file, or the directory at the
|
|
||||||
top of a tree. A group left with one copy is not reported. A missing `scan`
|
|
||||||
operand is still a fatal error (2026-11-11,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/173)
|
|
||||||
- The usage line of `scan` no longer ends in `[flags]`. It is the line
|
|
||||||
`README.md` shows under "Subcommands" (2026-11-11,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/169)
|
|
||||||
- `report` and `trees` leave out empty files. `report` writes no group of size
|
|
||||||
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
|
|
||||||
counts toward a tree's digest (2026-11-11,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/178)
|
|
||||||
- The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
|
|
||||||
command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
|
|
||||||
became `internal/sfdupes/cli.go`. Every other file kept its name. The install
|
|
||||||
path is now `sneak.berlin/go/sfdupes/cmd/sfdupes` (2026-11-10,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/157)
|
|
||||||
|
|
||||||
- New subcommand `dwim`. It runs what `sfdupes scan /` runs with no flags, then
|
|
||||||
what `sfdupes trees` runs. Its last line on stderr says that `sfdupes trees`
|
|
||||||
prints the report again (2026-11-06,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/154)
|
|
||||||
|
|
||||||
- `scan` resolves each operand to its real path, symlinks followed, before the
|
|
||||||
walk. A symlink operand is scanned at its target instead of skipped. With
|
|
||||||
`/srv/link` a symlink to a directory outside `/srv`, `scan /srv /srv/link/sub`
|
|
||||||
walks `sub` on its own instead of dropping it and deleting its records
|
|
||||||
(2026-11-06, https://git.eeqj.de/sneak/sfdupes/issues/144)
|
|
||||||
|
|
||||||
- `README.md` and a comment in `internal/sfdupes/scan.go` no longer say ZFS
|
|
||||||
counts a new file's blocks only seconds after the write. The count can land in
|
|
||||||
under a second. Both now say up to seconds after the write, as the comment on
|
|
||||||
`diskSizeOf` in `internal/sfdupes/scan_test.go` does (2026-11-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/152)
|
|
||||||
|
|
||||||
- The three disk size tests in `internal/sfdupes/scan_test.go` no longer fail at
|
|
||||||
random on ZFS. `unix.Sync` can return before ZFS has counted a file's blocks,
|
|
||||||
and one ZFS commit can take a file through several sizes. The test that needs
|
|
||||||
the size to change waits for it, for at most 30 seconds. Each test reads the
|
|
||||||
size before and after its scan. The rewrite test rescans until the two reads
|
|
||||||
match, then compares exactly. The other two fail only on a `disk_size` outside
|
|
||||||
the two reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150)
|
|
||||||
|
|
||||||
- `TODO.md` rewritten from scratch by the same rules as `README.md`, sentence by
|
|
||||||
sentence. Every completed step is kept, with its date and its issue or branch.
|
|
||||||
The policy-compliance checklist of 2026-07-22 is folded into its step
|
|
||||||
(2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
|
||||||
|
|
||||||
- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
|
|
||||||
it. It leads with what the program is and how to use it. A reference section
|
|
||||||
covers every setting and every subcommand. Development and testing come last.
|
|
||||||
Every statement was checked against the code on `next`. Five comments in
|
|
||||||
`internal/sfdupes/cli_test.go` cite "Settings", where their rules now live.
|
|
||||||
One comment each in these files cites what the README now says:
|
|
||||||
- `internal/sfdupes/scan_test.go`;
|
|
||||||
- `internal/sfdupes/trees_test.go`;
|
|
||||||
- `internal/sfdupes/exclude_test.go`.
|
|
||||||
|
|
||||||
(2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
|
||||||
|
|
||||||
- Each record holds `disk_size`, the bytes the file takes on disk. That is its
|
|
||||||
allocated blocks, `st_blocks` × 512, from the `lstat` call the walk already
|
|
||||||
makes. `trees` prints a `disk_size` column after `size`. It is the sum over
|
|
||||||
the `dupe` copy's files. On a compressed or sparse file `disk_size` differs
|
|
||||||
from `size`. A rescan updates the `disk_size` of an unchanged file whose size
|
|
||||||
on disk changed, without reading the file (2026-10-27,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/147)
|
|
||||||
|
|
||||||
- The database keeps the number of records and the sum of their sizes in the one
|
|
||||||
row of a `totals` table. Three triggers on `files` keep that row current.
|
|
||||||
`info` reads that row. So do `scan`'s `load` phase and the `read` phases of
|
|
||||||
`report` and `trees`, for their totals. No count reads every record any more.
|
|
||||||
The triggers make each write to `files` slower. `scan` keeps SQLite's
|
|
||||||
temporary data in memory. That wins back part of that time (2026-10-27,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/146)
|
|
||||||
|
|
||||||
- an operand at or inside a directory named `.zfs` is never scanned, whether
|
|
||||||
given alone or under another operand. `scan` warns about it, counts it as
|
|
||||||
skipped and does not walk it (2026-10-26,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/142)
|
|
||||||
|
|
||||||
- A file or directory removed while the scan runs is not an error, in the walk
|
|
||||||
or in the hash phase. The scan prints no line for it and does not count it as
|
|
||||||
skipped. When the first of several hard links to a file is gone, the hash
|
|
||||||
phase reads the next one instead. On a terminal every progress display starts
|
|
||||||
with the spinner, the hash phase's included. A phase with an exact total ends
|
|
||||||
its line with its percent and a 10-column bar at the terminal's right edge.
|
|
||||||
The bar no longer moves as the line's length changes. A line too wide for the
|
|
||||||
terminal now loses its rate before its bar. The time remaining is labelled
|
|
||||||
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
|
|
||||||
|
|
||||||
- The `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
|
|
||||||
bytes of a SHA-256. That is half the size of the hex text they held. The
|
|
||||||
hashes stay raw from the hash phase through the database to `report` and
|
|
||||||
`trees`. Reading a record whose hash is not 32 bytes is an error (2026-10-26,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/134)
|
|
||||||
|
|
||||||
- The hash phase gives three hashes to every file that the walk records. `head`
|
|
||||||
and `tail` are the SHA-256 of its first and last 64 KiB. `content` is the
|
|
||||||
SHA-256 of the whole file under 100 MiB, and of 1 MiB at each gigabyte from
|
|
||||||
100 MiB up. Gone are:
|
|
||||||
- the 10 MiB and 50 MiB thresholds;
|
|
||||||
- the size census, which counted the sizes of every walked file and of every
|
|
||||||
record outside the operands. A file was hashed only when the census showed
|
|
||||||
another file of its size;
|
|
||||||
- the records without hashes;
|
|
||||||
- the content phase.
|
|
||||||
|
|
||||||
A scan has four phases, and a rescan reads only new and changed files.
|
|
||||||
`info` no longer counts the records with `head` and `tail` hashes, nor those
|
|
||||||
with a content hash, since every record has all three (2026-10-25,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
|
||||||
|
|
||||||
- `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` were copied
|
|
||||||
again from `sneak/prompts` `next` at `0653579`. Two steps now come before
|
|
||||||
`ARG CACHEBUST`:
|
|
||||||
- the module download in the `lint`, `test` and `builder` stages of the
|
|
||||||
`Dockerfile`;
|
|
||||||
- the package install in `builder`.
|
|
||||||
|
|
||||||
Docker keeps them until one of these changes:
|
|
||||||
- the base image;
|
|
||||||
- the install line;
|
|
||||||
- `go.mod` or `go.sum`.
|
|
||||||
|
|
||||||
Every step that checks or compiles the code still comes after
|
|
||||||
`ARG CACHEBUST`. `golangci-lint run` also keeps the linter's own cache.
|
|
||||||
README says the same (2026-10-18,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/109)
|
|
||||||
|
|
||||||
- The release rule was set. Until 1.0, each tip of `main` that passes the build
|
|
||||||
gets the next `v0.0.N` tag.
|
|
||||||
`go install sneak.berlin/go/sfdupes/cmd/sfdupes@latest` then installs it.
|
|
||||||
`go.mod` retracts `v0.0.1`, the early build without a database. When the
|
|
||||||
linker set no version, `sfdupes --version` prints the version Go recorded in
|
|
||||||
the binary. That is the tag, or the pseudo-version of a `go install` build. It
|
|
||||||
prints `dev` only when neither is known. README "Install" documented:
|
|
||||||
- the rule;
|
|
||||||
- what `@main` and `v0.0.1` give;
|
|
||||||
- how to check an installed binary.
|
|
||||||
|
|
||||||
(2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/116)
|
|
||||||
|
|
||||||
- `scan` creates the database's directory where the database path leads. The
|
|
||||||
directory is the path with its last element cut off as text, so a `..` in it
|
|
||||||
stays. A `..` after a symlinked directory therefore goes through the symlink,
|
|
||||||
as it does when the file is opened. With
|
|
||||||
`SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a symlink to
|
|
||||||
`/data/x`, it now creates `/data/new` and the database in it. It used to
|
|
||||||
create `/srv/new` and then fail (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
|
||||||
|
|
||||||
- `scan` never records its own database. Once the database and its `.lock` file
|
|
||||||
are open, it notes the device and inode of each of these:
|
|
||||||
- the database file;
|
|
||||||
- its `-wal` and `-shm` files;
|
|
||||||
- the `.lock` file.
|
|
||||||
|
|
||||||
The walk passes over any file with the device and inode of one of them,
|
|
||||||
however it reaches the file. It prints no line for such a file and counts
|
|
||||||
nothing. A record an earlier scan left for one of them is deleted
|
|
||||||
(2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
|
||||||
|
|
||||||
- `scan` never records or reads anything on a filesystem that is not persistent,
|
|
||||||
such as `tmpfs` or `proc`. Such a filesystem is one the kernel makes up or
|
|
||||||
keeps in memory. `scan` matches them by device ID, however the walk or the
|
|
||||||
content phase reaches a path. As the scan starts, it lists the mounted
|
|
||||||
filesystems of those types, from `/proc/self/mountinfo` on Linux and from the
|
|
||||||
mount list on macOS. No flag turns that off. Below an operand, the walk does
|
|
||||||
not enter the default excludes either. So `sfdupes scan /` skips the kernel's
|
|
||||||
view of the running system and the temporary directories. On Linux those are:
|
|
||||||
|
|
||||||
```
|
|
||||||
/proc, /sys, /dev, /run, /tmp, /var/tmp
|
|
||||||
```
|
|
||||||
|
|
||||||
On macOS they are:
|
|
||||||
|
|
||||||
```
|
|
||||||
/dev, /private/tmp, /private/var/tmp, /private/var/vm
|
|
||||||
/System/Volumes/VM, /System/Volumes/Data
|
|
||||||
```
|
|
||||||
|
|
||||||
A Windows list exists in the code but is not used. An operand at or inside a
|
|
||||||
default exclude is still walked. The scan prints a line for each path it
|
|
||||||
leaves out, without counting it as skipped. The records there count as
|
|
||||||
outside the scanned operands, as for `-x`. `--no-default-excludes` turns off
|
|
||||||
the default excludes only, and `scan --help` lists them (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
|
||||||
|
|
||||||
- A progress display's rate covers the items completed over the last 5 seconds.
|
|
||||||
When stderr is not a terminal, it covers the items since the line before. It
|
|
||||||
falls to 0 while nothing completes. A burst after a wait no longer reads as a
|
|
||||||
steady rate of over 1,000 files a second. The time remaining still comes from
|
|
||||||
the phase's average. Once no item has completed for 10 seconds, the line shows
|
|
||||||
`stalled` and where the phase waits in place of the rate, as
|
|
||||||
`stalled disk/db`. The places are:
|
|
||||||
- `disk`;
|
|
||||||
- `db`;
|
|
||||||
- `out`, for stdout;
|
|
||||||
- `cpu`.
|
|
||||||
|
|
||||||
A phase that may wait in two places names both. Each phase's word is set in
|
|
||||||
the code that defines the phase. A line too wide for the terminal loses its
|
|
||||||
bar first, then its rate or those words. Four parts stay whole:
|
|
||||||
- the phase;
|
|
||||||
- the progress;
|
|
||||||
- the elapsed time;
|
|
||||||
- the ETA, the time remaining.
|
|
||||||
|
|
||||||
The content phase had one display for its checks and one for its reads. Both
|
|
||||||
lost those words on an 80-column terminal, at counts like those of README's
|
|
||||||
example. That happened for most elapsed times and ETAs in minutes. Every
|
|
||||||
other phase kept them whenever the elapsed time and the ETA were both in
|
|
||||||
minutes. The content phase writes its last partial batch, the records not
|
|
||||||
yet written, under a display of its own (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/125)
|
|
||||||
|
|
||||||
- New subcommand `info` prints the database's details on stdout, one per line as
|
|
||||||
a name, a tab and the value:
|
|
||||||
- the path and where it came from;
|
|
||||||
- the sizes of the database file and of its `-wal` file, when there is one;
|
|
||||||
- when the database file was last changed;
|
|
||||||
- the schema version;
|
|
||||||
- the number of records;
|
|
||||||
- how many records have `head` and `tail` hashes;
|
|
||||||
- how many records have a content hash;
|
|
||||||
- the sizes of all records added up.
|
|
||||||
|
|
||||||
It opens the database as `report` does and shows two phases on stderr,
|
|
||||||
`[1/2] open` and `[2/2] count`. The schema version and the counts come from
|
|
||||||
one read-only transaction, a read of the database at one moment. The counts
|
|
||||||
come from one query (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/117)
|
|
||||||
|
|
||||||
- `report` shows its work on stderr in two phases, `[1/2] open` and
|
|
||||||
`[2/2] read`, with the progress display `scan` and `trees` use. The display
|
|
||||||
runs from the moment `report` starts until the summary line. `open` says it is
|
|
||||||
opening the database, also while it waits on a busy one. `read` counts the
|
|
||||||
records, then reads them against that count in signature order. SQLite takes
|
|
||||||
that order from the `files_signature` index without sorting, so the first
|
|
||||||
record comes at once. `report` forms the duplicate groups itself and holds
|
|
||||||
only those of the size it is reading. It writes them ordered by first path
|
|
||||||
before it reads the next size. stdout and the summary line are unchanged, and
|
|
||||||
SQLite no longer needs a temporary file to sort in (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/114)
|
|
||||||
|
|
||||||
- `trees` shows its work on stderr in four phases, from `[1/4] open` to
|
|
||||||
`[4/4] write`, each with the progress display `scan` uses. The display runs
|
|
||||||
from the moment `trees` starts until the summary line. `open` says it is
|
|
||||||
opening the database, also while it waits on a busy one. `read` counts the
|
|
||||||
records, then reads them against that count. `group` counts the directories it
|
|
||||||
groups, and `write` the groups it writes. A display's rate names what its
|
|
||||||
phase counts:
|
|
||||||
- files;
|
|
||||||
- records;
|
|
||||||
- directories;
|
|
||||||
- groups.
|
|
||||||
|
|
||||||
stdout and the summary line are unchanged. When stdout and stderr are the
|
|
||||||
same file, as one terminal is, no line of stdout shares a line with the
|
|
||||||
display. When they are not, the display keeps being drawn while stdout waits
|
|
||||||
on a slow reader (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/113)
|
|
||||||
|
|
||||||
- Each `scan` phase's progress display names the phase and its place among the
|
|
||||||
five, as `[2/5] walk`. It runs from the moment that the phase starts. It
|
|
||||||
shows:
|
|
||||||
- the progress;
|
|
||||||
- the elapsed time;
|
|
||||||
- the ETA, the time remaining.
|
|
||||||
|
|
||||||
`load` counts the records first. The walk's total is estimated from the
|
|
||||||
records under the operands. The content phase takes its total from SQLite's
|
|
||||||
grouping alone. Until the first record is returned, it says it is searching
|
|
||||||
the database. On a terminal a phase with an exact total draws its line
|
|
||||||
followed by the bar of the progress-bar library. Any other phase draws a
|
|
||||||
spinner followed by the line. Every display is drawn again several times a
|
|
||||||
second. No line reaches the terminal's last column, however narrow the
|
|
||||||
terminal is, even after it is made narrower during a phase. A line too wide
|
|
||||||
leaves out its rate, then its bar. The rest stays whole. The line is cut
|
|
||||||
short only when the rest alone does not fit, as below about 70 columns late
|
|
||||||
in a large scan. When stderr is not a terminal, the line is printed every 5
|
|
||||||
seconds whether or not an item completed (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/112)
|
|
||||||
|
|
||||||
- `scan`, `report` and `trees` first print a line on stderr naming the database
|
|
||||||
they use, as an absolute path. The line says whether that path is the default
|
|
||||||
or comes from `SFDUPES_DATABASE`. `scan` says when the database is new. README
|
|
||||||
"Database" says when `scan` writes each file (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/111)
|
|
||||||
|
|
||||||
- The canonical files and model scripts were copied again from `sneak/prompts`
|
|
||||||
`next` at `e3f79c4`:
|
|
||||||
- `golangci-lint` v2.14.0;
|
|
||||||
- lint and test are phases of the `Dockerfile` that write no image, and
|
|
||||||
`make test` runs the suite under the race detector. Gone are
|
|
||||||
`Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race`;
|
|
||||||
- every `docker build` in `script/` passes a new `CACHEBUST` value. Every
|
|
||||||
stage of the `Dockerfile` declares it first, so every `RUN` step runs on
|
|
||||||
every build. The steps that compile keep Go's build cache, and the tests
|
|
||||||
run with `-count=1`;
|
|
||||||
- `prettier` runs on the host, from the `node` and `yarn` that
|
|
||||||
`script/bootstrap` installs. The `prettier` and `markdown` stages are
|
|
||||||
gone, and the build stage installs `git` and `make` itself;
|
|
||||||
- a new push cancels the workflow's older run on the same branch, and a run
|
|
||||||
stops after 20 minutes;
|
|
||||||
- a coding agent's settings file is deleted.
|
|
||||||
|
|
||||||
Since https://git.eeqj.de/sneak/sfdupes/issues/109 two steps come before
|
|
||||||
`ARG CACHEBUST`:
|
|
||||||
- the module download in `lint`, `test` and the build stage;
|
|
||||||
- the install of `git` and `make` in the build stage.
|
|
||||||
|
|
||||||
Docker keeps them between builds. Every step that checks or compiles the
|
|
||||||
code still runs on every build (2026-10-15,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95)
|
|
||||||
|
|
||||||
- With `-x`, an operand under another operand is no longer dropped when a
|
|
||||||
directory between the two, the inner one included, is on a different
|
|
||||||
filesystem than the outer one. The inner operand is then walked on its own.
|
|
||||||
For a regular-file operand the check starts at its directory. The records of
|
|
||||||
files gone from below it are deleted (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
|
||||||
|
|
||||||
- With `-x`, `scan` keeps the records below a directory on another filesystem
|
|
||||||
under its operand, instead of deleting them. It counts their sizes in the size
|
|
||||||
census (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
|
||||||
|
|
||||||
- The hash and content phases open each file without blocking and check that it
|
|
||||||
is still a regular file. A file that is now a FIFO or device node is warned
|
|
||||||
about and skipped instead of hanging the scan (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
|
||||||
|
|
||||||
- The scan lock file is named from the database's path with symlinks resolved,
|
|
||||||
after `scan` creates the database file if it is missing. A scan through a
|
|
||||||
symlink to the database therefore fails while another scan holds the lock
|
|
||||||
(2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/101)
|
|
||||||
|
|
||||||
- On a terminal, a warning for the last file of a phase no longer erases the
|
|
||||||
phase's finished bar. The bar is drawn again below the warning (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
|
||||||
|
|
||||||
- `scan` records the mtime to the nanosecond, as whole seconds in `mtime` plus
|
|
||||||
`mtime_nsec`, and compares it at that resolution. A same-size rewrite within
|
|
||||||
the same second is hashed again (2026-10-07,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/12)
|
|
||||||
|
|
||||||
- The narration was cut from `TODO.md` Completed Steps and from the comments in
|
|
||||||
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95). The Workflow section now
|
|
||||||
branches from and merges to `next` (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/49)
|
|
||||||
|
|
||||||
- `make test-race` ran the test suite under the race detector in a cgo-enabled
|
|
||||||
container, outside `make check`. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
|
|
||||||
under the race detector, in the `Dockerfile`'s Debian-based `test` phase.
|
|
||||||
`make test-race` is gone (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/18)
|
|
||||||
|
|
||||||
- A bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
|
|
||||||
rather than serve the gates, the lint and test steps, from cache. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
|
||||||
`REPO_POLICIES.md` requires, and may serve the gates from cache. The builds in
|
|
||||||
`script/` pass a new `CACHEBUST` value, so theirs always run (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/39)
|
|
||||||
|
|
||||||
- `make fmt` and `make fmt-check` run `prettier` over all Markdown, and the
|
|
||||||
Gitea workflow checks it. All Markdown was reformatted (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
|
||||||
|
|
||||||
- `script/lint` writes no image, so a run no longer leaves an untagged one
|
|
||||||
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
|
|
||||||
|
|
||||||
- Tests cover:
|
|
||||||
- a missing database;
|
|
||||||
- `scan` keeping stdout empty;
|
|
||||||
- its skip warning;
|
|
||||||
- the `report` and `trees` summary lines;
|
|
||||||
- every subcommand going through `runE`.
|
|
||||||
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
|
|
||||||
|
|
||||||
- `.golangci.yml` was replaced with the current canonical copy, which uses
|
|
||||||
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
|
||||||
|
|
||||||
- A test fails when either `hashWorker` cancellation check in
|
|
||||||
`internal/sfdupes/scan.go` is removed (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/83)
|
|
||||||
|
|
||||||
- A database path holding `?`, `#` or `%` opens exactly the file it names
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
|
||||||
|
|
||||||
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
|
||||||
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
|
||||||
|
|
||||||
- A test fails when either walk cancellation check in `internal/sfdupes/scan.go`
|
|
||||||
is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
|
||||||
|
|
||||||
- A test checks that `scan` refuses a database with another schema version
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64)
|
|
||||||
|
|
||||||
- Four inaccurate comments in `internal/sfdupes/cancel_test.go` were corrected,
|
|
||||||
and `walkCancelInFlightDirs` was renamed to `walkCancelInFlightFiles`
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/33)
|
|
||||||
|
|
||||||
- The `-x` filesystem-boundary rules in `subdirJob` are tested (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
|
||||||
|
|
||||||
- `scan` creates the schema in one transaction. A version-0 database with a
|
|
||||||
`files` table is refused with a clear schema-version error (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
|
||||||
|
|
||||||
- README documents install, Docker, a daily `cron` scan and how to read and
|
|
||||||
check the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
|
|
||||||
|
|
||||||
- The `Dockerfile` build stage kept the Go module cache out of `builder`'s home
|
|
||||||
and copied the sources with `--chown`, so no `chown -R` walked them. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
|
|
||||||
nothing changes owner. The build stage only compiles, as root, and the tests
|
|
||||||
run as `nobody` in the `test` phase (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/43)
|
|
||||||
|
|
||||||
- `--version` prints `sfdupes VERSION` to stdout. README documents it and
|
|
||||||
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
|
|
||||||
|
|
||||||
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: it commits what it has hashed,
|
|
||||||
deletes nothing more, and exits 1 (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/5)
|
|
||||||
|
|
||||||
- `report` and `trees` stream the records instead of holding them all in memory.
|
|
||||||
The schema gains the `files_signature` index (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/14)
|
|
||||||
|
|
||||||
- Progress prints at once on a non-terminal and uses a real terminal test.
|
|
||||||
Warnings print through a spinner instead of racing its redraw (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/13)
|
|
||||||
|
|
||||||
- These operands are warned about and skipped, and the records beneath them are
|
|
||||||
kept:
|
|
||||||
- symlinks;
|
|
||||||
- sockets;
|
|
||||||
- FIFOs;
|
|
||||||
- device nodes;
|
|
||||||
- `.zfs` directories.
|
|
||||||
|
|
||||||
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/9)
|
|
||||||
|
|
||||||
- `scan` holds a lock on a lock file beside the database for its whole run, so a
|
|
||||||
second `scan` fails at once with exit 1 (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/53)
|
|
||||||
|
|
||||||
- Tests cover stdout write failures in `report` and `trees`. README states that
|
|
||||||
`| head` ends `sfdupes` by `SIGPIPE` and that `>&-` writes to `/dev/null`
|
|
||||||
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
|
|
||||||
|
|
||||||
- `report` and `trees` open the database read-only, and `scan` leaves it out of
|
|
||||||
WAL mode, so reading needs only read access (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/8)
|
|
||||||
|
|
||||||
- Four characters are escaped in `report`, `trees` and warning paths:
|
|
||||||
- tab;
|
|
||||||
- newline;
|
|
||||||
- carriage return;
|
|
||||||
- backslash.
|
|
||||||
|
|
||||||
The root directory's path is `/` (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/7)
|
|
||||||
|
|
||||||
- A plain `docker build .` stamps the `git` tag or short commit instead of
|
|
||||||
`dev`. `.dockerignore` sends `.git` without `.git/config`. The build stage
|
|
||||||
stamps the `VERSION` build argument, else `git describe --tags --always`. It
|
|
||||||
fails if the context carries `.git` and the version is still empty, `dev` or
|
|
||||||
`unknown`. The Gitea workflow checks out the full history (`fetch-depth: 0`),
|
|
||||||
so it stamps the same value as `make build` (2026-10-02, branch `next`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/67)
|
|
||||||
|
|
||||||
- The 1 KiB end-window sampling was replaced with the hashes chosen by file
|
|
||||||
size, and README "Duplicate detection" documented every size threshold. A file
|
|
||||||
under 10 MiB was hashed in full, and its `head`, `tail` and `content` all held
|
|
||||||
that hash. A larger file got only its 64 KiB `head` and `tail` in the hash
|
|
||||||
phase. The content phase ran after the update phase. It read a file for
|
|
||||||
`content` only when its size, `head` and `tail` matched another record's, from
|
|
||||||
this scan or an earlier one. `content` was the whole file below 50 MiB, and
|
|
||||||
gigabyte-spaced 1 MiB samples at or above. It never read a file gone or
|
|
||||||
changed since its record was written. `report` and `trees` left out any record
|
|
||||||
without a `content` hash. The `content` column is part of the version 1
|
|
||||||
schema. Superseded by https://git.eeqj.de/sneak/sfdupes/issues/132
|
|
||||||
(2026-09-22, branch `next`, https://git.eeqj.de/sneak/sfdupes/issues/61)
|
|
||||||
|
|
||||||
- The dead `files.dat` references were removed from `Makefile`, `.gitignore` and
|
|
||||||
`.dockerignore` (2026-09-21, branch `next`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/22)
|
|
||||||
|
|
||||||
- The lint-image pin comments and `FROM` form in `Dockerfile` and
|
|
||||||
`Dockerfile.lint` were fixed. Both pins became the policy
|
|
||||||
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
|
|
||||||
without the false `(Debian-based)` note or the tag. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
|
|
||||||
holds the only `golangci-lint` pin, in that form. `Dockerfile.lint` is gone,
|
|
||||||
and so is `script/verify-lint-image-pin`, which compared the two pins
|
|
||||||
(2026-08-10, branch `next`, https://git.eeqj.de/sneak/sfdupes/issues/25)
|
|
||||||
|
|
||||||
- All linting ran in Docker, via `Dockerfile.lint` and `script/lint`. Per the
|
|
||||||
owner ruling the linter is never installed on a host, and
|
|
||||||
`golangci-lint config verify` runs before `golangci-lint run`.
|
|
||||||
`script/bootstrap` stopped installing or pinning the linter, and
|
|
||||||
`script/verify-linter-pin` was retired. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
|
|
||||||
`Dockerfile`'s `lint` phase. `script/lint` builds that phase alone, and the
|
|
||||||
build stage depends on it through `COPY --from=lint /src/go.sum /dev/null`.
|
|
||||||
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
|
|
||||||
an image build may run `docker`, so the phase calls `golangci-lint` directly
|
|
||||||
(2026-08-10, branch `next`, https://git.eeqj.de/sneak/sfdupes/issues/46)
|
|
||||||
|
|
||||||
- The Docker build stage's prerequisites were installed by running
|
|
||||||
`script/bootstrap` instead of `apk add --no-cache make` inline.
|
|
||||||
`script/verify-linter-pin` failed the build unless the linter copied from the
|
|
||||||
lint stage was the version that `script/bootstrap` pinned. Builds then took up
|
|
||||||
to 5m14s from an empty cache, mostly in a `chown -R` of the module cache,
|
|
||||||
filed as https://git.eeqj.de/sneak/sfdupes/issues/43. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
|
|
||||||
`make` with `apk add --no-cache` and only compiles. The `lint` and `test`
|
|
||||||
phases are the gates (2026-08-09, branch `dockerfile-bootstrap`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/42)
|
|
||||||
|
|
||||||
- The Docker layer cache is defeated for the gate steps, so `script/cibuild` and
|
|
||||||
`script/docker` cannot report a pass they did not earn. The `Dockerfile`
|
|
||||||
copies the tree before its gates. On an unchanged tree Docker served them from
|
|
||||||
cache, and the build exited 0 without running them. The fix was a
|
|
||||||
`CHECK_EPOCH` build argument. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
|
||||||
passes a new `CACHEBUST` value instead. Every stage of the `Dockerfile`
|
|
||||||
declares it before every step that checks or compiles the code. Run as root,
|
|
||||||
the tests fail `TestScanHardlinkRunFailsTogether`, because root reads through
|
|
||||||
the `chmod(0)` the test relies on. So the `test` phase runs them as `nobody`
|
|
||||||
(2026-08-09, branch `cibuild-cache-bust`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/32)
|
|
||||||
|
|
||||||
- `script/bootstrap` checked the installed `golangci-lint` version instead of
|
|
||||||
only its presence. The version lived only in `GOLANGCI_LINT_VERSION`, with the
|
|
||||||
`go install` module reference derived from it. Any installed version that was
|
|
||||||
not the pin was reinstalled, whether:
|
|
||||||
- older;
|
|
||||||
- newer;
|
|
||||||
- absent;
|
|
||||||
- unparseable.
|
|
||||||
|
|
||||||
`go install` writes into `GOBIN`, or `GOPATH/bin`, while `make lint` ran the
|
|
||||||
first `golangci-lint` on `PATH`. So bootstrap re-read the effective version
|
|
||||||
after installing. On a mismatch it printed both paths and both versions and
|
|
||||||
exited non-zero. It did not reorder `PATH` or delete anyone's binary. The
|
|
||||||
`--version` call kept its stderr and was bounded by `timeout(1)` where that
|
|
||||||
exists. `git`, `make` and `go` kept presence-only checks. The version check
|
|
||||||
was verified by bootstrapping this host from v2.10.1 to v2.12.2, and again
|
|
||||||
to a no-op. Stub runs of the script under `dash` covered:
|
|
||||||
- a thirteen-input version-parse matrix;
|
|
||||||
- a shadowed install, which must exit non-zero;
|
|
||||||
- an install destination not on `PATH`;
|
|
||||||
- `GOBIN` set;
|
|
||||||
- a wedged binary, which must hit the timeout.
|
|
||||||
|
|
||||||
`make check` and `make lint` were clean at v2.12.2, so v2.10.1 had not been
|
|
||||||
hiding findings on `main`. Superseded by
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/46, since the linter is no longer
|
|
||||||
installed on a host (2026-08-09, branch `bootstrap-version-check`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/24)
|
|
||||||
|
|
||||||
- The hash worker pool unwinds on the error path. The pool is an owned,
|
|
||||||
context-aware `hashPool`. Every blocking send in the feeder and the workers
|
|
||||||
selects on `ctx.Done()`, and `jobs` is closed on every path out. `hashPhase`
|
|
||||||
defers `pool.stop()`, which cancels and then drains `results` until the last
|
|
||||||
goroutine has exited. Draining is what frees a worker parked on a send. `ctx`
|
|
||||||
is passed from `cmd.Context()` as the first parameter, through:
|
|
||||||
- `runScan`;
|
|
||||||
- `syncScan`;
|
|
||||||
- both worker pools;
|
|
||||||
- the whole database layer.
|
|
||||||
|
|
||||||
The walk pool gets the same treatment, plus a `ctx.Err()` guard after the
|
|
||||||
walk. A cancelled walk yields a partial size census, and every file that it
|
|
||||||
never reached looks vanished to the update phase. That phase's own `BeginTx`
|
|
||||||
also fails on the cancelled context before deleting anything. The guard is
|
|
||||||
still the barrier once an interrupted scan may commit what it has. Tests
|
|
||||||
drive `run(scan)` against a database whose insert trigger aborts. They
|
|
||||||
assert that the scan fails instead of hanging, and that the goroutine count
|
|
||||||
returns to its pre-scan baseline. Other tests cancel a scan part-way through
|
|
||||||
the walk, deterministically, by counting the scan's own checks of
|
|
||||||
`ctx.Done()`. They assert that it stops at the guard with every record
|
|
||||||
intact. It then holds a partial size census and a still-populated record
|
|
||||||
index. Direct tests cover the remaining cancellation branches of both pools.
|
|
||||||
They test:
|
|
||||||
- `sendEvent`;
|
|
||||||
- the walk workers;
|
|
||||||
- `dispatchDirs`;
|
|
||||||
- `feedHashJobs`;
|
|
||||||
- `hashWorker`;
|
|
||||||
- `hashPhase`.
|
|
||||||
|
|
||||||
(2026-08-09, branch `hash-pool-cleanup`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/6)
|
|
||||||
|
|
||||||
- The database is closed on every fatal exit path. `fatalf` and its `os.Exit(1)`
|
|
||||||
are gone. So the deferred `db.Close()` runs when a subcommand fails, and with
|
|
||||||
it the SQLite WAL checkpoint. These return errors instead:
|
|
||||||
- `runScan`;
|
|
||||||
- `runReport`;
|
|
||||||
- `runTrees`;
|
|
||||||
- `loadRecords`;
|
|
||||||
- `resolveRoots`.
|
|
||||||
|
|
||||||
The single exit point is `run` in `internal/sfdupes/cli.go`. It maps a
|
|
||||||
`fatalError`, anything a subcommand returned, to exit 1, and cobra's own
|
|
||||||
argument and flag errors to exit 2. So a runtime failure is never reported
|
|
||||||
as a usage error, nor printed with the usage text.
|
|
||||||
`internal/sfdupes/cli_test.go` drives the CLI in-process and asserts the
|
|
||||||
exit codes from README "Error handling" and the stdout/stderr split. It also
|
|
||||||
checks that a fatal error after the database is open leaves no `-wal` or
|
|
||||||
`-shm` file behind, for `scan`, `report` or `trees` (2026-08-09, branch
|
|
||||||
`db-close-on-fatal`, https://git.eeqj.de/sneak/sfdupes/issues/4)
|
|
||||||
|
|
||||||
- `golangci-lint` was updated to v2.12.2 with the canonical config, merged as
|
|
||||||
`38a01bd`. The pinned linter in the `Dockerfile` lint stage and
|
|
||||||
`script/bootstrap` went from v2.12.1 to v2.12.2, and `.golangci.yml` was
|
|
||||||
replaced with the canonical file. Four thresholds now live under
|
|
||||||
`linters.settings`, as the v2 schema requires, so they are applied:
|
|
||||||
- `lll`;
|
|
||||||
- `funlen`;
|
|
||||||
- `cyclop`;
|
|
||||||
- `dupl`.
|
|
||||||
|
|
||||||
No new lint findings surfaced (2026-08-09, branch `golangci-v2.12.2`,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/3)
|
|
||||||
|
|
||||||
- `Makefile` targets were converted to scripts-to-rule-them-all `script/`
|
|
||||||
entrypoints, like the other managed repos. All 12 entrypoints exist:
|
|
||||||
- `bootstrap`, `setup` and `projectname`;
|
|
||||||
- `test`, `lint`, `fmt`, `fmt-check` and `check`;
|
|
||||||
- `docker`, `cibuild`, `precommit` and `install-precommit`.
|
|
||||||
|
|
||||||
Every `Makefile` target does nothing but call its script (2026-07-26, commit
|
|
||||||
`3abeacf`, https://git.eeqj.de/sneak/sfdupes/issues/1)
|
|
||||||
|
|
||||||
- The binary is the default `make` target. Plain `make` builds `sfdupes`, where
|
|
||||||
it used to run `check` plus `build`. `make build` remains as an alias
|
|
||||||
(2026-07-24, branch `make-default-target`)
|
|
||||||
|
|
||||||
- The phases became scan-wide, with concurrent operands and batched updates. All
|
|
||||||
operands seed the shared walk pool, and every pass runs once over the whole
|
|
||||||
scan. So the totals and ETAs cover the whole scan. The per-operand walk, hash
|
|
||||||
and update cycles and their stderr announcements are gone. The update pass
|
|
||||||
writes in batches, each in its own transaction. The filesystem is
|
|
||||||
authoritative and the database follows it, so one scan's changes need not land
|
|
||||||
all at once (2026-07-24, branch `scan-wide-phases`)
|
|
||||||
|
|
||||||
- The stat pass was split back out of the walk. Phases are strictly sequential
|
|
||||||
again, per operand, with parallelism only inside each phase:
|
|
||||||
- walk;
|
|
||||||
- stat;
|
|
||||||
- hash;
|
|
||||||
- update.
|
|
||||||
|
|
||||||
The walk enumerates paths with per-directory workers, and the stat pass
|
|
||||||
lstats them with per-file workers. That restores the exact total and ETA of
|
|
||||||
the stat bar (2026-07-24, branch `parallel-phases`)
|
|
||||||
|
|
||||||
- Each operand is announced on stderr before its passes. With per-operand walk,
|
|
||||||
hash and update cycles, a multi-operand run such as `scan /srv/*` showed pass
|
|
||||||
totals that looked like the whole run's (2026-07-24, branch
|
|
||||||
`scan-operand-progress`)
|
|
||||||
|
|
||||||
- The walk became parallel. The walk pass was a single goroutine and took hours
|
|
||||||
at about 20 million files on a busy pool. One ZFS server took 4 hours for 22
|
|
||||||
million files. It is now a walk by a pool of per-directory workers that
|
|
||||||
records size and mtime as it goes. That folds away the separate stat pass and
|
|
||||||
halves the metadata reads. Each `PATH` operand commits in its own transaction,
|
|
||||||
so an interrupted scan keeps completed operands (2026-07-24, branch
|
|
||||||
`parallel-walk`)
|
|
||||||
|
|
||||||
- `scan` gained a persistent database. It maintains an SQLite database that
|
|
||||||
survives between runs, with the absolute path as each record's key. The driver
|
|
||||||
is `modernc.org/sqlite`, pure Go, so cgo stays disabled. A rescan hashes only
|
|
||||||
new or changed files, by mtime and size. It deletes the records of files
|
|
||||||
vanished from under the scanned operands and leaves records outside them
|
|
||||||
untouched. So `scan` can run daily from `cron`. `report` and `trees` read the
|
|
||||||
database, with no positional arguments, instead of a scan stream. The database
|
|
||||||
is at `/var/lib/sfdupes/db.sqlite`, and `SFDUPES_DATABASE` overrides that. WAL
|
|
||||||
journaling plus a single-transaction update keep a report run during a scan
|
|
||||||
safe (2026-07-24, branch `persistent-database`)
|
|
||||||
|
|
||||||
- The `origin` remote, `git@git.eeqj.de:sneak/sfdupes.git`, was added. `v0.0.1`
|
|
||||||
was tagged, and `main` and the tags were pushed (2026-07-23)
|
|
||||||
|
|
||||||
- The `scan` command line was reworked. Required `PATH...` operands, parsed by
|
|
||||||
cobra, replace the `/srv` `-root` default. The new `-x`, `--one-file-system`
|
|
||||||
flag follows the GNU convention and stops at filesystem boundaries, which the
|
|
||||||
walk crosses by default (2026-07-23, branch `scan-required-paths`)
|
|
||||||
|
|
||||||
- The repo was brought into compliance with `REPO_POLICIES.md` (the version of
|
|
||||||
2026-07-06), the existing repo checklist and the Go styleguide. The audit of
|
|
||||||
2026-07-22 found these gaps, and the branch closed them:
|
|
||||||
- `.gitignore`, with the standard patterns for secrets and for OS, editor
|
|
||||||
and Go files. The compiled `sfdupes` binary and `files.dat` had been
|
|
||||||
sitting untracked in the tree;
|
|
||||||
- `.editorconfig`;
|
|
||||||
- `LICENSE` and the README License section. MIT was assumed from house
|
|
||||||
convention, for the owner to confirm;
|
|
||||||
- `REPO_POLICIES.md` in the repo root;
|
|
||||||
- the canonical `.golangci.yml`, with its 150 findings fixed, so that
|
|
||||||
`make lint` is clean;
|
|
||||||
- the `Makefile` targets the policy requires:
|
|
||||||
- `test` and `lint`;
|
|
||||||
- `fmt` and `fmt-check`;
|
|
||||||
- `docker` and `hooks`.
|
|
||||||
|
|
||||||
`check` depended on `build`, which wrote the binary. `make check` must
|
|
||||||
not modify files, so `check` no longer depends on `build`;
|
|
||||||
|
|
||||||
- a test suite, which the policy required with a 30-second timeout and the
|
|
||||||
conditional `-v` rerun. At 64% coverage, it covers:
|
|
||||||
- parsing and grouping;
|
|
||||||
- digests and suppression;
|
|
||||||
- hashing and the scan pipeline;
|
|
||||||
- a multistage `Dockerfile` and `.dockerignore`. The `Dockerfile` had:
|
|
||||||
- hash-pinned images;
|
|
||||||
- a lint stage that failed fast;
|
|
||||||
- a build stage that ran `make check`;
|
|
||||||
- `.gitea/workflows/check.yml`, running `docker build .` on push with the
|
|
||||||
checkout action pinned by commit SHA;
|
|
||||||
- the README sections the policy requires. The first is a Description whose
|
|
||||||
first line names the project, its purpose and category, its license and
|
|
||||||
its author. Then come Getting Started and Rationale, and at the end TODO,
|
|
||||||
License and Author;
|
|
||||||
- the stale README non-goal "no git repository setup and no CI" removed;
|
|
||||||
- the pre-commit hook, installed with `make hooks` once that target existed.
|
|
||||||
|
|
||||||
The code was already `gofmt`-clean, so no standalone formatting commit was
|
|
||||||
needed (2026-07-23, branch `repo-policy-compliance`)
|
|
||||||
|
|
||||||
- `git init` was run, with a README-only first commit. The code baseline was
|
|
||||||
committed on `main` (2026-07-22)
|
|
||||||
|
|
||||||
- The `scan`, `report` and `trees` subcommands were implemented (pre-git
|
|
||||||
history)
|
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Possible later features, out of scope per README, are a byte-for-byte check of
|
- Possible later features, out of scope per README, are a byte-for-byte check of
|
||||||
|
|||||||
+2
-4
@@ -5,10 +5,8 @@
|
|||||||
// KiB, and of the content. The content hash covers the whole file under
|
// KiB, and of the content. The content hash covers the whole file under
|
||||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||||
// file is never read in full. scan maintains a persistent SQLite
|
// file is never read in full. scan maintains a persistent SQLite
|
||||||
// database of file signatures that the other subcommands read. The
|
// database of file signatures (SFDUPES_DATABASE, default
|
||||||
// database is SFDUPES_DATABASE, or by default /var/lib/sfdupes/db.sqlite
|
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||||
// for root and sfdupes/db.sqlite in the XDG state directory for any
|
|
||||||
// other user.
|
|
||||||
//
|
//
|
||||||
// Usage:
|
// Usage:
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module sneak.berlin/go/sfdupes
|
|||||||
go 1.25.7
|
go 1.25.7
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/adrg/xdg v0.5.3
|
github.com/rivo/uniseg v0.4.7
|
||||||
github.com/schollz/progressbar/v3 v3.19.1
|
github.com/schollz/progressbar/v3 v3.19.1
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
golang.org/x/sys v0.46.0
|
golang.org/x/sys v0.46.0
|
||||||
@@ -19,7 +19,6 @@ require (
|
|||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
|
||||||
github.com/spf13/pflag v1.0.9 // indirect
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
modernc.org/libc v1.74.1 // indirect
|
modernc.org/libc v1.74.1 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
|
||||||
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
|
|
||||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
|
|||||||
@@ -358,8 +358,9 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
dir := buildSmokeTree(t)
|
dir := buildSmokeTree(t)
|
||||||
|
|
||||||
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
opts := scanOptions{workers: walkCancelWorkers}
|
||||||
false, nil)
|
|
||||||
|
err := runScan(t.Context(), []string{dir}, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -382,8 +383,7 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
|
||||||
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
|
err = runScan(cancelledContext(t), []string{dir}, opts)
|
||||||
false, false, nil)
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
awaitReturn(t, done, "runScan")
|
awaitReturn(t, done, "runScan")
|
||||||
@@ -410,8 +410,7 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
got, before)
|
got, before)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
err = runScan(t.Context(), []string{dir}, opts)
|
||||||
false, nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -437,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
|
|||||||
dir := buildWalkCancelTree(t)
|
dir := buildWalkCancelTree(t)
|
||||||
|
|
||||||
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
||||||
walkCancelWorkers, false, false, nil)
|
scanOptions{workers: walkCancelWorkers})
|
||||||
if !errors.Is(err, errInterrupted) {
|
if !errors.Is(err, errInterrupted) {
|
||||||
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
||||||
err, errInterrupted)
|
err, errInterrupted)
|
||||||
|
|||||||
+154
-63
@@ -32,17 +32,79 @@ const (
|
|||||||
cmdDwim = "dwim"
|
cmdDwim = "dwim"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// databaseHelp is the paragraph of the root and scan help that names
|
||||||
|
// the database (README "Location").
|
||||||
|
const databaseHelp = "scan and dwim store what they find in a database " +
|
||||||
|
"file. report, trees\nand info read it. The file is $" + databaseEnv +
|
||||||
|
" when that is set and\nnot empty, and otherwise " +
|
||||||
|
defaultDatabasePath + ", where creating it\nneeds root.\n"
|
||||||
|
|
||||||
|
// escapesHelp is the paragraph of the report and trees help on how
|
||||||
|
// paths are escaped (README "Report output format").
|
||||||
|
const escapesHelp = `Four characters in a path are escaped, so each row stays one line:
|
||||||
|
|
||||||
|
\\ a backslash
|
||||||
|
\t a tab
|
||||||
|
\n a newline
|
||||||
|
\r a carriage return
|
||||||
|
`
|
||||||
|
|
||||||
|
// reportHelp, treesHelp, infoHelp and dwimHelp follow the one-line
|
||||||
|
// summary in the help of their subcommands.
|
||||||
|
const (
|
||||||
|
reportHelp = `The report is tab-separated text on stdout: a header line, then one
|
||||||
|
row per duplicate file. The columns are:
|
||||||
|
|
||||||
|
first the path that sorts first, byte by byte, in its group of copies
|
||||||
|
dupe another copy of that file
|
||||||
|
size the file's size in bytes
|
||||||
|
|
||||||
|
Rows are ordered by size, largest first, then by path.
|
||||||
|
|
||||||
|
` + escapesHelp
|
||||||
|
|
||||||
|
treesHelp = `The report is tab-separated text on stdout: a header line, then one
|
||||||
|
row per duplicate directory tree. The columns are:
|
||||||
|
|
||||||
|
first the path that sorts first, byte by byte, in its group of copies
|
||||||
|
dupe another copy of that tree
|
||||||
|
files the number of regular files in one copy
|
||||||
|
size their total size in bytes
|
||||||
|
disk_size the bytes the dupe copy takes on disk
|
||||||
|
|
||||||
|
Rows are ordered by size, largest first, then by path.
|
||||||
|
|
||||||
|
` + escapesHelp
|
||||||
|
|
||||||
|
infoHelp = `The details go to stdout, one per line. Each line holds a name and its
|
||||||
|
value, separated by a tab:
|
||||||
|
|
||||||
|
database the path, and where it came from
|
||||||
|
database file size in bytes, then rounded, as 2147483648 (2.0 GiB)
|
||||||
|
-wal file size in bytes, then rounded; only if a -wal file exists
|
||||||
|
modified when the database file last changed, in UTC
|
||||||
|
schema version the version of the database's layout
|
||||||
|
records the number of records
|
||||||
|
size of recorded files their total size, in bytes, then rounded
|
||||||
|
`
|
||||||
|
|
||||||
|
dwimHelp = `dwim runs what "sfdupes scan /" runs with the same flags, then what
|
||||||
|
"sfdupes trees" runs. Run it as root, so that the scan can read every
|
||||||
|
file. The scan is stored in the database, as scan stores it, so
|
||||||
|
"sfdupes trees" prints the report again later.
|
||||||
|
`
|
||||||
|
)
|
||||||
|
|
||||||
// errNoSubcommand is returned by the root command when it is invoked
|
// errNoSubcommand is returned by the root command when it is invoked
|
||||||
// without a subcommand. That is a usage error, and the usage text
|
// without a subcommand. That is a usage error, and the usage text
|
||||||
// cobra prints for it is the whole message.
|
// cobra prints for it is the whole message.
|
||||||
var errNoSubcommand = errors.New("no subcommand")
|
var errNoSubcommand = errors.New("no subcommand")
|
||||||
|
|
||||||
// errWorkersBelowOne is the usage error for a scan --workers value
|
// errWorkersBelowOne is the usage error for a --workers value below 1.
|
||||||
// below 1.
|
|
||||||
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
||||||
|
|
||||||
// errEmptyExcludeValue is the usage error for an empty scan --exclude
|
// errEmptyExcludeValue is the usage error for an empty --exclude value,
|
||||||
// value, as from --exclude "$DIR" with DIR unset. Taken as a path, it
|
// as from --exclude "$DIR" with DIR unset. Taken as a path, it
|
||||||
// would name the working directory.
|
// would name the working directory.
|
||||||
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
|
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
|
||||||
|
|
||||||
@@ -122,14 +184,6 @@ func run(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseHelp says in sfdupes --help where the database is.
|
|
||||||
const databaseHelp = `The database is the file SFDUPES_DATABASE names.
|
|
||||||
When that is unset or empty, root uses /var/lib/sfdupes/db.sqlite. Any
|
|
||||||
other user uses sfdupes/db.sqlite in the XDG state directory:
|
|
||||||
$XDG_STATE_HOME, or ~/.local/state when that is unset or empty
|
|
||||||
(~/Library/Application Support on macOS). An XDG_STATE_HOME that is
|
|
||||||
not an absolute path is an error.`
|
|
||||||
|
|
||||||
// newRootCommand builds the command tree. Everything on stdout is
|
// newRootCommand builds the command tree. Everything on stdout is
|
||||||
// machine-readable data, the version line included; all human-facing
|
// machine-readable data, the version line included; all human-facing
|
||||||
// output (help, usage, errors) goes to stderr.
|
// output (help, usage, errors) goes to stderr.
|
||||||
@@ -148,12 +202,9 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
short := "Find candidate duplicate files by size and head/tail/content SHA-256"
|
|
||||||
|
|
||||||
root := &cobra.Command{
|
root := &cobra.Command{
|
||||||
Use: "sfdupes",
|
Use: "sfdupes",
|
||||||
Short: short,
|
Short: "Find candidate duplicate files by size and head/tail/content SHA-256",
|
||||||
Long: short + ".\n\n" + databaseHelp,
|
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if showVersion {
|
if showVersion {
|
||||||
@@ -204,76 +255,116 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
dwimCmd := &cobra.Command{
|
root.Long = root.Short + ".\n\n" + databaseHelp
|
||||||
Use: cmdDwim,
|
reportCmd.Long = reportCmd.Short + ".\n\n" + reportHelp
|
||||||
Short: "Scan / with the default excludes, then print the duplicate-tree report",
|
treesCmd.Long = treesCmd.Short + ".\n\n" + treesHelp
|
||||||
Args: cobra.NoArgs,
|
infoCmd.Long = infoCmd.Short + ".\n\n" + infoHelp
|
||||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
|
||||||
return runDwim(ctx, "/", stdout)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
|
|
||||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd,
|
||||||
|
newDwimCommand(stdout))
|
||||||
|
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|
||||||
// newScanCommand builds the scan subcommand, with its --workers, -x,
|
// scanOptions holds the values of the flags that scan and dwim share:
|
||||||
// --no-default-excludes and --exclude flags. Its help lists the default
|
// --workers, -x, --no-default-excludes and --exclude.
|
||||||
// excludes of the system it runs on, and the filesystem types it never
|
type scanOptions struct {
|
||||||
// records anything on.
|
workers int
|
||||||
|
oneFS bool
|
||||||
|
noDefaultExcludes bool
|
||||||
|
excludePaths []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// scanFlagsUsage names the flags that scan and dwim share, for their
|
||||||
|
// usage lines.
|
||||||
|
const scanFlagsUsage = "[--workers N] [-x] [--no-default-excludes] " +
|
||||||
|
"[--exclude PATH]..."
|
||||||
|
|
||||||
|
// addScanFlags adds the flags that scan and dwim share to cmd, which
|
||||||
|
// stores their values in opts.
|
||||||
|
func addScanFlags(cmd *cobra.Command, opts *scanOptions) {
|
||||||
|
cmd.Flags().IntVar(&opts.workers, "workers", runtime.NumCPU(),
|
||||||
|
"concurrent workers for the walk and hash phases")
|
||||||
|
cmd.Flags().BoolVarP(&opts.oneFS, "one-file-system", "x", false,
|
||||||
|
"do not cross filesystem boundaries")
|
||||||
|
cmd.Flags().BoolVar(&opts.noDefaultExcludes, "no-default-excludes",
|
||||||
|
false, "enter the default excludes, the paths listed above")
|
||||||
|
// A string array, not a slice, so a comma in a path is not a
|
||||||
|
// separator.
|
||||||
|
cmd.Flags().StringArrayVar(&opts.excludePaths, "exclude", nil,
|
||||||
|
"never enter or record `PATH` or anything below it; repeatable")
|
||||||
|
}
|
||||||
|
|
||||||
|
// check rejects the flag values that are usage errors, before the
|
||||||
|
// command runs. It is the PreRunE of scan and dwim.
|
||||||
|
func (o *scanOptions) check(cmd *cobra.Command, _ []string) error {
|
||||||
|
err := checkScanWorkers(cmd, o.workers)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return rejectEmptyExcludeValue(cmd, o.excludePaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newScanCommand builds the scan subcommand. Its help names the
|
||||||
|
// database, and lists the default excludes of the system it runs on and
|
||||||
|
// the filesystem types it never records anything on.
|
||||||
func newScanCommand() *cobra.Command {
|
func newScanCommand() *cobra.Command {
|
||||||
var (
|
var opts scanOptions
|
||||||
scanWorkers int
|
|
||||||
scanOneFS bool
|
|
||||||
scanNoDefaultExcludes bool
|
|
||||||
scanExclude []string
|
|
||||||
)
|
|
||||||
|
|
||||||
short := "Walk trees and synchronize the scan database"
|
short := "Walk trees and synchronize the scan database"
|
||||||
|
|
||||||
scanCmd := &cobra.Command{
|
scanCmd := &cobra.Command{
|
||||||
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] " +
|
Use: cmdScan + " " + scanFlagsUsage + " PATH...",
|
||||||
"[--exclude PATH]... PATH...",
|
|
||||||
// The Use line already names every flag, so cobra must not
|
// The Use line already names every flag, so cobra must not
|
||||||
// append its generic "[flags]" to it.
|
// append its generic "[flags]" to it.
|
||||||
DisableFlagsInUseLine: true,
|
DisableFlagsInUseLine: true,
|
||||||
Short: short,
|
Short: short,
|
||||||
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
Long: short + ".\n\n" + databaseHelp + "\n" +
|
||||||
Args: cobra.MinimumNArgs(1),
|
defaultExcludesHelp(runtime.GOOS),
|
||||||
PreRunE: func(cmd *cobra.Command, _ []string) error {
|
Args: cobra.MinimumNArgs(1),
|
||||||
err := checkScanWorkers(cmd, scanWorkers)
|
PreRunE: opts.check,
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rejectEmptyExcludeValue(cmd, scanExclude)
|
|
||||||
},
|
|
||||||
RunE: runE(func(ctx context.Context, args []string) error {
|
RunE: runE(func(ctx context.Context, args []string) error {
|
||||||
ctx, stop := interruptContext(ctx)
|
ctx, stop := interruptContext(ctx)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
return runScan(ctx, args, scanWorkers, scanOneFS,
|
return runScan(ctx, args, opts)
|
||||||
scanNoDefaultExcludes, scanExclude)
|
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
addScanFlags(scanCmd, &opts)
|
||||||
"concurrent workers for the walk and hash phases")
|
|
||||||
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
|
||||||
"do not cross filesystem boundaries")
|
|
||||||
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
|
|
||||||
false, "enter the default excludes, the paths listed above")
|
|
||||||
// A string array, not a slice, so a comma in a path is not a
|
|
||||||
// separator.
|
|
||||||
scanCmd.Flags().StringArrayVar(&scanExclude, "exclude", nil,
|
|
||||||
"never enter or record `PATH` or anything below it; repeatable")
|
|
||||||
|
|
||||||
return scanCmd
|
return scanCmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkScanWorkers rejects a scan --workers value below 1. That is a
|
// newDwimCommand builds the dwim subcommand, which takes the scan flags
|
||||||
// usage error reported in one line: cobra prints the returned message
|
// with the meaning they have for scan. Its help says what dwim runs,
|
||||||
// without the usage text, and run exits 2.
|
// then lists the default excludes and the filesystem types the scan
|
||||||
|
// never records anything on.
|
||||||
|
func newDwimCommand(stdout io.Writer) *cobra.Command {
|
||||||
|
var opts scanOptions
|
||||||
|
|
||||||
|
short := "Scan /, then print the duplicate-tree report"
|
||||||
|
|
||||||
|
dwimCmd := &cobra.Command{
|
||||||
|
Use: cmdDwim + " " + scanFlagsUsage,
|
||||||
|
DisableFlagsInUseLine: true,
|
||||||
|
Short: short,
|
||||||
|
Long: short + ".\n\n" + dwimHelp + "\n" +
|
||||||
|
defaultExcludesHelp(runtime.GOOS),
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
PreRunE: opts.check,
|
||||||
|
RunE: runE(func(ctx context.Context, _ []string) error {
|
||||||
|
return runDwim(ctx, "/", opts, stdout)
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
addScanFlags(dwimCmd, &opts)
|
||||||
|
|
||||||
|
return dwimCmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkScanWorkers rejects a --workers value below 1. That is a usage
|
||||||
|
// error reported in one line: cobra prints the returned message without
|
||||||
|
// the usage text, and run exits 2.
|
||||||
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
||||||
if workers >= 1 {
|
if workers >= 1 {
|
||||||
return nil
|
return nil
|
||||||
@@ -284,7 +375,7 @@ func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
|||||||
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
||||||
}
|
}
|
||||||
|
|
||||||
// rejectEmptyExcludeValue rejects an empty scan --exclude value. That
|
// rejectEmptyExcludeValue rejects an empty --exclude value. That
|
||||||
// is a usage error reported in one line, as for --workers below 1.
|
// is a usage error reported in one line, as for --workers below 1.
|
||||||
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
|
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
|
||||||
if !slices.Contains(excludePaths, "") {
|
if !slices.Contains(excludePaths, "") {
|
||||||
|
|||||||
+84
-101
@@ -15,7 +15,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/adrg/xdg"
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,6 +22,12 @@ import (
|
|||||||
// error prints and a runtime failure must not.
|
// error prints and a runtime failure must not.
|
||||||
const usageMarker = "Usage:"
|
const usageMarker = "Usage:"
|
||||||
|
|
||||||
|
// helpFlag asks for the help text.
|
||||||
|
const helpFlag = "--help"
|
||||||
|
|
||||||
|
// workersFlag is the scan and dwim flag that sets the worker count.
|
||||||
|
const workersFlag = "--workers"
|
||||||
|
|
||||||
// TestMain stops the tests at once, saying why, when their temporary
|
// TestMain stops the tests at once, saying why, when their temporary
|
||||||
// directory is not on a persistent filesystem: scan records nothing
|
// directory is not on a persistent filesystem: scan records nothing
|
||||||
// there (README "Rules for the walk"), so most scan tests would fail
|
// there (README "Rules for the walk"), so most scan tests would fail
|
||||||
@@ -554,17 +559,13 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
// line.
|
// line.
|
||||||
t.Setenv(databaseEnv, "")
|
t.Setenv(databaseEnv, "")
|
||||||
|
|
||||||
dbPath, err := defaultDatabasePath(os.Geteuid())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Whether scan calls the database new depends on that machine too.
|
// Whether scan calls the database new depends on that machine too.
|
||||||
scanWant := "scan: database " + dbPath + " (default)\n"
|
scanWant := "scan: database " + defaultDatabasePath + " (default)\n"
|
||||||
|
|
||||||
_, err = os.Stat(dbPath)
|
_, err := os.Stat(defaultDatabasePath)
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
scanWant = "scan: new database " + dbPath + " (default)\n"
|
scanWant = "scan: new database " + defaultDatabasePath +
|
||||||
|
" (default)\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
missing := filepath.Join(t.TempDir(), "nope")
|
missing := filepath.Join(t.TempDir(), "nope")
|
||||||
@@ -575,8 +576,8 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
cmdScan: {
|
cmdScan: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runScan(t.Context(), []string{missing}, 1, false, false,
|
return runScan(t.Context(), []string{missing},
|
||||||
nil)
|
scanOptions{workers: 1})
|
||||||
},
|
},
|
||||||
want: scanWant,
|
want: scanWant,
|
||||||
},
|
},
|
||||||
@@ -584,21 +585,21 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
run: func() error {
|
run: func() error {
|
||||||
return runReport(cancelledContext(t), io.Discard)
|
return runReport(cancelledContext(t), io.Discard)
|
||||||
},
|
},
|
||||||
want: "report: database " + dbPath + " (default)\n" +
|
want: "report: database " + defaultDatabasePath + " (default)\n" +
|
||||||
reportOpenLine + "\n" + reportOpenLine + "\n",
|
reportOpenLine + "\n" + reportOpenLine + "\n",
|
||||||
},
|
},
|
||||||
cmdTrees: {
|
cmdTrees: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runTrees(cancelledContext(t), io.Discard)
|
return runTrees(cancelledContext(t), io.Discard)
|
||||||
},
|
},
|
||||||
want: "trees: database " + dbPath + " (default)\n" +
|
want: "trees: database " + defaultDatabasePath + " (default)\n" +
|
||||||
treesOpenLine + "\n" + treesOpenLine + "\n",
|
treesOpenLine + "\n" + treesOpenLine + "\n",
|
||||||
},
|
},
|
||||||
cmdInfo: {
|
cmdInfo: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runInfo(cancelledContext(t), io.Discard)
|
return runInfo(cancelledContext(t), io.Discard)
|
||||||
},
|
},
|
||||||
want: "info: database " + dbPath + " (default)\n" +
|
want: "info: database " + defaultDatabasePath + " (default)\n" +
|
||||||
infoOpenLine + "\n" + infoOpenLine + "\n",
|
infoOpenLine + "\n" + infoOpenLine + "\n",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -667,7 +668,7 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
args := []string{cmdScan, "--workers", workers, t.TempDir()}
|
args := []string{cmdScan, workersFlag, workers, t.TempDir()}
|
||||||
|
|
||||||
code := run(args, &stdout, &stderr)
|
code := run(args, &stdout, &stderr)
|
||||||
if code != exitUsage {
|
if code != exitUsage {
|
||||||
@@ -698,7 +699,7 @@ func TestRunHelp(t *testing.T) {
|
|||||||
|
|
||||||
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
||||||
// empty.
|
// empty.
|
||||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
cases := [][]string{{helpFlag}, {"-h"}, {cmdScan, helpFlag}}
|
||||||
|
|
||||||
for _, args := range cases {
|
for _, args := range cases {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -717,22 +718,6 @@ func TestRunHelp(t *testing.T) {
|
|||||||
t.Errorf("run(%v) stdout = %q, want nothing (data only)",
|
t.Errorf("run(%v) stdout = %q, want nothing (data only)",
|
||||||
args, got)
|
args, got)
|
||||||
}
|
}
|
||||||
|
|
||||||
if args[0] == cmdScan {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// README §Location: sfdupes --help says where the database is,
|
|
||||||
// and that XDG_STATE_HOME must be an absolute path.
|
|
||||||
for _, want := range []string{
|
|
||||||
databaseEnv, rootDatabasePath, stateHomeEnv, "~/.local/state",
|
|
||||||
"not an absolute path is an error",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(stderr.String(), want) {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want it to name %q",
|
|
||||||
args, stderr.String(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The scan help lists --exclude with its argument.
|
// The scan help lists --exclude with its argument.
|
||||||
@@ -775,6 +760,73 @@ func TestRunScanUsageLine(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunHelpDescribesDatabaseAndOutput(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// README §Settings: the help names the database, and says what
|
||||||
|
// report, trees, info and dwim print.
|
||||||
|
database := []string{
|
||||||
|
"scan and dwim store", databaseEnv, defaultDatabasePath, "needs root",
|
||||||
|
}
|
||||||
|
escapes := []string{`\\`, `\t`, `\n`, `\r`}
|
||||||
|
|
||||||
|
// The subcommand whose help is printed; none for the root.
|
||||||
|
cases := []struct {
|
||||||
|
subcommand []string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{nil, database},
|
||||||
|
{[]string{cmdScan}, database},
|
||||||
|
{
|
||||||
|
[]string{cmdReport},
|
||||||
|
append([]string{
|
||||||
|
"tab-separated", "\n first ", "\n dupe ", "\n size ",
|
||||||
|
"largest first",
|
||||||
|
}, escapes...),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
[]string{cmdTrees},
|
||||||
|
append([]string{
|
||||||
|
"tab-separated", "\n first ", "\n dupe ", "\n files ",
|
||||||
|
"\n size ", "\n disk_size ", "largest first",
|
||||||
|
}, escapes...),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
[]string{cmdInfo},
|
||||||
|
[]string{
|
||||||
|
"\n database ", "\n database file size ",
|
||||||
|
"\n -wal file size ", "\n modified ",
|
||||||
|
"\n schema version ", "\n records ",
|
||||||
|
"\n size of recorded files ",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
[]string{cmdDwim},
|
||||||
|
[]string{
|
||||||
|
"sfdupes scan /", "as root", "sfdupes trees", "database",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
args := slices.Concat(tc.subcommand, []string{helpFlag})
|
||||||
|
|
||||||
|
code := run(args, &stdout, &stderr)
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("run(%v) = %d, want %d", args, code, exitOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, want := range tc.want {
|
||||||
|
if !strings.Contains(stderr.String(), want) {
|
||||||
|
t.Errorf("run(%v) help lacks %q:\n%s",
|
||||||
|
args, want, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunVersion(t *testing.T) {
|
func TestRunVersion(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1557,75 +1609,6 @@ func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunNonRootUsesStateDirectory(t *testing.T) {
|
|
||||||
// README §Location: without SFDUPES_DATABASE, a user other than
|
|
||||||
// root keeps the database in the XDG state directory. scan creates
|
|
||||||
// the missing directories with mode 0700, and report and trees read
|
|
||||||
// the same file.
|
|
||||||
if os.Geteuid() == 0 {
|
|
||||||
t.Skip("root uses " + rootDatabasePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(xdg.Reload)
|
|
||||||
|
|
||||||
state := filepath.Join(t.TempDir(), "state")
|
|
||||||
t.Setenv(stateHomeEnv, state)
|
|
||||||
t.Setenv(databaseEnv, "")
|
|
||||||
xdg.Reload()
|
|
||||||
|
|
||||||
assertRunsUseDatabase(t, filepath.Join(state, "sfdupes"), "db.sqlite")
|
|
||||||
|
|
||||||
for _, dir := range []string{state, filepath.Join(state, "sfdupes")} {
|
|
||||||
fi, err := os.Stat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := fi.Mode().Perm(); got != stateDirPerm {
|
|
||||||
t.Errorf("%s mode = %#o, want %#o", dir, got, stateDirPerm)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRelativeStateHomeFails(t *testing.T) {
|
|
||||||
// README §Location: without SFDUPES_DATABASE, a user other than
|
|
||||||
// root whose XDG_STATE_HOME is not an absolute path gets an error
|
|
||||||
// naming it from every subcommand, which exits 1 before it does
|
|
||||||
// anything else.
|
|
||||||
if os.Geteuid() == 0 {
|
|
||||||
t.Skip("root does not use " + stateHomeEnv)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv(stateHomeEnv, "relative/state")
|
|
||||||
t.Setenv(databaseEnv, "")
|
|
||||||
|
|
||||||
want := "sfdupes: " + stateHomeEnv +
|
|
||||||
" must be an absolute path, got \"relative/state\"\n"
|
|
||||||
|
|
||||||
// A scan that got past the check would stop at this missing
|
|
||||||
// operand, before it creates anything.
|
|
||||||
missing := filepath.Join(t.TempDir(), "missing")
|
|
||||||
|
|
||||||
for _, args := range [][]string{
|
|
||||||
{cmdScan, missing}, {cmdReport}, {cmdTrees}, {cmdInfo},
|
|
||||||
} {
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := run(args, &stdout, &stderr)
|
|
||||||
if code != exitFatal || stderr.String() != want {
|
|
||||||
t.Errorf("run(%v) = %d with stderr %q, want %d and %q",
|
|
||||||
args, code, stderr.String(), exitFatal, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// dwim scans /, so runDwim gets the missing operand instead.
|
|
||||||
err := runDwim(t.Context(), missing, io.Discard)
|
|
||||||
if !errors.Is(err, errStateHomeNotAbsolute) {
|
|
||||||
t.Errorf("runDwim: error = %v, want %v", err,
|
|
||||||
errStateHomeNotAbsolute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdScanLock takes the lock on the database at path, as a running
|
// holdScanLock takes the lock on the database at path, as a running
|
||||||
// scan does, and holds it until the test ends. It fails the test when
|
// scan does, and holds it until the test ends. It fails the test when
|
||||||
// the lock is already held.
|
// the lock is already held.
|
||||||
|
|||||||
+15
-68
@@ -15,40 +15,28 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/adrg/xdg"
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
|
// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
|
||||||
// disabled.
|
// disabled.
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
)
|
)
|
||||||
|
|
||||||
// rootDatabasePath is the database root uses when SFDUPES_DATABASE is
|
// defaultDatabasePath is where the persistent scan database lives when
|
||||||
// not set.
|
// SFDUPES_DATABASE is not set.
|
||||||
const rootDatabasePath = "/var/lib/sfdupes/db.sqlite"
|
const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
|
||||||
|
|
||||||
// databaseEnv is the environment variable that overrides the database
|
// databaseEnv is the environment variable that overrides the database
|
||||||
// path.
|
// path.
|
||||||
const databaseEnv = "SFDUPES_DATABASE"
|
const databaseEnv = "SFDUPES_DATABASE"
|
||||||
|
|
||||||
// stateHomeEnv is the environment variable that names the XDG state
|
|
||||||
// directory.
|
|
||||||
const stateHomeEnv = "XDG_STATE_HOME"
|
|
||||||
|
|
||||||
// schemaVersion is the database schema version this build reads and
|
// schemaVersion is the database schema version this build reads and
|
||||||
// writes, stored in PRAGMA user_version.
|
// writes, stored in PRAGMA user_version.
|
||||||
const schemaVersion = 1
|
const schemaVersion = 1
|
||||||
|
|
||||||
// dbDirPerm is the mode for a directory scan creates for root's
|
// dbDirPerm is the mode for a database parent directory created by
|
||||||
// default database or for SFDUPES_DATABASE.
|
// scan.
|
||||||
const dbDirPerm = 0o755
|
const dbDirPerm = 0o755
|
||||||
|
|
||||||
// stateDirPerm is the mode for a directory scan creates for the
|
|
||||||
// database at a user's default path in the XDG state directory. It is
|
|
||||||
// the mode the XDG Base Directory specification asks for, and it keeps
|
|
||||||
// other users from reading the database, which lists every scanned
|
|
||||||
// path.
|
|
||||||
const stateDirPerm = 0o700
|
|
||||||
|
|
||||||
// dbFilePerm is the mode for a database file created by scan, the mode
|
// dbFilePerm is the mode for a database file created by scan, the mode
|
||||||
// SQLite gives a database file it creates.
|
// SQLite gives a database file it creates.
|
||||||
const dbFilePerm = 0o644
|
const dbFilePerm = 0o644
|
||||||
@@ -156,48 +144,15 @@ var errScanRunning = errors.New("another scan is running")
|
|||||||
// 32 bytes of a SHA-256.
|
// 32 bytes of a SHA-256.
|
||||||
var errHashSize = errors.New("hash is not 32 bytes")
|
var errHashSize = errors.New("hash is not 32 bytes")
|
||||||
|
|
||||||
// errStateHomeNotAbsolute reports an XDG_STATE_HOME that is set to a
|
|
||||||
// path that is not absolute.
|
|
||||||
var errStateHomeNotAbsolute = errors.New(
|
|
||||||
stateHomeEnv + " must be an absolute path")
|
|
||||||
|
|
||||||
// defaultDatabasePath is the database used when SFDUPES_DATABASE is not
|
|
||||||
// set: rootDatabasePath for root (euid 0), and db.sqlite in the
|
|
||||||
// directory sfdupes under the XDG state directory for any other user.
|
|
||||||
// xdg reads XDG_STATE_HOME and HOME once, when the program starts. For
|
|
||||||
// any other user, an XDG_STATE_HOME that is set and not an absolute
|
|
||||||
// path is an error, where xdg would use its default instead. The path
|
|
||||||
// is joined as text, not cleaned, for the reason absolutePath gives.
|
|
||||||
func defaultDatabasePath(euid int) (string, error) {
|
|
||||||
if euid == 0 {
|
|
||||||
return rootDatabasePath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
stateHome := os.Getenv(stateHomeEnv)
|
|
||||||
if stateHome != "" && !filepath.IsAbs(stateHome) {
|
|
||||||
return "", fmt.Errorf("%w, got %q", errStateHomeNotAbsolute,
|
|
||||||
stateHome)
|
|
||||||
}
|
|
||||||
|
|
||||||
stateDir := strings.TrimSuffix(xdg.StateHome, "/")
|
|
||||||
|
|
||||||
return stateDir + "/sfdupes/db.sqlite", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||||
// set and non-empty, defaultDatabasePath otherwise. It returns the
|
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||||
// path, made absolute by absolutePath, and where the path came from,
|
// path, made absolute by absolutePath, and where the path came from,
|
||||||
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
||||||
// prints first on stderr.
|
// prints first on stderr.
|
||||||
func databasePath() (string, string, error) {
|
func databasePath() (string, string, error) {
|
||||||
path, source := os.Getenv(databaseEnv), "from "+databaseEnv
|
path, source := defaultDatabasePath, "default"
|
||||||
if path == "" {
|
if p := os.Getenv(databaseEnv); p != "" {
|
||||||
def, err := defaultDatabasePath(os.Geteuid())
|
path, source = p, "from "+databaseEnv
|
||||||
if err != nil {
|
|
||||||
return "", "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
path, source = def, "default"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
abs, err := absolutePath(path)
|
abs, err := absolutePath(path)
|
||||||
@@ -233,23 +188,15 @@ func absolutePath(path string) (string, error) {
|
|||||||
|
|
||||||
// createDatabaseDirectory creates the directory the database at path
|
// createDatabaseDirectory creates the directory the database at path
|
||||||
// goes in, and any directory above it, if missing. path is absolute, as
|
// goes in, and any directory above it, if missing. path is absolute, as
|
||||||
// databasePath returns it. Each directory gets mode dbDirPerm, or
|
// databasePath returns it. The directory is path with its last element
|
||||||
// stateDirPerm when databasePath uses a user's default path in the XDG
|
// cut off as text, by filepath.Split. filepath.Dir would clean it,
|
||||||
// state directory: the user is not root and SFDUPES_DATABASE is unset
|
// dropping each ".." together with the directory name before it; when
|
||||||
// or empty. The directory is path with its last element cut off as
|
// that name is a symlink, the cleaned path names another directory than
|
||||||
// text, by filepath.Split. filepath.Dir would clean it, dropping each
|
// the one the system reaches when it opens path.
|
||||||
// ".." together with the directory name before it; when that name is a
|
|
||||||
// symlink, the cleaned path names another directory than the one the
|
|
||||||
// system reaches when it opens path.
|
|
||||||
func createDatabaseDirectory(path string) error {
|
func createDatabaseDirectory(path string) error {
|
||||||
dir, _ := filepath.Split(path)
|
dir, _ := filepath.Split(path)
|
||||||
|
|
||||||
perm := os.FileMode(dbDirPerm)
|
err := os.MkdirAll(dir, dbDirPerm)
|
||||||
if os.Geteuid() != 0 && os.Getenv(databaseEnv) == "" {
|
|
||||||
perm = stateDirPerm
|
|
||||||
}
|
|
||||||
|
|
||||||
err := os.MkdirAll(dir, perm)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create database directory: %w", err)
|
return fmt.Errorf("create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,13 +9,10 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/adrg/xdg"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// testDBPath returns a database path inside a fresh temp dir, with
|
// testDBPath returns a database path inside a fresh temp dir, with
|
||||||
@@ -59,16 +56,11 @@ func TestDatabasePath(t *testing.T) {
|
|||||||
|
|
||||||
t.Chdir(dir)
|
t.Chdir(dir)
|
||||||
|
|
||||||
defaultPath, err := defaultDatabasePath(os.Geteuid())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fromEnv := "from " + databaseEnv
|
fromEnv := "from " + databaseEnv
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
env, path, source string
|
env, path, source string
|
||||||
}{
|
}{
|
||||||
{"", defaultPath, "default"},
|
{"", defaultDatabasePath, "default"},
|
||||||
{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
|
{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
|
||||||
{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
|
{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
|
||||||
fromEnv},
|
fromEnv},
|
||||||
@@ -97,48 +89,6 @@ func TestDatabasePath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDefaultDatabasePath(t *testing.T) {
|
|
||||||
// README §Location: root's database is /var/lib/sfdupes/db.sqlite.
|
|
||||||
// Any other user's is db.sqlite in sfdupes under XDG_STATE_HOME, or
|
|
||||||
// under ~/.local/state when XDG_STATE_HOME is unset or empty. For
|
|
||||||
// that user, an XDG_STATE_HOME that is not an absolute path is an
|
|
||||||
// error.
|
|
||||||
if runtime.GOOS != goosLinux {
|
|
||||||
t.Skip("macOS has another default state directory")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(xdg.Reload)
|
|
||||||
t.Setenv("HOME", "/home/someone")
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
euid int
|
|
||||||
stateHome string
|
|
||||||
want string
|
|
||||||
wantErr error
|
|
||||||
}{
|
|
||||||
{0, "", rootDatabasePath, nil},
|
|
||||||
{0, "/elsewhere/state", rootDatabasePath, nil},
|
|
||||||
{0, "relative/state", rootDatabasePath, nil},
|
|
||||||
{1000, "", "/home/someone/.local/state/sfdupes/db.sqlite", nil},
|
|
||||||
{1000, "relative/state", "", errStateHomeNotAbsolute},
|
|
||||||
{1000, "~/state", "", errStateHomeNotAbsolute},
|
|
||||||
{1000, "/elsewhere/state", "/elsewhere/state/sfdupes/db.sqlite", nil},
|
|
||||||
{1000, "/elsewhere/state/", "/elsewhere/state/sfdupes/db.sqlite", nil},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Setenv(stateHomeEnv, tc.stateHome)
|
|
||||||
xdg.Reload()
|
|
||||||
|
|
||||||
got, err := defaultDatabasePath(tc.euid)
|
|
||||||
if got != tc.want || !errors.Is(err, tc.wantErr) {
|
|
||||||
t.Errorf("defaultDatabasePath(%d) with %s=%q = %q, %v; want %q, %v",
|
|
||||||
tc.euid, stateHomeEnv, tc.stateHome, got, err, tc.want,
|
|
||||||
tc.wantErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenScanDatabaseCreates(t *testing.T) {
|
func TestOpenScanDatabaseCreates(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+10
-10
@@ -5,20 +5,20 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
||||||
// root" runs with no flags, then what "sfdupes trees" runs, and ends
|
// root" runs with the scan flags in opts, then what "sfdupes trees"
|
||||||
// with a line on stderr saying how to print the report again. The dwim
|
// runs, and ends with a line on stderr saying how to print the report
|
||||||
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
|
// again. The dwim command passes "/" as root. It catches SIGINT and
|
||||||
// does, but only during the scan. An error from the scan, an interrupt
|
// SIGTERM as scan does, but only during the scan. An error from the
|
||||||
// included, ends it before the report. During the report, a signal
|
// scan, an interrupt included, ends it before the report. During the
|
||||||
// ends it at once, as it ends trees.
|
// report, a signal ends it at once, as it ends trees.
|
||||||
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
|
func runDwim(
|
||||||
|
ctx context.Context, root string, opts scanOptions, stdout io.Writer,
|
||||||
|
) error {
|
||||||
scanCtx, stop := interruptContext(ctx)
|
scanCtx, stop := interruptContext(ctx)
|
||||||
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false,
|
err := runScan(scanCtx, []string{root}, opts)
|
||||||
nil)
|
|
||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ package sfdupes //nolint:testpackage // calls the unexported runDwim
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -28,7 +31,7 @@ func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
|||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
err := runDwim(t.Context(), root, &stdout)
|
err := runDwim(t.Context(), root, scanOptions{workers: 1}, &stdout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
}
|
}
|
||||||
@@ -67,7 +70,7 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
|||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
err := runDwim(t.Context(), missing, &stdout)
|
err := runDwim(t.Context(), missing, scanOptions{workers: 1}, &stdout)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
||||||
}
|
}
|
||||||
@@ -81,3 +84,127 @@ func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
|||||||
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunDwimExcludeLeavesTreeOut(t *testing.T) {
|
||||||
|
// dwim passes its --exclude paths to the scan: an excluded copy is
|
||||||
|
// not recorded, so its pair is not reported.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
root := realTempDir(t)
|
||||||
|
|
||||||
|
kept := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
||||||
|
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
||||||
|
}
|
||||||
|
writeFile(t, root, "b1/f.bin", pattern(2, 5000))
|
||||||
|
excluded := filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000)))
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
opts := scanOptions{workers: 1, excludePaths: []string{excluded}}
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), root, opts, &stdout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
||||||
|
if len(lines) != 2 {
|
||||||
|
t.Fatalf("stdout = %q, want a header and one group",
|
||||||
|
stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
want := kept[0] + "\t" + kept[1] + "\t1\t300\t"
|
||||||
|
if !strings.HasPrefix(lines[1], want) {
|
||||||
|
t.Errorf("group = %q, want it to start %q", lines[1], want)
|
||||||
|
}
|
||||||
|
|
||||||
|
skipped := "scan: skipped " + excluded + " (--exclude)\n"
|
||||||
|
if got := stderr(); !strings.Contains(got, skipped) {
|
||||||
|
t.Errorf("stderr = %q, want it to contain %q", got, skipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimTakesScanFlags(t *testing.T) {
|
||||||
|
// The dwim command parses and checks the scan flags as scan does.
|
||||||
|
// Each case stops before the walk of / begins, and before the
|
||||||
|
// database is created. "dwim --exclude /" excludes its own operand.
|
||||||
|
cases := map[string]struct {
|
||||||
|
args []string
|
||||||
|
code int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
"--exclude /": {
|
||||||
|
[]string{cmdDwim, excludeFlag, "/"},
|
||||||
|
exitFatal,
|
||||||
|
"sfdupes: scan /: operand excluded by --exclude /\n",
|
||||||
|
},
|
||||||
|
"--exclude empty": {
|
||||||
|
[]string{cmdDwim, excludeFlag, ""},
|
||||||
|
exitUsage,
|
||||||
|
"Error: --exclude must not be empty\n",
|
||||||
|
},
|
||||||
|
"--workers 0": {
|
||||||
|
[]string{cmdDwim, workersFlag, "0", "-x", "--no-default-excludes"},
|
||||||
|
exitUsage,
|
||||||
|
"Error: --workers must be at least 1, got 0\n",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
stdout := captureStdout(t)
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
code := run(tc.args, os.Stdout, os.Stderr)
|
||||||
|
if code != tc.code {
|
||||||
|
t.Errorf("run(%q) = %d, want %d", tc.args, code, tc.code)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasSuffix(stderr(), tc.want) || stdout() != "" {
|
||||||
|
t.Errorf("stdout, stderr = %q, %q, want \"\" and stderr "+
|
||||||
|
"ending %q", stdout(), stderr(), tc.want)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := os.Lstat(path)
|
||||||
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Errorf("lstat %s: %v, want no database", path, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimHelpListsScanFlags(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// README "Subcommands" shows this usage line for dwim.
|
||||||
|
const want = "\n sfdupes dwim [--workers N] [-x] [--no-default-excludes] " +
|
||||||
|
"[--exclude PATH]...\n"
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
run([]string{cmdDwim, "-h"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if !strings.Contains(stderr.String(), want) {
|
||||||
|
t.Errorf("dwim -h stderr = %q, want the usage line %q",
|
||||||
|
stderr.String(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
|
||||||
|
t.Errorf("dwim -h stderr = %q, want %s PATH", stderr.String(),
|
||||||
|
excludeFlag)
|
||||||
|
}
|
||||||
|
|
||||||
|
// README "Settings": the help lists the default excludes and the
|
||||||
|
// filesystem types the walk never records anything on.
|
||||||
|
help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
|
||||||
|
if !strings.Contains(stderr.String(), help) {
|
||||||
|
t.Errorf("dwim -h stderr = %q, want it to hold %q", stderr.String(),
|
||||||
|
help)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -254,7 +254,7 @@ func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
|
|||||||
return devices, lines.Err()
|
return devices, lines.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultExcludesHelp describes, for the scan command's help, the
|
// defaultExcludesHelp describes, for the scan and dwim help, the
|
||||||
// default excludes of the system goos names and, where the scan lists
|
// default excludes of the system goos names and, where the scan lists
|
||||||
// the mounted filesystems, the types it never records anything on.
|
// the mounted filesystems, the types it never records anything on.
|
||||||
func defaultExcludesHelp(goos string) string {
|
func defaultExcludesHelp(goos string) string {
|
||||||
|
|||||||
@@ -9,8 +9,10 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/rivo/uniseg"
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
)
|
)
|
||||||
@@ -116,7 +118,8 @@ type drawnCount struct {
|
|||||||
// terminal's width. A phase with an exact total ends the line with its
|
// terminal's width. A phase with an exact total ends the line with its
|
||||||
// percent and bar, at the terminal's right edge, when the terminal has
|
// percent and bar, at the terminal's right edge, when the terminal has
|
||||||
// room for them. It is drawn again every redrawInterval, so its elapsed
|
// room for them. It is drawn again every redrawInterval, so its elapsed
|
||||||
// time keeps moving while the phase waits for its next item. When
|
// time keeps moving while the phase waits for its next item. Each draw
|
||||||
|
// of the walk shows the path it reached last on the line below. When
|
||||||
// stderr is not a TTY it emits no ANSI redraws: it prints line as the
|
// stderr is not a TTY it emits no ANSI redraws: it prints line as the
|
||||||
// phase starts, every plainInterval while it runs, and as it ends.
|
// phase starts, every plainInterval while it runs, and as it ends.
|
||||||
//
|
//
|
||||||
@@ -141,6 +144,7 @@ type progress struct {
|
|||||||
estimate bool
|
estimate bool
|
||||||
waitingFor string
|
waitingFor string
|
||||||
stalledOn string
|
stalledOn string
|
||||||
|
path string // the path the walk reached last; "" for none
|
||||||
count int64
|
count int64
|
||||||
start time.Time // when the phase started, for the elapsed time
|
start time.Time // when the phase started, for the elapsed time
|
||||||
rateStart time.Time // when the count started, for the time remaining
|
rateStart time.Time // when the count started, for the time remaining
|
||||||
@@ -239,6 +243,82 @@ func (p *progress) draw() {
|
|||||||
// less the terminal's last column, which spinnerMargin keeps clear. A
|
// less the terminal's last column, which spinnerMargin keeps clear. A
|
||||||
// bar's cells take one column each but more than one byte.
|
// bar's cells take one column each but more than one byte.
|
||||||
p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
|
p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
|
||||||
|
|
||||||
|
if p.path != "" {
|
||||||
|
// The library draws the path on the line below and returns to
|
||||||
|
// the phase's line. The path is escaped as warnings are, and its
|
||||||
|
// other control characters are written out, so that the terminal
|
||||||
|
// shows them instead of acting on them. It stays clear of the
|
||||||
|
// last column, so that no terminal wraps it.
|
||||||
|
shown := escapeControlChars(escapePath(p.path))
|
||||||
|
_ = p.bar.AddDetail(cutToColumns(shown, width-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// escapeControlChars returns s with each control character written as
|
||||||
|
// \x and its code in two hex digits, as \x1b for the escape character.
|
||||||
|
// The control characters are U+0000 to U+001F, U+007F and U+0080 to
|
||||||
|
// U+009F, as unicode.IsControl reports them. A byte from 0x80 to 0x9F
|
||||||
|
// that is not part of a valid UTF-8 character is written the same way,
|
||||||
|
// as \x9b, since a terminal not in UTF-8 mode acts on it. Every other
|
||||||
|
// byte is kept as it is.
|
||||||
|
func escapeControlChars(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
for rest := s; rest != ""; {
|
||||||
|
char, size := utf8.DecodeRuneInString(rest)
|
||||||
|
if char == utf8.RuneError && size == 1 {
|
||||||
|
// Not part of a valid UTF-8 character. A terminal not in
|
||||||
|
// UTF-8 mode reads the byte as the character with its code,
|
||||||
|
// so that 0x9b is a control character there and 0xe9 is not.
|
||||||
|
char = rune(rest[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
if unicode.IsControl(char) {
|
||||||
|
fmt.Fprintf(&b, `\x%02x`, char)
|
||||||
|
} else {
|
||||||
|
b.WriteString(rest[:size])
|
||||||
|
}
|
||||||
|
|
||||||
|
rest = rest[size:]
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// cutToColumns returns s cut to at most columns terminal columns, at a
|
||||||
|
// character boundary. A wide character, as in Chinese, takes two
|
||||||
|
// columns.
|
||||||
|
func cutToColumns(s string, columns int) string {
|
||||||
|
used, end, state := 0, 0, -1
|
||||||
|
|
||||||
|
for rest := s; rest != ""; {
|
||||||
|
char, next, width, nextState := uniseg.FirstGraphemeClusterInString(
|
||||||
|
rest, state)
|
||||||
|
if used+width > columns {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
used += width
|
||||||
|
end += len(char)
|
||||||
|
rest, state = next, nextState
|
||||||
|
}
|
||||||
|
|
||||||
|
return s[:end]
|
||||||
|
}
|
||||||
|
|
||||||
|
// clear erases the display from the terminal: its line, and the path on
|
||||||
|
// the line below when one is shown. p.mu must be held.
|
||||||
|
func (p *progress) clear() {
|
||||||
|
if p.bar == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.path != "" {
|
||||||
|
_ = p.bar.AddDetail("")
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = p.bar.Clear()
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBar has the library make what draw draws: its spinner, followed by
|
// newBar has the library make what draw draws: its spinner, followed by
|
||||||
@@ -263,6 +343,8 @@ func (p *progress) newBar(width int) {
|
|||||||
// The text carries the elapsed time and the time remaining.
|
// The text carries the elapsed time and the time remaining.
|
||||||
progressbar.OptionSetElapsedTime(false),
|
progressbar.OptionSetElapsedTime(false),
|
||||||
progressbar.OptionSetPredictTime(false),
|
progressbar.OptionSetPredictTime(false),
|
||||||
|
// The line below the phase's line, for the walk's path.
|
||||||
|
progressbar.OptionSetMaxDetailRow(1),
|
||||||
)
|
)
|
||||||
p.widest = 0
|
p.widest = 0
|
||||||
}
|
}
|
||||||
@@ -314,6 +396,19 @@ func (p *progress) increment() {
|
|||||||
p.lastItem = now
|
p.lastItem = now
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setPath records the path the walk reached last, which a display on a
|
||||||
|
// terminal shows at its next draw.
|
||||||
|
func (p *progress) setPath(path string) {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
|
p.path = path
|
||||||
|
}
|
||||||
|
|
||||||
// warnf prints a one-line warning to stderr without corrupting the
|
// warnf prints a one-line warning to stderr without corrupting the
|
||||||
// display. The whole message is escaped like a report's path columns,
|
// display. The whole message is escaped like a report's path columns,
|
||||||
// so a path holding a newline cannot split the warning.
|
// so a path holding a newline cannot split the warning.
|
||||||
@@ -327,11 +422,9 @@ func (p *progress) warnf(format string, args ...any) {
|
|||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
if p.bar != nil {
|
// The warning takes the display's line; the next draw shows the
|
||||||
// The warning takes the display's line; the next draw shows
|
// display again, on the line below.
|
||||||
// the display again, on the line below.
|
p.clear()
|
||||||
_ = p.bar.Clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr, msg)
|
fmt.Fprintln(os.Stderr, msg)
|
||||||
}
|
}
|
||||||
@@ -392,9 +485,7 @@ func (w *progressWriter) Flush() error {
|
|||||||
w.p.mu.Lock()
|
w.p.mu.Lock()
|
||||||
defer w.p.mu.Unlock()
|
defer w.p.mu.Unlock()
|
||||||
|
|
||||||
if w.p.bar != nil {
|
w.p.clear()
|
||||||
_ = w.p.bar.Clear()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := w.out.Write(w.buf)
|
_, err := w.out.Write(w.buf)
|
||||||
@@ -414,6 +505,13 @@ func (p *progress) finish() {
|
|||||||
close(p.stop)
|
close(p.stop)
|
||||||
<-p.done
|
<-p.done
|
||||||
|
|
||||||
|
// The walk's path is only for someone watching the walk. Its line is
|
||||||
|
// erased, and the next phase is drawn on it.
|
||||||
|
p.mu.Lock()
|
||||||
|
p.clear()
|
||||||
|
p.path = ""
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
p.draw()
|
p.draw()
|
||||||
|
|
||||||
if p.width != nil {
|
if p.width != nil {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -1138,3 +1139,164 @@ func TestWarningAfterLastItemKeepsBar(t *testing.T) {
|
|||||||
t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
|
t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCutToColumns(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
s string
|
||||||
|
columns int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"/srv/a", 10, "/srv/a"},
|
||||||
|
{"/srv/photos", 8, "/srv/pho"},
|
||||||
|
// Each of these two fullwidth letters takes two columns.
|
||||||
|
{"/AB/a", 4, "/A"},
|
||||||
|
{"/AB/a", 5, "/AB"},
|
||||||
|
// An accent that combines with the letter before stays with it.
|
||||||
|
{"/café/a", 5, "/café"},
|
||||||
|
{"/srv", 0, ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := cutToColumns(tc.s, tc.columns); got != tc.want {
|
||||||
|
t.Errorf("cutToColumns(%q, %d) = %q, want %q",
|
||||||
|
tc.s, tc.columns, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEscapeControlChars(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := map[string]string{
|
||||||
|
"/srv/music": "/srv/music",
|
||||||
|
"/a\x1b[2Jb": `/a\x1b[2Jb`,
|
||||||
|
"/a\x00\a\x7fb": `/a\x00\x07\x7fb`,
|
||||||
|
// U+009B takes two bytes in UTF-8.
|
||||||
|
"/a\u009b2Jb": `/a\x9b2Jb`,
|
||||||
|
// Byte 0x9b alone is not valid UTF-8. A terminal not in UTF-8
|
||||||
|
// mode acts on it, as on every byte from 0x80 to 0x9f.
|
||||||
|
"/a\x9b2Jb": `/a\x9b2Jb`,
|
||||||
|
"/a\x80\x9f\xa0b": `/a\x80\x9f` + "\xa0b",
|
||||||
|
// In ě, which is c4 9b in UTF-8, byte 0x9b is part of a valid
|
||||||
|
// character.
|
||||||
|
"/aě2Jb": "/aě2Jb",
|
||||||
|
"/café/AB": "/café/AB",
|
||||||
|
"/latin1-caf\xe9": "/latin1-caf\xe9",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := escapeControlChars(in); got != want {
|
||||||
|
t.Errorf("escapeControlChars(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pathLines returns the text of each line the library wrote below a
|
||||||
|
// display's line. It writes one as an erase to the end of the line, a
|
||||||
|
// carriage return, the text and a newline.
|
||||||
|
func pathLines(out string) []string {
|
||||||
|
matches := regexp.MustCompile("\x1b\\[K\r([^\n]*)\n").
|
||||||
|
FindAllStringSubmatch(out, -1)
|
||||||
|
lines := make([]string, 0, len(matches))
|
||||||
|
|
||||||
|
for _, m := range matches {
|
||||||
|
lines = append(lines, m[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTerminalWalkShowsPathBelow checks that the walk's display on a
|
||||||
|
// terminal 30 columns wide shows the path it reached last on the line
|
||||||
|
// below, cut to 29 columns. That line is erased before a warning is
|
||||||
|
// printed, and again when the walk ends, so the next phase is drawn on
|
||||||
|
// an empty line.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
|
func TestTerminalWalkShowsPathBelow(t *testing.T) {
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
p := startProgress(progressParams{
|
||||||
|
label: scanPhase("walk"), unit: scanUnit, total: -1,
|
||||||
|
}, func() int { return 30 }, testRedraw, time.Now)
|
||||||
|
|
||||||
|
const shown = "/srv/photos/2026/holiday/beac"
|
||||||
|
|
||||||
|
p.setPath("/srv/photos/2026/holiday/beach.jpg")
|
||||||
|
p.increment()
|
||||||
|
awaitOutput(t, stderr, shown, 1)
|
||||||
|
|
||||||
|
p.warnf("warning")
|
||||||
|
|
||||||
|
before, _, _ := strings.Cut(stderr(), "warning")
|
||||||
|
if lines := pathLines(before); lines[len(lines)-1] != "" {
|
||||||
|
t.Errorf("line below shows %q as the warning is printed, want it "+
|
||||||
|
"empty", lines[len(lines)-1])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drawn again below the warning.
|
||||||
|
awaitOutput(t, stderr, shown, strings.Count(before, shown)+1)
|
||||||
|
p.finish()
|
||||||
|
|
||||||
|
lines := pathLines(stderr())
|
||||||
|
if !slices.Contains(lines, shown) {
|
||||||
|
t.Errorf("lines below the display = %q, want %q among them",
|
||||||
|
lines, shown)
|
||||||
|
}
|
||||||
|
|
||||||
|
if last := lines[len(lines)-1]; last != "" {
|
||||||
|
t.Errorf("line below the display shows %q after the walk, want it "+
|
||||||
|
"empty", last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTerminalWalkEscapesControlChars checks that a walked path holding
|
||||||
|
// escape characters, which would clear the screen and set the window
|
||||||
|
// title, puts none on the line below the walk's display. The line shows
|
||||||
|
// each control character written out, as \x1b.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
|
func TestTerminalWalkEscapesControlChars(t *testing.T) {
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
p := startProgress(progressParams{
|
||||||
|
label: scanPhase("walk"), unit: scanUnit, total: -1,
|
||||||
|
}, func() int { return 80 }, testRedraw, time.Now)
|
||||||
|
|
||||||
|
p.setPath("/srv/\x1b[2J\x1b]0;title\a/beach.jpg")
|
||||||
|
p.increment()
|
||||||
|
awaitOutput(t, stderr, "beach.jpg", 1)
|
||||||
|
p.finish()
|
||||||
|
|
||||||
|
lines := pathLines(stderr())
|
||||||
|
for _, line := range lines {
|
||||||
|
if strings.Contains(line, "\x1b") {
|
||||||
|
t.Errorf("line below the display = %q, want no escape "+
|
||||||
|
"character", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const shown = `/srv/\x1b[2J\x1b]0;title\x07/beach.jpg`
|
||||||
|
if !slices.Contains(lines, shown) {
|
||||||
|
t.Errorf("lines below the display = %q, want %q among them",
|
||||||
|
lines, shown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPlainWalkShowsNoPath checks that the walk's display prints no path
|
||||||
|
// when stderr is not a terminal.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
|
func TestPlainWalkShowsNoPath(t *testing.T) {
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
p := startProgress(progressParams{
|
||||||
|
label: scanPhase("walk"), unit: scanUnit, total: -1,
|
||||||
|
}, nil, testRedraw, time.Now)
|
||||||
|
|
||||||
|
p.setPath("/srv/a")
|
||||||
|
p.increment()
|
||||||
|
p.finish()
|
||||||
|
|
||||||
|
if strings.Contains(stderr(), "/srv/a") {
|
||||||
|
t.Errorf("stderr = %q, want no path", stderr())
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-14
@@ -98,17 +98,16 @@ func scanPhase(name string) string {
|
|||||||
// runs, and the lock is released after it. When ctx is cancelled, as by
|
// runs, and the lock is released after it. When ctx is cancelled, as by
|
||||||
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
||||||
// what it has hashed (see syncScan), prints how many files its walk
|
// what it has hashed (see syncScan), prints how many files its walk
|
||||||
// reached, and returns errInterrupted. workers must be at least 1; the
|
// reached, and returns errInterrupted. opts.workers must be at least 1;
|
||||||
// scan command rejects anything less. Before the lock is taken, the
|
// the scan and dwim commands reject anything less. Before the lock is
|
||||||
// scan lists the filesystems that are not persistent, on which the walk
|
// taken, the scan lists the filesystems that are not persistent, on
|
||||||
// records nothing; unless noDefaultExcludes is set, the walk does not
|
// which the walk records nothing; unless opts.noDefaultExcludes is set,
|
||||||
// enter the default excludes either. The walk neither enters nor
|
// the walk does not enter the default excludes either. The walk neither
|
||||||
// records the paths in excludePaths, the --exclude values; an operand
|
// enters nor records the paths in opts.excludePaths, the --exclude
|
||||||
// at or below one is a fatal error. Once the database is open, the scan
|
// values; an operand at or below one is a fatal error. Once the
|
||||||
// notes its files, which the walk passes over.
|
// database is open, the scan notes its files, which the walk passes
|
||||||
func runScan(ctx context.Context, roots []string, workers int,
|
// over.
|
||||||
oneFS, noDefaultExcludes bool, excludePaths []string,
|
func runScan(ctx context.Context, roots []string, opts scanOptions) error {
|
||||||
) error {
|
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -123,7 +122,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
|
|
||||||
var rules walkRules
|
var rules walkRules
|
||||||
|
|
||||||
rules.excludePaths, err = excludePathInodes(excludePaths)
|
rules.excludePaths, err = excludePathInodes(opts.excludePaths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("--exclude: %w", err)
|
return fmt.Errorf("--exclude: %w", err)
|
||||||
}
|
}
|
||||||
@@ -138,7 +137,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
return fmt.Errorf("list the mounted filesystems: %w", err)
|
return fmt.Errorf("list the mounted filesystems: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !noDefaultExcludes {
|
if !opts.noDefaultExcludes {
|
||||||
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
|
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,7 +166,7 @@ func runScan(ctx context.Context, roots []string, workers int,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
|
st, err := syncScan(ctx, db, roots, opts.workers, opts.oneFS, rules)
|
||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
return interrupted(st.walked)
|
return interrupted(st.walked)
|
||||||
}
|
}
|
||||||
@@ -563,6 +562,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
|
|
||||||
s.st.walked++
|
s.st.walked++
|
||||||
|
|
||||||
|
prog.setPath(ev.rec.path)
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
old, ok := s.existing[ev.rec.path]
|
old, ok := s.existing[ev.rec.path]
|
||||||
|
|||||||
@@ -2196,7 +2196,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) {
|
|||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{
|
code := run([]string{
|
||||||
cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir,
|
cmdScan, workersFlag, strconv.Itoa(hashLeakWorkers), dir,
|
||||||
}, io.Discard, &stderr)
|
}, io.Discard, &stderr)
|
||||||
if code != exitFatal {
|
if code != exitFatal {
|
||||||
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
||||||
|
|||||||
Reference in new issue
Block a user