Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62b9cfc7ae |
No files matched your search
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
|
|||||||
|
|
||||||
BINARY := sfdupes
|
BINARY := sfdupes
|
||||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||||
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
LDFLAGS := -X main.Version=$(VERSION)
|
||||||
|
|
||||||
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
|||||||
# Default target: build the binary. Phony so go build (which has its
|
# Default target: build the binary. Phony so go build (which has its
|
||||||
# own build cache) always decides what to recompile.
|
# own build cache) always decides what to recompile.
|
||||||
sfdupes:
|
sfdupes:
|
||||||
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes
|
go build -ldflags "$(LDFLAGS)" -o $(BINARY)
|
||||||
|
|
||||||
build: sfdupes
|
build: sfdupes
|
||||||
|
|
||||||
|
|||||||
@@ -1,33 +1,537 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
- Take an issue from the `1.0.0` milestone on the tracker. Work that is not on
|
- take an issue from the `1.0.0` milestone on the tracker; work not yet on the
|
||||||
the tracker yet is filed as an issue first.
|
tracker gets filed as an issue first
|
||||||
- Branch from `next`.
|
- branch from `next`
|
||||||
- Do the work, with tests, in small focused commits.
|
- do the work, with tests, in small focused commits
|
||||||
- Delete Next Step from `TODO.md`, in the same commit as the work.
|
- record it at the top of Completed Steps (`TODO.md` changes in the same commit
|
||||||
- Push the branch and open a PR against `next`. Its title ends with
|
as the work)
|
||||||
` (closes #N)`.
|
- push the branch and open a PR against `next` whose title ends with
|
||||||
- An independent review gates each merge to `next`. Every finding is addressed
|
` (closes #N)`
|
||||||
or rebutted on the PR.
|
- an independent review gates each merge to `next`; every finding is addressed
|
||||||
- Only the owner merges `next` into `main`.
|
or explicitly rebutted on the PR
|
||||||
|
- only the owner merges `next` to `main`
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
- Pre-1.0.
|
- pre-1.0
|
||||||
- The tracker is authoritative for the pre-1.0 backlog. The open issues under
|
- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues
|
||||||
the `1.0.0` milestone are what remains before the tag.
|
under the `1.0.0` milestone are what remains before the tag, and this file
|
||||||
|
records history and process, not the queue
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
- Take the next issue from the `1.0.0` milestone,
|
- take the next issue from the `1.0.0` milestone on the tracker:
|
||||||
https://git.eeqj.de/sneak/sfdupes/milestone/17. The issues are not restated
|
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source
|
||||||
here, because a copy drifts as soon as the tracker moves.
|
of truth for what is left before 1.0.0. Individual issues are deliberately not
|
||||||
|
restated here; a copy in this file drifts out of date the moment the tracker
|
||||||
|
moves
|
||||||
|
|
||||||
|
# Completed Steps
|
||||||
|
|
||||||
|
- the hash phase gives every file the walk records three hashes: `head` and
|
||||||
|
`tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
|
||||||
|
of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
|
||||||
|
The 10 MiB and 50 MiB thresholds, the size census, the records stored without
|
||||||
|
hashes and the content phase are gone, so a scan has four phases and a rescan
|
||||||
|
reads only new and changed files. `info` no longer counts the records with
|
||||||
|
hashes, since every record has them (2026-10-25,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
||||||
|
|
||||||
|
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
|
||||||
|
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
|
||||||
|
stages of the `Dockerfile` the module download, and in `builder` the
|
||||||
|
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
|
||||||
|
until the base image, the install line, `go.mod` or `go.sum` changes, and
|
||||||
|
every step that lints, tests or compiles still comes after it;
|
||||||
|
`golangci-lint run` also keeps golangci-lint's cache. README says the same
|
||||||
|
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
|
||||||
|
|
||||||
|
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
|
||||||
|
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
|
||||||
|
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
|
||||||
|
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
|
||||||
|
early build without a database. When the linker set no version,
|
||||||
|
`sfdupes --version` prints the main module's version that Go recorded in the
|
||||||
|
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
|
||||||
|
when neither is known (2026-10-17,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/116)
|
||||||
|
|
||||||
|
- `scan` creates the database's directory where the database path leads: the
|
||||||
|
path with its last element cut off as text, never cleaned, so a `..` after a
|
||||||
|
symlinked directory is followed through the symlink, as when the file is
|
||||||
|
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
|
||||||
|
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
|
||||||
|
used to create `/srv/new` and then fail (2026-10-17,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
||||||
|
|
||||||
|
- `scan` never records its own database: once the database and its `.lock` file
|
||||||
|
are open, it notes the device and inode of the database file, its `-wal` and
|
||||||
|
`-shm` files and its `.lock` file, and the walk passes over a file with one of
|
||||||
|
them wherever the operands reach it, through a symlinked directory included,
|
||||||
|
printing no line and counting nothing. A record an earlier scan left for one
|
||||||
|
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
||||||
|
|
||||||
|
- `scan` never records or reads anything that is not on a persistent filesystem
|
||||||
|
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
|
||||||
|
matched by the device ID of each such filesystem, listed as the scan starts
|
||||||
|
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
|
||||||
|
however the walk or the content phase reaches it; no flag turns that off.
|
||||||
|
Below an operand, its walk does not enter the default excludes either, so
|
||||||
|
`sfdupes scan /` skips the kernel's view of the running system and the
|
||||||
|
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
|
||||||
|
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
|
||||||
|
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
|
||||||
|
Windows list, not used yet. An operand at or inside a default exclude is still
|
||||||
|
walked. The scan prints a line for each path it leaves out, without counting
|
||||||
|
it as skipped, and the records there count as outside the scanned operands, as
|
||||||
|
for `-x`. `--no-default-excludes` turns off the default excludes only, and
|
||||||
|
`scan --help` lists them (2026-10-17,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
||||||
|
|
||||||
|
- the rate a progress display shows covers the items completed over the last 5
|
||||||
|
seconds, or since the line before when stderr is not a terminal, so it falls
|
||||||
|
to 0 while nothing completes, and a burst after a wait no longer reads as a
|
||||||
|
steady rate of over 1,000 files a second; the ETA still comes from the phase's
|
||||||
|
average. Once no item has completed for 10 seconds, the word `stalled` and
|
||||||
|
where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
|
||||||
|
`db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
|
||||||
|
either, each phase giving its word where it is defined. A line too wide for
|
||||||
|
the terminal loses its bar first, then its rate or those words, so the phase,
|
||||||
|
progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
|
||||||
|
like those of README's example, the content phase's checks and reads lose
|
||||||
|
those words at most times in minutes, and every other phase keeps them
|
||||||
|
whenever the elapsed time and the ETA are both in minutes. The content phase's
|
||||||
|
last partial batch is now written under a display of its own (2026-10-16,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/125)
|
||||||
|
|
||||||
|
- new subcommand `info` prints the database's details on stdout, one per line as
|
||||||
|
a name, a tab and the value: its path and where it came from, the sizes of the
|
||||||
|
database file and of its `-wal` file when there is one, when the database file
|
||||||
|
was last changed, the schema version, the number of records, how many have
|
||||||
|
head and tail hashes and how many a content hash, and the sizes of all records
|
||||||
|
added up. It opens the database as `report` does, and shows two phases on
|
||||||
|
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
|
||||||
|
from one read-only transaction, the counts from one query (2026-10-16,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/117)
|
||||||
|
|
||||||
|
- `report` shows its work on stderr in two phases, `[1/2] open` and
|
||||||
|
`[2/2] read`, with the progress display `scan` and `trees` use, from the
|
||||||
|
moment it starts until the summary line: `open` says it is opening the
|
||||||
|
database, also while it waits on a busy one; `read` counts the records, then
|
||||||
|
reads them against that count in signature order, which SQLite takes from the
|
||||||
|
`files_signature` index without sorting, so the first record comes at once.
|
||||||
|
`report` forms the duplicate groups itself, holding only those of the size it
|
||||||
|
is reading, and writes them ordered by first path before it reads the next
|
||||||
|
size. stdout and the summary line are unchanged, and SQLite no longer needs a
|
||||||
|
temporary file to sort in (2026-10-16,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/114)
|
||||||
|
|
||||||
|
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
|
||||||
|
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
|
||||||
|
from the moment it starts until the summary line: `open` says it is opening
|
||||||
|
the database, also while it waits on a busy one; `read` counts the records,
|
||||||
|
then reads them against that count; `group` counts the directories it groups,
|
||||||
|
and `write` the groups it writes. A display's rate names what its phase
|
||||||
|
counts: files, records, directories or groups. stdout and the summary line are
|
||||||
|
unchanged. When stdout and stderr are the same file, as one terminal is, none
|
||||||
|
of stdout's lines shares a line with the display; when they are not, the
|
||||||
|
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/113)
|
||||||
|
|
||||||
|
- each `scan` phase's progress display names the phase and its place among the
|
||||||
|
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
|
||||||
|
time and ETA: `load` counts the records first, the walk's total is estimated
|
||||||
|
from the records under the operands, and the content phase counts the records
|
||||||
|
it checks from SQLite's grouping alone, then says it is searching the database
|
||||||
|
until the first one is returned. On a terminal a phase with an exact total
|
||||||
|
draws its line followed by the library's bar, and any other a spinner followed
|
||||||
|
by the line; every display is drawn again several times a second. No line
|
||||||
|
reaches the terminal's last column, however narrow, also after the terminal is
|
||||||
|
made narrower during a phase: a line too wide leaves out its rate, then its
|
||||||
|
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
|
||||||
|
only when those alone do not fit, as below about 70 columns late in a large
|
||||||
|
scan. In plain mode the line is printed every 5 seconds whether or not an item
|
||||||
|
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
|
||||||
|
|
||||||
|
- `scan`, `report` and `trees` first print a line on stderr naming the database
|
||||||
|
they use, as an absolute path, and whether that path is the default or comes
|
||||||
|
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
|
||||||
|
"Database" says when `scan` writes each file (2026-10-16,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/111)
|
||||||
|
|
||||||
|
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
||||||
|
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
||||||
|
that write no image, and `make test` runs the suite under the race detector,
|
||||||
|
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
||||||
|
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
||||||
|
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
|
||||||
|
every build, while the steps that compile keep Go's build cache and the tests
|
||||||
|
run with `-count=1`; prettier runs on the host, from the node and yarn
|
||||||
|
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
|
||||||
|
and the build stage installs `git` and `make` itself; a new push cancels the
|
||||||
|
workflow's older run on the same branch, and a run stops after 20 minutes;
|
||||||
|
`.claude/settings.json` is deleted (2026-10-15,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
|
||||||
|
`test` and the build stage, and the install of `git` and `make` in the build
|
||||||
|
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
|
||||||
|
every step that lints, tests or compiles still runs on every build
|
||||||
|
|
||||||
|
- with `-x`, an operand under another operand is walked on its own instead of
|
||||||
|
being dropped when a directory from it (from its directory, for a regular
|
||||||
|
file) up to but not including that operand is on a different filesystem than
|
||||||
|
that operand, whose walk then cannot reach it, and the records of files gone
|
||||||
|
from below it are deleted (2026-10-14,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
||||||
|
|
||||||
|
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
||||||
|
under its operand, and counts their sizes in the size census, instead of
|
||||||
|
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
||||||
|
|
||||||
|
- the hash and content phases open each file without blocking and check that it
|
||||||
|
is still a regular file, so one replaced by a FIFO or device node after it was
|
||||||
|
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
||||||
|
|
||||||
|
- the scan lock file is named from the database's path with symlinks resolved,
|
||||||
|
after `scan` creates the database file if it is missing, so a scan through a
|
||||||
|
symlink to the database fails while another scan holds the lock (2026-10-14,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
||||||
|
|
||||||
|
- on a terminal, a warning for the last file of a phase no longer erases the
|
||||||
|
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
||||||
|
|
||||||
|
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
||||||
|
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
||||||
|
within the same second is re-hashed (2026-10-07,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/12)
|
||||||
|
|
||||||
|
- cut the narration from `TODO.md` Completed Steps and from the comments in
|
||||||
|
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
|
||||||
|
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
|
||||||
|
|
||||||
|
- `make test-race` ran the test suite under the race detector in a cgo-enabled
|
||||||
|
container, outside `make check` (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/18). Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
|
||||||
|
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
|
||||||
|
`make test-race` is gone
|
||||||
|
|
||||||
|
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
|
||||||
|
rather than serve the gates from cache (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
||||||
|
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
||||||
|
`script/` pass a new `CACHEBUST` value, so theirs always run
|
||||||
|
|
||||||
|
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
||||||
|
it; all Markdown reformatted (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
||||||
|
|
||||||
|
- `script/lint` writes no image, so a run no longer leaves an untagged one
|
||||||
|
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
|
||||||
|
|
||||||
|
- tests cover a missing database, `scan` keeping stdout empty, its skip warning,
|
||||||
|
the `report` and `trees` summary lines, and every subcommand going through
|
||||||
|
`runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
|
||||||
|
|
||||||
|
- `.golangci.yml` replaced with the current canonical copy, which uses
|
||||||
|
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
||||||
|
|
||||||
|
- a test fails when either `hashWorker` cancellation check in `scan.go` is
|
||||||
|
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
|
||||||
|
|
||||||
|
- a database path holding `?`, `#` or `%` opens exactly the file it names
|
||||||
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
||||||
|
|
||||||
|
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
||||||
|
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
||||||
|
|
||||||
|
- a test fails when either walk cancellation check in `scan.go` is removed
|
||||||
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
||||||
|
|
||||||
|
- test that `scan` refuses a database with another schema version (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/64)
|
||||||
|
|
||||||
|
- correct four inaccurate comments in `cancel_test.go` and rename
|
||||||
|
`walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
||||||
|
|
||||||
|
- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
||||||
|
|
||||||
|
- `scan` creates the schema in one transaction; a version-0 database with a
|
||||||
|
`files` table is refused with a clear schema-version error (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
||||||
|
|
||||||
|
- README documents install, Docker, a daily cron scan and how to read and check
|
||||||
|
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
|
||||||
|
|
||||||
|
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home
|
||||||
|
and copied the sources with `--chown`, so no `chown -R` walked them
|
||||||
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
|
||||||
|
nothing changes owner: the build stage only compiles, as root, and the tests
|
||||||
|
run as `nobody` in the `test` phase
|
||||||
|
|
||||||
|
- `--version` prints `sfdupes VERSION` to stdout; README documents it and
|
||||||
|
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
|
||||||
|
|
||||||
|
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed,
|
||||||
|
deletes nothing more, exits 1 (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/5)
|
||||||
|
|
||||||
|
- `report` and `trees` stream the records instead of holding them all in memory;
|
||||||
|
the schema gains the `files_signature` index (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/14)
|
||||||
|
|
||||||
|
- progress prints at once on a non-terminal, uses a real terminal test, and
|
||||||
|
prints warnings through a spinner instead of racing its redraw (2026-10-03,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/13)
|
||||||
|
|
||||||
|
- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping
|
||||||
|
the records beneath them (2026-10-03,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/9)
|
||||||
|
|
||||||
|
- `scan` holds a lock on a lock file beside the database for its whole run, so a
|
||||||
|
second `scan` fails at once with exit 1 (2026-10-03,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/53)
|
||||||
|
|
||||||
|
- test stdout write failures in `report` and `trees`; README states that
|
||||||
|
`| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null`
|
||||||
|
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
|
||||||
|
|
||||||
|
- `report` and `trees` open the database read-only, and `scan` leaves it out of
|
||||||
|
WAL mode, so reading needs only read access (2026-10-03, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/8)
|
||||||
|
|
||||||
|
- escape tabs, newlines, carriage returns and backslashes in report, trees and
|
||||||
|
warning paths; the root directory's path is `/` (2026-10-03,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/7)
|
||||||
|
|
||||||
|
- stamp the git tag or short commit in a plain `docker build .` instead of `dev`
|
||||||
|
(2026-10-02, branch `next`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git`
|
||||||
|
without `.git/config`; the build stage stamps the `VERSION` build argument,
|
||||||
|
else `git describe --tags --always`, and fails if the context carries `.git`
|
||||||
|
and the version is still empty, `dev` or `unknown`. CI checks out the full
|
||||||
|
history (`fetch-depth: 0`) so it stamps the same value as `make build`.
|
||||||
|
|
||||||
|
- replace the 1 KiB end-window sampling with the head/tail plus content-hash
|
||||||
|
ladder (2026-09-22, branch `next`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection"
|
||||||
|
documents every rung. A file under 10 MiB is hashed in full, and its `head`,
|
||||||
|
`tail` and `content` all hold that hash. A larger file gets only its 64 KiB
|
||||||
|
`head` and `tail` in the hash phase; the content phase, after the update
|
||||||
|
phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1
|
||||||
|
MiB samples at or above) only when its size, `head` and `tail` match another
|
||||||
|
record's from this scan or an earlier one, and never reads a file gone or
|
||||||
|
changed since its record was written. `report` and `trees` leave out any
|
||||||
|
record without a `content` hash. The `content` column is part of the version 1
|
||||||
|
schema.
|
||||||
|
|
||||||
|
- remove the dead `files.dat` references from `Makefile`, `.gitignore` and
|
||||||
|
`.dockerignore` (2026-09-21, branch `next`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/22)
|
||||||
|
|
||||||
|
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and
|
||||||
|
`Dockerfile.lint` (2026-08-10, branch `next`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy
|
||||||
|
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
|
||||||
|
without the false `(Debian-based)` note or the tag. Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
|
||||||
|
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and
|
||||||
|
`script/verify-lint-image-pin`, which compared the two pins, are gone.
|
||||||
|
|
||||||
|
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
|
||||||
|
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
|
||||||
|
owner ruling the linter is never installed on a host, and
|
||||||
|
`golangci-lint config verify` runs before `golangci-lint run`.
|
||||||
|
`script/bootstrap` stopped installing or pinning the linter, and
|
||||||
|
`script/verify-linter-pin` was retired. Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
|
||||||
|
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
|
||||||
|
stage depends on through `COPY --from=lint /src/go.sum /dev/null`;
|
||||||
|
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
|
||||||
|
an image build may run docker, so the phase calls `golangci-lint` directly.
|
||||||
|
|
||||||
|
- install the Docker build stage's prerequisites by running `script/bootstrap`
|
||||||
|
instead of `apk add --no-cache make` inline (2026-08-09, branch
|
||||||
|
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42),
|
||||||
|
with `script/verify-linter-pin` failing the build unless the linter copied
|
||||||
|
from the lint stage was the version `script/bootstrap` pinned. Builds then
|
||||||
|
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/43. Since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
|
||||||
|
`make` with `apk add --no-cache` and only compiles; the `lint` and `test`
|
||||||
|
phases are the gates
|
||||||
|
- bust the Docker layer cache for the gate steps, so `script/cibuild` and
|
||||||
|
`script/docker` cannot report a green they did not earn (2026-08-09, branch
|
||||||
|
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
|
||||||
|
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
|
||||||
|
served them from cache and the build exited 0 having run nothing. The fix was
|
||||||
|
a `CHECK_EPOCH` build argument; since
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
||||||
|
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
||||||
|
declares before every step that lints, tests or compiles. Run as root, the
|
||||||
|
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
|
||||||
|
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
|
||||||
|
- check the installed golangci-lint version in `script/bootstrap` instead of
|
||||||
|
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
|
||||||
|
`GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and
|
||||||
|
any installed version that is not the pin — older, newer, absent or
|
||||||
|
unparseable — is reinstalled. `go install` writes into `GOBIN` (or
|
||||||
|
`GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so
|
||||||
|
bootstrap re-reads the effective version after installing and, on a mismatch,
|
||||||
|
prints both paths and both versions and exits non-zero; it does not reorder
|
||||||
|
`PATH` or delete anyone's binary. The `--version` call keeps its stderr and is
|
||||||
|
bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep
|
||||||
|
presence-only checks. Verified by bootstrapping this host from v2.10.1 to
|
||||||
|
v2.12.2 and again to a no-op, and by stub runs of the script under `dash`
|
||||||
|
covering a thirteen-input version-parse matrix, a shadowed install that must
|
||||||
|
exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged
|
||||||
|
binary that must hit the timeout; `make check` and `make lint` are clean at
|
||||||
|
v2.12.2, so v2.10.1 was not hiding any findings on `main`
|
||||||
|
- unwind the hash worker pool on the error path (2026-08-09, branch
|
||||||
|
`hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the
|
||||||
|
pool is now an owned, context-aware `hashPool`: every blocking send in the
|
||||||
|
feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path
|
||||||
|
out, and `hashPhase` defers `pool.stop()`, which cancels and then drains
|
||||||
|
`results` until the last goroutine has exited — draining is what frees a
|
||||||
|
worker already parked on a send. `ctx` is threaded from `cmd.Context()`
|
||||||
|
through `runScan`, `syncScan`, both worker pools and the whole database layer,
|
||||||
|
as the first parameter everywhere. The walk pool gets the same treatment plus
|
||||||
|
a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size
|
||||||
|
census, and every file it never reached looks vanished to the update phase.
|
||||||
|
That phase's own `BeginTx` also fails on the cancelled context before deleting
|
||||||
|
anything, but the guard is the barrier that still holds once an interrupted
|
||||||
|
scan may commit what it has. Tests drive `run(scan)` against a database whose
|
||||||
|
insert trigger aborts and assert that the scan fails instead of hanging and
|
||||||
|
that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others
|
||||||
|
cancel a scan part-way through the walk, deterministically, by counting its
|
||||||
|
own consultations of `ctx.Done()`, and assert that it stops at the guard
|
||||||
|
holding a partial census and a still-populated record index, with every record
|
||||||
|
intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`,
|
||||||
|
`feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation
|
||||||
|
branches of both pools
|
||||||
|
- guarantee the database is closed on every fatal exit path (2026-08-09, branch
|
||||||
|
`db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4):
|
||||||
|
`fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and
|
||||||
|
with it the SQLite WAL checkpoint — now actually runs when a subcommand fails;
|
||||||
|
`runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return
|
||||||
|
errors instead. The single exit point is `run` in `main.go`: it maps a
|
||||||
|
`fatalError` (anything a subcommand returned) to exit 1 and cobra's own
|
||||||
|
argument and flag errors to exit 2, which keeps a runtime failure from being
|
||||||
|
reported as a usage error or printing the usage text. New `main_test.go`
|
||||||
|
drives the CLI in-process and asserts the exit codes from README §Error
|
||||||
|
handling plus the stdout/stderr split, including that a fatal error raised
|
||||||
|
after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`,
|
||||||
|
`report` or `trees`
|
||||||
|
- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch
|
||||||
|
`golangci-v2.12.2`, merged as `38a01bd`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the
|
||||||
|
`Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and
|
||||||
|
replaced `.golangci.yml` with the canonical file — the linter settings (`lll`,
|
||||||
|
`funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per
|
||||||
|
the v2 schema, so they are actually applied; no new lint findings surfaced
|
||||||
|
- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints
|
||||||
|
like the other managed repos (2026-07-26, commit `3abeacf`, closes
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints
|
||||||
|
exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
|
||||||
|
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`)
|
||||||
|
and every Makefile target is now a thin shim over them
|
||||||
|
- make the binary the default Make target (2026-07-24, branch
|
||||||
|
`make-default-target`): plain `make` now builds `sfdupes` (previously it ran
|
||||||
|
`check` plus `build`); `make build` remains as an alias
|
||||||
|
- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch
|
||||||
|
`scan-wide-phases`): all operands seed the shared walk pool and every pass
|
||||||
|
runs once over the whole scan, so totals and ETAs are scan-global; the
|
||||||
|
per-operand walk/hash/update cycles and their stderr announcements are gone;
|
||||||
|
the update pass commits in batched transactions — the filesystem is
|
||||||
|
authoritative and the database an eventually-consistent reflection, so
|
||||||
|
scan-level atomicity is not required
|
||||||
|
- split the stat pass back out of the walk (2026-07-24, branch
|
||||||
|
`parallel-phases`): phases are strictly sequential again — walk, stat, hash,
|
||||||
|
update per operand — with parallelism only inside each phase; the walk
|
||||||
|
enumerates paths with per-directory workers and the stat pass lstats them with
|
||||||
|
per-file workers, restoring the exact total/ETA stat bar
|
||||||
|
- announce each operand on stderr before its passes (2026-07-24, branch
|
||||||
|
`scan-operand-progress`): with per-operand walk/hash/update cycles, a
|
||||||
|
multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the
|
||||||
|
whole run's
|
||||||
|
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single
|
||||||
|
goroutine and took hours at ~20M files on a busy pool (observed: 22M files in
|
||||||
|
4h on a ZFS server); it is now a per-directory worker-pool traversal that
|
||||||
|
records size/mtime during the walk (folding away the separate stat pass,
|
||||||
|
halving metadata I/O), and each `PATH` operand commits in its own transaction
|
||||||
|
so an interrupted scan keeps completed operands
|
||||||
|
|
||||||
|
- persistent scan database (2026-07-24, branch `persistent-database`): `scan`
|
||||||
|
now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays
|
||||||
|
disabled) keyed by absolute path that survives between runs — a rescan hashes
|
||||||
|
only new or changed files (by mtime/size), deletes records for files vanished
|
||||||
|
from under the scanned operands, and leaves records outside them untouched, so
|
||||||
|
`scan` can be cronned daily; `report` and `trees` read the database (no
|
||||||
|
positional arguments) instead of a scan stream. Database at
|
||||||
|
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL
|
||||||
|
journaling plus a single-transaction update keep a report run during a scan
|
||||||
|
safe
|
||||||
|
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`,
|
||||||
|
and push `main` plus tags (2026-07-23)
|
||||||
|
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
|
||||||
|
`PATH...` operands via cobra flags replacing the `/srv` `-root` default; new
|
||||||
|
`-x`/`--one-file-system` flag (GNU convention) to stop at filesystem
|
||||||
|
boundaries, which are crossed by default
|
||||||
|
- bring the repo into full policy compliance (2026-07-23, branch
|
||||||
|
`repo-policy-compliance`; checklist below)
|
||||||
|
- `git init` with README-only first commit; code baseline committed on `main`
|
||||||
|
(2026-07-22)
|
||||||
|
- implement `scan`, `report`, and `trees` subcommands (pre-git history)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Possible later features, out of scope per README, are a byte-for-byte check of
|
- possible later features (explicitly out of scope per README): full-content
|
||||||
candidates and helper scripts for removal.
|
verification of candidates, removal-script helpers
|
||||||
|
|
||||||
# Accepted Divergences
|
# Repo Policy Compliance
|
||||||
|
|
||||||
None.
|
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo
|
||||||
|
checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone
|
||||||
|
formatting commit is needed.
|
||||||
|
|
||||||
|
- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit
|
||||||
|
untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns
|
||||||
|
- [x] `.editorconfig` missing
|
||||||
|
- [x] `LICENSE` missing and README has no License section (MIT assumed from
|
||||||
|
house convention — user to confirm)
|
||||||
|
- [x] `REPO_POLICIES.md` missing from repo root
|
||||||
|
- [x] `.golangci.yml` missing (install canonical copy); code must then pass
|
||||||
|
`make lint` (150 findings fixed; `make lint` is clean)
|
||||||
|
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`,
|
||||||
|
`docker`, `hooks`; `check` currently depends on `build`, which writes the
|
||||||
|
binary (`make check` must not modify files)
|
||||||
|
- [x] no tests — `go test ./...` has nothing to run; policy requires real tests
|
||||||
|
with a 30-second timeout and the conditional `-v` rerun pattern (suite
|
||||||
|
covers parsing, grouping, digests, suppression, hashing, and the scan
|
||||||
|
pipeline; 64% coverage)
|
||||||
|
- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast
|
||||||
|
lint stage, build stage running `make check`
|
||||||
|
- [x] `.dockerignore` missing
|
||||||
|
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout
|
||||||
|
action pinned by commit SHA)
|
||||||
|
- [x] README lacks required sections: Description first line
|
||||||
|
(name/purpose/category/license/author), Getting Started, Rationale, TODO,
|
||||||
|
License, Author
|
||||||
|
- [x] README non-goal "no git repository setup and no CI" is stale now that the
|
||||||
|
repo is under git with CI
|
||||||
|
- [x] pre-commit hook not installed (`make hooks` once the target exists)
|
||||||
|
|
||||||
|
Accepted divergences (no action):
|
||||||
|
|
||||||
|
- flat single-package layout with `.go` files in the repo root — fine for a
|
||||||
|
small single-binary tool per the Go styleguide; the tracker audit agrees
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -358,9 +357,8 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
dir := buildSmokeTree(t)
|
dir := buildSmokeTree(t)
|
||||||
|
|
||||||
opts := scanOptions{workers: walkCancelWorkers}
|
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
||||||
|
false)
|
||||||
err := runScan(t.Context(), []string{dir}, opts)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -383,7 +381,8 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
|
||||||
err = runScan(cancelledContext(t), []string{dir}, opts)
|
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
|
||||||
|
false, false)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
awaitReturn(t, done, "runScan")
|
awaitReturn(t, done, "runScan")
|
||||||
@@ -410,7 +409,8 @@ func TestRunScanInterrupted(t *testing.T) {
|
|||||||
got, before)
|
got, before)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = runScan(t.Context(), []string{dir}, opts)
|
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
|
||||||
|
false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -436,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
|
|||||||
dir := buildWalkCancelTree(t)
|
dir := buildWalkCancelTree(t)
|
||||||
|
|
||||||
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
|
||||||
scanOptions{workers: walkCancelWorkers})
|
walkCancelWorkers, false, false)
|
||||||
if !errors.Is(err, errInterrupted) {
|
if !errors.Is(err, errInterrupted) {
|
||||||
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
|
||||||
err, errInterrupted)
|
err, errInterrupted)
|
||||||
@@ -762,9 +762,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
|
|||||||
|
|
||||||
var hashed atomic.Bool
|
var hashed atomic.Bool
|
||||||
|
|
||||||
hash := func(path string, size int64) (
|
hash := func(path string, size int64) (string, string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
hashed.Store(true)
|
hashed.Store(true)
|
||||||
|
|
||||||
return hashSignature(path, size)
|
return hashSignature(path, size)
|
||||||
@@ -804,9 +802,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
|
|||||||
|
|
||||||
// The scan is cancelled while the worker hashes, so the worker has
|
// The scan is cancelled while the worker hashes, so the worker has
|
||||||
// already passed the check that drops queued runs.
|
// already passed the check that drops queued runs.
|
||||||
hash := func(path string, size int64) (
|
hash := func(path string, size int64) (string, string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
return hashSignature(path, size)
|
return hashSignature(path, size)
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
// Command sfdupes quickly identifies candidate duplicate files across
|
|
||||||
// very large filesystems without reading every byte of every file.
|
|
||||||
// Files are considered duplicates when their sizes are equal and they
|
|
||||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
|
||||||
// KiB, and of the content. The content hash covers the whole file under
|
|
||||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
|
||||||
// file is never read in full. scan maintains a persistent SQLite
|
|
||||||
// database of file signatures (SFDUPES_DATABASE, default
|
|
||||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
|
||||||
//
|
|
||||||
// Usage:
|
|
||||||
//
|
|
||||||
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
|
||||||
// sfdupes report > dupes.tsv
|
|
||||||
// sfdupes trees > dupetrees.tsv
|
|
||||||
// sfdupes info
|
|
||||||
// sfdupes dwim
|
|
||||||
// sfdupes --version
|
|
||||||
//
|
|
||||||
// See README.md for the complete specification.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"sneak.berlin/go/sfdupes/internal/sfdupes"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() { os.Exit(sfdupes.Main()) }
|
|
||||||
+71
-215
@@ -1,8 +1,7 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -47,20 +46,18 @@ const dbFilePerm = 0o644
|
|||||||
const lockFilePerm = 0o600
|
const lockFilePerm = 0o600
|
||||||
|
|
||||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
|
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||||
// disk_size holds the bytes the file takes on disk. mtime holds whole
|
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||||
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
|
// second.
|
||||||
// tail and content each hold the raw 32 bytes of a SHA-256.
|
|
||||||
const createTableSQL = `
|
const createTableSQL = `
|
||||||
CREATE TABLE files (
|
CREATE TABLE files (
|
||||||
path BLOB PRIMARY KEY,
|
path BLOB PRIMARY KEY,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
disk_size INTEGER NOT NULL,
|
|
||||||
mtime INTEGER NOT NULL,
|
mtime INTEGER NOT NULL,
|
||||||
mtime_nsec INTEGER NOT NULL,
|
mtime_nsec INTEGER NOT NULL,
|
||||||
head BLOB NOT NULL,
|
head TEXT NOT NULL,
|
||||||
tail BLOB NOT NULL,
|
tail TEXT NOT NULL,
|
||||||
content BLOB NOT NULL
|
content TEXT NOT NULL
|
||||||
) WITHOUT ROWID
|
) WITHOUT ROWID
|
||||||
`
|
`
|
||||||
|
|
||||||
@@ -70,59 +67,14 @@ const createIndexSQL = `
|
|||||||
CREATE INDEX files_signature ON files (size, head, tail, content)
|
CREATE INDEX files_signature ON files (size, head, tail, content)
|
||||||
`
|
`
|
||||||
|
|
||||||
// createTotalsSQL creates the totals table, which holds one row: the
|
|
||||||
// number of records and the sizes of all records added up. Counting
|
|
||||||
// them over files would read every record. The three triggers below
|
|
||||||
// keep the row current, in the transaction of each change to files.
|
|
||||||
const createTotalsSQL = `
|
|
||||||
CREATE TABLE totals (
|
|
||||||
records INTEGER NOT NULL,
|
|
||||||
size INTEGER NOT NULL
|
|
||||||
)
|
|
||||||
`
|
|
||||||
|
|
||||||
// insertTotalsSQL adds the one row of totals, for an empty files table.
|
|
||||||
const insertTotalsSQL = `
|
|
||||||
INSERT INTO totals (records, size) VALUES (0, 0)
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterInsertSQL counts each inserted record into totals.
|
|
||||||
const totalsAfterInsertSQL = `
|
|
||||||
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
|
|
||||||
UPDATE totals SET records = records + 1, size = size + NEW.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterDeleteSQL takes each deleted record out of totals.
|
|
||||||
const totalsAfterDeleteSQL = `
|
|
||||||
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
|
|
||||||
UPDATE totals SET records = records - 1, size = size - OLD.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterUpdateSQL replaces a record's old size in totals with its
|
|
||||||
// new one. scan's upsert of an existing record is such an update.
|
|
||||||
const totalsAfterUpdateSQL = `
|
|
||||||
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
|
|
||||||
UPDATE totals SET size = size + NEW.size - OLD.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// recordCountSQL reads the number of records from totals.
|
|
||||||
const recordCountSQL = `
|
|
||||||
SELECT records FROM totals
|
|
||||||
`
|
|
||||||
|
|
||||||
// upsertSQL inserts one file record, replacing any existing record for
|
// upsertSQL inserts one file record, replacing any existing record for
|
||||||
// the same path.
|
// the same path.
|
||||||
const upsertSQL = `
|
const upsertSQL = `
|
||||||
INSERT INTO files (
|
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
|
||||||
path, size, disk_size, mtime, mtime_nsec, head, tail, content
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT (path) DO UPDATE SET
|
ON CONFLICT (path) DO UPDATE SET
|
||||||
size = excluded.size, disk_size = excluded.disk_size,
|
size = excluded.size, mtime = excluded.mtime,
|
||||||
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
head = excluded.head, tail = excluded.tail,
|
head = excluded.head, tail = excluded.tail,
|
||||||
content = excluded.content
|
content = excluded.content
|
||||||
`
|
`
|
||||||
@@ -140,50 +92,35 @@ var errSchemaVersion = errors.New("unsupported database schema version")
|
|||||||
// database.
|
// database.
|
||||||
var errScanRunning = errors.New("another scan is running")
|
var errScanRunning = errors.New("another scan is running")
|
||||||
|
|
||||||
// errHashSize reports a record whose head, tail or content is not the
|
|
||||||
// 32 bytes of a SHA-256.
|
|
||||||
var errHashSize = errors.New("hash is not 32 bytes")
|
|
||||||
|
|
||||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||||
// path, made absolute by absolutePath, and where the path came from,
|
// path, with the working directory, its symlinks resolved, put in front
|
||||||
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
// of it when it is relative, and where the path came from, "default" or
|
||||||
// prints first on stderr.
|
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
|
||||||
|
// stderr.
|
||||||
func databasePath() (string, string, error) {
|
func databasePath() (string, string, error) {
|
||||||
path, source := defaultDatabasePath, "default"
|
path, source := defaultDatabasePath, "default"
|
||||||
if p := os.Getenv(databaseEnv); p != "" {
|
if p := os.Getenv(databaseEnv); p != "" {
|
||||||
path, source = p, "from "+databaseEnv
|
path, source = p, "from "+databaseEnv
|
||||||
}
|
}
|
||||||
|
|
||||||
abs, err := absolutePath(path)
|
if filepath.IsAbs(path) {
|
||||||
|
return path, source, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The working directory as the kernel reports it, with no symlink
|
||||||
|
// in it, as README "Database" says. os.Getwd may return PWD
|
||||||
|
// instead, which can name it through a symlink.
|
||||||
|
wd, err := unix.Getwd()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return abs, source, nil
|
// Not filepath.Abs or filepath.Join: they clean the path, which
|
||||||
}
|
// drops each ".." together with the directory name before it. When
|
||||||
|
// that directory is a symlink, the cleaned path names another file.
|
||||||
// absolutePath returns path unchanged when it is absolute, and
|
|
||||||
// otherwise the working directory, a "/" and path. The working
|
|
||||||
// directory is the one the kernel reports, with no symlink in it;
|
|
||||||
// os.Getwd may return PWD instead, which can name it through a symlink.
|
|
||||||
// The result is not cleaned, as filepath.Abs and filepath.Join would
|
|
||||||
// clean it. Cleaning drops each ".." together with the directory name
|
|
||||||
// before it. When that name is a symlink, the cleaned path names
|
|
||||||
// another file than the one the system reaches by following the symlink
|
|
||||||
// first.
|
|
||||||
func absolutePath(path string) (string, error) {
|
|
||||||
if filepath.IsAbs(path) {
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
wd, err := unix.Getwd()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The working directory ends in "/" only when it is the root.
|
// The working directory ends in "/" only when it is the root.
|
||||||
return strings.TrimSuffix(wd, "/") + "/" + path, nil
|
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createDatabaseDirectory creates the directory the database at path
|
// createDatabaseDirectory creates the directory the database at path
|
||||||
@@ -207,14 +144,9 @@ func createDatabaseDirectory(path string) error {
|
|||||||
// scanParams are the connection parameters for scan: read-write, with
|
// scanParams are the connection parameters for scan: read-write, with
|
||||||
// WAL journaling and a busy timeout, so a report can run while a cron
|
// WAL journaling and a busy timeout, so a report can run while a cron
|
||||||
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
||||||
//
|
|
||||||
// Each write to files fires a trigger, so SQLite keeps a statement
|
|
||||||
// journal for it. temp_store(MEMORY) keeps that journal in memory,
|
|
||||||
// which costs far less than a temporary file.
|
|
||||||
const scanParams = "_pragma=busy_timeout(10000)" +
|
const scanParams = "_pragma=busy_timeout(10000)" +
|
||||||
"&_pragma=journal_mode(WAL)" +
|
"&_pragma=journal_mode(WAL)" +
|
||||||
"&_pragma=synchronous(NORMAL)" +
|
"&_pragma=synchronous(NORMAL)"
|
||||||
"&_pragma=temp_store(MEMORY)"
|
|
||||||
|
|
||||||
// reportParams are the connection parameters for report, trees and
|
// reportParams are the connection parameters for report, trees and
|
||||||
// info: read-only, with the same busy timeout. They set no journal
|
// info: read-only, with the same busy timeout. They set no journal
|
||||||
@@ -488,14 +420,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
|
|||||||
|
|
||||||
defer func() { _ = tx.Rollback() }()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
for _, stmt := range []string{
|
_, err = tx.ExecContext(ctx, createTableSQL)
|
||||||
createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
|
|
||||||
totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
|
|
||||||
} {
|
|
||||||
_, err = tx.ExecContext(ctx, stmt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create schema: %w", err)
|
return fmt.Errorf("create schema: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_, err = tx.ExecContext(ctx, createIndexSQL)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create schema: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = tx.ExecContext(ctx,
|
_, err = tx.ExecContext(ctx,
|
||||||
@@ -542,7 +474,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
|
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("count records: %w", err)
|
return fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -550,8 +482,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
total(n)
|
total(n)
|
||||||
|
|
||||||
rows, err := tx.QueryContext(ctx,
|
rows, err := tx.QueryContext(ctx,
|
||||||
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
|
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
|
||||||
"content FROM files ORDER BY path")
|
"FROM files ORDER BY path")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -560,25 +492,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path, head, tail, content []byte
|
path []byte
|
||||||
sec, nsec int64
|
sec, nsec int64
|
||||||
r scanRec
|
r scanRec
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
|
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
|
||||||
&head, &tail, &content)
|
&r.content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
r.path = string(path)
|
r.path = string(path)
|
||||||
r.mtime = time.Unix(sec, nsec)
|
r.mtime = time.Unix(sec, nsec)
|
||||||
|
|
||||||
err = r.setHashes(head, tail, content)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn(r)
|
fn(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,11 +527,10 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
|
|||||||
`
|
`
|
||||||
|
|
||||||
// loadSignatureRows gives total the number of records, then streams
|
// loadSignatureRows gives total the number of records, then streams
|
||||||
// every record's signature and path to fn, without its mtime or its
|
// every record's signature and path to fn, without its mtime, in the
|
||||||
// disk_size, in the order of signatureRowsSQL. The count and the
|
// order of signatureRowsSQL. The count and the records are read in one
|
||||||
// records are read in one transaction, so they agree while a scan is
|
// transaction, so they agree while a scan is committing. An error from
|
||||||
// committing. An error from fn stops the reading and is returned as it
|
// fn stops the reading and is returned as it is.
|
||||||
// is.
|
|
||||||
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||||
total func(n int64), fn func(r scanRec) error,
|
total func(n int64), fn func(r scanRec) error,
|
||||||
) error {
|
) error {
|
||||||
@@ -620,7 +545,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("count records: %w", err)
|
return fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -636,22 +561,17 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path, head, tail, content []byte
|
path []byte
|
||||||
r scanRec
|
r scanRec
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&r.size, &head, &tail, &content, &path)
|
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
r.path = string(path)
|
r.path = string(path)
|
||||||
|
|
||||||
err = r.setHashes(head, tail, content)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = fn(r)
|
err = fn(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -666,27 +586,11 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// setHashes sets r's head, tail and content to the values read from
|
|
||||||
// those columns. Each must be the raw 32 bytes of a SHA-256.
|
|
||||||
func (r *scanRec) setHashes(head, tail, content []byte) error {
|
|
||||||
for _, b := range [][]byte{head, tail, content} {
|
|
||||||
if len(b) != sha256.Size {
|
|
||||||
return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
r.head = [sha256.Size]byte(head)
|
|
||||||
r.tail = [sha256.Size]byte(tail)
|
|
||||||
r.content = [sha256.Size]byte(content)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// countRecords returns the number of records in the database.
|
// countRecords returns the number of records in the database.
|
||||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("count records: %w", err)
|
return 0, fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -694,10 +598,12 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseCountsSQL reads the number of records and the sizes of all
|
// databaseCountsSQL counts the records and adds up their sizes, in one
|
||||||
// records added up from totals.
|
// pass over the table. A SUM over no rows is NULL, so on an empty table
|
||||||
|
// the SUM falls back to 0.
|
||||||
const databaseCountsSQL = `
|
const databaseCountsSQL = `
|
||||||
SELECT records, size FROM totals
|
SELECT COUNT(*), COALESCE(SUM(size), 0)
|
||||||
|
FROM files
|
||||||
`
|
`
|
||||||
|
|
||||||
// databaseCounts is what info reads from the database: its schema
|
// databaseCounts is what info reads from the database: its schema
|
||||||
@@ -740,15 +646,15 @@ func readDatabaseCounts(ctx context.Context,
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFileMeta streams every record's path, size, disk_size and mtime
|
// loadFileMeta streams every record's path, size and mtime to fn. Scan
|
||||||
// to fn. Scan change detection needs no hash values, and skipping the
|
// change detection needs no hash values, and skipping the hash columns
|
||||||
// hash columns keeps the scan's in-memory index small on
|
// keeps the scan's in-memory index small on multi-million-file
|
||||||
// multi-million-file databases.
|
// databases.
|
||||||
func loadFileMeta(ctx context.Context, db *sql.DB,
|
func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||||
fn func(path string, m fileMeta),
|
fn func(path string, size int64, mtime time.Time),
|
||||||
) error {
|
) error {
|
||||||
rows, err := db.QueryContext(ctx,
|
rows, err := db.QueryContext(ctx,
|
||||||
"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
|
"SELECT path, size, mtime, mtime_nsec FROM files")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -758,18 +664,15 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path []byte
|
path []byte
|
||||||
m fileMeta
|
size, sec, nsec int64
|
||||||
sec, nsec int64
|
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
|
err = rows.Scan(&path, &size, &sec, &nsec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.mtime = time.Unix(sec, nsec)
|
fn(string(path), size, time.Unix(sec, nsec))
|
||||||
|
|
||||||
fn(string(path), m)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
@@ -787,21 +690,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
// WAL small and let concurrent reports observe progress.
|
// WAL small and let concurrent reports observe progress.
|
||||||
const updateBatchSize = 10000
|
const updateBatchSize = 10000
|
||||||
|
|
||||||
// applyChanges writes upserts for new and changed files and deletes for
|
// applyChanges writes one scan's database changes — upserts for new and
|
||||||
// vanished ones, in batched transactions. Progress is rendered on prog
|
// changed files, deletes for vanished ones — in batched transactions.
|
||||||
// (one increment per change).
|
// Progress is rendered on prog (one increment per change).
|
||||||
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, batch, nil, nil, prog)
|
err := applyBatch(ctx, db, batch, nil, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, nil, nil, batch, prog)
|
err := applyBatch(ctx, db, nil, batch, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -810,26 +713,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyDiskSizes writes the disk_size of each of recs into its record,
|
// applyBatch commits one batch of upserts and deletes in a single
|
||||||
// in batched transactions, and leaves the rest of the record alone.
|
// transaction.
|
||||||
// Progress is rendered on prog (one increment per record).
|
|
||||||
func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
for batch := range slices.Chunk(recs, updateBatchSize) {
|
|
||||||
err := applyBatch(ctx, db, nil, batch, nil, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyBatch commits one batch of upserts, disk_size updates and
|
|
||||||
// deletes in a single transaction.
|
|
||||||
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
diskSizes []fileRec, deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
tx, err := db.BeginTx(ctx, nil)
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -843,11 +730,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = execDiskSizes(ctx, tx, diskSizes, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = execDeletes(ctx, tx, deletes, prog)
|
err = execDeletes(ctx, tx, deletes, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -873,9 +755,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
defer func() { _ = st.Close() }()
|
defer func() { _ = st.Close() }()
|
||||||
|
|
||||||
for _, r := range upserts {
|
for _, r := range upserts {
|
||||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
|
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
|
||||||
r.head[:], r.tail[:], r.content[:])
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("upsert %s: %w", r.path, err)
|
return fmt.Errorf("upsert %s: %w", r.path, err)
|
||||||
}
|
}
|
||||||
@@ -886,31 +767,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// execDiskSizes sets the disk_size of the record of each of recs. It
|
|
||||||
// does not touch size, so the totals triggers do not fire.
|
|
||||||
func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
st, err := tx.PrepareContext(ctx,
|
|
||||||
"UPDATE files SET disk_size = ? WHERE path = ?")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("prepare disk_size update: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = st.Close() }()
|
|
||||||
|
|
||||||
for _, r := range recs {
|
|
||||||
_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("update disk_size of %s: %w", r.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prog.increment()
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// execDeletes removes the records for paths no longer present.
|
// execDeletes removes the records for paths no longer present.
|
||||||
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
package sfdupes //nolint:testpackage // calls the unexported database openers
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -474,73 +473,17 @@ func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
|
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
||||||
// content is not the 32 bytes of a SHA-256: either one byte, or the 64
|
// from the files_signature index alone, with nothing to sort: the
|
||||||
// bytes of a SHA-256 written as hex text. loadFileRows and
|
// first record comes at once, so report's read phase counts each
|
||||||
// loadSignatureRows must each fail on it with an error naming its path.
|
// record as it comes rather than waiting on SQLite for all of them.
|
||||||
func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
|
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hexText := strings.Repeat("ab", sha256.Size)
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
update string
|
|
||||||
value any
|
|
||||||
}{
|
|
||||||
{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
|
|
||||||
{"hex-text head", "UPDATE files SET head = ?", hexText},
|
|
||||||
{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
|
|
||||||
{"hex-text tail", "UPDATE files SET tail = ?", hexText},
|
|
||||||
{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
|
|
||||||
{"hex-text content", "UPDATE files SET content = ?", hexText},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := openTestDB(t)
|
db := openTestDB(t)
|
||||||
|
|
||||||
err := applyChanges(t.Context(), db,
|
rows, err := db.QueryContext(t.Context(),
|
||||||
[]scanRec{{path: "/a/f"}}, nil, nil)
|
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ExecContext(t.Context(), c.update, c.value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
loads := map[string]func() error{
|
|
||||||
"loadFileRows": func() error {
|
|
||||||
return loadFileRows(t.Context(), db, func(int64) {},
|
|
||||||
func(scanRec) {})
|
|
||||||
},
|
|
||||||
"loadSignatureRows": func() error {
|
|
||||||
return loadSignatureRows(t.Context(), db, func(int64) {},
|
|
||||||
func(scanRec) error { return nil })
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, load := range loads {
|
|
||||||
err := load()
|
|
||||||
if !errors.Is(err, errHashSize) ||
|
|
||||||
!strings.Contains(err.Error(), "/a/f") {
|
|
||||||
t.Errorf("%s: err = %v, want %v naming /a/f",
|
|
||||||
name, err, errHashSize)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
|
|
||||||
// PLAN for query.
|
|
||||||
func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -568,164 +511,12 @@ func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return plan
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
|
||||||
// from the files_signature index alone, with nothing to sort: the
|
|
||||||
// first record comes at once, so report's read phase counts each
|
|
||||||
// record as it comes rather than waiting on SQLite for all of them.
|
|
||||||
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
plan := queryPlan(t, db, signatureRowsSQL)
|
|
||||||
|
|
||||||
want := []string{"SCAN files USING COVERING INDEX files_signature"}
|
want := []string{"SCAN files USING COVERING INDEX files_signature"}
|
||||||
if !slices.Equal(plan, want) {
|
if !slices.Equal(plan, want) {
|
||||||
t.Errorf("query plan = %q, want %q", plan, want)
|
t.Errorf("query plan = %q, want %q", plan, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCountsReadOnlyTotals checks that SQLite answers the queries for
|
|
||||||
// the number of records and their total size from the one row of
|
|
||||||
// totals. A plan that reads files or its index reads every record.
|
|
||||||
func TestCountsReadOnlyTotals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
for _, query := range []string{recordCountSQL, databaseCountsSQL} {
|
|
||||||
plan := queryPlan(t, db, query)
|
|
||||||
|
|
||||||
want := []string{"SCAN totals"}
|
|
||||||
if !slices.Equal(plan, want) {
|
|
||||||
t.Errorf("query plan of %q = %q, want %q", query, plan, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCountsComeFromTotals sets the row of totals to numbers that files
|
|
||||||
// does not hold, then checks that scan's load phase, the read phases of
|
|
||||||
// trees and report, and info each report the numbers from totals.
|
|
||||||
func TestCountsComeFromTotals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// files is empty, so a count over files gives 0 records of 0 bytes.
|
|
||||||
const records, size = 7, 9000
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
_, err := db.ExecContext(t.Context(),
|
|
||||||
"UPDATE totals SET records = ?, size = ?", records, size)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
scanTotal, err := countRecords(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var treesTotal, reportTotal int64
|
|
||||||
|
|
||||||
err = loadFileRows(t.Context(), db,
|
|
||||||
func(n int64) { treesTotal = n }, func(scanRec) {})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = loadSignatureRows(t.Context(), db,
|
|
||||||
func(n int64) { reportTotal = n },
|
|
||||||
func(scanRec) error { return nil })
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
info, err := readDatabaseCounts(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, got := range map[string]int64{
|
|
||||||
"scan's load": scanTotal,
|
|
||||||
"trees' read": treesTotal,
|
|
||||||
"report's read": reportTotal,
|
|
||||||
"info": info.records,
|
|
||||||
} {
|
|
||||||
if got != records {
|
|
||||||
t.Errorf("%s: %d records, want %d from totals",
|
|
||||||
name, got, records)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.size != size {
|
|
||||||
t.Errorf("info: size %d, want %d from totals", info.size, size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertTotalsMatchFiles fails unless the row of totals holds the
|
|
||||||
// number of records and the sizes of all records added up, as counted
|
|
||||||
// over files.
|
|
||||||
func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got, err := readDatabaseCounts(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var records, size int64
|
|
||||||
|
|
||||||
err = db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
|
|
||||||
Scan(&records, &size)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.records != records || got.size != size {
|
|
||||||
t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
|
|
||||||
got.records, got.size, records, size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
|
|
||||||
// change size, and rescans it after two files are removed. After each
|
|
||||||
// scan the row of totals must agree with files.
|
|
||||||
func TestTotalsMatchFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := buildSmokeTree(t)
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
|
|
||||||
writeFile(t, dir, "a/one.bin", pattern(10, 2500))
|
|
||||||
writeFile(t, dir, "t1/f1", pattern(30, 1000))
|
|
||||||
|
|
||||||
if st := syncTree(t, db, dir); st.updated != 2 {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 2 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
|
|
||||||
for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
|
|
||||||
err := os.Remove(filepath.Join(dir, name))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if st := syncTree(t, db, dir); st.removed != 2 {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 2 removed", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplyChangesRoundTrip(t *testing.T) {
|
func TestApplyChangesRoundTrip(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -736,13 +527,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
|
|||||||
// written.
|
// written.
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{
|
{
|
||||||
size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
|
size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
|
||||||
tail: testHash("t2"), content: testHash("c2"),
|
content: "c2", path: "/a/tab\tnew\nline",
|
||||||
path: "/a/tab\tnew\nline",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
|
size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
|
||||||
tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
|
content: "c1", path: "/a/x",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -763,8 +553,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
|
|||||||
// An upsert for an existing path updates in place; a delete
|
// An upsert for an existing path updates in place; a delete
|
||||||
// removes exactly its path.
|
// removes exactly its path.
|
||||||
upd := scanRec{
|
upd := scanRec{
|
||||||
size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
|
size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
|
||||||
tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
|
path: "/a/x",
|
||||||
}
|
}
|
||||||
|
|
||||||
err = applyChanges(t.Context(), db, []scanRec{upd},
|
err = applyChanges(t.Context(), db, []scanRec{upd},
|
||||||
@@ -791,8 +581,8 @@ func TestApplyChangesBatching(t *testing.T) {
|
|||||||
recs := make([]scanRec, 0, n)
|
recs := make([]scanRec, 0, n)
|
||||||
for i := range n {
|
for i := range n {
|
||||||
recs = append(recs, scanRec{
|
recs = append(recs, scanRec{
|
||||||
size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
|
size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
|
||||||
tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
|
path: fmt.Sprintf("/batch/%07d", i),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
// This file holds three rules for the walk (README "Rules for the
|
// This file holds two rules for the walk (README "Rules for the walk"):
|
||||||
// walk"): the paths given with --exclude, which the scan never enters;
|
|
||||||
// the default excludes, the paths a scan's walk does not enter below an
|
// the default excludes, the paths a scan's walk does not enter below an
|
||||||
// operand unless scan is given --no-default-excludes; and the
|
// operand unless scan is given --no-default-excludes, and the
|
||||||
// filesystems that are not persistent, on which the scan never records
|
// filesystems that are not persistent, on which the scan never records
|
||||||
// or reads anything.
|
// or reads anything.
|
||||||
|
|
||||||
@@ -13,7 +12,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -45,23 +43,15 @@ const typesPerHelpLine = 6
|
|||||||
// readMountInfo reads.
|
// readMountInfo reads.
|
||||||
var errMountInfoLine = errors.New("unexpected line")
|
var errMountInfoLine = errors.New("unexpected line")
|
||||||
|
|
||||||
// errExcludedOperand reports an operand at or below a path given with
|
|
||||||
// --exclude.
|
|
||||||
var errExcludedOperand = errors.New("operand excluded by --exclude")
|
|
||||||
|
|
||||||
// walkRules holds the rules for the walk that a scan sets up as it
|
// walkRules holds the rules for the walk that a scan sets up as it
|
||||||
// starts. excludePaths are the paths given with --exclude, each with
|
// starts. excludes are the default excludes, or none with
|
||||||
// the device ID and inode number of the file it names (see
|
// --no-default-excludes. nonPersistent maps the device ID of every
|
||||||
// excludePathInodes), which the walk neither enters nor records,
|
// mounted filesystem that is not persistent to its type; the scan
|
||||||
// whatever path leads to them. defaultExcludes are the default
|
// records and reads nothing on one of those, whatever the flags.
|
||||||
// excludes, or none with --no-default-excludes. nonPersistent maps the
|
// databaseFiles are the scan's own database files, which the walk
|
||||||
// device ID of every mounted filesystem that is not persistent to its
|
// passes over (see databaseFiles in db.go).
|
||||||
// type; the scan records and reads nothing on one of those, whatever
|
|
||||||
// the flags. databaseFiles are the scan's own database files, which the
|
|
||||||
// walk passes over (see databaseFiles in db.go).
|
|
||||||
type walkRules struct {
|
type walkRules struct {
|
||||||
excludePaths []fileRec
|
excludes []string
|
||||||
defaultExcludes []string
|
|
||||||
nonPersistent map[uint64]string
|
nonPersistent map[uint64]string
|
||||||
databaseFiles []fileRec
|
databaseFiles []fileRec
|
||||||
}
|
}
|
||||||
@@ -72,31 +62,14 @@ type walkRules struct {
|
|||||||
func defaultExcludes(goos string) []string {
|
func defaultExcludes(goos string) []string {
|
||||||
switch goos {
|
switch goos {
|
||||||
case goosLinux:
|
case goosLinux:
|
||||||
// The kernel's view of the running system, temporary files, and
|
return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
|
||||||
// caches that programs and package managers fill again on their
|
|
||||||
// own: the system's, root's and those in each home in /home.
|
|
||||||
return []string{
|
|
||||||
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
|
||||||
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
|
||||||
"/root/go/pkg/mod",
|
|
||||||
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
|
||||||
"/home/*/go/pkg/mod",
|
|
||||||
}
|
|
||||||
case goosDarwin:
|
case goosDarwin:
|
||||||
// A walk of / reaches everything in /System/Volumes/Data through
|
// A walk of / reaches everything in /System/Volumes/Data through
|
||||||
// the firmlinks, so walking it as well would record each of
|
// the firmlinks, so walking it as well would record each of
|
||||||
// those files twice. /private/var/folders holds each user's
|
// those files twice.
|
||||||
// temporary files and caches. The walk reaches root's home,
|
|
||||||
// /var/root, as /private/var/root.
|
|
||||||
return []string{
|
return []string{
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
"/private/var/folders", "/Library/Caches",
|
|
||||||
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
|
||||||
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
|
||||||
"/private/var/root/go/pkg/mod",
|
|
||||||
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
|
||||||
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
|
||||||
}
|
}
|
||||||
case goosWindows:
|
case goosWindows:
|
||||||
// Relative to the root of the drive an operand lies on. The
|
// Relative to the root of the drive an operand lies on. The
|
||||||
@@ -124,88 +97,6 @@ func isDefaultExclude(path string, excludes []string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// excludePathInodes returns each of paths, the --exclude values, as its
|
|
||||||
// real path (see realPaths) with the device ID and inode number of the
|
|
||||||
// file there. The walk compares what it reaches with those, as it does
|
|
||||||
// with the database files, so it also leaves out an --exclude path
|
|
||||||
// spelled another way, such as in another letter case on macOS.
|
|
||||||
func excludePathInodes(paths []string) ([]fileRec, error) {
|
|
||||||
resolved, err := realPaths(paths)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
files := make([]fileRec, 0, len(resolved))
|
|
||||||
|
|
||||||
for _, path := range resolved {
|
|
||||||
fi, err := os.Lstat(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ino := inodeOfInfo(fi)
|
|
||||||
files = append(files, fileRec{path: path, dev: dev, ino: ino})
|
|
||||||
}
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// matchingExcludePath returns the path of the one of excludePaths with
|
|
||||||
// the device ID and inode number of fi, an lstat result, or "" when
|
|
||||||
// there is none.
|
|
||||||
func matchingExcludePath(fi fs.FileInfo, excludePaths []fileRec) string {
|
|
||||||
dev, ino := inodeOfInfo(fi)
|
|
||||||
f := fileRec{dev: dev, ino: ino}
|
|
||||||
|
|
||||||
for _, exclude := range excludePaths {
|
|
||||||
if sameInode(exclude, f) {
|
|
||||||
return exclude.path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkOperandsNotExcluded returns an error when one of roots, the
|
|
||||||
// operands, or a directory above it is one of excludePaths, the
|
|
||||||
// --exclude paths, by device ID and inode number. The error names the
|
|
||||||
// operand and the --exclude path. The roots are absolute, so the
|
|
||||||
// directories above each one end at /. A path here that lstat cannot
|
|
||||||
// read is an error too.
|
|
||||||
func checkOperandsNotExcluded(roots []string, excludePaths []fileRec) error {
|
|
||||||
for _, root := range roots {
|
|
||||||
for path := root; ; path = filepath.Dir(path) {
|
|
||||||
fi, err := os.Lstat(path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("scan %s: %w", root, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
exclude := matchingExcludePath(fi, excludePaths)
|
|
||||||
if exclude != "" {
|
|
||||||
return fmt.Errorf("scan %s: %w %s", root, errExcludedOperand,
|
|
||||||
exclude)
|
|
||||||
}
|
|
||||||
|
|
||||||
if path == "/" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// excludePathLine returns the line the scan prints for path when fi,
|
|
||||||
// its lstat, makes it one of rules.excludePaths, and "" when it does
|
|
||||||
// not.
|
|
||||||
func (rules walkRules) excludePathLine(path string, fi fs.FileInfo) string {
|
|
||||||
if matchingExcludePath(fi, rules.excludePaths) == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("scan: skipped %s (--exclude)", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// nonPersistentFilesystemTypes returns the filesystem types that are
|
// nonPersistentFilesystemTypes returns the filesystem types that are
|
||||||
// not persistent: filesystems that the kernel makes up or that are kept
|
// not persistent: filesystems that the kernel makes up or that are kept
|
||||||
// in memory. devfs is the macOS filesystem of /dev; the others are
|
// in memory. devfs is the macOS filesystem of /dev; the others are
|
||||||
@@ -271,7 +162,7 @@ func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
|
|||||||
return devices, lines.Err()
|
return devices, lines.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultExcludesHelp describes, for the scan and dwim help, the
|
// defaultExcludesHelp describes, for the scan command's help, the
|
||||||
// default excludes of the system goos names and, where the scan lists
|
// default excludes of the system goos names and, where the scan lists
|
||||||
// the mounted filesystems, the types it never records anything on.
|
// the mounted filesystems, the types it never records anything on.
|
||||||
func defaultExcludesHelp(goos string) string {
|
func defaultExcludesHelp(goos string) string {
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"slices"
|
"slices"
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -14,32 +14,15 @@ import (
|
|||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
// excludeFlag is the scan flag that names a path the walk never enters.
|
|
||||||
const excludeFlag = "--exclude"
|
|
||||||
|
|
||||||
func TestDefaultExcludes(t *testing.T) {
|
func TestDefaultExcludes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README "Rules for the walk": the Linux and macOS lists, and none
|
// README "Rules for the walk": each system's list.
|
||||||
// on any other system. The Windows list exists in the code only: the
|
|
||||||
// binary does not build for Windows, so the README leaves it out.
|
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {
|
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
||||||
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
|
||||||
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
|
||||||
"/root/go/pkg/mod",
|
|
||||||
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
|
||||||
"/home/*/go/pkg/mod",
|
|
||||||
},
|
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
"/private/var/folders", "/Library/Caches",
|
|
||||||
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
|
||||||
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
|
||||||
"/private/var/root/go/pkg/mod",
|
|
||||||
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
|
||||||
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
|
||||||
},
|
},
|
||||||
goosWindows: {
|
goosWindows: {
|
||||||
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
||||||
@@ -75,29 +58,6 @@ func TestIsDefaultExclude(t *testing.T) {
|
|||||||
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The Linux list matches the cache directories in /root and in each
|
|
||||||
// home in /home, but nothing below them and no .cache directory
|
|
||||||
// elsewhere.
|
|
||||||
linux := map[string]bool{
|
|
||||||
"/home/u/.cache": true,
|
|
||||||
"/home/u/go/pkg/mod": true,
|
|
||||||
"/root/.cache": true,
|
|
||||||
"/var/cache": true,
|
|
||||||
"/home/u/.cache/pip": false,
|
|
||||||
"/home/u/src/.cache": false,
|
|
||||||
"/srv/home/u/.cache": false,
|
|
||||||
"/home/u/.cargo/bin": false,
|
|
||||||
"/home/u/go/pkg/mod/cache": false,
|
|
||||||
}
|
|
||||||
|
|
||||||
for path, want := range linux {
|
|
||||||
got := isDefaultExclude(path, defaultExcludes(goosLinux))
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path,
|
|
||||||
got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReadMountInfo(t *testing.T) {
|
func TestReadMountInfo(t *testing.T) {
|
||||||
@@ -226,7 +186,7 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
|||||||
unseen := writeFile(t, excluded, "new", pattern(2, 10))
|
unseen := writeFile(t, excluded, "new", pattern(2, 10))
|
||||||
|
|
||||||
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
|
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
|
||||||
walkRules{defaultExcludes: []string{excluded}})
|
walkRules{excludes: []string{excluded}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -272,7 +232,7 @@ func TestScanWalksOperandInDefaultExclude(t *testing.T) {
|
|||||||
db := openTestDB(t)
|
db := openTestDB(t)
|
||||||
|
|
||||||
st, err := syncScan(t.Context(), db, roots, 1, false,
|
st, err := syncScan(t.Context(), db, roots, 1, false,
|
||||||
walkRules{defaultExcludes: []string{excluded}})
|
walkRules{excludes: []string{excluded}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -300,7 +260,7 @@ func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
|
|||||||
f := writeFile(t, excluded, "f", pattern(1, 10))
|
f := writeFile(t, excluded, "f", pattern(1, 10))
|
||||||
|
|
||||||
st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
|
st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
|
||||||
walkRules{defaultExcludes: []string{excluded}})
|
walkRules{excludes: []string{excluded}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -324,8 +284,8 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
if !insideDefaultExclude(dir, defaultExcludes(runtime.GOOS)) {
|
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
||||||
t.Skip(dir + " is not inside a default exclude")
|
t.Skip(dir + " is not inside a default exclude")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,217 +354,6 @@ func TestRunScanNoDefaultExcludes(t *testing.T) {
|
|||||||
assertRecorded(t, path, f, true)
|
assertRecorded(t, path, f, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanExcludeNested scans a tree through the command with an
|
|
||||||
// --exclude path two levels below the operand. The walk records nothing
|
|
||||||
// at or below it and says so. The record an earlier scan left there is
|
|
||||||
// kept although its file is gone.
|
|
||||||
func TestRunScanExcludeNested(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
excluded := filepath.Join(dir, "a", "b")
|
|
||||||
kept := writeFile(t, dir, "a/f", pattern(1, 10))
|
|
||||||
gone := writeFile(t, excluded, "gone", pattern(2, 10))
|
|
||||||
|
|
||||||
scanOK(t, dir)
|
|
||||||
|
|
||||||
err := os.Remove(gone)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unseen := writeFile(t, excluded, "c/new", pattern(3, 10))
|
|
||||||
|
|
||||||
stderr := scanOK(t, excludeFlag, excluded, dir)
|
|
||||||
|
|
||||||
line := "scan: skipped " + excluded + " (--exclude)\n"
|
|
||||||
if !strings.Contains(stderr, line) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertRecorded(t, path, kept, true)
|
|
||||||
assertRecorded(t, path, gone, true)
|
|
||||||
assertRecorded(t, path, unseen, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanExcludeRepeated gives --exclude twice, once for a
|
|
||||||
// directory and once for a file, the file through a symlink to it. The
|
|
||||||
// walk records neither, and records the file beside them.
|
|
||||||
func TestRunScanExcludeRepeated(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
inDir := writeFile(t, dir, "skip/f", pattern(1, 10))
|
|
||||||
file := writeFile(t, dir, "skip-file", pattern(2, 10))
|
|
||||||
kept := writeFile(t, dir, "keep", pattern(3, 10))
|
|
||||||
|
|
||||||
link := filepath.Join(t.TempDir(), "link")
|
|
||||||
symlink(t, file, link)
|
|
||||||
|
|
||||||
scanOK(t, excludeFlag, filepath.Dir(inDir), excludeFlag, link, dir)
|
|
||||||
|
|
||||||
assertRecorded(t, path, inDir, false)
|
|
||||||
assertRecorded(t, path, file, false)
|
|
||||||
assertRecorded(t, path, kept, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanExcludeReachedByAnotherPath gives --exclude a directory
|
|
||||||
// through a symlink and a file through "..", so the walk reaches
|
|
||||||
// neither by the path given. The walk also reaches the file through a
|
|
||||||
// hard link. It records none of these, prints a line naming each, and
|
|
||||||
// records the file beside them.
|
|
||||||
func TestRunScanExcludeReachedByAnotherPath(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
inDir := writeFile(t, dir, "a/cache/f", pattern(1, 10))
|
|
||||||
file := writeFile(t, dir, "a/big", pattern(2, 10))
|
|
||||||
kept := writeFile(t, dir, "keep", pattern(3, 10))
|
|
||||||
|
|
||||||
hardLink := filepath.Join(dir, "big-link")
|
|
||||||
|
|
||||||
err := os.Link(file, hardLink)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
link := filepath.Join(t.TempDir(), "link")
|
|
||||||
symlink(t, filepath.Join(dir, "a"), link)
|
|
||||||
|
|
||||||
// Joined as text, since filepath.Join would drop the "..".
|
|
||||||
dotDot := dir + "/a/cache/../big"
|
|
||||||
|
|
||||||
stderr := scanOK(t, excludeFlag, filepath.Join(link, "cache"),
|
|
||||||
excludeFlag, dotDot, dir)
|
|
||||||
|
|
||||||
for _, skipped := range []string{filepath.Dir(inDir), file, hardLink} {
|
|
||||||
line := "scan: skipped " + skipped + " (--exclude)\n"
|
|
||||||
if !strings.Contains(stderr, line) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assertRecorded(t, path, inDir, false)
|
|
||||||
assertRecorded(t, path, file, false)
|
|
||||||
assertRecorded(t, path, hardLink, false)
|
|
||||||
assertRecorded(t, path, kept, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanExcludedOperand checks that an operand at or below an
|
|
||||||
// --exclude path is a fatal error that names both, and so is an
|
|
||||||
// --exclude path that does not exist. An operand that is a hard link to
|
|
||||||
// an --exclude file is at it. Neither scan writes a database.
|
|
||||||
func TestRunScanExcludedOperand(t *testing.T) {
|
|
||||||
dir := realTempDir(t)
|
|
||||||
sub := filepath.Join(dir, "sub")
|
|
||||||
f := writeFile(t, sub, "f", pattern(1, 10))
|
|
||||||
|
|
||||||
hardLink := filepath.Join(dir, "f-link")
|
|
||||||
|
|
||||||
err := os.Link(f, hardLink)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cases := map[string]struct {
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"at it": {
|
|
||||||
[]string{excludeFlag, sub, sub},
|
|
||||||
"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
|
|
||||||
},
|
|
||||||
"below it": {
|
|
||||||
[]string{excludeFlag, dir, sub},
|
|
||||||
"scan " + sub + ": operand excluded by --exclude " + dir + "\n",
|
|
||||||
},
|
|
||||||
"after an operand it does not exclude": {
|
|
||||||
[]string{dir, excludeFlag, sub, sub},
|
|
||||||
"scan " + sub + ": operand excluded by --exclude " + sub + "\n",
|
|
||||||
},
|
|
||||||
"a hard link to it": {
|
|
||||||
[]string{excludeFlag, f, hardLink},
|
|
||||||
"scan " + hardLink + ": operand excluded by --exclude " + f + "\n",
|
|
||||||
},
|
|
||||||
"exclude missing": {
|
|
||||||
[]string{excludeFlag, filepath.Join(dir, "nope"), dir},
|
|
||||||
"--exclude: resolve " + filepath.Join(dir, "nope") + ": ",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
stdout := captureStdout(t)
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
code := run(append([]string{cmdScan}, tc.args...), os.Stdout,
|
|
||||||
os.Stderr)
|
|
||||||
if code != exitFatal {
|
|
||||||
t.Errorf("run(scan %q) = %d, want %d", tc.args, code,
|
|
||||||
exitFatal)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertFatalOutput(t, stderr(), stdout())
|
|
||||||
|
|
||||||
if !strings.Contains(stderr(), "sfdupes: "+tc.want) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr(), tc.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := os.Lstat(path)
|
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Errorf("lstat %s: %v, want no database", path, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanRejectsEmptyExclude checks that an empty --exclude value,
|
|
||||||
// given as its own argument, after "=" or after another --exclude, is
|
|
||||||
// a usage error reported in one line before anything is scanned
|
|
||||||
// (README "Settings").
|
|
||||||
func TestRunScanRejectsEmptyExclude(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
cases := map[string][]string{
|
|
||||||
"separate": {excludeFlag, "", dir},
|
|
||||||
"joined by =": {excludeFlag + "=", dir},
|
|
||||||
"after a path": {excludeFlag, t.TempDir(), excludeFlag, "", dir},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, args := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := run(append([]string{cmdScan}, args...), &stdout,
|
|
||||||
&stderr)
|
|
||||||
if code != exitUsage {
|
|
||||||
t.Errorf("run(scan %q) = %d, want %d", args, code,
|
|
||||||
exitUsage)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := "Error: --exclude must not be empty\n"
|
|
||||||
if stdout.String() != "" || stderr.String() != want {
|
|
||||||
t.Errorf("stdout, stderr = %q, %q, want \"\", %q",
|
|
||||||
stdout.String(), stderr.String(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := os.Lstat(path)
|
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Errorf("lstat %s: %v, want no database", path, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanSkipsNonPersistentOperand scans, through the command, a
|
// TestRunScanSkipsNonPersistentOperand scans, through the command, a
|
||||||
// directory on a filesystem that is not persistent, with and without
|
// directory on a filesystem that is not persistent, with and without
|
||||||
// --no-default-excludes: either way it is skipped, with a line saying
|
// --no-default-excludes: either way it is skipped, with a line saying
|
||||||
@@ -746,20 +495,6 @@ func shmTempDir(t *testing.T) (string, string, walkRules) {
|
|||||||
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
||||||
}
|
}
|
||||||
|
|
||||||
// insideDefaultExclude reports whether path, which is absolute and
|
|
||||||
// clean, or a directory above it is one of excludes.
|
|
||||||
func insideDefaultExclude(path string, excludes []string) bool {
|
|
||||||
for dir := path; ; dir = filepath.Dir(dir) {
|
|
||||||
if isDefaultExclude(dir, excludes) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if dir == "/" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRecorded checks that the database at dbPath holds a record for
|
// assertRecorded checks that the database at dbPath holds a record for
|
||||||
// path when want is set, and none when it is not.
|
// path when want is set, and none when it is not.
|
||||||
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -26,11 +26,11 @@ func infoPhase(name string) string {
|
|||||||
// it. Any database problem, including a missing database, is fatal.
|
// it. Any database problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in two phases, each under a progress display of its
|
// The work runs in two phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database) and count (read the number of
|
// own on stderr: open (open the database) and count (count the records,
|
||||||
// records and their total size from the totals table). Each display
|
// with one query). Each display ends before the next phase starts, and
|
||||||
// ends before the next phase starts, and before an error is returned
|
// before an error is returned to be printed. The details are written
|
||||||
// to be printed. The details are written once the last display has
|
// once the last display has ended, so no display is drawn while they
|
||||||
// ended, so no display is drawn while they are written.
|
// are written.
|
||||||
func runInfo(ctx context.Context, stdout io.Writer) error {
|
func runInfo(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
|
|||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
// One row holds both counts, so the phase never has a total.
|
// One query counts everything, so the phase never has a total.
|
||||||
prog = newProgress(progressParams{
|
prog = newProgress(progressParams{
|
||||||
label: infoPhase("count"), total: -1,
|
label: infoPhase("count"), total: -1,
|
||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -12,25 +12,22 @@ import (
|
|||||||
// infoOpenLine is the plain line of info's open phase when it opens the
|
// infoOpenLine is the plain line of info's open phase when it opens the
|
||||||
// database at once; it prints it as the phase starts and as it ends.
|
// database at once; it prints it as the phase starts and as it ends.
|
||||||
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// infoCountLine is the plain line of info's count phase when the count
|
// infoCountLine is the plain line of info's count phase when the count
|
||||||
// takes under a second; it prints it as the phase starts and as it
|
// takes under a second; it prints it as the phase starts and as it
|
||||||
// ends.
|
// ends.
|
||||||
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
|
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// infoRecs are four records: a duplicate pair, a file of 20 MiB and
|
// infoRecs are four records: a duplicate pair, a file of 20 MiB and
|
||||||
// another small file.
|
// another small file.
|
||||||
func infoRecs() []scanRec {
|
func infoRecs() []scanRec {
|
||||||
h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
|
|
||||||
bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/a"},
|
||||||
{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/b"},
|
||||||
{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
|
{size: 20 << 20, head: "bh", tail: "bt", content: "bc", path: "/d/big"},
|
||||||
{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
|
{size: 7, head: "u", tail: "u", content: "u", path: "/d/unique"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
package sfdupes
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
|
||||||
// root" runs with the scan flags in opts, then what "sfdupes trees"
|
|
||||||
// runs, and ends with a line on stderr saying how to print the report
|
|
||||||
// again. The dwim command passes "/" as root. It catches SIGINT and
|
|
||||||
// SIGTERM as scan does, but only during the scan. An error from the
|
|
||||||
// scan, an interrupt included, ends it before the report. During the
|
|
||||||
// report, a signal ends it at once, as it ends trees.
|
|
||||||
func runDwim(
|
|
||||||
ctx context.Context, root string, opts scanOptions, stdout io.Writer,
|
|
||||||
) error {
|
|
||||||
scanCtx, stop := interruptContext(ctx)
|
|
||||||
err := runScan(scanCtx, []string{root}, opts)
|
|
||||||
|
|
||||||
stop()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = runTrees(ctx, stdout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr,
|
|
||||||
`dwim: run "sfdupes trees" to print this report again`)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,210 +0,0 @@
|
|||||||
package sfdupes //nolint:testpackage // calls the unexported runDwim
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
|
||||||
// dwim scans its root, then prints the trees report, the larger of
|
|
||||||
// two duplicate-tree pairs first, and ends by saying how to print
|
|
||||||
// the report again.
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
root := realTempDir(t)
|
|
||||||
|
|
||||||
small := []string{
|
|
||||||
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
|
||||||
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
|
||||||
}
|
|
||||||
large := []string{
|
|
||||||
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
|
|
||||||
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
|
|
||||||
}
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
err := runDwim(t.Context(), root, scanOptions{workers: 1}, &stdout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
|
||||||
}
|
|
||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
|
||||||
if len(lines) != 3 {
|
|
||||||
t.Fatalf("stdout = %q, want a header and two groups",
|
|
||||||
stdout.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
|
|
||||||
if !strings.HasPrefix(lines[1], wantFirst) {
|
|
||||||
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
|
|
||||||
if !strings.HasPrefix(lines[2], wantSecond) {
|
|
||||||
t.Errorf("second group = %q, want it to start %q",
|
|
||||||
lines[2], wantSecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
|
|
||||||
if got := stderr(); !strings.HasSuffix(got, last) {
|
|
||||||
t.Errorf("stderr = %q, want it to end %q", got, last)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
|
||||||
// A fatal scan error ends dwim before the report: a root that does
|
|
||||||
// not exist fails the scan as it fails scan itself.
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
missing := filepath.Join(realTempDir(t), "missing")
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
err := runDwim(t.Context(), missing, scanOptions{workers: 1}, &stdout)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := stdout.String(); got != "" {
|
|
||||||
t.Errorf("stdout = %q, want no report", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := stderr(); strings.Contains(got, "trees:") ||
|
|
||||||
strings.Contains(got, "dwim:") {
|
|
||||||
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDwimExcludeLeavesTreeOut(t *testing.T) {
|
|
||||||
// dwim passes its --exclude paths to the scan: an excluded copy is
|
|
||||||
// not recorded, so its pair is not reported.
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
root := realTempDir(t)
|
|
||||||
|
|
||||||
kept := []string{
|
|
||||||
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
|
||||||
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
|
||||||
}
|
|
||||||
writeFile(t, root, "b1/f.bin", pattern(2, 5000))
|
|
||||||
excluded := filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000)))
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
opts := scanOptions{workers: 1, excludePaths: []string{excluded}}
|
|
||||||
|
|
||||||
err := runDwim(t.Context(), root, opts, &stdout)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
|
||||||
}
|
|
||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
|
||||||
if len(lines) != 2 {
|
|
||||||
t.Fatalf("stdout = %q, want a header and one group",
|
|
||||||
stdout.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
want := kept[0] + "\t" + kept[1] + "\t1\t300\t"
|
|
||||||
if !strings.HasPrefix(lines[1], want) {
|
|
||||||
t.Errorf("group = %q, want it to start %q", lines[1], want)
|
|
||||||
}
|
|
||||||
|
|
||||||
skipped := "scan: skipped " + excluded + " (--exclude)\n"
|
|
||||||
if got := stderr(); !strings.Contains(got, skipped) {
|
|
||||||
t.Errorf("stderr = %q, want it to contain %q", got, skipped)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDwimTakesScanFlags(t *testing.T) {
|
|
||||||
// The dwim command parses and checks the scan flags as scan does.
|
|
||||||
// Each case stops before the walk of / begins, and before the
|
|
||||||
// database is created. "dwim --exclude /" excludes its own operand.
|
|
||||||
cases := map[string]struct {
|
|
||||||
args []string
|
|
||||||
code int
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"--exclude /": {
|
|
||||||
[]string{cmdDwim, excludeFlag, "/"},
|
|
||||||
exitFatal,
|
|
||||||
"sfdupes: scan /: operand excluded by --exclude /\n",
|
|
||||||
},
|
|
||||||
"--exclude empty": {
|
|
||||||
[]string{cmdDwim, excludeFlag, ""},
|
|
||||||
exitUsage,
|
|
||||||
"Error: --exclude must not be empty\n",
|
|
||||||
},
|
|
||||||
"--workers 0": {
|
|
||||||
[]string{cmdDwim, workersFlag, "0", "-x", "--no-default-excludes"},
|
|
||||||
exitUsage,
|
|
||||||
"Error: --workers must be at least 1, got 0\n",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
stdout := captureStdout(t)
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
code := run(tc.args, os.Stdout, os.Stderr)
|
|
||||||
if code != tc.code {
|
|
||||||
t.Errorf("run(%q) = %d, want %d", tc.args, code, tc.code)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasSuffix(stderr(), tc.want) || stdout() != "" {
|
|
||||||
t.Errorf("stdout, stderr = %q, %q, want \"\" and stderr "+
|
|
||||||
"ending %q", stdout(), stderr(), tc.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := os.Lstat(path)
|
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Errorf("lstat %s: %v, want no database", path, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunDwimHelpListsScanFlags(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// README "Subcommands" shows this usage line for dwim.
|
|
||||||
const want = "\n sfdupes dwim [--workers N] [-x] [--no-default-excludes] " +
|
|
||||||
"[--exclude PATH]...\n"
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
run([]string{cmdDwim, "-h"}, &stdout, &stderr)
|
|
||||||
|
|
||||||
if !strings.Contains(stderr.String(), want) {
|
|
||||||
t.Errorf("dwim -h stderr = %q, want the usage line %q",
|
|
||||||
stderr.String(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
|
|
||||||
t.Errorf("dwim -h stderr = %q, want %s PATH", stderr.String(),
|
|
||||||
excludeFlag)
|
|
||||||
}
|
|
||||||
|
|
||||||
// README "Settings": the help lists the default excludes and the
|
|
||||||
// filesystem types the walk never records anything on.
|
|
||||||
help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
|
|
||||||
if !strings.Contains(stderr.String(), help) {
|
|
||||||
t.Errorf("dwim -h stderr = %q, want it to hold %q", stderr.String(),
|
|
||||||
help)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,23 @@
|
|||||||
// Package sfdupes implements the sfdupes command and the database its
|
// Command sfdupes quickly identifies candidate duplicate files across
|
||||||
// subcommands share.
|
// very large filesystems without reading every byte of every file.
|
||||||
package sfdupes
|
// Files are considered duplicates when their sizes are equal and they
|
||||||
|
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
||||||
|
// KiB, and of the content. The content hash covers the whole file under
|
||||||
|
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||||
|
// file is never read in full. scan maintains a persistent SQLite
|
||||||
|
// database of file signatures (SFDUPES_DATABASE, default
|
||||||
|
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||||
|
// sfdupes report > dupes.tsv
|
||||||
|
// sfdupes trees > dupetrees.tsv
|
||||||
|
// sfdupes info
|
||||||
|
// sfdupes --version
|
||||||
|
//
|
||||||
|
// See README.md for the complete specification.
|
||||||
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -10,7 +27,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"slices"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
@@ -29,70 +45,6 @@ const (
|
|||||||
cmdReport = "report"
|
cmdReport = "report"
|
||||||
cmdTrees = "trees"
|
cmdTrees = "trees"
|
||||||
cmdInfo = "info"
|
cmdInfo = "info"
|
||||||
cmdDwim = "dwim"
|
|
||||||
)
|
|
||||||
|
|
||||||
// databaseHelp is the paragraph of the root and scan help that names
|
|
||||||
// the database (README "Location").
|
|
||||||
const databaseHelp = "scan and dwim store what they find in a database " +
|
|
||||||
"file. report, trees\nand info read it. The file is $" + databaseEnv +
|
|
||||||
" when that is set and\nnot empty, and otherwise " +
|
|
||||||
defaultDatabasePath + ", where creating it\nneeds root.\n"
|
|
||||||
|
|
||||||
// escapesHelp is the paragraph of the report and trees help on how
|
|
||||||
// paths are escaped (README "Report output format").
|
|
||||||
const escapesHelp = `Four characters in a path are escaped, so each row stays one line:
|
|
||||||
|
|
||||||
\\ a backslash
|
|
||||||
\t a tab
|
|
||||||
\n a newline
|
|
||||||
\r a carriage return
|
|
||||||
`
|
|
||||||
|
|
||||||
// reportHelp, treesHelp, infoHelp and dwimHelp follow the one-line
|
|
||||||
// summary in the help of their subcommands.
|
|
||||||
const (
|
|
||||||
reportHelp = `The report is tab-separated text on stdout: a header line, then one
|
|
||||||
row per duplicate file. The columns are:
|
|
||||||
|
|
||||||
first the path that sorts first, byte by byte, in its group of copies
|
|
||||||
dupe another copy of that file
|
|
||||||
size the file's size in bytes
|
|
||||||
|
|
||||||
Rows are ordered by size, largest first, then by path.
|
|
||||||
|
|
||||||
` + escapesHelp
|
|
||||||
|
|
||||||
treesHelp = `The report is tab-separated text on stdout: a header line, then one
|
|
||||||
row per duplicate directory tree. The columns are:
|
|
||||||
|
|
||||||
first the path that sorts first, byte by byte, in its group of copies
|
|
||||||
dupe another copy of that tree
|
|
||||||
files the number of regular files in one copy
|
|
||||||
size their total size in bytes
|
|
||||||
disk_size the bytes the dupe copy takes on disk
|
|
||||||
|
|
||||||
Rows are ordered by size, largest first, then by path.
|
|
||||||
|
|
||||||
` + escapesHelp
|
|
||||||
|
|
||||||
infoHelp = `The details go to stdout, one per line. Each line holds a name and its
|
|
||||||
value, separated by a tab:
|
|
||||||
|
|
||||||
database the path, and where it came from
|
|
||||||
database file size in bytes, then rounded, as 2147483648 (2.0 GiB)
|
|
||||||
-wal file size in bytes, then rounded; only if a -wal file exists
|
|
||||||
modified when the database file last changed, in UTC
|
|
||||||
schema version the version of the database's layout
|
|
||||||
records the number of records
|
|
||||||
size of recorded files their total size, in bytes, then rounded
|
|
||||||
`
|
|
||||||
|
|
||||||
dwimHelp = `dwim runs what "sfdupes scan /" runs with the same flags, then what
|
|
||||||
"sfdupes trees" runs. Run it as root, so that the scan can read every
|
|
||||||
file. The scan is stored in the database, as scan stores it, so
|
|
||||||
"sfdupes trees" prints the report again later.
|
|
||||||
`
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// errNoSubcommand is returned by the root command when it is invoked
|
// errNoSubcommand is returned by the root command when it is invoked
|
||||||
@@ -100,14 +52,10 @@ file. The scan is stored in the database, as scan stores it, so
|
|||||||
// cobra prints for it is the whole message.
|
// cobra prints for it is the whole message.
|
||||||
var errNoSubcommand = errors.New("no subcommand")
|
var errNoSubcommand = errors.New("no subcommand")
|
||||||
|
|
||||||
// errWorkersBelowOne is the usage error for a --workers value below 1.
|
// errWorkersBelowOne is the usage error for a scan --workers value
|
||||||
|
// below 1.
|
||||||
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
||||||
|
|
||||||
// errEmptyExcludeValue is the usage error for an empty --exclude value,
|
|
||||||
// as from --exclude "$DIR" with DIR unset. Taken as a path, it
|
|
||||||
// would name the working directory.
|
|
||||||
var errEmptyExcludeValue = errors.New("--exclude must not be empty")
|
|
||||||
|
|
||||||
// Version is the build version, injected at link time via -ldflags
|
// Version is the build version, injected at link time via -ldflags
|
||||||
// (see the Makefile); empty when the linker set none, as for a binary
|
// (see the Makefile); empty when the linker set none, as for a binary
|
||||||
// built by go install. buildVersion turns it into what --version prints.
|
// built by go install. buildVersion turns it into what --version prints.
|
||||||
@@ -136,22 +84,20 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
|
|||||||
return info.Main.Version
|
return info.Main.Version
|
||||||
}
|
}
|
||||||
|
|
||||||
// Main runs the command line in os.Args and returns the process exit
|
func main() {
|
||||||
// code.
|
|
||||||
func Main() int {
|
|
||||||
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
||||||
// head", the Go runtime ends the process with SIGPIPE on the next
|
// head", the Go runtime ends the process with SIGPIPE on the next
|
||||||
// write instead of returning an error (README "Error handling").
|
// write instead of returning an error (README "Error handling").
|
||||||
// Registering for SIGPIPE with os/signal would change that.
|
// Registering for SIGPIPE with os/signal would change that.
|
||||||
return run(os.Args[1:], os.Stdout, os.Stderr)
|
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
||||||
}
|
}
|
||||||
|
|
||||||
// run executes args against the command tree and returns the process
|
// run executes args against the command tree and returns the process
|
||||||
// exit code. It is the program's single exit point: the subcommands
|
// exit code. It is the program's single exit point: the subcommands
|
||||||
// return their errors instead of exiting, so every deferred cleanup —
|
// return their errors instead of exiting, so every deferred cleanup —
|
||||||
// above all closing the database, which checkpoints the SQLite WAL —
|
// above all closing the database, which checkpoints the SQLite WAL —
|
||||||
// runs before the process ends. The report, trees, info and dwim
|
// runs before the process ends. The report, trees and info subcommands
|
||||||
// subcommands write their data to stdout.
|
// write their data to stdout.
|
||||||
func run(args []string, stdout, stderr io.Writer) int {
|
func run(args []string, stdout, stderr io.Writer) int {
|
||||||
// A nil slice makes cobra fall back to os.Args, which would let a
|
// A nil slice makes cobra fall back to os.Args, which would let a
|
||||||
// test binary's own flags reach the command tree.
|
// test binary's own flags reach the command tree.
|
||||||
@@ -255,116 +201,53 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
root.Long = root.Short + ".\n\n" + databaseHelp
|
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
|
||||||
reportCmd.Long = reportCmd.Short + ".\n\n" + reportHelp
|
|
||||||
treesCmd.Long = treesCmd.Short + ".\n\n" + treesHelp
|
|
||||||
infoCmd.Long = infoCmd.Short + ".\n\n" + infoHelp
|
|
||||||
|
|
||||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd,
|
|
||||||
newDwimCommand(stdout))
|
|
||||||
|
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|
||||||
// scanOptions holds the values of the flags that scan and dwim share:
|
// newScanCommand builds the scan subcommand, with its --workers, -x and
|
||||||
// --workers, -x, --no-default-excludes and --exclude.
|
// --no-default-excludes flags. Its help lists the default excludes of
|
||||||
type scanOptions struct {
|
// the system it runs on, and the filesystem types it never records
|
||||||
workers int
|
// anything on.
|
||||||
oneFS bool
|
|
||||||
noDefaultExcludes bool
|
|
||||||
excludePaths []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanFlagsUsage names the flags that scan and dwim share, for their
|
|
||||||
// usage lines.
|
|
||||||
const scanFlagsUsage = "[--workers N] [-x] [--no-default-excludes] " +
|
|
||||||
"[--exclude PATH]..."
|
|
||||||
|
|
||||||
// addScanFlags adds the flags that scan and dwim share to cmd, which
|
|
||||||
// stores their values in opts.
|
|
||||||
func addScanFlags(cmd *cobra.Command, opts *scanOptions) {
|
|
||||||
cmd.Flags().IntVar(&opts.workers, "workers", runtime.NumCPU(),
|
|
||||||
"concurrent workers for the walk and hash phases")
|
|
||||||
cmd.Flags().BoolVarP(&opts.oneFS, "one-file-system", "x", false,
|
|
||||||
"do not cross filesystem boundaries")
|
|
||||||
cmd.Flags().BoolVar(&opts.noDefaultExcludes, "no-default-excludes",
|
|
||||||
false, "enter the default excludes, the paths listed above")
|
|
||||||
// A string array, not a slice, so a comma in a path is not a
|
|
||||||
// separator.
|
|
||||||
cmd.Flags().StringArrayVar(&opts.excludePaths, "exclude", nil,
|
|
||||||
"never enter or record `PATH` or anything below it; repeatable")
|
|
||||||
}
|
|
||||||
|
|
||||||
// check rejects the flag values that are usage errors, before the
|
|
||||||
// command runs. It is the PreRunE of scan and dwim.
|
|
||||||
func (o *scanOptions) check(cmd *cobra.Command, _ []string) error {
|
|
||||||
err := checkScanWorkers(cmd, o.workers)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rejectEmptyExcludeValue(cmd, o.excludePaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newScanCommand builds the scan subcommand. Its help names the
|
|
||||||
// database, and lists the default excludes of the system it runs on and
|
|
||||||
// the filesystem types it never records anything on.
|
|
||||||
func newScanCommand() *cobra.Command {
|
func newScanCommand() *cobra.Command {
|
||||||
var opts scanOptions
|
var (
|
||||||
|
scanWorkers int
|
||||||
|
scanOneFS bool
|
||||||
|
scanNoDefaultExcludes bool
|
||||||
|
)
|
||||||
|
|
||||||
short := "Walk trees and synchronize the scan database"
|
short := "Walk trees and synchronize the scan database"
|
||||||
|
|
||||||
scanCmd := &cobra.Command{
|
scanCmd := &cobra.Command{
|
||||||
Use: cmdScan + " " + scanFlagsUsage + " PATH...",
|
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
|
||||||
// The Use line already names every flag, so cobra must not
|
|
||||||
// append its generic "[flags]" to it.
|
|
||||||
DisableFlagsInUseLine: true,
|
|
||||||
Short: short,
|
Short: short,
|
||||||
Long: short + ".\n\n" + databaseHelp + "\n" +
|
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
|
||||||
defaultExcludesHelp(runtime.GOOS),
|
|
||||||
Args: cobra.MinimumNArgs(1),
|
Args: cobra.MinimumNArgs(1),
|
||||||
PreRunE: opts.check,
|
PreRunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
return checkScanWorkers(cmd, scanWorkers)
|
||||||
|
},
|
||||||
RunE: runE(func(ctx context.Context, args []string) error {
|
RunE: runE(func(ctx context.Context, args []string) error {
|
||||||
ctx, stop := interruptContext(ctx)
|
ctx, stop := interruptContext(ctx)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
return runScan(ctx, args, opts)
|
return runScan(ctx, args, scanWorkers, scanOneFS,
|
||||||
|
scanNoDefaultExcludes)
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
addScanFlags(scanCmd, &opts)
|
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
||||||
|
"concurrent workers for the walk and hash phases")
|
||||||
|
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
||||||
|
"do not cross filesystem boundaries")
|
||||||
|
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
|
||||||
|
false, "enter the default excludes, the paths listed above")
|
||||||
|
|
||||||
return scanCmd
|
return scanCmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDwimCommand builds the dwim subcommand, which takes the scan flags
|
// checkScanWorkers rejects a scan --workers value below 1. That is a
|
||||||
// with the meaning they have for scan. Its help says what dwim runs,
|
// usage error reported in one line: cobra prints the returned message
|
||||||
// then lists the default excludes and the filesystem types the scan
|
// without the usage text, and run exits 2.
|
||||||
// never records anything on.
|
|
||||||
func newDwimCommand(stdout io.Writer) *cobra.Command {
|
|
||||||
var opts scanOptions
|
|
||||||
|
|
||||||
short := "Scan /, then print the duplicate-tree report"
|
|
||||||
|
|
||||||
dwimCmd := &cobra.Command{
|
|
||||||
Use: cmdDwim + " " + scanFlagsUsage,
|
|
||||||
DisableFlagsInUseLine: true,
|
|
||||||
Short: short,
|
|
||||||
Long: short + ".\n\n" + dwimHelp + "\n" +
|
|
||||||
defaultExcludesHelp(runtime.GOOS),
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
PreRunE: opts.check,
|
|
||||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
|
||||||
return runDwim(ctx, "/", opts, stdout)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
addScanFlags(dwimCmd, &opts)
|
|
||||||
|
|
||||||
return dwimCmd
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkScanWorkers rejects a --workers value below 1. That is a usage
|
|
||||||
// error reported in one line: cobra prints the returned message without
|
|
||||||
// the usage text, and run exits 2.
|
|
||||||
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
||||||
if workers >= 1 {
|
if workers >= 1 {
|
||||||
return nil
|
return nil
|
||||||
@@ -375,18 +258,6 @@ func checkScanWorkers(cmd *cobra.Command, workers int) error {
|
|||||||
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
|
||||||
}
|
}
|
||||||
|
|
||||||
// rejectEmptyExcludeValue rejects an empty --exclude value. That
|
|
||||||
// is a usage error reported in one line, as for --workers below 1.
|
|
||||||
func rejectEmptyExcludeValue(cmd *cobra.Command, excludePaths []string) error {
|
|
||||||
if !slices.Contains(excludePaths, "") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.SilenceUsage = true
|
|
||||||
|
|
||||||
return errEmptyExcludeValue
|
|
||||||
}
|
|
||||||
|
|
||||||
// runE adapts a subcommand implementation, or the version print, to
|
// runE adapts a subcommand implementation, or the version print, to
|
||||||
// cobra's RunE. Cobra prints the error and the command's usage text for
|
// cobra's RunE. Cobra prints the error and the command's usage text for
|
||||||
// every error RunE returns, but a subcommand that ran and failed has no
|
// every error RunE returns, but a subcommand that ran and failed has no
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes //nolint:testpackage // drives the unexported run
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -14,20 +14,12 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// usageMarker is the first line of cobra's usage text, which a usage
|
// usageMarker is the first line of cobra's usage text, which a usage
|
||||||
// error prints and a runtime failure must not.
|
// error prints and a runtime failure must not.
|
||||||
const usageMarker = "Usage:"
|
const usageMarker = "Usage:"
|
||||||
|
|
||||||
// helpFlag asks for the help text.
|
|
||||||
const helpFlag = "--help"
|
|
||||||
|
|
||||||
// workersFlag is the scan and dwim flag that sets the worker count.
|
|
||||||
const workersFlag = "--workers"
|
|
||||||
|
|
||||||
// TestMain stops the tests at once, saying why, when their temporary
|
// TestMain stops the tests at once, saying why, when their temporary
|
||||||
// directory is not on a persistent filesystem: scan records nothing
|
// directory is not on a persistent filesystem: scan records nothing
|
||||||
// there (README "Rules for the walk"), so most scan tests would fail
|
// there (README "Rules for the walk"), so most scan tests would fail
|
||||||
@@ -188,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// brokenDatabase writes a database that opens cleanly and passes the
|
// brokenDatabase writes a database that opens cleanly and passes the
|
||||||
// schema-version check but has no tables, so the first query fails
|
// schema-version check but has no files table, so the first query
|
||||||
// with the database already open: a fatal error on a path that owns an
|
// fails with the database already open: a fatal error on a path that
|
||||||
// open database. It closes the database the way scan does.
|
// owns an open database. It closes the database the way scan does.
|
||||||
func brokenDatabase(t *testing.T) string {
|
func brokenDatabase(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -273,9 +265,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
|
|||||||
assertFatalOutput(t, stderr.String(), stdout())
|
assertFatalOutput(t, stderr.String(), stdout())
|
||||||
|
|
||||||
// Proof that the failure happened after the open: only a
|
// Proof that the failure happened after the open: only a
|
||||||
// query against the opened database can report this. Each
|
// query against the opened database can report this.
|
||||||
// subcommand's first query reads totals.
|
if !strings.Contains(stderr.String(), "no such table: files") {
|
||||||
if !strings.Contains(stderr.String(), "no such table: totals") {
|
|
||||||
t.Errorf("stderr = %q, want the failure to come from a "+
|
t.Errorf("stderr = %q, want the failure to come from a "+
|
||||||
"query on the open database", stderr.String())
|
"query on the open database", stderr.String())
|
||||||
}
|
}
|
||||||
@@ -402,7 +393,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
||||||
// README §Settings: the path names the file exactly. With link a
|
// README §Database: the path names the file exactly. With link a
|
||||||
// symlink to the directory real/sub, link/../db.sqlite names
|
// symlink to the directory real/sub, link/../db.sqlite names
|
||||||
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
||||||
// use that file, relative or absolute, and its first line must name
|
// use that file, relative or absolute, and its first line must name
|
||||||
@@ -576,8 +567,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
cmdScan: {
|
cmdScan: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runScan(t.Context(), []string{missing},
|
return runScan(t.Context(), []string{missing}, 1, false, false)
|
||||||
scanOptions{workers: 1})
|
|
||||||
},
|
},
|
||||||
want: scanWant,
|
want: scanWant,
|
||||||
},
|
},
|
||||||
@@ -629,7 +619,6 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
||||||
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
||||||
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
||||||
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
|
|
||||||
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
||||||
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
||||||
}
|
}
|
||||||
@@ -659,7 +648,7 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||||
// README §Settings: --workers below 1 is a usage error reported in
|
// README §scan mode: --workers below 1 is a usage error reported in
|
||||||
// one line on stderr, before the scan opens the database.
|
// one line on stderr, before the scan opens the database.
|
||||||
for _, workers := range []string{"0", "-1"} {
|
for _, workers := range []string{"0", "-1"} {
|
||||||
t.Run(workers, func(t *testing.T) {
|
t.Run(workers, func(t *testing.T) {
|
||||||
@@ -668,7 +657,7 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
args := []string{cmdScan, workersFlag, workers, t.TempDir()}
|
args := []string{cmdScan, "--workers", workers, t.TempDir()}
|
||||||
|
|
||||||
code := run(args, &stdout, &stderr)
|
code := run(args, &stdout, &stderr)
|
||||||
if code != exitUsage {
|
if code != exitUsage {
|
||||||
@@ -697,9 +686,9 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
func TestRunHelp(t *testing.T) {
|
func TestRunHelp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
// README §Subcommands: help goes to stderr, exits 0, and leaves
|
||||||
// empty.
|
// stdout empty.
|
||||||
cases := [][]string{{helpFlag}, {"-h"}, {cmdScan, helpFlag}}
|
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||||
|
|
||||||
for _, args := range cases {
|
for _, args := range cases {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -719,119 +708,13 @@ func TestRunHelp(t *testing.T) {
|
|||||||
args, got)
|
args, got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The scan help lists --exclude with its argument.
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
run([]string{cmdScan, "-h"}, &stdout, &stderr)
|
|
||||||
|
|
||||||
if !strings.Contains(stderr.String(), excludeFlag+" PATH") {
|
|
||||||
t.Errorf("scan -h stderr = %q, want %s PATH", stderr.String(),
|
|
||||||
excludeFlag)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunScanUsageLine(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// README §Subcommands shows this usage line for scan, and the help
|
|
||||||
// and the usage error both print it.
|
|
||||||
const want = "\n sfdupes scan [--workers N] [-x] [--no-default-excludes] " +
|
|
||||||
"[--exclude PATH]... PATH...\n"
|
|
||||||
|
|
||||||
cases := map[string][]string{
|
|
||||||
"help": {cmdScan, "-h"},
|
|
||||||
"missing paths": {cmdScan},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, args := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
run(args, &stdout, &stderr)
|
|
||||||
|
|
||||||
if !strings.Contains(stderr.String(), want) {
|
|
||||||
t.Errorf("run(%v) stderr = %q, want the usage line %q",
|
|
||||||
args, stderr.String(), want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunHelpDescribesDatabaseAndOutput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// README §Settings: the help names the database, and says what
|
|
||||||
// report, trees, info and dwim print.
|
|
||||||
database := []string{
|
|
||||||
"scan and dwim store", databaseEnv, defaultDatabasePath, "needs root",
|
|
||||||
}
|
|
||||||
escapes := []string{`\\`, `\t`, `\n`, `\r`}
|
|
||||||
|
|
||||||
// The subcommand whose help is printed; none for the root.
|
|
||||||
cases := []struct {
|
|
||||||
subcommand []string
|
|
||||||
want []string
|
|
||||||
}{
|
|
||||||
{nil, database},
|
|
||||||
{[]string{cmdScan}, database},
|
|
||||||
{
|
|
||||||
[]string{cmdReport},
|
|
||||||
append([]string{
|
|
||||||
"tab-separated", "\n first ", "\n dupe ", "\n size ",
|
|
||||||
"largest first",
|
|
||||||
}, escapes...),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[]string{cmdTrees},
|
|
||||||
append([]string{
|
|
||||||
"tab-separated", "\n first ", "\n dupe ", "\n files ",
|
|
||||||
"\n size ", "\n disk_size ", "largest first",
|
|
||||||
}, escapes...),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[]string{cmdInfo},
|
|
||||||
[]string{
|
|
||||||
"\n database ", "\n database file size ",
|
|
||||||
"\n -wal file size ", "\n modified ",
|
|
||||||
"\n schema version ", "\n records ",
|
|
||||||
"\n size of recorded files ",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[]string{cmdDwim},
|
|
||||||
[]string{
|
|
||||||
"sfdupes scan /", "as root", "sfdupes trees", "database",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
args := slices.Concat(tc.subcommand, []string{helpFlag})
|
|
||||||
|
|
||||||
code := run(args, &stdout, &stderr)
|
|
||||||
if code != exitOK {
|
|
||||||
t.Fatalf("run(%v) = %d, want %d", args, code, exitOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, want := range tc.want {
|
|
||||||
if !strings.Contains(stderr.String(), want) {
|
|
||||||
t.Errorf("run(%v) help lacks %q:\n%s",
|
|
||||||
args, want, stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunVersion(t *testing.T) {
|
func TestRunVersion(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Settings: the version is one line on stdout, with nothing
|
// README §Subcommands: the version is one line on stdout, with
|
||||||
// on stderr, and exits 0.
|
// nothing on stderr, and exits 0.
|
||||||
info, _ := debug.ReadBuildInfo()
|
info, _ := debug.ReadBuildInfo()
|
||||||
|
|
||||||
for _, arg := range []string{"--version", "-v"} {
|
for _, arg := range []string{"--version", "-v"} {
|
||||||
@@ -890,8 +773,8 @@ func TestBuildVersion(t *testing.T) {
|
|||||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Error handling: a failed write to stdout is a fatal error,
|
// README §Error handling: a stdout write failure exits 1, reported
|
||||||
// exit 1, reported on stderr starting with "sfdupes:".
|
// in one line on stderr.
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
||||||
@@ -913,7 +796,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
|||||||
func scanFixture(t *testing.T) []string {
|
func scanFixture(t *testing.T) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
|
|
||||||
dupes := []string{
|
dupes := []string{
|
||||||
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
||||||
@@ -934,23 +817,6 @@ func scanFixture(t *testing.T) []string {
|
|||||||
return dupes
|
return dupes
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedDiskSize returns the disk_size of the record for path in the
|
|
||||||
// database at dbPath. A test of a trees run compares with this rather
|
|
||||||
// than with a new lstat, which on ZFS can differ from the scan's (see
|
|
||||||
// diskSizeOf).
|
|
||||||
func storedDiskSize(t *testing.T, dbPath, path string) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := openReportDatabase(t.Context(), dbPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
return recordByPath(t, dbRecords(t, db), path).diskSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
||||||
// nothing on stdout, and returns everything it printed to stderr.
|
// nothing on stdout, and returns everything it printed to stderr.
|
||||||
func scanOK(t *testing.T, operands ...string) string {
|
func scanOK(t *testing.T, operands ...string) string {
|
||||||
@@ -983,7 +849,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
writeFile(t, dir, "a.bin", pattern(1, 300))
|
writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||||
|
|
||||||
// Same size as a.bin, so the scan reads it, and the read fails.
|
// Same size as a.bin, so the scan reads it, and the read fails.
|
||||||
@@ -1011,292 +877,75 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
assertNoSidecars(t, path)
|
assertNoSidecars(t, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// realTempDir returns a new temporary directory by its real path, so
|
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
|
||||||
// tests hold where the temporary directory is reached through a
|
|
||||||
// symlink.
|
|
||||||
func realTempDir(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// symlink creates link pointing at target.
|
|
||||||
func symlink(t *testing.T, target, link string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := os.Symlink(target, link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
|
|
||||||
// The operand is scanned at its real path, and its file is recorded
|
|
||||||
// there.
|
|
||||||
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
link := filepath.Join(dir, "link")
|
||||||
symlink(t, filepath.Join(dir, "target"), link)
|
|
||||||
|
|
||||||
stderr := scanOK(t, link)
|
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||||
|
if err != nil {
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
t.Fatal(err)
|
||||||
"0 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
// Scanning a directory through the symlink stores a record beneath
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
// the symlink's own path for a file beneath its target.
|
||||||
}
|
scanOK(t, filepath.Join(link, "sub"))
|
||||||
|
|
||||||
|
assertOperandSkipped(t, path, link, "symlink",
|
||||||
|
filepath.Join(link, "sub", "f"))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
|
|
||||||
// with link a symlink to target. Both resolve under target, so
|
|
||||||
// target/sub is walked once, as part of target.
|
|
||||||
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
link := filepath.Join(dir, "link")
|
||||||
symlink(t, filepath.Join(dir, "target"), link)
|
|
||||||
|
|
||||||
stderr := scanOK(t, link, filepath.Join(link, "sub"))
|
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
|
|
||||||
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
|
|
||||||
// with dir/link a symlink to a directory outside dir. The walk of dir
|
|
||||||
// does not follow link, so dir/link/sub is walked on its own at its
|
|
||||||
// real path, and the record an earlier scan stored there survives.
|
|
||||||
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
a := writeFile(t, dir, "a", pattern(1, 10))
|
|
||||||
|
|
||||||
outside := realTempDir(t)
|
|
||||||
f := writeFile(t, outside, "sub/f", pattern(2, 10))
|
|
||||||
|
|
||||||
symlink(t, outside, filepath.Join(dir, "link"))
|
|
||||||
|
|
||||||
scanOK(t, filepath.Join(outside, "sub"))
|
|
||||||
|
|
||||||
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
|
|
||||||
|
|
||||||
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"1 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{a, f}
|
|
||||||
slices.Sort(want)
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
|
||||||
t.Errorf("records = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
|
|
||||||
// link, outside dir, a symlink to dir/real. link resolves inside dir,
|
|
||||||
// so it is dropped as covered by dir, and its file is recorded once.
|
|
||||||
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
f := writeFile(t, dir, "real/f", pattern(1, 10))
|
|
||||||
|
|
||||||
link := filepath.Join(realTempDir(t), "link")
|
|
||||||
symlink(t, filepath.Join(dir, "real"), link)
|
|
||||||
|
|
||||||
stderr := scanOK(t, dir, link)
|
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
|
|
||||||
// a symlink whose target does not exist. The scan fails with exit 1 and
|
|
||||||
// an error naming the symlink as given, before it walks anything, so no
|
|
||||||
// database is created.
|
|
||||||
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
writeFile(t, dir, "f", pattern(1, 10))
|
|
||||||
|
|
||||||
dangling := filepath.Join(realTempDir(t), "dangling")
|
|
||||||
symlink(t, filepath.Join(dir, "missing"), dangling)
|
|
||||||
|
|
||||||
stdout := captureStdout(t)
|
|
||||||
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
|
|
||||||
if code != exitFatal {
|
|
||||||
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
|
|
||||||
exitFatal)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertFatalOutput(t, stderr.String(), stdout())
|
|
||||||
|
|
||||||
want := "sfdupes: resolve " + dangling + ": "
|
|
||||||
if !strings.Contains(stderr.String(), want) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := os.Stat(path)
|
|
||||||
if !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
t.Errorf("database: stat error %v, want it missing", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
|
|
||||||
// after a cd through home/proj, a symlink to data/proj. The file is
|
|
||||||
// recorded under data/proj, its real path.
|
|
||||||
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
|
||||||
|
|
||||||
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
// link is dropped as a symlink, but link/sub must still be scanned,
|
||||||
filepath.Join(dir, "home", "proj"))
|
// not dropped as lying under link.
|
||||||
|
scanOK(t, link, filepath.Join(link, "sub"))
|
||||||
|
|
||||||
// t.Chdir also sets PWD to the path through the symlink, as a
|
db, err := openDB(path, reportParams)
|
||||||
// shell does after cd.
|
|
||||||
t.Chdir(filepath.Join(dir, "home", "proj"))
|
|
||||||
|
|
||||||
scanOK(t, ".")
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
|
|
||||||
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
|
|
||||||
// a cd through home/proj. As with realpath, the ".." counts from
|
|
||||||
// data/proj, so data is scanned and home is not.
|
|
||||||
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
|
|
||||||
for _, tc := range []struct{ wd, operand string }{
|
|
||||||
{"", "home/proj/.."},
|
|
||||||
{"home/proj", ".."},
|
|
||||||
} {
|
|
||||||
t.Run(tc.operand, func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
|
||||||
g := writeFile(t, dir, "data/g", pattern(2, 10))
|
|
||||||
writeFile(t, dir, "home/h", pattern(3, 10))
|
|
||||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
|
||||||
filepath.Join(dir, "home", "proj"))
|
|
||||||
|
|
||||||
// t.Chdir also sets PWD to the path given, as a shell does
|
|
||||||
// after cd.
|
|
||||||
t.Chdir(filepath.Join(dir, tc.wd))
|
|
||||||
|
|
||||||
scanOK(t, tc.operand)
|
|
||||||
|
|
||||||
want := []string{f, g}
|
|
||||||
slices.Sort(want)
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
|
||||||
t.Errorf("records = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
|
|
||||||
// The scan warns about the FIFO by its real path, counts it as skipped,
|
|
||||||
// and exits 0.
|
|
||||||
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
fifo := filepath.Join(dir, "fifo")
|
|
||||||
|
|
||||||
err := unix.Mkfifo(fifo, 0o600)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
symlink(t, fifo, link)
|
|
||||||
|
|
||||||
stderr := scanOK(t, link)
|
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
|
||||||
|
|
||||||
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
|
||||||
if !strings.Contains(stderr, warning) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||||
zfs := filepath.Join(realTempDir(t), ".zfs")
|
path := testDBPath(t)
|
||||||
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
|
||||||
|
|
||||||
// No scan records a file inside a .zfs directory, so the record
|
|
||||||
// beneath the operand is written straight into the database.
|
|
||||||
path := seedDatabase(t, []scanRec{{size: 10, path: f}})
|
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
zfs := filepath.Join(t.TempDir(), ".zfs")
|
||||||
|
snapshot := filepath.Join(zfs, "snapshot", "hourly")
|
||||||
|
f := writeFile(t, snapshot, "f", pattern(1, 10))
|
||||||
|
|
||||||
|
// An operand beneath a .zfs directory is walked, because it is not
|
||||||
|
// itself named .zfs.
|
||||||
|
scanOK(t, snapshot)
|
||||||
|
|
||||||
assertOperandSkipped(t, path, zfs, ".zfs directory", f)
|
assertOperandSkipped(t, path, zfs, ".zfs directory", f)
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertOperandSkipped scans operand alone and checks that it is skipped
|
// assertOperandSkipped scans operand alone and checks that it is skipped
|
||||||
// as kind: a warning naming it, one skip in the summary, exit 0, and the
|
// as kind: a warning naming it, one skip in the summary, exit 0, and the
|
||||||
// record for kept, stored beneath operand before this scan, still in the
|
// record for kept, which an earlier scan stored beneath operand, still
|
||||||
// database at dbPath.
|
// in the database at dbPath.
|
||||||
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
||||||
kept string,
|
kept string,
|
||||||
) {
|
) {
|
||||||
@@ -1325,107 +974,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
|||||||
recordByPath(t, dbRecords(t, db), kept)
|
recordByPath(t, dbRecords(t, db), kept)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
|
|
||||||
// directory, first alone and then under another operand. Each time it
|
|
||||||
// is warned about and counted as skipped, and nothing in it is walked.
|
|
||||||
// The other operand is scanned as usual.
|
|
||||||
func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
|
||||||
writeFile(t, hourly, "f", pattern(1, 10))
|
|
||||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
|
||||||
|
|
||||||
warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
operands []string
|
|
||||||
summary string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
[]string{hourly},
|
|
||||||
"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[]string{dir, hourly},
|
|
||||||
"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
stderr := scanOK(t, tc.operands...)
|
|
||||||
if !strings.Contains(stderr, warning) ||
|
|
||||||
!strings.Contains(stderr, tc.summary) {
|
|
||||||
t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
|
|
||||||
stderr, warning, tc.summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
|
||||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
|
||||||
// by its real path, counted as skipped, and not walked.
|
|
||||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
|
||||||
|
|
||||||
snaps := filepath.Join(dir, "snaps")
|
|
||||||
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
|
||||||
|
|
||||||
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
|
||||||
|
|
||||||
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
|
||||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n"
|
|
||||||
|
|
||||||
if !strings.Contains(stderr, warning) ||
|
|
||||||
!strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
|
||||||
// a symlink to pool/data. The operand is scanned as usual, and its file
|
|
||||||
// is recorded by its real path.
|
|
||||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
|
||||||
|
|
||||||
data := filepath.Join(dir, "data")
|
|
||||||
symlink(t, filepath.Join(dir, "pool", "data"), data)
|
|
||||||
|
|
||||||
stderr := scanOK(t, filepath.Join(data, "daily"))
|
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunReportSucceeds(t *testing.T) {
|
func TestRunReportSucceeds(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
@@ -1480,9 +1028,8 @@ func TestRunTreesSucceeds(t *testing.T) {
|
|||||||
|
|
||||||
// The two directories holding the duplicate pair are duplicate
|
// The two directories holding the duplicate pair are duplicate
|
||||||
// trees of each other.
|
// trees of each other.
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want := "first\tdupe\tfiles\tsize\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
|
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1516,10 +1063,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
|
|||||||
cases := map[string]string{
|
cases := map[string]string{
|
||||||
cmdReport: "first\tdupe\tsize\n" +
|
cmdReport: "first\tdupe\tsize\n" +
|
||||||
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
||||||
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
cmdTrees: "first\tdupe\tfiles\tsize\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
||||||
"\t1\t300\t" +
|
"\t1\t300\n",
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, want := range cases {
|
for name, want := range cases {
|
||||||
@@ -1554,11 +1100,8 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
t.Errorf("report stdout = %q, want %q", got, want)
|
t.Errorf("report stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
|
want = "first\tdupe\tfiles\tsize\n" +
|
||||||
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||||
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
|
||||||
strconv.FormatInt(diskSize, 10) + "\n"
|
|
||||||
if got := runStdout(t, cmdTrees); got != want {
|
if got := runStdout(t, cmdTrees); got != want {
|
||||||
t.Errorf("trees stdout = %q, want %q", got, want)
|
t.Errorf("trees stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1579,7 +1122,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
||||||
// README §Settings: the path names the database file exactly. In
|
// README §Database: the path names the database file exactly. In
|
||||||
// SQLite's connection string an unescaped ? or # would end the file
|
// SQLite's connection string an unescaped ? or # would end the file
|
||||||
// name and % would start an escape, and a path starting with //
|
// name and % would start an escape, and a path starting with //
|
||||||
// could be read as a host name. %25 is a valid escape, so unescaped
|
// could be read as a host name. %25 is a valid escape, so unescaped
|
||||||
@@ -1624,8 +1167,9 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: a second scan that finds the lock held fails
|
// README §Database: while one scan holds the lock, a second scan
|
||||||
// at once, naming the lock file, without opening the database.
|
// fails at once, naming the lock file, without opening the
|
||||||
|
// database.
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
@@ -1684,7 +1228,7 @@ func assertSecondScanFails(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||||
// README §Database: a path through a symlink names the same lock
|
// README §Database: every path to one database names the same lock
|
||||||
// file, so while a scan holds it through one path, a scan through
|
// file, so while a scan holds it through one path, a scan through
|
||||||
// another fails, naming the lock file beside the database's real
|
// another fails, naming the lock file beside the database's real
|
||||||
// path.
|
// path.
|
||||||
@@ -1789,8 +1333,8 @@ func TestRunReportsDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
||||||
// README §Error handling: a failed write to stdout is a fatal error,
|
// README §Error handling: a stdout write failure exits 1, reported
|
||||||
// exit 1, reported on stderr starting with "sfdupes:".
|
// in one line on stderr.
|
||||||
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
@@ -35,17 +34,10 @@ const stallTime = 10 * time.Second
|
|||||||
// spinnerType selects the progressbar library's spinner style.
|
// spinnerType selects the progressbar library's spinner style.
|
||||||
const spinnerType = 14
|
const spinnerType = 14
|
||||||
|
|
||||||
// barWidth is the width of a phase's bar between its two ends, as in
|
// barWidth is the width of the progressbar library's bar between its
|
||||||
// the example line in README "Progress".
|
// two ends, as in the example line in README "Progress".
|
||||||
const barWidth = 10
|
const barWidth = 10
|
||||||
|
|
||||||
// barCell is one column of the filled part of a bar.
|
|
||||||
const barCell = "█"
|
|
||||||
|
|
||||||
// barColumns is the width of what percentBar returns: the percent in
|
|
||||||
// four columns and its sign, a space, and the bar with its two ends.
|
|
||||||
const barColumns = barWidth + 8
|
|
||||||
|
|
||||||
// percentScale converts a fraction to a percentage.
|
// percentScale converts a fraction to a percentage.
|
||||||
const percentScale = 100
|
const percentScale = 100
|
||||||
|
|
||||||
@@ -53,14 +45,19 @@ const percentScale = 100
|
|||||||
// read, as the progressbar library takes it.
|
// read, as the progressbar library takes it.
|
||||||
const fallbackWidth = 80
|
const fallbackWidth = 80
|
||||||
|
|
||||||
// spinnerMargin is the columns of a line besides its text: the spinner
|
// spinnerMargin is the columns of a spinner's line besides its text:
|
||||||
// and the three spaces the library draws around the text, and the
|
// the spinner and the three spaces the library draws around the text,
|
||||||
// terminal's last column, which every line stays clear of so that no
|
// and the terminal's last column, which every line stays clear of so
|
||||||
// terminal wraps it.
|
// that no terminal wraps it.
|
||||||
const spinnerMargin = 5
|
const spinnerMargin = 5
|
||||||
|
|
||||||
|
// barMargin is the columns of a bar's line besides its text: the
|
||||||
|
// percent and the bar the library draws after the text, the spaces
|
||||||
|
// around them, and the terminal's last column.
|
||||||
|
const barMargin = barWidth + 11
|
||||||
|
|
||||||
// countingRecords is what the display of a phase that reads the records
|
// countingRecords is what the display of a phase that reads the records
|
||||||
// says until it has the number of records, its total.
|
// says while it counts them, before it has its total.
|
||||||
const countingRecords = "counting the records"
|
const countingRecords = "counting the records"
|
||||||
|
|
||||||
// openPhase is the name of the phase in which report, trees and info
|
// openPhase is the name of the phase in which report, trees and info
|
||||||
@@ -112,13 +109,13 @@ type drawnCount struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// progress renders one phase's progress on stderr. On a TTY the
|
// progress renders one phase's progress on stderr. On a TTY the
|
||||||
// progressbar library draws its spinner and then line, fitted to the
|
// progressbar library draws line, fitted to the terminal's width: after
|
||||||
// terminal's width. A phase with an exact total ends the line with its
|
// the library's spinner, or, for a phase with an exact total, before the
|
||||||
// percent and bar, at the terminal's right edge, when the terminal has
|
// library's bar when the terminal has room for the bar beside it. It is
|
||||||
// room for them. It is drawn again every redrawInterval, so its elapsed
|
// drawn again every redrawInterval, so its elapsed time keeps moving
|
||||||
// time keeps moving while the phase waits for its next item. When
|
// while the phase waits for its next item. When stderr is not a TTY it
|
||||||
// stderr is not a TTY it emits no ANSI redraws: it prints line as the
|
// emits no ANSI redraws: it prints line as the phase starts, every
|
||||||
// phase starts, every plainInterval while it runs, and as it ends.
|
// plainInterval while it runs, and as it ends.
|
||||||
//
|
//
|
||||||
// The draws come from a goroutine of their own, so mu guards the fields
|
// The draws come from a goroutine of their own, so mu guards the fields
|
||||||
// below it. Every method must be called from the main goroutine. A nil
|
// below it. Every method must be called from the main goroutine. A nil
|
||||||
@@ -134,6 +131,7 @@ type progress struct {
|
|||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
bar *progressbar.ProgressBar // what the library draws; nil until then
|
bar *progressbar.ProgressBar // what the library draws; nil until then
|
||||||
|
withBar bool // bar is the library's bar, not its spinner
|
||||||
widest int // the widest line bar has drawn, in columns
|
widest int // the widest line bar has drawn, in columns
|
||||||
label string
|
label string
|
||||||
unit string
|
unit string
|
||||||
@@ -143,7 +141,7 @@ type progress struct {
|
|||||||
stalledOn string
|
stalledOn string
|
||||||
count int64
|
count int64
|
||||||
start time.Time // when the phase started, for the elapsed time
|
start time.Time // when the phase started, for the elapsed time
|
||||||
rateStart time.Time // when the count started, for the time remaining
|
rateStart time.Time // when the count started, for the ETA
|
||||||
lastItem time.Time // when the last item was completed, or the count started
|
lastItem time.Time // when the last item was completed, or the count started
|
||||||
// drawn holds the count at each draw over the last rateWindow, oldest
|
// drawn holds the count at each draw over the last rateWindow, oldest
|
||||||
// first; the rate counts from the first. It keeps the draw before the
|
// first; the rate counts from the first. It keeps the draw before the
|
||||||
@@ -217,34 +215,55 @@ func (p *progress) draw() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if p.width == nil {
|
if p.width == nil {
|
||||||
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
|
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, true))
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
width := p.width()
|
width := p.width()
|
||||||
text := p.line(now, width-spinnerMargin, true)
|
|
||||||
|
|
||||||
// Before each draw the library erases its line with as many spaces as
|
// A phase with an exact total gets the library's bar when its whole
|
||||||
// the widest line it has drawn. A display gets a new one when the
|
// line, without the percent the bar shows, fits beside the bar: the
|
||||||
// terminal has been made too narrow for that erase, which would wrap
|
// bar gives way before any part of the line. A total of 0 leaves the
|
||||||
// and leave a line behind at every draw.
|
// bar nothing to fill.
|
||||||
if p.bar == nil || p.widest >= width {
|
text, withBar := "", false
|
||||||
p.newBar(width)
|
if p.total > 0 && !p.estimate {
|
||||||
|
text = p.line(now, math.MaxInt, false)
|
||||||
|
withBar = len(text) <= width-barMargin
|
||||||
|
}
|
||||||
|
|
||||||
|
margin := barMargin
|
||||||
|
if !withBar {
|
||||||
|
margin = spinnerMargin
|
||||||
|
text = p.line(now, width-margin, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The library makes a bar or a spinner, and before each draw erases
|
||||||
|
// its line with as many spaces as the widest line it has drawn. A
|
||||||
|
// display gets a new one when it turns from one to the other, as its
|
||||||
|
// line grows or the terminal is resized, and when the terminal has
|
||||||
|
// been made too narrow for that erase, which would wrap and leave a
|
||||||
|
// line behind at every draw.
|
||||||
|
if p.bar == nil || p.withBar != withBar || p.widest >= width {
|
||||||
|
p.newBar(withBar, width)
|
||||||
|
}
|
||||||
|
|
||||||
|
if withBar {
|
||||||
|
_ = p.bar.Set64(p.count)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.bar.Describe(text)
|
p.bar.Describe(text)
|
||||||
|
|
||||||
// The library drew the spinner, the text and the spaces around it,
|
// The library drew the text and its margin, less the terminal's last
|
||||||
// less the terminal's last column, which spinnerMargin keeps clear. A
|
// column, which the margin keeps clear.
|
||||||
// bar's cells take one column each but more than one byte.
|
p.widest = max(p.widest, len(text)+margin-1)
|
||||||
p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBar has the library make what draw draws: its spinner, followed by
|
// newBar has the library make what draw draws: a bar for the phase's
|
||||||
// the text draw gives it and nothing of the library's own. p.mu must be
|
// total when withBar is set, and a spinner when not. Either shows the
|
||||||
// held.
|
// text draw gives it, and nothing of its own besides the spinner, or
|
||||||
func (p *progress) newBar(width int) {
|
// the bar and its percent. p.mu must be held.
|
||||||
|
func (p *progress) newBar(withBar bool, width int) {
|
||||||
if p.bar != nil {
|
if p.bar != nil {
|
||||||
// Erase what was drawn before, short of the last column of a
|
// Erase what was drawn before, short of the last column of a
|
||||||
// terminal that may have been made narrower since: the library's
|
// terminal that may have been made narrower since: the library's
|
||||||
@@ -253,36 +272,29 @@ func (p *progress) newBar(width int) {
|
|||||||
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
|
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
|
||||||
}
|
}
|
||||||
|
|
||||||
// A total of -1 gives the library's spinner.
|
total := int64(-1) // the library's spinner
|
||||||
p.bar = progressbar.NewOptions64(-1,
|
if withBar {
|
||||||
|
total = p.total
|
||||||
|
}
|
||||||
|
|
||||||
|
p.bar = progressbar.NewOptions64(total,
|
||||||
progressbar.OptionSetWriter(os.Stderr),
|
progressbar.OptionSetWriter(os.Stderr),
|
||||||
|
progressbar.OptionSetWidth(barWidth),
|
||||||
progressbar.OptionSpinnerType(spinnerType),
|
progressbar.OptionSpinnerType(spinnerType),
|
||||||
// The spinner turns at each draw, not on a timer of the
|
// The spinner turns at each draw, not on a timer of the
|
||||||
// library's own.
|
// library's own.
|
||||||
progressbar.OptionSetSpinnerChangeInterval(0),
|
progressbar.OptionSetSpinnerChangeInterval(0),
|
||||||
// The text carries the elapsed time and the time remaining.
|
// The text carries the elapsed time and the ETA.
|
||||||
progressbar.OptionSetElapsedTime(false),
|
progressbar.OptionSetElapsedTime(false),
|
||||||
progressbar.OptionSetPredictTime(false),
|
progressbar.OptionSetPredictTime(false),
|
||||||
)
|
)
|
||||||
p.widest = 0
|
p.withBar, p.widest = withBar, 0
|
||||||
}
|
|
||||||
|
|
||||||
// percentBar returns the percent and the bar that end the text of a
|
|
||||||
// phase with an exact total, as " 25% |██ |": the bar's
|
|
||||||
// barWidth columns fill in proportion to the items done. A count past
|
|
||||||
// the total, as when a scan adds records while trees reads them, shows
|
|
||||||
// as 100%. p.mu must be held.
|
|
||||||
func (p *progress) percentBar() string {
|
|
||||||
done := min(float64(p.count)/float64(p.total), 1)
|
|
||||||
filled := strings.Repeat(barCell, int(done*barWidth))
|
|
||||||
|
|
||||||
return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// setTotal gives a phase that started without a total its total, once
|
// setTotal gives a phase that started without a total its total, once
|
||||||
// it is known. The rate, the time remaining and the time with no item
|
// it is known. The rate, the ETA and the time with no item completed
|
||||||
// completed count from then on: the time spent waiting for the total
|
// count from then on: the time spent waiting for the total says nothing
|
||||||
// says nothing about how fast items go.
|
// about how fast items go.
|
||||||
func (p *progress) setTotal(total int64) {
|
func (p *progress) setTotal(total int64) {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return
|
return
|
||||||
@@ -426,26 +438,22 @@ func (p *progress) finish() {
|
|||||||
// line formats the phase's progress as of now in at most room columns,
|
// line formats the phase's progress as of now in at most room columns,
|
||||||
// as
|
// as
|
||||||
//
|
//
|
||||||
// [3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
|
// [3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
|
||||||
//
|
//
|
||||||
// where "files" is the phase's unit. A phase with no total shows the
|
// where "files" is the phase's unit, without the percent when percent
|
||||||
// items done instead, or what it waits for. The rate is that since the
|
// is false, as before the library's bar, which shows its own. A phase
|
||||||
// first draw in drawn, and the time remaining comes from the rate since
|
// with no total shows the items done instead, or what it waits for. The
|
||||||
// the count started. Once no item has been completed for stallTime, the
|
// rate is that since the first draw in drawn, and the ETA comes from
|
||||||
// rate, which would read 0, gives way to the word "stalled" and where
|
// the rate since the count started. Once no item has been completed for
|
||||||
// the phase waits, as "stalled disk/db".
|
// stallTime, the rate, which would read 0, gives way to the word
|
||||||
|
// "stalled" and where the phase waits, as "stalled disk/db".
|
||||||
//
|
//
|
||||||
// When bar is true, as on a terminal, a phase with an exact total ends
|
// The phase and its place, the progress, the elapsed time and the ETA
|
||||||
// the line with its percent and bar, in place of the percent after its
|
// are kept whole. A line wider than room leaves out the rate, the words
|
||||||
// count. A total of 0 leaves the bar nothing to fill.
|
// of a stalled phase, or what the phase waits for; only if it is still
|
||||||
//
|
// wider is it cut at room. Every part is ASCII, so its length is its
|
||||||
// The phase and its place, the progress, the elapsed time and the time
|
// width in columns. p.mu must be held.
|
||||||
// remaining are kept whole. A line wider than room first leaves out the
|
func (p *progress) line(now time.Time, room int, percent bool) string {
|
||||||
// rate, the words of a stalled phase, or what the phase waits for. Next
|
|
||||||
// it leaves out the bar. Only if it is still wider is it cut at room.
|
|
||||||
// Every part but the bar is ASCII, so its length is its width in
|
|
||||||
// columns. p.mu must be held.
|
|
||||||
func (p *progress) line(now time.Time, room int, bar bool) string {
|
|
||||||
elapsed := now.Sub(p.start).Round(time.Second)
|
elapsed := now.Sub(p.start).Round(time.Second)
|
||||||
|
|
||||||
// head and tail hold the parts kept whole; extra, between them, is
|
// head and tail hold the parts kept whole; extra, between them, is
|
||||||
@@ -454,41 +462,21 @@ func (p *progress) line(now time.Time, room int, bar bool) string {
|
|||||||
|
|
||||||
if p.total < 0 && p.waitingFor != "" {
|
if p.total < 0 && p.waitingFor != "" {
|
||||||
head, extra = p.label+": ", p.waitingFor+", "
|
head, extra = p.label+": ", p.waitingFor+", "
|
||||||
tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
|
tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
|
||||||
} else {
|
} else {
|
||||||
since := p.drawn[0]
|
since := p.drawn[0]
|
||||||
rate := perSecond(p.count-since.count, now.Sub(since.at))
|
rate := perSecond(p.count-since.count, now.Sub(since.at))
|
||||||
average := perSecond(p.count, now.Sub(p.rateStart))
|
average := perSecond(p.count, now.Sub(p.rateStart))
|
||||||
|
|
||||||
head = p.label + ": " + p.itemsDone(true) + ", "
|
head = p.label + ": " + p.itemsDone(percent) + ", "
|
||||||
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
|
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
|
||||||
tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
|
tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
|
||||||
|
|
||||||
if now.Sub(p.lastItem) >= stallTime {
|
if now.Sub(p.lastItem) >= stallTime {
|
||||||
extra = "stalled " + p.stalledOn + ", "
|
extra = "stalled " + p.stalledOn + ", "
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if bar && p.total > 0 && !p.estimate {
|
|
||||||
// Beside the bar, which shows the percent, the count shows none.
|
|
||||||
// The line is padded to the room the bar leaves it, so the bar
|
|
||||||
// stays at room's right edge however long the line is.
|
|
||||||
beside := room - barColumns
|
|
||||||
barHead := p.label + ": " + p.itemsDone(false) + ", "
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case len(barHead+extra+tail) <= beside:
|
|
||||||
return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
|
|
||||||
p.percentBar())
|
|
||||||
case len(barHead+tail) <= beside:
|
|
||||||
return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bar gives way. The rate, or the words of a stalled phase,
|
|
||||||
// gave way before it and stay out.
|
|
||||||
extra = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
full, kept := head+extra+tail, head+tail
|
full, kept := head+extra+tail, head+tail
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
@@ -538,10 +526,10 @@ func (p *progress) itemsDone(percent bool) string {
|
|||||||
return fmt.Sprintf("%s %.0f%%", done, pct)
|
return fmt.Sprintf("%s %.0f%%", done, pct)
|
||||||
}
|
}
|
||||||
|
|
||||||
// remain returns the time the phase has left at rate items a second, or
|
// eta returns the time the phase has left at rate items a second, or
|
||||||
// "unknown": with no total, before the first item, and once the count
|
// "unknown": with no total, before the first item, and once the count
|
||||||
// passes an estimate. p.mu must be held.
|
// passes an estimate. p.mu must be held.
|
||||||
func (p *progress) remain(rate float64) string {
|
func (p *progress) eta(rate float64) string {
|
||||||
switch {
|
switch {
|
||||||
case p.total < 0, p.estimate && p.count > p.total:
|
case p.total < 0, p.estimate && p.count > p.total:
|
||||||
return "unknown"
|
return "unknown"
|
||||||
@@ -551,8 +539,8 @@ func (p *progress) remain(rate float64) string {
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
left := time.Duration(
|
remaining := time.Duration(
|
||||||
float64(p.total-p.count) / rate * float64(time.Second))
|
float64(p.total-p.count) / rate * float64(time.Second))
|
||||||
|
|
||||||
return left.Round(time.Second).String()
|
return remaining.Round(time.Second).String()
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes //nolint:testpackage // drives the unexported startProgress
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -117,24 +117,24 @@ func TestProgressLine(t *testing.T) {
|
|||||||
label: scanPhase("load"), total: -1,
|
label: scanPhase("load"), total: -1,
|
||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
},
|
},
|
||||||
want: "[1/4] load: counting the records, elapsed 10s, remain unknown",
|
want: "[1/4] load: counting the records, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
|
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
|
||||||
},
|
},
|
||||||
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
|
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
|
||||||
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, remain unknown",
|
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
|
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
|
||||||
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, remain 0s",
|
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
||||||
estimate: true, count: 20,
|
estimate: true, count: 20,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
// Past the estimate the walk cannot tell how long it has left.
|
// Past the estimate the walk cannot tell how long it has left.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
@@ -142,13 +142,13 @@ func TestProgressLine(t *testing.T) {
|
|||||||
estimate: true, count: 120,
|
estimate: true, count: 120,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
|
want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
|
||||||
"remain unknown",
|
"eta unknown",
|
||||||
}, {
|
}, {
|
||||||
// A first scan's walk has no estimate.
|
// A first scan's walk has no estimate.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
|
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, remain unknown",
|
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
// The rate is in what the phase counts.
|
// The rate is in what the phase counts.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
@@ -156,14 +156,14 @@ func TestProgressLine(t *testing.T) {
|
|||||||
count: 20,
|
count: 20,
|
||||||
},
|
},
|
||||||
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
|
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
|
||||||
"remain 40s",
|
"eta 40s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
tc.p.start, tc.p.rateStart, tc.p.lastItem = ago, ago, now
|
tc.p.start, tc.p.rateStart, tc.p.lastItem = ago, ago, now
|
||||||
tc.p.drawn = []drawnCount{{at: ago}}
|
tc.p.drawn = []drawnCount{{at: ago}}
|
||||||
|
|
||||||
if got := tc.p.line(now, math.MaxInt, false); got != tc.want {
|
if got := tc.p.line(now, math.MaxInt, true); got != tc.want {
|
||||||
t.Errorf("line = %q, want %q", got, tc.want)
|
t.Errorf("line = %q, want %q", got, tc.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -200,47 +200,46 @@ func TestLineFitsRoom(t *testing.T) {
|
|||||||
cases := []struct {
|
cases := []struct {
|
||||||
p *progress
|
p *progress
|
||||||
room int
|
room int
|
||||||
bar bool
|
percent bool
|
||||||
want string
|
want string
|
||||||
}{{
|
}{{
|
||||||
p: hash, room: 60,
|
p: hash, room: 57, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: hash, room: 59,
|
p: hash, room: 56, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40s",
|
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: hash, room: 48,
|
p: hash, room: 45, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40",
|
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40",
|
||||||
}, {
|
}, {
|
||||||
// The bar shows the percent.
|
// Before the library's bar, which shows the percent.
|
||||||
p: hash, room: 74, bar: true,
|
p: hash, room: 57, percent: false,
|
||||||
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, remain 40s" +
|
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
|
||||||
" 20% |██ |",
|
|
||||||
}, {
|
}, {
|
||||||
p: load, room: 43,
|
p: load, room: 40, percent: true,
|
||||||
want: "[1/4] load: elapsed 10s, remain unknown",
|
want: "[1/4] load: elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: stalled, room: 68,
|
p: stalled, room: 65, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
|
want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
|
||||||
"remain 3m20s",
|
"eta 3m20s",
|
||||||
}, {
|
}, {
|
||||||
p: stalled, room: 67,
|
p: stalled, room: 64, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 50s, remain 3m20s",
|
want: "[3/4] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
if got := tc.p.line(now, tc.room, tc.bar); got != tc.want {
|
if got := tc.p.line(now, tc.room, tc.percent); got != tc.want {
|
||||||
t.Errorf("line(%d, %t) = %q, want %q", tc.room, tc.bar, got,
|
t.Errorf("line(%d, %t) = %q, want %q",
|
||||||
tc.want)
|
tc.room, tc.percent, got, tc.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
|
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
|
||||||
// its time remaining and its time with no record read from when the
|
// its ETA and its time with no record read from when the total becomes
|
||||||
// total becomes known, not from when the phase started waiting for it:
|
// known, not from when the phase started waiting for it: after 20
|
||||||
// after 20 seconds of waiting for the total, the phase has not stalled 5
|
// seconds of waiting for the total, the phase has not stalled 5 seconds
|
||||||
// seconds after it, and has 10 seconds after it with no record read.
|
// after it, and has 10 seconds after it with no record read.
|
||||||
func TestSetTotalStartsRate(t *testing.T) {
|
func TestSetTotalStartsRate(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -256,15 +255,15 @@ func TestSetTotalStartsRate(t *testing.T) {
|
|||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
|
|
||||||
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, remain unknown"
|
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
|
|
||||||
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, remain unknown"
|
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -273,8 +272,8 @@ func TestSetTotalStartsRate(t *testing.T) {
|
|||||||
p.increment()
|
p.increment()
|
||||||
}
|
}
|
||||||
|
|
||||||
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, remain 40s"
|
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -317,7 +316,7 @@ func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
|
|||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
||||||
|
|
||||||
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown"
|
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
|
||||||
if len(lines) != 2 || lines[0] != want ||
|
if len(lines) != 2 || lines[0] != want ||
|
||||||
!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
|
!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
|
||||||
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
|
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
|
||||||
@@ -337,7 +336,7 @@ func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
|
|||||||
}, nil, testRedraw, time.Now)
|
}, nil, testRedraw, time.Now)
|
||||||
|
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[1/4] load: counting the records, elapsed 0s, remain unknown\n", 3)
|
"[1/4] load: counting the records, elapsed 0s, eta unknown\n", 3)
|
||||||
p.finish()
|
p.finish()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -367,23 +366,23 @@ func TestPlainProgressStalledProducer(t *testing.T) {
|
|||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, remain 20s\n", 3)
|
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
|
||||||
|
|
||||||
clock.Add(time.Second)
|
clock.Add(time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, remain 24s\n", 3)
|
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
|
||||||
|
|
||||||
clock.Add(4 * time.Second)
|
clock.Add(4 * time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, remain 40s\n",
|
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
|
||||||
3)
|
3)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
|
// TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
|
||||||
// little over plainInterval after the line before, as a late tick
|
// little over plainInterval after the line before, as a late tick
|
||||||
// does, through a stall and a burst of files after it: the rate covers
|
// does, through a stall and a burst of files after it: the rate covers
|
||||||
// the files read since the line before, while the time remaining comes
|
// the files read since the line before, while the ETA comes from the
|
||||||
// from the phase's average.
|
// phase's average.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestPlainProgressRateAfterStall(t *testing.T) {
|
func TestPlainProgressRateAfterStall(t *testing.T) {
|
||||||
@@ -411,10 +410,10 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
|
|||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
want := []string{
|
want := []string{
|
||||||
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown",
|
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
|
||||||
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, remain unknown",
|
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
|
||||||
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, remain unknown",
|
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
|
||||||
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, remain 2m15s",
|
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
||||||
@@ -427,8 +426,8 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
|
|||||||
// redrawInterval, as a terminal does, through a stall of stallTime and
|
// redrawInterval, as a terminal does, through a stall of stallTime and
|
||||||
// a burst of files after it: the rate covers the files read over the
|
// a burst of files after it: the rate covers the files read over the
|
||||||
// last rateWindow, and falls to 0 once the burst is older than that,
|
// last rateWindow, and falls to 0 once the burst is older than that,
|
||||||
// while the time remaining comes from the phase's average. Plain lines
|
// while the ETA comes from the phase's average. Plain lines show each
|
||||||
// show each draw.
|
// draw.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
||||||
@@ -458,8 +457,8 @@ func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
|||||||
redraw(rateWindow + redrawInterval)
|
redraw(rateWindow + redrawInterval)
|
||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, remain 4m45s\n" +
|
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
|
||||||
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, remain 4m47s\n"
|
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
|
||||||
if !strings.Contains(stderr(), want) {
|
if !strings.Contains(stderr(), want) {
|
||||||
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
|
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
|
||||||
}
|
}
|
||||||
@@ -481,10 +480,10 @@ func lastFrame(out string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
|
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
|
||||||
// phase, with a bar or without, keeps being drawn while the phase waits
|
// phase, spinner or bar, keeps being drawn while the phase waits for its
|
||||||
// for its next item, with the elapsed time as of now, and shows every
|
// next item, with the elapsed time as of now, and shows every item
|
||||||
// item completed before the wait. The terminal is 120 columns wide, so
|
// completed before the wait. stderr is not a terminal here, so the
|
||||||
// the hash phase's display has room for its bar.
|
// displays take it to be fallbackWidth wide.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
||||||
@@ -507,8 +506,7 @@ func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
|||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
before := len(stderr())
|
before := len(stderr())
|
||||||
since := func() string { return stderr()[before:] }
|
since := func() string { return stderr()[before:] }
|
||||||
p := startProgress(tc.params, func() int { return 120 }, testRedraw,
|
p := startProgress(tc.params, terminalWidth, testRedraw, time.Now)
|
||||||
time.Now)
|
|
||||||
|
|
||||||
for range 50 {
|
for range 50 {
|
||||||
p.increment()
|
p.increment()
|
||||||
@@ -554,13 +552,13 @@ func drawLate(stderr func() string, params progressParams, width int,
|
|||||||
return stderr()[before:]
|
return stderr()[before:]
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalLinesKeepTimeRemaining checks the line each display shows
|
// TestTerminalLinesKeepETA checks the line each display shows on an
|
||||||
// on an 80-column terminal at a large scan's counts and times, and the
|
// 80-column terminal at a large scan's counts and times, and the line
|
||||||
// line of trees' read phase at the same counts: the rate is left out,
|
// of trees' read phase at the same counts: the rate is left out, so
|
||||||
// so that the elapsed time and the time remaining stay whole.
|
// that the elapsed time and the ETA stay whole.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
func TestTerminalLinesKeepETA(t *testing.T) {
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -573,19 +571,19 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
|||||||
estimate: true,
|
estimate: true,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
|
want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
|
||||||
"remain unknown",
|
"eta unknown",
|
||||||
}, {
|
}, {
|
||||||
params: progressParams{
|
params: progressParams{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
||||||
},
|
},
|
||||||
want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
||||||
"remain 17h30m0s",
|
"eta 17h30m0s",
|
||||||
}, {
|
}, {
|
||||||
params: progressParams{
|
params: progressParams{
|
||||||
label: treesPhase("read"), unit: "records", total: 9876543,
|
label: treesPhase("read"), unit: "records", total: 9876543,
|
||||||
},
|
},
|
||||||
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
||||||
"remain 17h30m0s",
|
"eta 17h30m0s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -596,32 +594,30 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalStalledLines draws every phase's line once the phase has
|
// TestTerminalStalledLines draws every phase's line on an 80-column
|
||||||
// stalled, its last item done 42 seconds before. At an ordinary scan's
|
// terminal once the phase has stalled, its last item done 42 seconds
|
||||||
// counts and times, 12345 of 45678 items done four and a half minutes
|
// before. At an ordinary scan's counts and times, 12345 of 45678 items
|
||||||
// in, on a terminal 100 columns wide, the line shows where the phase
|
// done four and a half minutes in, the line shows where the phase waits
|
||||||
// waits whole, beside the phase, progress, elapsed time and time
|
// whole, beside the phase, progress, elapsed time and ETA. At a large
|
||||||
// remaining. At a large scan's, 1234567 of 9876543 two and a half hours
|
// scan's, 1234567 of 9876543 two and a half hours in, those four still
|
||||||
// in, on an 80-column terminal, those four still show whole, as the
|
// show whole, as the words give way first where they do not fit.
|
||||||
// words give way first where they do not fit.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalStalledLines(t *testing.T) {
|
func TestTerminalStalledLines(t *testing.T) {
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
// drawStalled draws the display params describes on a terminal width
|
// drawStalled draws the display params describes as it starts and
|
||||||
// columns wide as it starts and once more as it ends, with count of
|
// once more as it ends, with count of total items done elapsed into
|
||||||
// total items done elapsed into the phase, and returns what the
|
// the phase, and returns what the terminal shows.
|
||||||
// terminal shows.
|
drawStalled := func(params progressParams, total, count int64,
|
||||||
drawStalled := func(params progressParams, width int, total,
|
elapsed time.Duration,
|
||||||
count int64, elapsed time.Duration,
|
|
||||||
) string {
|
) string {
|
||||||
before := len(stderr())
|
before := len(stderr())
|
||||||
clock := &testClock{now: time.Now()}
|
clock := &testClock{now: time.Now()}
|
||||||
params.total = total
|
params.total = total
|
||||||
|
|
||||||
// No timed draws come between the two.
|
// No timed draws come between the two.
|
||||||
p := startProgress(params, func() int { return width }, time.Hour,
|
p := startProgress(params, func() int { return 80 }, time.Hour,
|
||||||
clock.Now)
|
clock.Now)
|
||||||
|
|
||||||
clock.Add(elapsed - 42*time.Second)
|
clock.Add(elapsed - 42*time.Second)
|
||||||
@@ -642,9 +638,9 @@ func TestTerminalStalledLines(t *testing.T) {
|
|||||||
{label: scanPhase("hash"), stalledOn: hashStalledOn},
|
{label: scanPhase("hash"), stalledOn: hashStalledOn},
|
||||||
{label: scanPhase("update"), stalledOn: "db"},
|
{label: scanPhase("update"), stalledOn: "db"},
|
||||||
{label: treesPhase("read"), stalledOn: "db"},
|
{label: treesPhase("read"), stalledOn: "db"},
|
||||||
{label: treesPhase("group"), stalledOn: "cpu/disk"},
|
{label: treesPhase("group"), stalledOn: "cpu"},
|
||||||
{label: treesPhase("write"), stalledOn: "out"},
|
{label: treesPhase("write"), stalledOn: "out"},
|
||||||
{label: reportPhase("read"), stalledOn: "db/disk/out"},
|
{label: reportPhase("read"), stalledOn: "db/out"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, params := range phases {
|
for _, params := range phases {
|
||||||
@@ -653,22 +649,19 @@ func TestTerminalStalledLines(t *testing.T) {
|
|||||||
approx = "~"
|
approx = "~"
|
||||||
}
|
}
|
||||||
|
|
||||||
// The percent is in the line, or beside the bar.
|
shown := drawStalled(params, 45678, 12345,
|
||||||
shown := drawStalled(params, 100, 45678, 12345,
|
|
||||||
4*time.Minute+30*time.Second)
|
4*time.Minute+30*time.Second)
|
||||||
for _, want := range []string{
|
want := params.label + ": [12345/" + approx + "45678] 27%, stalled " +
|
||||||
params.label + ": [12345/" + approx + "45678]",
|
params.stalledOn + ", elapsed 4m30s, eta 12m9s"
|
||||||
", stalled " + params.stalledOn + ", elapsed 4m30s, remain 12m9s",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(shown, want) {
|
if !strings.Contains(shown, want) {
|
||||||
t.Errorf("terminal shows %q, want %q", shown, want)
|
t.Errorf("terminal shows %q, want %q", shown, want)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
shown = drawStalled(params, 80, 9876543, 1234567, 150*time.Minute)
|
shown = drawStalled(params, 9876543, 1234567, 150*time.Minute)
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
params.label + ": [1234567/" + approx + "9876543] 12%, ",
|
params.label + ": [1234567/" + approx + "9876543] 12%, ",
|
||||||
", elapsed 2h30m0s, remain 17h30m0s",
|
", elapsed 2h30m0s, eta 17h30m0s",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(shown, want) {
|
if !strings.Contains(shown, want) {
|
||||||
t.Errorf("terminal shows %q, want %q", shown, want)
|
t.Errorf("terminal shows %q, want %q", shown, want)
|
||||||
@@ -713,164 +706,13 @@ func TestTerminalLinesFitWidth(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalLinesStartWithSpinner draws a display with no total and
|
|
||||||
// one with a bar, two and a half hours into the phase, on a terminal 120
|
|
||||||
// columns wide. Each line must start with the library's spinner in the
|
|
||||||
// first column, then a space and the phase's label.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalLinesStartWithSpinner(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
for _, params := range []progressParams{
|
|
||||||
{label: scanPhase("walk"), unit: scanUnit, total: -1},
|
|
||||||
{label: scanPhase("hash"), unit: scanUnit, total: 9876543},
|
|
||||||
} {
|
|
||||||
shown := lastFrame(drawLate(stderr, params, 120, 1234567))
|
|
||||||
spinner, rest, _ := strings.Cut(shown, " ")
|
|
||||||
|
|
||||||
if utf8.RuneCountInString(spinner) != 1 ||
|
|
||||||
!strings.HasPrefix(rest, params.label+": ") {
|
|
||||||
t.Errorf("terminal shows %q, want the spinner, a space and %q",
|
|
||||||
shown, params.label)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.total > 0 && !strings.Contains(shown, "% |") {
|
|
||||||
t.Errorf("terminal shows %q, want a bar", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalBarStaysAtRightEdge draws the hash phase's display two
|
|
||||||
// and a half hours in, at two counts whose lines differ in length, on
|
|
||||||
// terminals 110 and 120 columns wide. The bar is partly filled at the
|
|
||||||
// first count and full at the second. It must start the same number of
|
|
||||||
// columns from the terminal's right edge each time, and end the same
|
|
||||||
// number of columns from it: a line that grows or shrinks, as its time
|
|
||||||
// remaining does, never moves the bar, and nor do the bar's filled
|
|
||||||
// cells.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalBarStaysAtRightEdge(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
widths := []int{110, 120}
|
|
||||||
counts := []int64{1234567, 9876543}
|
|
||||||
startFromEdge := make([]int, 0, len(widths)*len(counts))
|
|
||||||
endFromEdge := make([]int, 0, len(widths)*len(counts))
|
|
||||||
|
|
||||||
for _, width := range widths {
|
|
||||||
for _, count := range counts {
|
|
||||||
shown := lastFrame(drawLate(stderr, params, width, count))
|
|
||||||
|
|
||||||
beforeBar, _, found := strings.Cut(shown, "% |")
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("terminal %d wide shows %q, want a bar", width, shown)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bar ends with the last "|".
|
|
||||||
throughBar := shown[:strings.LastIndex(shown, "|")+1]
|
|
||||||
|
|
||||||
startFromEdge = append(startFromEdge,
|
|
||||||
width-utf8.RuneCountInString(beforeBar))
|
|
||||||
endFromEdge = append(endFromEdge,
|
|
||||||
width-utf8.RuneCountInString(throughBar))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Min(startFromEdge) != slices.Max(startFromEdge) {
|
|
||||||
t.Errorf("bar starts %v columns from the right edge, want the same "+
|
|
||||||
"every time", startFromEdge)
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Min(endFromEdge) != slices.Max(endFromEdge) {
|
|
||||||
t.Errorf("bar ends %v columns from the right edge, want the same "+
|
|
||||||
"every time", endFromEdge)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalRateGivesWayBeforeBar draws the hash phase's display two
|
|
||||||
// and a half hours in. On a terminal 90 columns wide the line fits
|
|
||||||
// beside the bar without its rate, but not with it: the bar is drawn
|
|
||||||
// and the rate is not. On a terminal 85 columns wide the line does not
|
|
||||||
// fit beside the bar even without its rate: neither is drawn, though
|
|
||||||
// the rate would fit where the bar was.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalRateGivesWayBeforeBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
shown := lastFrame(drawLate(stderr, params, 90, 1234567))
|
|
||||||
if !strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
|
|
||||||
t.Errorf("terminal 90 wide shows %q, want the bar and no rate", shown)
|
|
||||||
}
|
|
||||||
|
|
||||||
shown = lastFrame(drawLate(stderr, params, 85, 1234567))
|
|
||||||
if strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
|
|
||||||
t.Errorf("terminal 85 wide shows %q, want neither bar nor rate", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalSpinnerTurnsBesideBar draws the hash phase's display, its
|
|
||||||
// bar partly filled, several times on a terminal 120 columns wide. The
|
|
||||||
// spinner must turn at every draw. A filled cell of the bar takes one
|
|
||||||
// column but three bytes. A line measured in bytes would seem too wide
|
|
||||||
// for the terminal, and every draw would start the display over, its
|
|
||||||
// spinner included.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalSpinnerTurnsBesideBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
// No timed draws: the test draws.
|
|
||||||
p := startProgress(params, func() int { return 120 }, time.Hour,
|
|
||||||
time.Now)
|
|
||||||
|
|
||||||
p.mu.Lock()
|
|
||||||
p.count = 1234567
|
|
||||||
p.mu.Unlock()
|
|
||||||
|
|
||||||
const draws = 4
|
|
||||||
|
|
||||||
spinners := make([]string, 0, draws)
|
|
||||||
|
|
||||||
for range draws {
|
|
||||||
before := len(stderr())
|
|
||||||
|
|
||||||
p.draw()
|
|
||||||
|
|
||||||
spinner, _, _ := strings.Cut(lastFrame(stderr()[before:]), " ")
|
|
||||||
spinners = append(spinners, spinner)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.finish()
|
|
||||||
|
|
||||||
for i := 1; i < len(spinners); i++ {
|
|
||||||
if spinners[i] == spinners[i-1] {
|
|
||||||
t.Errorf("spinners at successive draws are %q, want each to "+
|
|
||||||
"differ from the one before", spinners)
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
|
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
|
||||||
// and a half hours into the phase, on a terminal 120 columns wide, then
|
// and a half hours into the phase, on a terminal 120 columns wide, then
|
||||||
// makes the terminal narrower than that line and draws it again a few
|
// makes the terminal narrower than that line and draws it again a few
|
||||||
// times. Nothing written after that, the erase before each redraw
|
// times. Nothing written after that, the erase before each redraw
|
||||||
// included, may reach the narrower terminal's last column: it would
|
// included, may reach the narrower terminal's last column: it would
|
||||||
// wrap, and every redraw would leave a line behind. At the narrower
|
// wrap, and every redraw would leave a line behind. At these widths the
|
||||||
// width the hash phase's bar gives way.
|
// spinner stays a spinner and the bar gives way to the spinner.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
||||||
@@ -888,7 +730,7 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
|||||||
progressParams{
|
progressParams{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
||||||
},
|
},
|
||||||
80,
|
90,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -928,10 +770,10 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProgressWarningsOnOwnLines drives the terminal displays, with a
|
// TestProgressWarningsOnOwnLines drives the terminal displays, spinner
|
||||||
// bar and without, through a run of warnings with no items between
|
// and bar, through a run of warnings with no items between them, as when
|
||||||
// them, as when the walk meets many unreadable paths, for many redraws:
|
// the walk meets many unreadable paths, for many redraws: every warning
|
||||||
// every warning must land on a line of its own, never inside a redraw.
|
// must land on a line of its own, never inside a redraw.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressWarningsOnOwnLines(t *testing.T) {
|
func TestProgressWarningsOnOwnLines(t *testing.T) {
|
||||||
@@ -977,12 +819,11 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProgressWriterRowsOnOwnLines drives the terminal displays, with a
|
// TestProgressWriterRowsOnOwnLines drives the terminal displays,
|
||||||
// bar and without, through a run of rows written through the display's
|
// spinner and bar, through a run of rows written through the display's
|
||||||
// writer to the same terminal, as stdout is in a plain "sfdupes trees",
|
// writer to the same terminal, as stdout is in a plain "sfdupes trees",
|
||||||
// each written out at once, for many redraws: the terminal must show
|
// each written out at once, for many redraws: the terminal must show
|
||||||
// every row whole, in order, on a line of its own. The terminal is 120
|
// every row whole, in order, on a line of its own.
|
||||||
// columns wide, so the write phase's display has room for its bar.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
||||||
@@ -999,8 +840,7 @@ func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
|||||||
var want []string
|
var want []string
|
||||||
|
|
||||||
for _, params := range cases {
|
for _, params := range cases {
|
||||||
p := startProgress(params, func() int { return 120 }, testRedraw,
|
p := startProgress(params, terminalWidth, testRedraw, time.Now)
|
||||||
time.Now)
|
|
||||||
out := p.writer(os.Stderr)
|
out := p.writer(os.Stderr)
|
||||||
|
|
||||||
// No pause between rows: one is garbled only if a redraw lands
|
// No pause between rows: one is garbled only if a redraw lands
|
||||||
@@ -1,17 +1,13 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -24,15 +20,13 @@ const minGroupSize = 2
|
|||||||
|
|
||||||
// scanRec is one file record from the database. The signature (size,
|
// scanRec is one file record from the database. The signature (size,
|
||||||
// head, tail, content) is the duplicate key; mtime is informational
|
// head, tail, content) is the duplicate key; mtime is informational
|
||||||
// only and used by scan for change detection. diskSize is the bytes the
|
// only and used by scan for change detection.
|
||||||
// file takes on disk, which trees adds up.
|
|
||||||
type scanRec struct {
|
type scanRec struct {
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
|
||||||
mtime time.Time
|
mtime time.Time
|
||||||
head [sha256.Size]byte
|
head string
|
||||||
tail [sha256.Size]byte
|
tail string
|
||||||
content [sha256.Size]byte
|
content string
|
||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,17 +38,17 @@ func reportPhase(name string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runReport implements the report subcommand: it prints the file-level
|
// runReport implements the report subcommand: it prints the file-level
|
||||||
// duplicates report as TSV on stdout. Of the scanned filesystem it
|
// duplicates report as TSV on stdout. It never touches the scanned
|
||||||
// only checks, with pathExists, that each path in a duplicate group is
|
// filesystem; its only I/O is the database, stdout, and stderr. It
|
||||||
// still there. It names the database on stderr before it opens it. Any
|
// names the database on stderr before it opens it. Any database
|
||||||
// database problem, including a missing database, is fatal.
|
// problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in two phases, each under a progress display of its
|
// The work runs in two phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database) and read (read the number of
|
// own on stderr: open (open the database) and read (count the records,
|
||||||
// records from the totals table, then read the records in signature
|
// then read them in signature order, writing each size's duplicate
|
||||||
// order, writing each size's duplicate groups to stdout once that size
|
// groups to stdout once that size is read; see readReport). Each
|
||||||
// is read; see readReport). Each display ends before the next phase
|
// display ends before the next phase starts, and before an error is
|
||||||
// starts, and before an error is returned to be printed.
|
// returned to be printed.
|
||||||
func runReport(ctx context.Context, stdout io.Writer) error {
|
func runReport(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -80,14 +74,13 @@ func runReport(ctx context.Context, stdout io.Writer) error {
|
|||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
// The read phase checks the paths of each duplicate group on disk
|
// The read phase writes the report to stdout while it reads the
|
||||||
// and writes the report to stdout while it reads the records, so it
|
// records, so it may wait on either.
|
||||||
// may wait on any of the three.
|
|
||||||
prog = newProgress(progressParams{
|
prog = newProgress(progressParams{
|
||||||
label: reportPhase("read"), unit: "records", total: -1,
|
label: reportPhase("read"), unit: "records", total: -1,
|
||||||
waitingFor: countingRecords, stalledOn: "db/disk/out",
|
waitingFor: countingRecords, stalledOn: "db/out",
|
||||||
})
|
})
|
||||||
report, err := readReport(ctx, db, dbPath, stdout, pathExists, prog)
|
report, err := readReport(ctx, db, dbPath, stdout, prog)
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
@@ -108,15 +101,14 @@ func runReport(ctx context.Context, stdout io.Writer) error {
|
|||||||
// dbPath, as its total, then reads every record in signature order,
|
// dbPath, as its total, then reads every record in signature order,
|
||||||
// counting each on prog, and writes the report to stdout: the header
|
// counting each on prog, and writes the report to stdout: the header
|
||||||
// line, then the rows of each size's duplicate groups once that size is
|
// line, then the rows of each size's duplicate groups once that size is
|
||||||
// read. A path for which exists returns false is left out of its group.
|
// read. stdout may be the terminal prog is drawn on, so the lines go
|
||||||
// stdout may be the terminal prog is drawn on, so the lines go through
|
// through prog's writer. It returns what it wrote, with the totals for
|
||||||
// prog's writer. It returns what it wrote, with the totals for the
|
// the summary line.
|
||||||
// summary line.
|
|
||||||
func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
||||||
stdout io.Writer, exists func(path string) bool, prog *progress,
|
stdout io.Writer, prog *progress,
|
||||||
) (*reportWriter, error) {
|
) (*reportWriter, error) {
|
||||||
out := prog.writer(stdout)
|
out := prog.writer(stdout)
|
||||||
report := &reportWriter{out: out, exists: exists}
|
report := &reportWriter{out: out}
|
||||||
|
|
||||||
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
|
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -163,10 +155,6 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
|||||||
type reportWriter struct {
|
type reportWriter struct {
|
||||||
out io.Writer
|
out io.Writer
|
||||||
|
|
||||||
// exists reports whether a path is still on disk. A path that is
|
|
||||||
// gone is left out of its group.
|
|
||||||
exists func(path string) bool
|
|
||||||
|
|
||||||
sig scanRec // the signature being read; its path is not used
|
sig scanRec // the signature being read; its path is not used
|
||||||
paths []string // the paths read so far with sig
|
paths []string // the paths read so far with sig
|
||||||
groups [][]string // the duplicate groups of sig.size, each sorted
|
groups [][]string // the duplicate groups of sig.size, each sorted
|
||||||
@@ -203,20 +191,11 @@ func (w *reportWriter) add(r scanRec) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// endGroup ends the paths read with w.sig: two or more that are still
|
// endGroup ends the paths read with w.sig: two or more are a duplicate
|
||||||
// on disk are a duplicate group, held with its paths sorted until
|
// group, held with its paths sorted until writeGroups writes it.
|
||||||
// writeGroups writes it. Empty files form no group, since removing one
|
|
||||||
// frees nothing. Only the paths of a possible group are checked, never
|
|
||||||
// a path with no duplicate or an empty file.
|
|
||||||
func (w *reportWriter) endGroup() {
|
func (w *reportWriter) endGroup() {
|
||||||
if len(w.paths) >= minGroupSize && w.sig.size > 0 {
|
if len(w.paths) < minGroupSize {
|
||||||
w.paths = slices.DeleteFunc(w.paths, func(p string) bool {
|
// A path with no duplicate is dropped, and its slice reused.
|
||||||
return !w.exists(p)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(w.paths) < minGroupSize || w.sig.size == 0 {
|
|
||||||
// The paths are dropped, and their slice reused.
|
|
||||||
w.paths = w.paths[:0]
|
w.paths = w.paths[:0]
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -263,16 +242,6 @@ func (w *reportWriter) finish() error {
|
|||||||
return w.writeGroups()
|
return w.writeGroups()
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathExists reports whether path is still on disk, for report and
|
|
||||||
// trees. It is false only when lstat finds that the path, or a
|
|
||||||
// directory above it, is gone. Any other error counts as present, so
|
|
||||||
// a user who may not look inside a directory still sees its copies.
|
|
||||||
func pathExists(path string) bool {
|
|
||||||
_, err := os.Lstat(path)
|
|
||||||
|
|
||||||
return !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ENOTDIR)
|
|
||||||
}
|
|
||||||
|
|
||||||
// escapePath returns a path as it is written in a report column (README
|
// escapePath returns a path as it is written in a report column (README
|
||||||
// "Report output format"): a backslash, tab, newline or carriage return
|
// "Report output format"): a backslash, tab, newline or carriage return
|
||||||
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
|
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -19,27 +18,14 @@ import (
|
|||||||
// awkwardDir is a directory name holding every byte the reports escape.
|
// awkwardDir is a directory name holding every byte the reports escape.
|
||||||
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
|
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
|
||||||
|
|
||||||
// testHash returns a stand-in hash for a test record: the bytes of s,
|
|
||||||
// then zeros. Stand-ins sort as their strings do, so a test can choose
|
|
||||||
// the order in which report reads its records.
|
|
||||||
func testHash(s string) [sha256.Size]byte {
|
|
||||||
var h [sha256.Size]byte
|
|
||||||
|
|
||||||
copy(h[:], s)
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
|
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
|
||||||
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
|
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
|
||||||
// sorts after it, so awkwardDir coming first shows that sorting uses
|
// sorts after it, so awkwardDir coming first shows that sorting uses
|
||||||
// the raw path.
|
// the raw path.
|
||||||
func awkwardPairRecs() []scanRec {
|
func awkwardPairRecs() []scanRec {
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
|
||||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,37 +58,15 @@ type dupeGroup struct {
|
|||||||
paths []string
|
paths []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// everyPathExists stands in for pathExists where a test's records name
|
|
||||||
// made-up paths that are not on disk.
|
|
||||||
func everyPathExists(string) bool { return true }
|
|
||||||
|
|
||||||
// recsOnDisk returns a new directory and recs with each path moved
|
|
||||||
// under it, after creating an empty file at each moved path: report and
|
|
||||||
// trees then find every path still on disk.
|
|
||||||
func recsOnDisk(t *testing.T, recs []scanRec) (string, []scanRec) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir := realTempDir(t)
|
|
||||||
moved := make([]scanRec, len(recs))
|
|
||||||
|
|
||||||
for i, r := range recs {
|
|
||||||
r.path = writeFile(t, dir, r.path, nil)
|
|
||||||
moved[i] = r
|
|
||||||
}
|
|
||||||
|
|
||||||
return dir, moved
|
|
||||||
}
|
|
||||||
|
|
||||||
// dupeGroups returns the duplicate groups report reads from db, in
|
// dupeGroups returns the duplicate groups report reads from db, in
|
||||||
// report order, from the rows it writes: a group's rows come together,
|
// report order, from the rows it writes: a group's rows come together,
|
||||||
// each naming the group's first path. Every path counts as still on
|
// each naming the group's first path.
|
||||||
// disk.
|
|
||||||
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
|
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
|
|
||||||
_, err := readReport(t.Context(), db, "", &out, everyPathExists, nil)
|
_, err := readReport(t.Context(), db, "", &out, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -149,8 +113,7 @@ func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunReportEscapesPaths(t *testing.T) {
|
func TestRunReportEscapesPaths(t *testing.T) {
|
||||||
dir, recs := recsOnDisk(t, awkwardPairRecs())
|
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
|
||||||
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -161,7 +124,7 @@ func TestRunReportEscapesPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
want := "first\tdupe\tsize\n" +
|
want := "first\tdupe\tsize\n" +
|
||||||
dir + `/d/\tone\ntwo\rthree\\four/f` + "\t" + dir + "/d/A/f\t5\n"
|
`/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n"
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -180,8 +143,8 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
recs := make([]scanRec, 2*ioBufSize/len(dir))
|
recs := make([]scanRec, 2*ioBufSize/len(dir))
|
||||||
for i := range recs {
|
for i := range recs {
|
||||||
recs[i] = scanRec{
|
recs[i] = scanRec{
|
||||||
size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
|
size: int64(i/2) + 1, head: "h", tail: "t", content: "c",
|
||||||
content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
|
path: fmt.Sprintf("%s/%d", dir, i),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,8 +158,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
// Counts the records read, and draws nothing.
|
// Counts the records read, and draws nothing.
|
||||||
prog := &progress{now: time.Now}
|
prog := &progress{now: time.Now}
|
||||||
|
|
||||||
_, err = readReport(t.Context(), db, "", failingWriter{},
|
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
|
||||||
everyPathExists, prog)
|
|
||||||
if !errors.Is(err, errWriteFailed) ||
|
if !errors.Is(err, errWriteFailed) ||
|
||||||
!strings.HasPrefix(err.Error(), "write stdout: ") {
|
!strings.HasPrefix(err.Error(), "write stdout: ") {
|
||||||
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
|
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
|
||||||
@@ -211,7 +173,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
// reportOpenLine is the plain line of report's open phase when it opens
|
// reportOpenLine is the plain line of report's open phase when it opens
|
||||||
// the database at once; it prints it as the phase starts and as it ends.
|
// the database at once; it prints it as the phase starts and as it ends.
|
||||||
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// smokeReport is report's stdout for the smoke-test records.
|
// smokeReport is report's stdout for the smoke-test records.
|
||||||
const smokeReport = "first\tdupe\tsize\n" +
|
const smokeReport = "first\tdupe\tsize\n" +
|
||||||
@@ -227,8 +189,7 @@ const smokeReport = "first\tdupe\tsize\n" +
|
|||||||
// it ends; read counts the 6 records before it reads them. The summary
|
// it ends; read counts the 6 records before it reads them. The summary
|
||||||
// comes last, and stdout holds the report alone.
|
// comes last, and stdout holds the report alone.
|
||||||
func TestRunReportProgressNamesEachPhase(t *testing.T) {
|
func TestRunReportProgressNamesEachPhase(t *testing.T) {
|
||||||
dir, recs := recsOnDisk(t, smokeTreeRecs())
|
path := seedDatabase(t, smokeTreeRecs())
|
||||||
path := seedDatabase(t, recs)
|
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
@@ -241,9 +202,8 @@ func TestRunReportProgressNamesEachPhase(t *testing.T) {
|
|||||||
code, exitOK, stderr())
|
code, exitOK, stderr())
|
||||||
}
|
}
|
||||||
|
|
||||||
report := strings.ReplaceAll(smokeReport, "/d/", dir+"/d/")
|
if got := stdout.String(); got != smokeReport {
|
||||||
if got := stdout.String(); got != report {
|
t.Errorf("stdout = %q, want %q", got, smokeReport)
|
||||||
t.Errorf("stdout = %q, want %q", got, report)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// How each line starts; the rate at the end of a phase varies.
|
// How each line starts; the rate at the end of a phase varies.
|
||||||
@@ -251,7 +211,7 @@ func TestRunReportProgressNamesEachPhase(t *testing.T) {
|
|||||||
"report: database " + path + " (from " + databaseEnv + ")",
|
"report: database " + path + " (from " + databaseEnv + ")",
|
||||||
reportOpenLine,
|
reportOpenLine,
|
||||||
reportOpenLine,
|
reportOpenLine,
|
||||||
"[2/2] read: counting the records, elapsed 0s, remain unknown",
|
"[2/2] read: counting the records, elapsed 0s, eta unknown",
|
||||||
"[2/2] read: [6/6] 100%, ",
|
"[2/2] read: [6/6] 100%, ",
|
||||||
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
|
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
|
||||||
"6.1 KiB reclaimable",
|
"6.1 KiB reclaimable",
|
||||||
@@ -292,7 +252,7 @@ func TestReadReportOnTerminal(t *testing.T) {
|
|||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
}, terminalWidth, time.Hour, time.Now)
|
}, terminalWidth, time.Hour, time.Now)
|
||||||
|
|
||||||
_, err = readReport(t.Context(), db, "", os.Stderr, everyPathExists, p)
|
_, err = readReport(t.Context(), db, "", os.Stderr, p)
|
||||||
|
|
||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
@@ -317,18 +277,14 @@ func TestReadReportOnTerminal(t *testing.T) {
|
|||||||
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
|
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
|
||||||
// README §Constraints: identical database contents give identical
|
// README §Constraints: identical database contents give identical
|
||||||
// output, whatever order the records were inserted in.
|
// output, whatever order the records were inserted in.
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
|
|
||||||
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
|
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
|
||||||
recs = append(recs,
|
recs = append(recs,
|
||||||
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
|
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
|
||||||
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
|
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
|
||||||
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
|
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
|
||||||
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
|
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
|
||||||
)
|
)
|
||||||
|
|
||||||
_, recs = recsOnDisk(t, recs)
|
|
||||||
|
|
||||||
reversed := slices.Clone(recs)
|
reversed := slices.Clone(recs)
|
||||||
slices.Reverse(reversed)
|
slices.Reverse(reversed)
|
||||||
|
|
||||||
@@ -429,19 +385,16 @@ func TestWarnfEscapes(t *testing.T) {
|
|||||||
func TestDupeGroups(t *testing.T) {
|
func TestDupeGroups(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
|
|
||||||
u := testHash("u")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
|
||||||
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
|
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
|
||||||
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
|
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
|
||||||
// Same size as the /z group but a different head hash.
|
// Same size as the /z group but a different head hash.
|
||||||
{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
|
{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
|
||||||
// A singleton signature must not form a group.
|
// A singleton signature must not form a group.
|
||||||
{size: 7, head: u, tail: u, content: u, path: "/lonely"},
|
{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -462,150 +415,15 @@ func TestDupeGroups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
|
|
||||||
// duplicate group, while 1-byte files still do.
|
|
||||||
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
e, x := sha256.Sum256(nil), testHash("x")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
|
|
||||||
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
|
|
||||||
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
|
|
||||||
}
|
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
|
||||||
if len(groups) != 1 || groups[0].size != 1 ||
|
|
||||||
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
|
|
||||||
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReadReportChecksOnlyPossibleDuplicates checks that report asks
|
|
||||||
// whether a path is on disk only when its signature has two or more
|
|
||||||
// records, and never for an empty file.
|
|
||||||
func TestReadReportChecksOnlyPossibleDuplicates(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
a, u, e := testHash("a"), testHash("u"), sha256.Sum256(nil)
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/copy/1"},
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/copy/2"},
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/copy/3"},
|
|
||||||
{size: 9, head: u, tail: u, content: u, path: "/lonely"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/empty1"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/empty2"},
|
|
||||||
}
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
err := applyChanges(t.Context(), db, recs, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var checked []string
|
|
||||||
|
|
||||||
exists := func(path string) bool {
|
|
||||||
checked = append(checked, path)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = readReport(t.Context(), db, "", io.Discard, exists, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.Sort(checked)
|
|
||||||
|
|
||||||
want := []string{"/copy/1", "/copy/2", "/copy/3"}
|
|
||||||
if !slices.Equal(checked, want) {
|
|
||||||
t.Errorf("paths checked on disk = %q, want %q", checked, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReadReportDropsGroupLeftWithOnePath checks that a group of two
|
|
||||||
// paths, one of them gone, is no duplicate group. If kept, it would
|
|
||||||
// write no row, but the summary line would count it.
|
|
||||||
func TestReadReportDropsGroupLeftWithOnePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
a := testHash("a")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/a/kept"},
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/a/gone"},
|
|
||||||
}
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
err := applyChanges(t.Context(), db, recs, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
exists := func(path string) bool { return path != "/a/gone" }
|
|
||||||
|
|
||||||
report, err := readReport(t.Context(), db, "", io.Discard, exists, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report.dupeGroups != 0 {
|
|
||||||
t.Errorf("duplicate groups = %d, want 0", report.dupeGroups)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReadReportKeepsGroupLeftWithTwoPaths checks that a group of three
|
|
||||||
// paths, one of them gone, is a duplicate group of the other two.
|
|
||||||
func TestReadReportKeepsGroupLeftWithTwoPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
a := testHash("a")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/b/kept1"},
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/b/gone"},
|
|
||||||
{size: 9, head: a, tail: a, content: a, path: "/b/kept2"},
|
|
||||||
}
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
err := applyChanges(t.Context(), db, recs, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
exists := func(path string) bool { return path != "/b/gone" }
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
report, err := readReport(t.Context(), db, "", &out, exists, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report.dupeGroups != 1 {
|
|
||||||
t.Errorf("duplicate groups = %d, want 1", report.dupeGroups)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := "first\tdupe\tsize\n/b/kept1\t/b/kept2\t9\n"
|
|
||||||
if got := out.String(); got != want {
|
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDupeGroupsContentSeparates(t *testing.T) {
|
func TestDupeGroupsContentSeparates(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Same size, head, and tail: /b differs from the others only in its
|
// Same size, head, and tail: /b differs from the others only in its
|
||||||
// content hash, so it joins no group, while /a and /c group.
|
// content hash, so it joins no group, while /a and /c group.
|
||||||
h, tl := testHash("h"), testHash("t")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
|
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
|
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
|
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -624,15 +442,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
|
|||||||
|
|
||||||
// mtime is informational only; records differing only in mtime
|
// mtime is informational only; records differing only in mtime
|
||||||
// still group together.
|
// still group together.
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{
|
{
|
||||||
size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
|
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
|
||||||
content: c, path: "/m/1",
|
content: "c", path: "/m/1",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
|
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
|
||||||
content: c, path: "/m/2",
|
content: "c", path: "/m/2",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -647,12 +464,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
|
|||||||
|
|
||||||
// The hashes sort opposite to the first paths, so ordering the
|
// The hashes sort opposite to the first paths, so ordering the
|
||||||
// groups by hash instead of by first path fails this test.
|
// groups by hash instead of by first path fails this test.
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -674,16 +490,15 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
|
|||||||
// /z, /k, which is neither the order of their first paths nor its
|
// /z, /k, which is neither the order of their first paths nor its
|
||||||
// reverse, and must write them ordered by first path. The group of
|
// reverse, and must write them ordered by first path. The group of
|
||||||
// size 7 comes after them all, though its first path sorts first.
|
// size 7 comes after them all, though its first path sorts first.
|
||||||
a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/k/2"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/k/1"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/z/1"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/z/2"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
|
||||||
{size: 50, head: c, tail: c, content: c, path: "/a/2"},
|
{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
|
||||||
{size: 50, head: c, tail: c, content: c, path: "/a/1"},
|
{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
|
||||||
{size: 7, head: d, tail: d, content: d, path: "/0/2"},
|
{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
|
||||||
{size: 7, head: d, tail: d, content: d, path: "/0/1"},
|
{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -702,12 +517,11 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
|
|||||||
func TestDupeGroupsDeterministic(t *testing.T) {
|
func TestDupeGroupsDeterministic(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 1, head: a, tail: a, content: a, path: "/p/1"},
|
{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
|
||||||
{size: 1, head: a, tail: a, content: a, path: "/p/2"},
|
{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
|
||||||
{size: 2, head: b, tail: b, content: b, path: "/q/1"},
|
{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
|
||||||
{size: 2, head: b, tail: b, content: b, path: "/q/2"},
|
{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
|
||||||
}
|
}
|
||||||
|
|
||||||
forward := dupeGroupsOf(t, recs)
|
forward := dupeGroupsOf(t, recs)
|
||||||
+143
-254
@@ -1,10 +1,11 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -52,25 +53,22 @@ const workQueueDepth = 1024
|
|||||||
// has already printed its line, so run prints nothing more.
|
// has already printed its line, so run prints nothing more.
|
||||||
var errInterrupted = errors.New("scan interrupted")
|
var errInterrupted = errors.New("scan interrupted")
|
||||||
|
|
||||||
// fileRec carries one statted file between the scan phases. diskSize is
|
// fileRec carries one statted file between the scan phases. dev and
|
||||||
// the bytes the file takes on disk. dev and ino identify the underlying
|
// ino identify the underlying inode so hard-linked paths can share
|
||||||
// inode so hard-linked paths can share one read; both are zero when the
|
// one read; both are zero when the platform exposes no inode.
|
||||||
// platform exposes no inode.
|
|
||||||
type fileRec struct {
|
type fileRec struct {
|
||||||
path string
|
path string
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
|
||||||
mtime time.Time
|
mtime time.Time
|
||||||
dev uint64
|
dev uint64
|
||||||
ino uint64
|
ino uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// fileMeta is the in-memory index entry for one existing database
|
// fileMeta is the in-memory index entry for one existing database
|
||||||
// record: what the walk compares the file with. Hashes stay on disk; at
|
// record: just enough for change detection. Hashes stay on disk; at
|
||||||
// tens of millions of records they would dominate the scan's memory.
|
// tens of millions of records they would dominate the scan's memory.
|
||||||
type fileMeta struct {
|
type fileMeta struct {
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
|
||||||
mtime time.Time
|
mtime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,47 +96,44 @@ func scanPhase(name string) string {
|
|||||||
// runs, and the lock is released after it. When ctx is cancelled, as by
|
// runs, and the lock is released after it. When ctx is cancelled, as by
|
||||||
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
// the SIGINT or SIGTERM that interruptContext catches, the scan keeps
|
||||||
// what it has hashed (see syncScan), prints how many files its walk
|
// what it has hashed (see syncScan), prints how many files its walk
|
||||||
// reached, and returns errInterrupted. opts.workers must be at least 1;
|
// reached, and returns errInterrupted. workers must be at least 1; the
|
||||||
// the scan and dwim commands reject anything less. Before the lock is
|
// scan command rejects anything less. Before the lock is taken, the
|
||||||
// taken, the scan lists the filesystems that are not persistent, on
|
// scan lists the filesystems that are not persistent, on which the walk
|
||||||
// which the walk records nothing; unless opts.noDefaultExcludes is set,
|
// records nothing; unless noDefaultExcludes is set, the walk does not
|
||||||
// the walk does not enter the default excludes either. The walk neither
|
// enter the default excludes either. Once the database is open, the
|
||||||
// enters nor records the paths in opts.excludePaths, the --exclude
|
// scan notes its files, which the walk passes over.
|
||||||
// values; an operand at or below one is a fatal error. Once the
|
func runScan(ctx context.Context, roots []string, workers int,
|
||||||
// database is open, the scan notes its files, which the walk passes
|
oneFS, noDefaultExcludes bool,
|
||||||
// over.
|
) error {
|
||||||
func runScan(ctx context.Context, roots []string, opts scanOptions) error {
|
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
printDatabasePath(dbPath, source)
|
// A database file missing now is created by lockScanDatabase below.
|
||||||
|
what := "database"
|
||||||
|
|
||||||
roots, err = realPaths(roots)
|
_, err = os.Stat(dbPath)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
what = "new database"
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
|
||||||
|
source)
|
||||||
|
|
||||||
|
roots, err = resolveRoots(roots)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var rules walkRules
|
nonPersistent, err := nonPersistentFilesystems()
|
||||||
|
|
||||||
rules.excludePaths, err = excludePathInodes(opts.excludePaths)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("--exclude: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = checkOperandsNotExcluded(roots, rules.excludePaths)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
rules.nonPersistent, err = nonPersistentFilesystems()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("list the mounted filesystems: %w", err)
|
return fmt.Errorf("list the mounted filesystems: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !opts.noDefaultExcludes {
|
rules := walkRules{nonPersistent: nonPersistent}
|
||||||
rules.defaultExcludes = defaultExcludes(runtime.GOOS)
|
if !noDefaultExcludes {
|
||||||
|
rules.excludes = defaultExcludes(runtime.GOOS)
|
||||||
}
|
}
|
||||||
|
|
||||||
lock, err := lockScanDatabase(dbPath)
|
lock, err := lockScanDatabase(dbPath)
|
||||||
@@ -166,7 +161,7 @@ func runScan(ctx context.Context, roots []string, opts scanOptions) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
st, err := syncScan(ctx, db, roots, opts.workers, opts.oneFS, rules)
|
st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
|
||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
return interrupted(st.walked)
|
return interrupted(st.walked)
|
||||||
}
|
}
|
||||||
@@ -184,21 +179,6 @@ func runScan(ctx context.Context, roots []string, opts scanOptions) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// printDatabasePath prints on stderr the line that names the database
|
|
||||||
// at dbPath, with source, saying whether it is new. A database file
|
|
||||||
// missing now is new: lockScanDatabase creates it.
|
|
||||||
func printDatabasePath(dbPath, source string) {
|
|
||||||
what := "database"
|
|
||||||
|
|
||||||
_, err := os.Stat(dbPath)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
what = "new database"
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
|
|
||||||
source)
|
|
||||||
}
|
|
||||||
|
|
||||||
// interruptContext returns a copy of ctx that the first SIGINT or
|
// interruptContext returns a copy of ctx that the first SIGINT or
|
||||||
// SIGTERM cancels; the scan command runs the scan under it. stop
|
// SIGTERM cancels; the scan command runs the scan under it. stop
|
||||||
// releases the signals.
|
// releases the signals.
|
||||||
@@ -227,31 +207,29 @@ func interrupted(walked int) error {
|
|||||||
return errInterrupted
|
return errInterrupted
|
||||||
}
|
}
|
||||||
|
|
||||||
// realPaths returns the real path of each of paths, the PATH operands
|
// resolveRoots converts each PATH operand to an absolute, lexically
|
||||||
// or the --exclude values: absolute, with every symlink in it followed.
|
// cleaned path (symlinks are not resolved) and verifies that it
|
||||||
// Database records are keyed by that path, so scan results depend
|
// exists. Database records are keyed by absolute path, so scan results
|
||||||
// neither on the working directory nor on the symlinks an operand was
|
// must not depend on the working directory.
|
||||||
// given through. A path that cannot be resolved, because it or a
|
func resolveRoots(roots []string) ([]string, error) {
|
||||||
// symlink's target does not exist or cannot be read, is a fatal error
|
abs := make([]string, 0, len(roots))
|
||||||
// before any scanning.
|
|
||||||
func realPaths(paths []string) ([]string, error) {
|
|
||||||
resolved := make([]string, 0, len(paths))
|
|
||||||
|
|
||||||
for _, path := range paths {
|
for _, root := range roots {
|
||||||
a, err := absolutePath(path)
|
a, err := filepath.Abs(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("resolve %s: %w", path, err)
|
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
r, err := filepath.EvalSymlinks(a)
|
// A nonexistent operand is a fatal error before any scanning.
|
||||||
|
_, err = os.Lstat(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("resolve %s: %w", path, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
resolved = append(resolved, r)
|
abs = append(abs, a)
|
||||||
}
|
}
|
||||||
|
|
||||||
return resolved, nil
|
return abs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneRoots drops operands already covered by another operand:
|
// pruneRoots drops operands already covered by another operand:
|
||||||
@@ -314,7 +292,7 @@ func walkReaches(above, r string, oneFS bool, rules walkRules,
|
|||||||
}
|
}
|
||||||
|
|
||||||
for ; dir != above; dir = filepath.Dir(dir) {
|
for ; dir != above; dir = filepath.Dir(dir) {
|
||||||
if isDefaultExclude(dir, rules.defaultExcludes) {
|
if isDefaultExclude(dir, rules.excludes) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,7 +343,6 @@ type scanState struct {
|
|||||||
db *sql.DB
|
db *sql.DB
|
||||||
existing map[string]fileMeta
|
existing map[string]fileMeta
|
||||||
toHash []fileRec // new and changed files: must be read
|
toHash []fileRec // new and changed files: must be read
|
||||||
diskSizes []fileRec // unchanged files whose disk_size changed
|
|
||||||
batch []scanRec
|
batch []scanRec
|
||||||
st scanStats
|
st scanStats
|
||||||
}
|
}
|
||||||
@@ -407,8 +384,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
) error {
|
) error {
|
||||||
// Types and filesystems are checked before pruning so that an
|
// Types and filesystems are checked before pruning so that an
|
||||||
// operand under a dropped one is still scanned, not dropped as lying
|
// operand under a dropped one is still scanned, not dropped as lying
|
||||||
// under it, and so that an operand inside a .zfs directory gets its
|
// under it.
|
||||||
// warning even when it lies under another operand.
|
|
||||||
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
|
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
|
||||||
deviceOfPath)
|
deviceOfPath)
|
||||||
|
|
||||||
@@ -436,17 +412,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// walkableRoots returns the operands the walk can start from: regular
|
// walkableRoots returns the operands the walk can start from: regular
|
||||||
// files, and directories not named .zfs, none of them inside a
|
// files, and directories not named .zfs. Every other operand is warned
|
||||||
// directory named .zfs. The operands are real paths from realPaths,
|
// about, counted as skipped, and dropped. A dropped operand is no
|
||||||
// so these checks apply to what a symlink operand points at. Every
|
// longer a root, so the records stored beneath it are treated as if it
|
||||||
// other operand is warned about, counted as skipped, and dropped. A
|
// had not been given: those under no other root count as outside the
|
||||||
// dropped operand is no longer a root, so the records stored beneath it
|
// roots and are not deleted as unverified. An operand on a filesystem
|
||||||
// are treated as if it had not been given: those under no other root
|
// in rules that is not persistent is dropped the same way, with a line
|
||||||
// count as outside the roots and are not deleted as unverified. An
|
// saying so that is not a warning, and is not counted as skipped. An
|
||||||
// operand on a filesystem in rules that is not persistent is dropped
|
// operand that fails lstat here is kept, and the walk warns about it.
|
||||||
// the same way, with a line saying so that is not a warning, and is not
|
|
||||||
// counted as skipped. An operand that fails lstat here is kept, and the
|
|
||||||
// walk warns about it unless it no longer exists.
|
|
||||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||||
kept := make([]string, 0, len(roots))
|
kept := make([]string, 0, len(roots))
|
||||||
|
|
||||||
@@ -481,8 +454,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
|||||||
// included, so its display counts against the number of records.
|
// included, so its display counts against the number of records.
|
||||||
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||||
// Indexing tens of millions of records takes real time; without a
|
// Indexing tens of millions of records takes real time; without a
|
||||||
// display the scan looks hung before the walk begins. The number of
|
// display the scan looks hung before the walk begins. Counting them
|
||||||
// records, read first, gives the display its total.
|
// first gives the display its total.
|
||||||
prog := newProgress(progressParams{
|
prog := newProgress(progressParams{
|
||||||
label: scanPhase("load"), unit: scanUnit, total: -1,
|
label: scanPhase("load"), unit: scanUnit, total: -1,
|
||||||
waitingFor: countingRecords, stalledOn: "db",
|
waitingFor: countingRecords, stalledOn: "db",
|
||||||
@@ -498,25 +471,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
|||||||
|
|
||||||
s.existing = make(map[string]fileMeta)
|
s.existing = make(map[string]fileMeta)
|
||||||
|
|
||||||
return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
|
return loadFileMeta(ctx, s.db,
|
||||||
|
func(path string, size int64, mtime time.Time) {
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
if underAnyRoot(path, roots) {
|
if underAnyRoot(path, roots) {
|
||||||
s.existing[path] = m
|
s.existing[path] = fileMeta{size: size, mtime: mtime}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkPhase drains the walk and queues every new or changed file for
|
// walkPhase drains the walk and queues every new or changed file for
|
||||||
// the hash phase. An unchanged file is not read, and its record counts
|
// the hash phase. An unchanged file needs nothing more, and its record
|
||||||
// as verified. When its disk_size differs from the record's, it is
|
// counts as verified. A record at or below a path the walk left out,
|
||||||
// queued for the update phase, which writes the new disk_size. A record
|
// because -x stopped at it, it is a default exclude, or it is not on a
|
||||||
// at or below a path the walk left out, because -x stopped at it, it is
|
// persistent filesystem, counts as outside the roots: it is not
|
||||||
// an --exclude path or a default exclude, or it is not on a persistent
|
// deleted. That does not hold below a root at or inside that path,
|
||||||
// filesystem, counts as outside the roots: it is not deleted. That does
|
// since that root's own walk verifies the records below it; nor is a
|
||||||
// not hold below a root at or inside that path, since that root's own
|
// line printed for such a root.
|
||||||
// walk verifies the records below it; nor is a line printed for such a
|
|
||||||
// root.
|
|
||||||
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||||
// The records the last scan left under the roots estimate how many
|
// The records the last scan left under the roots estimate how many
|
||||||
// files the walk will find. A first scan has none, and no estimate.
|
// files the walk will find. A first scan has none, and no estimate.
|
||||||
@@ -565,7 +537,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
old, ok := s.existing[ev.rec.path]
|
old, ok := s.existing[ev.rec.path]
|
||||||
if !ok || old.size != ev.rec.size || !ev.rec.mtime.Equal(old.mtime) {
|
if !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime) {
|
||||||
s.toHash = append(s.toHash, ev.rec)
|
s.toHash = append(s.toHash, ev.rec)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@@ -574,13 +546,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
delete(s.existing, ev.rec.path)
|
delete(s.existing, ev.rec.path)
|
||||||
|
|
||||||
s.st.unchanged++
|
s.st.unchanged++
|
||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
|
||||||
// up to seconds after the write.
|
|
||||||
if old.diskSize != ev.rec.diskSize {
|
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
@@ -665,9 +630,8 @@ func isDatabaseFile(f fileRec, rules walkRules) bool {
|
|||||||
// (an interrupted scan resumes cheaply: the next run skips everything
|
// (an interrupted scan resumes cheaply: the next run skips everything
|
||||||
// already recorded). A run that fails to hash is warned about and
|
// already recorded). A run that fails to hash is warned about and
|
||||||
// counted as skipped; stale records for its paths, if any, are deleted
|
// counted as skipped; stale records for its paths, if any, are deleted
|
||||||
// by the update phase. A run whose file no longer exists gets no record
|
// by the update phase. The total counts actual reads, so the display
|
||||||
// and no warning, and is not counted as skipped. The total counts actual
|
// shows a real ETA.
|
||||||
// reads, so the display's time remaining is meaningful.
|
|
||||||
//
|
//
|
||||||
// Returning early — a failed database write, or a cancelled scan — must
|
// Returning early — a failed database write, or a cancelled scan — must
|
||||||
// not strand the pool: the feeder would park forever on a full jobs
|
// not strand the pool: the feeder would park forever on a full jobs
|
||||||
@@ -699,11 +663,6 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
|
|||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
// A file removed since the walk is no error: it gets no record.
|
|
||||||
if errors.Is(r.err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
s.st.skipped += len(r.run)
|
s.st.skipped += len(r.run)
|
||||||
|
|
||||||
@@ -730,7 +689,6 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
|
|||||||
|
|
||||||
s.batch = append(s.batch, scanRec{
|
s.batch = append(s.batch, scanRec{
|
||||||
size: rec.size,
|
size: rec.size,
|
||||||
diskSize: rec.diskSize,
|
|
||||||
mtime: rec.mtime,
|
mtime: rec.mtime,
|
||||||
head: r.head,
|
head: r.head,
|
||||||
tail: r.tail,
|
tail: r.tail,
|
||||||
@@ -751,7 +709,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
|
err := applyBatch(ctx, s.db, s.batch, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -762,10 +720,9 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// updatePhase writes the scan's tail under one progress display: the
|
// updatePhase writes the scan's tail under one progress display: the
|
||||||
// final partial batch of hashed records, the new disk_size of every
|
// final partial batch of hashed records, and deletions for every record
|
||||||
// unchanged file whose disk_size changed, and deletions for every
|
// the scan did not verify (vanished files, plus paths that failed to
|
||||||
// record the scan did not verify (vanished files, plus paths that
|
// stat or hash).
|
||||||
// failed to stat or hash).
|
|
||||||
func (s *scanState) updatePhase(ctx context.Context) error {
|
func (s *scanState) updatePhase(ctx context.Context) error {
|
||||||
deletes := make([]string, 0, len(s.existing))
|
deletes := make([]string, 0, len(s.existing))
|
||||||
for path := range s.existing {
|
for path := range s.existing {
|
||||||
@@ -777,7 +734,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.st.removed = len(deletes)
|
s.st.removed = len(deletes)
|
||||||
|
|
||||||
total := len(s.batch) + len(s.diskSizes) + len(deletes)
|
total := len(s.batch) + len(deletes)
|
||||||
prog := newProgress(progressParams{
|
prog := newProgress(progressParams{
|
||||||
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
||||||
stalledOn: "db",
|
stalledOn: "db",
|
||||||
@@ -792,12 +749,17 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.batch = nil
|
s.batch = nil
|
||||||
|
|
||||||
err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
|
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||||
if err != nil {
|
}
|
||||||
return err
|
|
||||||
|
// mtimeAfter reports whether mtime a is later than mtime b.
|
||||||
|
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
|
||||||
|
func mtimeAfter(a, b time.Time) bool {
|
||||||
|
if a.Unix() != b.Unix() {
|
||||||
|
return a.Unix() > b.Unix()
|
||||||
}
|
}
|
||||||
|
|
||||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
return a.Nanosecond() > b.Nanosecond()
|
||||||
}
|
}
|
||||||
|
|
||||||
// underAnyRoot reports whether path is any of the roots or lies under
|
// underAnyRoot reports whether path is any of the roots or lies under
|
||||||
@@ -854,9 +816,9 @@ type dirJob struct {
|
|||||||
// file's statted record, a warning when fail is set, or a path the walk
|
// file's statted record, a warning when fail is set, or a path the walk
|
||||||
// left out: when otherFSDir is set, a directory -x did not enter
|
// left out: when otherFSDir is set, a directory -x did not enter
|
||||||
// because it is on another filesystem than its operand, and when
|
// because it is on another filesystem than its operand, and when
|
||||||
// leftOut is set, an --exclude path, a default exclude or a directory or
|
// leftOut is set, a default exclude or a directory or file on a
|
||||||
// file on a filesystem that is not persistent, with warn holding the
|
// filesystem that is not persistent, with warn holding the line that
|
||||||
// line that says so.
|
// says so.
|
||||||
type walkEvent struct {
|
type walkEvent struct {
|
||||||
rec fileRec
|
rec fileRec
|
||||||
warn string
|
warn string
|
||||||
@@ -868,12 +830,10 @@ type walkEvent struct {
|
|||||||
// startWalk seeds every root into the shared walk worker pool and
|
// startWalk seeds every root into the shared walk worker pool and
|
||||||
// returns the event stream: one record per regular file, one warning
|
// returns the event stream: one record per regular file, one warning
|
||||||
// event per per-path error, and one event per path -x or rules kept the
|
// event per per-path error, and one event per path -x or rules kept the
|
||||||
// walk from entering or recording. A path that no longer exists when
|
// walk from entering or recording. The channel is closed when the walk
|
||||||
// the walk reaches it is no error, and gets no event. The channel is
|
// completes, and also when ctx is cancelled — every goroutine in the
|
||||||
// closed when the walk completes, and also when ctx is cancelled —
|
// pool abandons its blocking send in that case, so the consumer sees a
|
||||||
// every goroutine in the pool abandons its blocking send in that case,
|
// truncated but properly terminated stream instead of a stalled one.
|
||||||
// so the consumer sees a truncated but properly terminated stream
|
|
||||||
// instead of a stalled one.
|
|
||||||
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
||||||
rules walkRules, workers int,
|
rules walkRules, workers int,
|
||||||
) <-chan walkEvent {
|
) <-chan walkEvent {
|
||||||
@@ -906,18 +866,9 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
|||||||
|
|
||||||
// operandWarning returns the one-line warning for an operand the walk
|
// operandWarning returns the one-line warning for an operand the walk
|
||||||
// does not start from, naming the path and what it is, or "" for one it
|
// does not start from, naming the path and what it is, or "" for one it
|
||||||
// does: a regular file, or a directory not named .zfs, that is not
|
// does: a regular file, or a directory not named .zfs. Symlinks are
|
||||||
// inside a directory named .zfs. realPaths has already followed the
|
// never followed, including as operands.
|
||||||
// operand's symlinks, so a symlink here is an operand replaced by one
|
|
||||||
// after it was resolved, and is not followed.
|
|
||||||
func operandWarning(root string, fi fs.FileInfo) string {
|
func operandWarning(root string, fi fs.FileInfo) string {
|
||||||
// The walk never enters a .zfs directory, so it starts from nothing
|
|
||||||
// inside one either.
|
|
||||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
|
||||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
|
||||||
root)
|
|
||||||
}
|
|
||||||
|
|
||||||
var kind string
|
var kind string
|
||||||
|
|
||||||
switch mode := fi.Mode(); {
|
switch mode := fi.Mode(); {
|
||||||
@@ -949,19 +900,13 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
|||||||
// one of the scan's own database files in rules, and a directory
|
// one of the scan's own database files in rules, and a directory
|
||||||
// operand becomes an initial job. walkableRoots has already dropped
|
// operand becomes an initial job. walkableRoots has already dropped
|
||||||
// every other operand. One that has changed into something else since
|
// every other operand. One that has changed into something else since
|
||||||
// is warned about and skipped here, and one removed since is skipped
|
// is warned about and skipped here; it is still a root, so the records
|
||||||
// without a warning. Either is still a root, so the records stored
|
// stored beneath it, other than those under another root, are deleted
|
||||||
// beneath it, other than those under another root, are deleted as
|
// as unverified.
|
||||||
// unverified.
|
|
||||||
func seedRoot(ctx context.Context, root string, rules walkRules,
|
func seedRoot(ctx context.Context, root string, rules walkRules,
|
||||||
events chan<- walkEvent,
|
events chan<- walkEvent,
|
||||||
) []dirJob {
|
) []dirJob {
|
||||||
fi, err := os.Lstat(root)
|
fi, err := os.Lstat(root)
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
// Removed since the scan started: no error, and no files.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", root, err),
|
warn: fmt.Sprintf("walk %s: %v", root, err),
|
||||||
@@ -988,7 +933,6 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
|
|||||||
rec := fileRec{
|
rec := fileRec{
|
||||||
path: root,
|
path: root,
|
||||||
size: fi.Size(),
|
size: fi.Size(),
|
||||||
diskSize: diskSizeOfInfo(fi),
|
|
||||||
mtime: fi.ModTime(),
|
mtime: fi.ModTime(),
|
||||||
dev: dev,
|
dev: dev,
|
||||||
ino: ino,
|
ino: ino,
|
||||||
@@ -1080,17 +1024,11 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
|
|||||||
|
|
||||||
// walkOneDir reads one directory, emitting an event per regular-file
|
// walkOneDir reads one directory, emitting an event per regular-file
|
||||||
// entry and a warning event per unreadable one, and returns the
|
// entry and a warning event per unreadable one, and returns the
|
||||||
// subdirectories to descend into. A directory or entry removed since
|
// subdirectories to descend into.
|
||||||
// the walk found it gets no event.
|
|
||||||
func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
||||||
rules walkRules, events chan<- walkEvent,
|
rules walkRules, events chan<- walkEvent,
|
||||||
) []dirJob {
|
) []dirJob {
|
||||||
entries, err := os.ReadDir(job.path)
|
entries, err := os.ReadDir(job.path)
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
// Removed since its parent was read: no error, and no files.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", job.path, err),
|
warn: fmt.Sprintf("walk %s: %v", job.path, err),
|
||||||
@@ -1133,23 +1071,16 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
|||||||
|
|
||||||
// emitFile stats one regular directory entry and emits its record.
|
// emitFile stats one regular directory entry and emits its record.
|
||||||
// The lstat happens here in the walk worker, while the directory's
|
// The lstat happens here in the walk worker, while the directory's
|
||||||
// metadata is still hot; a path that fails to stat between the
|
// metadata is still hot; a path that fails to stat (or stops being a
|
||||||
// directory read and the lstat is warned about and skipped, unless it
|
// regular file) between the directory read and the lstat is warned
|
||||||
// no longer exists, and one that has stopped being a regular file is
|
// about and skipped. A file on a filesystem in rules that is not
|
||||||
// skipped. A file on a filesystem in rules that is not persistent, as
|
// persistent, as through a bind mount of one file, is not recorded: it
|
||||||
// through a bind mount of one file, is not recorded: it gets an event
|
// gets an event naming it instead. One of the scan's own database files
|
||||||
// naming it instead, and so does a file given with --exclude, reached
|
// in rules gets no event at all.
|
||||||
// by any path, a hard link to it included. One of the scan's own
|
|
||||||
// database files in rules gets no event at all.
|
|
||||||
func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
||||||
rules walkRules, events chan<- walkEvent,
|
rules walkRules, events chan<- walkEvent,
|
||||||
) {
|
) {
|
||||||
info, err := e.Info()
|
info, err := e.Info()
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
// Removed since its directory was read: no error.
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("stat %s: %v", p, err),
|
warn: fmt.Sprintf("stat %s: %v", p, err),
|
||||||
@@ -1164,10 +1095,6 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
}
|
}
|
||||||
|
|
||||||
line := rules.nonPersistentLine(p, info)
|
line := rules.nonPersistentLine(p, info)
|
||||||
if line == "" {
|
|
||||||
line = rules.excludePathLine(p, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line != "" {
|
if line != "" {
|
||||||
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
|
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
|
||||||
|
|
||||||
@@ -1178,7 +1105,6 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
rec := fileRec{
|
rec := fileRec{
|
||||||
path: p,
|
path: p,
|
||||||
size: info.Size(),
|
size: info.Size(),
|
||||||
diskSize: diskSizeOfInfo(info),
|
|
||||||
mtime: info.ModTime(),
|
mtime: info.ModTime(),
|
||||||
dev: dev,
|
dev: dev,
|
||||||
ino: ino,
|
ino: ino,
|
||||||
@@ -1192,11 +1118,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
// subdirJob applies the descent rules to directory p: never enter
|
// subdirJob applies the descent rules to directory p: never enter
|
||||||
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
||||||
// snapshot), never enter a directory on a filesystem in rules that is
|
// snapshot), never enter a directory on a filesystem in rules that is
|
||||||
// not persistent, nor an --exclude path or a default exclude in rules,
|
// not persistent, nor a default exclude in rules, and with -x never
|
||||||
// and with -x never enter a directory on a different filesystem than
|
// enter a directory on a different filesystem than its operand. For the
|
||||||
// its operand. For all but .zfs it sends an event that names the
|
// last three it sends an event that names the directory, so the scan
|
||||||
// directory, so the scan keeps the records below it, except those below
|
// keeps the records below it, except those below an operand at or
|
||||||
// an operand at or inside it.
|
// inside it.
|
||||||
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||||
parent dirJob, oneFS bool, rules walkRules,
|
parent dirJob, oneFS bool, rules walkRules,
|
||||||
events chan<- walkEvent,
|
events chan<- walkEvent,
|
||||||
@@ -1206,11 +1132,6 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
}
|
}
|
||||||
|
|
||||||
info, err := e.Info()
|
info, err := e.Info()
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
// Removed since its parent was read: no error, and no files.
|
|
||||||
return dirJob{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", p, err),
|
warn: fmt.Sprintf("walk %s: %v", p, err),
|
||||||
@@ -1221,17 +1142,13 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
}
|
}
|
||||||
|
|
||||||
line := rules.nonPersistentLine(p, info)
|
line := rules.nonPersistentLine(p, info)
|
||||||
if line == "" {
|
|
||||||
line = rules.excludePathLine(p, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line != "" {
|
if line != "" {
|
||||||
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
|
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
|
||||||
|
|
||||||
return dirJob{}, false
|
return dirJob{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
if isDefaultExclude(p, rules.defaultExcludes) {
|
if isDefaultExclude(p, rules.excludes) {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
leftOut: p,
|
leftOut: p,
|
||||||
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
|
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
|
||||||
@@ -1277,31 +1194,14 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
|
|||||||
return statDev(st), st.Ino
|
return statDev(st), st.Ino
|
||||||
}
|
}
|
||||||
|
|
||||||
// statBlockSize is the unit of st_blocks in an lstat result on Linux and
|
|
||||||
// macOS, whatever the filesystem's own block size.
|
|
||||||
const statBlockSize = 512
|
|
||||||
|
|
||||||
// diskSizeOfInfo returns the bytes a file takes on disk: its allocated
|
|
||||||
// blocks, st_blocks × 512. On a compressed or sparse file it is less
|
|
||||||
// than the file's size. It is 0 when the platform exposes no block
|
|
||||||
// count.
|
|
||||||
func diskSizeOfInfo(fi fs.FileInfo) int64 {
|
|
||||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
|
||||||
if !ok {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.Blocks * statBlockSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// hashResult carries the hashes computed for one inode run (or the
|
// hashResult carries the hashes computed for one inode run (or the
|
||||||
// error that prevented computing them) from the pool's workers to the
|
// error that prevented computing them) from the pool's workers to the
|
||||||
// hash phase. Its run leaves out the paths found removed.
|
// hash phase.
|
||||||
type hashResult struct {
|
type hashResult struct {
|
||||||
run []fileRec
|
run []fileRec
|
||||||
head [sha256.Size]byte
|
head string
|
||||||
tail [sha256.Size]byte
|
tail string
|
||||||
content [sha256.Size]byte
|
content string
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1321,8 +1221,7 @@ type hashPool struct {
|
|||||||
// hash each run's first path with hash (all paths in a run are hard
|
// hash each run's first path with hash (all paths in a run are hard
|
||||||
// links to the same inode) and write one result per run.
|
// links to the same inode) and write one result per run.
|
||||||
func startHashPool(ctx context.Context, runs [][]fileRec, workers int,
|
func startHashPool(ctx context.Context, runs [][]fileRec, workers int,
|
||||||
hash func(path string, size int64) (
|
hash func(path string, size int64) (string, string, string, error),
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error),
|
|
||||||
) *hashPool {
|
) *hashPool {
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
|
||||||
@@ -1387,8 +1286,7 @@ func feedHashJobs(ctx context.Context, runs [][]fileRec,
|
|||||||
// the hash of only delays that.
|
// the hash of only delays that.
|
||||||
func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
||||||
results chan<- hashResult,
|
results chan<- hashResult,
|
||||||
hash func(path string, size int64) (
|
hash func(path string, size int64) (string, string, string, error),
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error),
|
|
||||||
) {
|
) {
|
||||||
for run := range jobs {
|
for run := range jobs {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
@@ -1397,13 +1295,6 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
|||||||
|
|
||||||
head, tail, content, err := hash(run[0].path, run[0].size)
|
head, tail, content, err := hash(run[0].path, run[0].size)
|
||||||
|
|
||||||
// A path removed since the walk is dropped from its run. The
|
|
||||||
// next path is a hard link to the same file, and is read instead.
|
|
||||||
for errors.Is(err, fs.ErrNotExist) && len(run) > 1 {
|
|
||||||
run = run[1:]
|
|
||||||
head, tail, content, err = hash(run[0].path, run[0].size)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case results <- hashResult{
|
case results <- hashResult{
|
||||||
run: run, head: head, tail: tail, content: content, err: err,
|
run: run, head: head, tail: tail, content: content, err: err,
|
||||||
@@ -1446,38 +1337,37 @@ func openRegularFile(path string) (*os.File, error) {
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
|
||||||
|
// tail, and content hash of every zero-length file.
|
||||||
|
const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
|
||||||
|
"27ae41e4649b934ca495991b7852b855"
|
||||||
|
|
||||||
// hashSignature computes the three hashes the hash phase records for a
|
// hashSignature computes the three hashes the hash phase records for a
|
||||||
// file: head and tail (see hashEnds) and content (see hashContent).
|
// file: head and tail (see hashEnds) and content (see hashContent).
|
||||||
// With the file size they form its duplicate signature, and two files
|
// With the file size they form its duplicate signature, and two files
|
||||||
// are duplicates only when all four agree. size is the value recorded
|
// are duplicates only when all four agree. size is the value recorded
|
||||||
// when the file was statted; a zero-length file has constant hashes and
|
// when the file was statted; a zero-length file has constant hashes and
|
||||||
// is never opened. On an error every hash is left zero.
|
// is never opened.
|
||||||
func hashSignature(path string, size int64) (
|
func hashSignature(path string, size int64) (string, string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
var zero [sha256.Size]byte
|
|
||||||
|
|
||||||
if size == 0 {
|
if size == 0 {
|
||||||
empty := sha256.Sum256(nil)
|
return emptyHash, emptyHash, emptyHash, nil
|
||||||
|
|
||||||
return empty, empty, empty, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := openRegularFile(path)
|
f, err := openRegularFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return zero, zero, zero, err
|
return "", "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
head, tail, err := hashEnds(f, size)
|
head, tail, err := hashEnds(f, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return zero, zero, zero, err
|
return "", "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
content, err := hashContent(f, size)
|
content, err := hashContent(f, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return zero, zero, zero, err
|
return "", "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return head, tail, content, nil
|
return head, tail, content, nil
|
||||||
@@ -1487,32 +1377,31 @@ func hashSignature(path string, size int64) (
|
|||||||
// bytes of f. In a file no larger than headTailWindow, each window is
|
// bytes of f. In a file no larger than headTailWindow, each window is
|
||||||
// the whole file. Reading fewer bytes than asked means the file shrank
|
// the whole file. Reading fewer bytes than asked means the file shrank
|
||||||
// between the stat and the hash, and is an error.
|
// between the stat and the hash, and is an error.
|
||||||
func hashEnds(f *os.File, size int64) (
|
func hashEnds(f *os.File, size int64) (string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
var zero [sha256.Size]byte
|
|
||||||
|
|
||||||
buf := make([]byte, min(int64(headTailWindow), size))
|
buf := make([]byte, min(int64(headTailWindow), size))
|
||||||
|
|
||||||
_, err := f.ReadAt(buf, 0)
|
_, err := f.ReadAt(buf, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return zero, zero, err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
head := sha256.Sum256(buf)
|
h := sha256.Sum256(buf)
|
||||||
|
head := hex.EncodeToString(h[:])
|
||||||
|
|
||||||
_, err = f.ReadAt(buf, size-int64(len(buf)))
|
_, err = f.ReadAt(buf, size-int64(len(buf)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return zero, zero, err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return head, sha256.Sum256(buf), nil
|
t := sha256.Sum256(buf)
|
||||||
|
|
||||||
|
return head, hex.EncodeToString(t[:]), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// hashContent returns the content hash of f: the SHA-256 of the whole
|
// hashContent returns the content hash of f: the SHA-256 of the whole
|
||||||
// file when it is smaller than wholeFileMax, or of sampled windows when
|
// file when it is smaller than wholeFileMax, or of sampled windows when
|
||||||
// it is that size or larger.
|
// it is that size or larger.
|
||||||
func hashContent(f *os.File, size int64) ([sha256.Size]byte, error) {
|
func hashContent(f *os.File, size int64) (string, error) {
|
||||||
if size >= int64(wholeFileMax) {
|
if size >= int64(wholeFileMax) {
|
||||||
return hashSamples(f, size)
|
return hashSamples(f, size)
|
||||||
}
|
}
|
||||||
@@ -1525,20 +1414,20 @@ func hashContent(f *os.File, size int64) ([sha256.Size]byte, error) {
|
|||||||
// reads. Reading fewer than size bytes means the file shrank between
|
// reads. Reading fewer than size bytes means the file shrank between
|
||||||
// the stat and the hash; that is an error rather than a hash of content
|
// the stat and the hash; that is an error rather than a hash of content
|
||||||
// that no longer matches the recorded size.
|
// that no longer matches the recorded size.
|
||||||
func hashWhole(f *os.File, size int64) ([sha256.Size]byte, error) {
|
func hashWhole(f *os.File, size int64) (string, error) {
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
|
|
||||||
n, err := io.Copy(h, io.NewSectionReader(f, 0, size))
|
n, err := io.Copy(h, io.NewSectionReader(f, 0, size))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return [sha256.Size]byte{}, err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if n != size {
|
if n != size {
|
||||||
return [sha256.Size]byte{}, fmt.Errorf("read %d of %d bytes: %w",
|
return "", fmt.Errorf("read %d of %d bytes: %w", n, size,
|
||||||
n, size, io.ErrUnexpectedEOF)
|
io.ErrUnexpectedEOF)
|
||||||
}
|
}
|
||||||
|
|
||||||
return [sha256.Size]byte(h.Sum(nil)), nil
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset
|
// hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset
|
||||||
@@ -1548,7 +1437,7 @@ func hashWhole(f *os.File, size int64) ([sha256.Size]byte, error) {
|
|||||||
// without every byte being read. Because size is part of the signature,
|
// without every byte being read. Because size is part of the signature,
|
||||||
// only files of one size are compared, so their sample boundaries
|
// only files of one size are compared, so their sample boundaries
|
||||||
// always align.
|
// always align.
|
||||||
func hashSamples(f *os.File, size int64) ([sha256.Size]byte, error) {
|
func hashSamples(f *os.File, size int64) (string, error) {
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
buf := make([]byte, sampleWindow)
|
buf := make([]byte, sampleWindow)
|
||||||
|
|
||||||
@@ -1557,11 +1446,11 @@ func hashSamples(f *os.File, size int64) ([sha256.Size]byte, error) {
|
|||||||
|
|
||||||
_, err := f.ReadAt(buf[:n], off)
|
_, err := f.ReadAt(buf[:n], off)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return [sha256.Size]byte{}, err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
h.Write(buf[:n])
|
h.Write(buf[:n])
|
||||||
}
|
}
|
||||||
|
|
||||||
return [sha256.Size]byte(h.Sum(nil)), nil
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,10 +1,11 @@
|
|||||||
package sfdupes //nolint:testpackage // drives the unexported syncTree
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -42,28 +43,11 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
// hexSum returns the lowercase-hex SHA-256 of data.
|
||||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's
|
func hexSum(data []byte) string {
|
||||||
// blocks only when it commits the write, up to seconds later, and one
|
s := sha256.Sum256(data)
|
||||||
// commit can take the file through several growing sizes. So this value
|
|
||||||
// can change at any moment. Neither fsync nor unix.Sync hastens that,
|
|
||||||
// but a unix.Sync call often takes long enough that the blocks are
|
|
||||||
// counted when it returns. A test that needs the value to change waits
|
|
||||||
// for the change. A test that compares it with a scan's disk_size reads
|
|
||||||
// it before and after the scan. The rewrite test rescans until the two
|
|
||||||
// reads match, then compares exactly. The other two tests fail only on
|
|
||||||
// a disk_size outside the two reads.
|
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var st syscall.Stat_t
|
return hex.EncodeToString(s[:])
|
||||||
|
|
||||||
err := syscall.Lstat(path, &st)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.Blocks * 512
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pattern returns n bytes of deterministic content seeded by tag.
|
// pattern returns n bytes of deterministic content seeded by tag.
|
||||||
@@ -78,9 +62,7 @@ func pattern(tag byte, n int) []byte {
|
|||||||
|
|
||||||
// sig returns a file's full signature (head, tail, content), failing the
|
// sig returns a file's full signature (head, tail, content), failing the
|
||||||
// test on any error.
|
// test on any error.
|
||||||
func sig(t *testing.T, path string, size int64) (
|
func sig(t *testing.T, path string, size int64) (string, string, string) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
head, tail, content, err := hashSignature(path, size)
|
head, tail, content, err := hashSignature(path, size)
|
||||||
@@ -118,12 +100,12 @@ func TestHashSignature(t *testing.T) {
|
|||||||
head, tail, content := sig(t, p, int64(len(c.data)))
|
head, tail, content := sig(t, p, int64(len(c.data)))
|
||||||
|
|
||||||
window := min(len(c.data), headTailWindow)
|
window := min(len(c.data), headTailWindow)
|
||||||
wantHead := sha256.Sum256(c.data[:window])
|
wantHead := hexSum(c.data[:window])
|
||||||
wantTail := sha256.Sum256(c.data[len(c.data)-window:])
|
wantTail := hexSum(c.data[len(c.data)-window:])
|
||||||
wantContent := sha256.Sum256(c.data)
|
wantContent := hexSum(c.data)
|
||||||
|
|
||||||
if head != wantHead || tail != wantTail || content != wantContent {
|
if head != wantHead || tail != wantTail || content != wantContent {
|
||||||
t.Errorf("head=%x tail=%x content=%x, want %x %x %x",
|
t.Errorf("head=%s tail=%s content=%s, want %s %s %s",
|
||||||
head, tail, content, wantHead, wantTail, wantContent)
|
head, tail, content, wantHead, wantTail, wantContent)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -188,25 +170,22 @@ func TestHashSignatureErrors(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
var zero [sha256.Size]byte
|
// A missing file: an error, and every hash left empty.
|
||||||
|
|
||||||
// A missing file: an error, and every hash left zero.
|
|
||||||
head, tail, content, err := hashSignature(filepath.Join(dir, "missing"), 1)
|
head, tail, content, err := hashSignature(filepath.Join(dir, "missing"), 1)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("no error for a missing file")
|
t.Error("no error for a missing file")
|
||||||
}
|
}
|
||||||
|
|
||||||
if head != zero || tail != zero || content != zero {
|
if head != "" || tail != "" || content != "" {
|
||||||
t.Errorf("missing file returned hashes: %x %x %x", head, tail, content)
|
t.Errorf("missing file returned hashes: %q %q %q", head, tail, content)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A zero-length file has constant hashes and is never opened: even
|
// A zero-length file has constant hashes and is never opened: even
|
||||||
// a missing path succeeds.
|
// a missing path succeeds.
|
||||||
empty := sha256.Sum256(nil)
|
|
||||||
|
|
||||||
head, tail, content, err = hashSignature(filepath.Join(dir, "missing"), 0)
|
head, tail, content, err = hashSignature(filepath.Join(dir, "missing"), 0)
|
||||||
if err != nil || head != empty || tail != empty || content != empty {
|
if err != nil ||
|
||||||
t.Errorf("empty: head=%x tail=%x content=%x err=%v, "+
|
head != emptyHash || tail != emptyHash || content != emptyHash {
|
||||||
|
t.Errorf("empty: head=%q tail=%q content=%q err=%v, "+
|
||||||
"want constant hashes", head, tail, content, err)
|
"want constant hashes", head, tail, content, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,8 +198,8 @@ func TestHashSignatureErrors(t *testing.T) {
|
|||||||
t.Error("no error when the stat size exceeds the file size")
|
t.Error("no error when the stat size exceeds the file size")
|
||||||
}
|
}
|
||||||
|
|
||||||
if head != zero || tail != zero || content != zero {
|
if head != "" || tail != "" || content != "" {
|
||||||
t.Errorf("shrunk file returned hashes: %x %x %x", head, tail, content)
|
t.Errorf("shrunk file returned hashes: %q %q %q", head, tail, content)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -707,108 +686,6 @@ func TestSubdirJobStatError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWalkPassesOverRemovedPaths removes a file and a directory after
|
|
||||||
// their parent was read, as when they are deleted while a scan runs.
|
|
||||||
// The walk then reaches them, the directory also as one to read and as
|
|
||||||
// an operand. None of that is an error: the walk sends no event.
|
|
||||||
func TestWalkPassesOverRemovedPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
file := writeFile(t, dir, "file", []byte("x"))
|
|
||||||
writeFile(t, dir, "sub/f", []byte("y"))
|
|
||||||
sub := filepath.Join(dir, "sub")
|
|
||||||
fileEntry := dirEntryFor(t, dir, "file")
|
|
||||||
subEntry := dirEntryFor(t, dir, "sub")
|
|
||||||
|
|
||||||
for _, p := range []string{file, sub} {
|
|
||||||
err := os.RemoveAll(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := t.Context()
|
|
||||||
events := make(chan walkEvent, 4)
|
|
||||||
|
|
||||||
emitFile(ctx, file, fileEntry, walkRules{}, events)
|
|
||||||
|
|
||||||
_, descend := subdirJob(ctx, sub, subEntry, dirJob{path: dir}, false,
|
|
||||||
walkRules{}, events)
|
|
||||||
subs := walkOneDir(ctx, dirJob{path: sub}, false, walkRules{}, events)
|
|
||||||
jobs := seedRoot(ctx, sub, walkRules{}, events)
|
|
||||||
|
|
||||||
close(events)
|
|
||||||
|
|
||||||
for ev := range events {
|
|
||||||
t.Errorf("event %+v for a removed path, want none", ev)
|
|
||||||
}
|
|
||||||
|
|
||||||
if descend || len(subs) != 0 || len(jobs) != 0 {
|
|
||||||
t.Errorf("descend = %t, subdirectories %+v, jobs %+v, want none",
|
|
||||||
descend, subs, jobs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanWarnsAboutUnreadablePaths checks that a permission error is
|
|
||||||
// still warned about and counted as skipped, unlike a removed path. The
|
|
||||||
// walk reaches an unreadable directory, and a file in a directory that
|
|
||||||
// can be read but not searched. Another such file is an operand.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
|
||||||
if os.Geteuid() == 0 {
|
|
||||||
t.Skip("root ignores file permissions")
|
|
||||||
}
|
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
tree := filepath.Join(dir, "tree")
|
|
||||||
walked := writeFile(t, tree, "unsearchable/f", []byte("x"))
|
|
||||||
operand := writeFile(t, dir, "unsearchable/f", []byte("y"))
|
|
||||||
unreadable := filepath.Join(tree, "unreadable")
|
|
||||||
|
|
||||||
err := os.Mkdir(unreadable, 0o750)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
modes := map[string]fs.FileMode{
|
|
||||||
unreadable: 0,
|
|
||||||
filepath.Dir(walked): 0o400,
|
|
||||||
filepath.Dir(operand): 0o400,
|
|
||||||
}
|
|
||||||
|
|
||||||
for p, mode := range modes {
|
|
||||||
err = os.Chmod(p, mode)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Runs before t.TempDir's own cleanup, which must delete the files.
|
|
||||||
t.Cleanup(func() {
|
|
||||||
//nolint:gosec // removing the files needs the search bit back
|
|
||||||
_ = os.Chmod(p, 0o700)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncTree(t, openTestDB(t), tree, operand)
|
|
||||||
if st != (scanStats{skipped: 3}) {
|
|
||||||
t.Errorf("stats = %+v, want 3 skipped and nothing else", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := stderr()
|
|
||||||
for _, warning := range []string{
|
|
||||||
"walk " + unreadable + ": open " + unreadable + ": permission denied\n",
|
|
||||||
"stat " + walked + ": lstat " + walked + ": permission denied\n",
|
|
||||||
"walk " + operand + ": lstat " + operand + ": permission denied\n",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(got, warning) {
|
|
||||||
t.Errorf("stderr = %q, want %q", got, warning)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// syncOneFilesystem scans roots into db with -x and returns the scan
|
// syncOneFilesystem scans roots into db with -x and returns the scan
|
||||||
// stats. Every device pruneRoots looks up, and the device each
|
// stats. Every device pruneRoots looks up, and the device each
|
||||||
// operand's walk compares its subdirectories with, comes from devices
|
// operand's walk compares its subdirectories with, comes from devices
|
||||||
@@ -1088,13 +965,12 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildSmokeTree recreates the tree of the test under README
|
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||||
// "Definition of done" with deterministic content and returns the tree
|
// with deterministic content and returns the tree root.
|
||||||
// root.
|
|
||||||
func buildSmokeTree(t *testing.T) string {
|
func buildSmokeTree(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := t.TempDir()
|
||||||
one := pattern(10, 2000)
|
one := pattern(10, 2000)
|
||||||
f1 := pattern(30, 3000)
|
f1 := pattern(30, 3000)
|
||||||
f2 := pattern(40, 100)
|
f2 := pattern(40, 100)
|
||||||
@@ -1185,12 +1061,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
groups := dupeGroups(t, db)
|
groups := dupeGroups(t, db)
|
||||||
if len(groups) != 4 {
|
if len(groups) != 5 {
|
||||||
t.Fatalf("len(groups) = %d, want 4", len(groups))
|
t.Fatalf("len(groups) = %d, want 5", len(groups))
|
||||||
}
|
}
|
||||||
|
|
||||||
// The empty files empty1 and empty2 form no group.
|
wantSizes := []int64{3000, 2000, 100, 1, 0}
|
||||||
wantSizes := []int64{3000, 2000, 100, 1}
|
|
||||||
for i, g := range groups {
|
for i, g := range groups {
|
||||||
if g.size != wantSizes[i] {
|
if g.size != wantSizes[i] {
|
||||||
t.Errorf("groups[%d].size = %d, want %d",
|
t.Errorf("groups[%d].size = %d, want %d",
|
||||||
@@ -1215,7 +1090,7 @@ func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) {
|
|||||||
|
|
||||||
super, dirs := dbTree(t, db)
|
super, dirs := dbTree(t, db)
|
||||||
|
|
||||||
tg := collectTreeGroups(dirs, super, everyPathExists, nil)
|
tg := collectTreeGroups(dirs, super, nil)
|
||||||
if len(tg) != 1 {
|
if len(tg) != 1 {
|
||||||
t.Fatalf("len(tree groups) = %d, want 1", len(tg))
|
t.Fatalf("len(tree groups) = %d, want 1", len(tg))
|
||||||
}
|
}
|
||||||
@@ -1254,9 +1129,9 @@ func TestScanPipeline(t *testing.T) {
|
|||||||
// TestScanProgressNamesEachPhase checks the plain progress lines of a
|
// TestScanProgressNamesEachPhase checks the plain progress lines of a
|
||||||
// first scan, and of a rescan of one operand holding two of the
|
// first scan, and of a rescan of one operand holding two of the
|
||||||
// database's records. Each phase's first line names it with its place
|
// database's records. Each phase's first line names it with its place
|
||||||
// among the four: load gets the number of records before reading them,
|
// among the four: load counts the records before reading them, and the
|
||||||
// and the walk estimates its total from the records under its operand,
|
// walk estimates its total from the records under its operand, or says
|
||||||
// or says its time remaining is unknown when there are none.
|
// its ETA is unknown when there are none.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanProgressNamesEachPhase(t *testing.T) {
|
func TestScanProgressNamesEachPhase(t *testing.T) {
|
||||||
@@ -1271,14 +1146,14 @@ func TestScanProgressNamesEachPhase(t *testing.T) {
|
|||||||
syncTree(t, db, filepath.Join(dir, "a"))
|
syncTree(t, db, filepath.Join(dir, "a"))
|
||||||
|
|
||||||
rescan := stderr()[len(first):]
|
rescan := stderr()[len(first):]
|
||||||
load := "[1/4] load: counting the records, elapsed 0s, remain unknown"
|
load := "[1/4] load: counting the records, elapsed 0s, eta unknown"
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
first: {
|
first: {
|
||||||
load, "[2/4] walk: 0 files, 0 files/s, elapsed 0s, remain unknown",
|
load, "[2/4] walk: 0 files, 0 files/s, elapsed 0s, eta unknown",
|
||||||
"[3/4] hash: [0/", "[4/4] update: [0/",
|
"[3/4] hash: [0/", "[4/4] update: [0/",
|
||||||
},
|
},
|
||||||
rescan: {
|
rescan: {
|
||||||
load, "[2/4] walk: [0/~2] 0%, 0 files/s, elapsed 0s, remain unknown",
|
load, "[2/4] walk: [0/~2] 0%, 0 files/s, elapsed 0s, eta unknown",
|
||||||
"[3/4] hash: [0/", "[4/4] update: [0/",
|
"[3/4] hash: [0/", "[4/4] update: [0/",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1336,10 +1211,8 @@ func TestSyncScanUnchangedReuse(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sentinel := sha256.Sum256([]byte("sentinel"))
|
|
||||||
|
|
||||||
_, err := db.ExecContext(context.Background(),
|
_, err := db.ExecContext(context.Background(),
|
||||||
"UPDATE files SET head = ? WHERE path = ?", sentinel[:], []byte(a))
|
"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1350,9 +1223,9 @@ func TestSyncScanUnchangedReuse(t *testing.T) {
|
|||||||
st)
|
st)
|
||||||
}
|
}
|
||||||
|
|
||||||
if r := recordByPath(t, dbRecords(t, db), a); r.head != sentinel {
|
if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
|
||||||
t.Fatalf("head = %x, want the sentinel %x (file must not be re-read)",
|
t.Fatalf("head = %q, want sentinel (file must not be re-read)",
|
||||||
r.head, sentinel)
|
r.head)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1384,47 +1257,6 @@ func TestSyncScanMtimeBump(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSyncScanMtimeMovedBack replaces a file with other content of the
|
|
||||||
// same size and an older mtime, as cp -p or a restore from backup does.
|
|
||||||
// The next scan must hash the file again.
|
|
||||||
func TestSyncScanMtimeMovedBack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
a := writeFile(t, dir, "a.bin", pattern(1, 500))
|
|
||||||
|
|
||||||
recorded := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
err := os.Chtimes(a, recorded, recorded)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
replaced := pattern(3, 500)
|
|
||||||
writeFile(t, dir, "a.bin", replaced)
|
|
||||||
|
|
||||||
older := recorded.Add(-time.Hour)
|
|
||||||
|
|
||||||
err = os.Chtimes(a, older, older)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertWholeFileHashed(t, db, a, replaced)
|
|
||||||
|
|
||||||
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(older) {
|
|
||||||
t.Fatalf("mtime = %v, want %v", r.mtime, older)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertWholeFileHashed fails unless the record for path holds the
|
// assertWholeFileHashed fails unless the record for path holds the
|
||||||
// whole-file hash of data as its head, tail, and content.
|
// whole-file hash of data as its head, tail, and content.
|
||||||
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
||||||
@@ -1433,209 +1265,13 @@ func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
r := recordByPath(t, dbRecords(t, db), path)
|
r := recordByPath(t, dbRecords(t, db), path)
|
||||||
if want := sha256.Sum256(data); r.head != want || r.tail != want ||
|
if want := hexSum(data); r.head != want || r.tail != want ||
|
||||||
r.content != want {
|
r.content != want {
|
||||||
t.Fatalf("head, tail, content = %x, %x, %x, want %x for each",
|
t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
|
||||||
r.head, r.tail, r.content, want)
|
r.head, r.tail, r.content, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestScanStoresRawHashes checks what the database holds for a file
|
|
||||||
// larger than both end windows, whose three hashes all differ: head,
|
|
||||||
// tail and content are each a BLOB of the 32 bytes of the SHA-256 the
|
|
||||||
// scan computed.
|
|
||||||
func TestScanStoresRawHashes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
data := pattern(3, 3*headTailWindow)
|
|
||||||
p := writeFile(t, dir, "f.bin", data)
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
var (
|
|
||||||
blobs bool
|
|
||||||
head, tail, content []byte
|
|
||||||
)
|
|
||||||
|
|
||||||
err := db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT typeof(head) = 'blob' AND typeof(tail) = 'blob' AND "+
|
|
||||||
"typeof(content) = 'blob', head, tail, content "+
|
|
||||||
"FROM files WHERE path = ?", []byte(p)).
|
|
||||||
Scan(&blobs, &head, &tail, &content)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !blobs {
|
|
||||||
t.Error("head, tail and content are not all BLOBs")
|
|
||||||
}
|
|
||||||
|
|
||||||
wantHead := sha256.Sum256(data[:headTailWindow])
|
|
||||||
wantTail := sha256.Sum256(data[len(data)-headTailWindow:])
|
|
||||||
wantContent := sha256.Sum256(data)
|
|
||||||
|
|
||||||
if !bytes.Equal(head, wantHead[:]) || !bytes.Equal(tail, wantTail[:]) ||
|
|
||||||
!bytes.Equal(content, wantContent[:]) {
|
|
||||||
t.Errorf("head, tail, content = %x, %x, %x, want %x, %x, %x",
|
|
||||||
head, tail, content, wantHead, wantTail, wantContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
|
||||||
// disk than its length, once through its directory and once as the
|
|
||||||
// operand itself. Each scan stores the disk_size of the file's lstat.
|
|
||||||
func TestScanStoresDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const length = 1024 * 1024
|
|
||||||
|
|
||||||
p := sparseFile(t, t.TempDir(), "sparse", length)
|
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
size := diskSizeOf(t, p)
|
|
||||||
if size == 0 || size >= length {
|
|
||||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
|
||||||
"and fewer than its length %d", size, length)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, operand := range []string{filepath.Dir(p), p} {
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
// ZFS may count the blocks during the scan; see diskSizeOf.
|
|
||||||
before := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
syncTree(t, db, operand)
|
|
||||||
|
|
||||||
after := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got < before || got > after {
|
|
||||||
t.Errorf("scan of %s: disk_size = %d, want %d to %d",
|
|
||||||
operand, got, before, after)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
|
|
||||||
// sparse file and moves its mtime on. The rescan hashes the file again
|
|
||||||
// and rewrites its record with the new disk_size.
|
|
||||||
func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := sparseFile(t, dir, "sparse", 1024*1024)
|
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
recorded := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
|
|
||||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
|
||||||
|
|
||||||
// The write may leave the mtime the scan recorded; a later one makes
|
|
||||||
// the rescan read the file again.
|
|
||||||
later := time.Now().Add(time.Hour)
|
|
||||||
|
|
||||||
err := os.Chtimes(p, later, later)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for ZFS to count the write; see diskSizeOf.
|
|
||||||
deadline := time.Now().Add(30 * time.Second)
|
|
||||||
for diskSizeOf(t, p) == recorded {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
|
|
||||||
"after the write into its hole", recorded)
|
|
||||||
}
|
|
||||||
|
|
||||||
unix.Sync()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ZFS may still be counting, so the size may change during a rescan.
|
|
||||||
// Only a rescan with the same size on both sides of it can be compared
|
|
||||||
// exactly. A rescan that keeps the old size still fails, because the
|
|
||||||
// wait made before differ from it.
|
|
||||||
for {
|
|
||||||
before := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
|
|
||||||
after := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
if before == after {
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got != after {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, after)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("the size on disk was still changing 30 seconds after "+
|
|
||||||
"the write into its hole: %d, then %d", before, after)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later mtime makes the next rescan read the file again.
|
|
||||||
later = later.Add(time.Second)
|
|
||||||
|
|
||||||
err = os.Chtimes(p, later, later)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
|
|
||||||
// disk_size the file does not have, as when ZFS had not yet counted the
|
|
||||||
// file's blocks at the last scan. The rescan writes the file's own
|
|
||||||
// disk_size into the record without reading the file again.
|
|
||||||
func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := writeFile(t, dir, "a.bin", pattern(1, 500))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
// lstat counts 512-byte units, so no file takes 1 byte on disk.
|
|
||||||
_, err := db.ExecContext(t.Context(),
|
|
||||||
"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
// ZFS may count the blocks during the rescan; see diskSizeOf.
|
|
||||||
before := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
// A file read again would count as updated.
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
|
|
||||||
after := diskSizeOf(t, p)
|
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got < before || got > after {
|
|
||||||
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
|
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
|
||||||
// size with an mtime later in the same second as the recorded one: the
|
// size with an mtime later in the same second as the recorded one: the
|
||||||
// next scan must notice the change and re-hash the file. The operand is
|
// next scan must notice the change and re-hash the file. The operand is
|
||||||
@@ -1931,8 +1567,7 @@ func TestScanHardlinksReadOnce(t *testing.T) {
|
|||||||
ra := recordByPath(t, recs, a)
|
ra := recordByPath(t, recs, a)
|
||||||
rb := recordByPath(t, recs, b)
|
rb := recordByPath(t, recs, b)
|
||||||
|
|
||||||
if ra.head == [sha256.Size]byte{} || ra.head != rb.head ||
|
if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
|
||||||
ra.tail != rb.tail {
|
|
||||||
t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
|
t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2007,55 +1642,6 @@ func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHashPhasePassesOverRemovedFiles removes two walked files before
|
|
||||||
// the hash phase reads them: one with no other path, and the first of
|
|
||||||
// two hard links to one file. Neither is an error. The phase prints
|
|
||||||
// nothing but its progress, counts nothing as skipped, and records only
|
|
||||||
// the hard link still there.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestHashPhasePassesOverRemovedFiles(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
alone := writeFile(t, dir, "alone.bin", pattern(1, 300))
|
|
||||||
a := writeFile(t, dir, "a.bin", pattern(2, 300))
|
|
||||||
b := filepath.Join(dir, "b.bin")
|
|
||||||
|
|
||||||
err := os.Link(a, b)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
walked, _ := collectWalk(t, []string{dir}, false, 1)
|
|
||||||
|
|
||||||
// a.bin sorts before b.bin, so the run of the two reads it first.
|
|
||||||
for _, p := range []string{alone, a} {
|
|
||||||
err = os.Remove(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &scanState{db: openTestDB(t), toHash: walked}
|
|
||||||
|
|
||||||
err = s.hashPhase(t.Context(), 1)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hashPhase = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.st.skipped != 0 || len(s.batch) != 1 || s.batch[0].path != b {
|
|
||||||
t.Errorf("skipped = %d, records %+v, want none skipped and a "+
|
|
||||||
"record for %s alone", s.st.skipped, s.batch, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
for line := range strings.Lines(stderr()) {
|
|
||||||
if !strings.HasPrefix(line, scanPhase("hash")+": ") {
|
|
||||||
t.Errorf("stderr holds %q, want the hash phase's progress alone",
|
|
||||||
line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// injectedWriteFailure is the message the injected database trigger
|
// injectedWriteFailure is the message the injected database trigger
|
||||||
// aborts with, so the test can recognize its own failure in the error
|
// aborts with, so the test can recognize its own failure in the error
|
||||||
// the scan reports.
|
// the scan reports.
|
||||||
@@ -2196,7 +1782,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) {
|
|||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{
|
code := run([]string{
|
||||||
cmdScan, workersFlag, strconv.Itoa(hashLeakWorkers), dir,
|
cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir,
|
||||||
}, io.Discard, &stderr)
|
}, io.Discard, &stderr)
|
||||||
if code != exitFatal {
|
if code != exitFatal {
|
||||||
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
t.Fatalf("run(scan) = %d, want %d; stderr: %s",
|
||||||
@@ -2289,7 +1875,7 @@ func TestPruneRoots(t *testing.T) {
|
|||||||
|
|
||||||
// The walk of /p does not enter /p/q when it is a default exclude, so
|
// The walk of /p does not enter /p/q when it is a default exclude, so
|
||||||
// /p/q is kept, without -x too; /p/q/r lies under it and is dropped.
|
// /p/q is kept, without -x too; /p/q/r lies under it and is dropped.
|
||||||
got = pruneRoots(nested, false, walkRules{defaultExcludes: []string{pqRoot}},
|
got = pruneRoots(nested, false, walkRules{excludes: []string{pqRoot}},
|
||||||
device)
|
device)
|
||||||
if !slices.Equal(got, want) {
|
if !slices.Equal(got, want) {
|
||||||
t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want)
|
t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want)
|
||||||
@@ -2307,86 +1893,23 @@ func TestPruneRoots(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunReportsLeaveOutRemovedCopies scans the smoke tree, then
|
func TestReportsNeverTouchFilesystem(t *testing.T) {
|
||||||
// removes copies the reports name. report leaves out each removed file,
|
|
||||||
// and a group left with one path. trees checks only the root of each
|
|
||||||
// duplicate tree: with a file under t2 removed it still names t2, and
|
|
||||||
// with t2 removed it names nothing.
|
|
||||||
func TestRunReportsLeaveOutRemovedCopies(t *testing.T) {
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
dir := buildSmokeTree(t)
|
|
||||||
p := func(name string) string { return filepath.Join(dir, name) }
|
|
||||||
|
|
||||||
scanOK(t, dir)
|
|
||||||
|
|
||||||
for _, name := range []string{"b/copy2.bin", "tiny2", "t2/sub/f2"} {
|
|
||||||
err := os.Remove(p(name))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
want := "first\tdupe\tsize\n" +
|
|
||||||
p("t1/f1") + "\t" + p("t2/f1") + "\t3000\n" +
|
|
||||||
p("t1/f1") + "\t" + p("t3/f1") + "\t3000\n" +
|
|
||||||
p("a/one.bin") + "\t" + p("b/copy.bin") + "\t2000\n" +
|
|
||||||
p("t1/sub/f2") + "\t" + p("t3/sub/f2renamed") + "\t100\n"
|
|
||||||
if got := runStdout(t, cmdReport); got != want {
|
|
||||||
t.Errorf("report stdout = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// disk_size, the last column, depends on the filesystem.
|
|
||||||
header := "first\tdupe\tfiles\tsize\tdisk_size\n"
|
|
||||||
|
|
||||||
want = header + p("t1") + "\t" + p("t2") + "\t2\t3100\t"
|
|
||||||
if got := runStdout(t, cmdTrees); !strings.HasPrefix(got, want) ||
|
|
||||||
strings.Count(got, "\n") != 2 {
|
|
||||||
t.Errorf("trees stdout = %q, want one row starting %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := os.RemoveAll(p("t2"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := runStdout(t, cmdTrees); got != header {
|
|
||||||
t.Errorf("trees stdout after removing t2 = %q, want %q", got, header)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathExists(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := buildSmokeTree(t)
|
||||||
file := writeFile(t, dir, "file", nil)
|
db := openTestDB(t)
|
||||||
locked := writeFile(t, dir, "locked/file", nil)
|
|
||||||
|
|
||||||
// Without search permission on its directory, a file cannot be
|
syncTree(t, db, dir)
|
||||||
// looked at by anyone but root.
|
|
||||||
err := os.Chmod(filepath.Dir(locked), 0)
|
// Remove the scanned tree entirely; the analysis must be
|
||||||
|
// unaffected because it reads the database alone.
|
||||||
|
err := os.RemoveAll(dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs before t.TempDir's own cleanup, which must delete the files.
|
assertSmokeDupeGroups(t, dir, db)
|
||||||
t.Cleanup(func() {
|
assertSmokeTreeGroups(t, dir, db)
|
||||||
//nolint:gosec // removing the directory needs its search bit back
|
|
||||||
_ = os.Chmod(filepath.Dir(locked), 0o700)
|
|
||||||
})
|
|
||||||
|
|
||||||
cases := map[string]bool{
|
|
||||||
dir: true,
|
|
||||||
file: true,
|
|
||||||
locked: true,
|
|
||||||
filepath.Join(dir, "gone"): false,
|
|
||||||
filepath.Join(file, "under-a-file"): false,
|
|
||||||
}
|
|
||||||
for path, want := range cases {
|
|
||||||
if got := pathExists(path); got != want {
|
|
||||||
t.Errorf("pathExists(%q) = %v, want %v", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUnderRoot(t *testing.T) {
|
func TestUnderRoot(t *testing.T) {
|
||||||
@@ -2415,24 +1938,33 @@ func TestUnderRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
||||||
// database, and in one case also a hard link to the database file. The
|
// database, reached directly and through a symlinked directory. The
|
||||||
// walk knows the database's files by device ID and inode number, so it
|
// walk passes over the database file, its -wal and -shm files and its
|
||||||
// passes over the database file, its -wal and -shm files, its .lock
|
// .lock file with no line and no count, and the record an earlier scan
|
||||||
// file and the hard link with no line and no count. The record an
|
// left for the .lock file is deleted. A rescan of the unchanged tree
|
||||||
// earlier scan left for the .lock file is deleted. A rescan of the
|
// updates nothing, and report lists none of the database's files.
|
||||||
// unchanged tree updates nothing, and report lists none of the
|
|
||||||
// database's files.
|
|
||||||
func TestRunScanPassesOverDatabase(t *testing.T) {
|
func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||||
for _, viaHardLink := range []bool{false, true} {
|
for _, viaSymlink := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
|
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
|
||||||
tree := realTempDir(t)
|
base := t.TempDir()
|
||||||
hardLink := filepath.Join(tree, "hardlink")
|
tree := filepath.Join(base, "real", "tree")
|
||||||
|
operand := tree
|
||||||
|
|
||||||
writeFile(t, tree, "a", pattern(1, 100))
|
writeFile(t, tree, "a", pattern(1, 100))
|
||||||
writeFile(t, tree, "b", pattern(1, 100))
|
writeFile(t, tree, "b", pattern(1, 100))
|
||||||
writeFile(t, tree, "empty1", nil)
|
writeFile(t, tree, "empty1", nil)
|
||||||
writeFile(t, tree, "empty2", nil)
|
writeFile(t, tree, "empty2", nil)
|
||||||
|
|
||||||
|
if viaSymlink {
|
||||||
|
err := os.Symlink(filepath.Join(base, "real"),
|
||||||
|
filepath.Join(base, "link"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
operand = filepath.Join(base, "link", "tree")
|
||||||
|
}
|
||||||
|
|
||||||
dbPath := filepath.Join(tree, "db.sqlite")
|
dbPath := filepath.Join(tree, "db.sqlite")
|
||||||
t.Setenv(databaseEnv, dbPath)
|
t.Setenv(databaseEnv, dbPath)
|
||||||
|
|
||||||
@@ -2442,7 +1974,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = applyChanges(t.Context(), db, []scanRec{
|
err = applyChanges(t.Context(), db, []scanRec{
|
||||||
{path: filepath.Join(tree, "db.sqlite.lock")},
|
{path: filepath.Join(operand, "db.sqlite.lock")},
|
||||||
}, nil, nil)
|
}, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -2450,30 +1982,17 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
|
|
||||||
closeScanDatabase(t.Context(), db, dbPath)
|
closeScanDatabase(t.Context(), db, dbPath)
|
||||||
|
|
||||||
if viaHardLink {
|
assertScanSummary(t,
|
||||||
err = os.Link(dbPath, hardLink)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stderr := assertScanSummary(t,
|
|
||||||
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
||||||
tree)
|
operand)
|
||||||
stderr += assertScanSummary(t,
|
assertScanSummary(t,
|
||||||
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
||||||
tree)
|
operand)
|
||||||
|
|
||||||
if strings.Contains(stderr, hardLink) {
|
|
||||||
t.Errorf("scan stderr = %q, want no line naming %s",
|
|
||||||
stderr, hardLink)
|
|
||||||
}
|
|
||||||
|
|
||||||
report := runStdout(t, cmdReport)
|
report := runStdout(t, cmdReport)
|
||||||
if strings.Contains(report, "db.sqlite") ||
|
if strings.Contains(report, "db.sqlite") ||
|
||||||
strings.Contains(report, hardLink) ||
|
!strings.Contains(report, filepath.Join(operand, "empty2")) {
|
||||||
!strings.Contains(report, filepath.Join(tree, "b")) {
|
t.Errorf("report = %q, want the empty files and none of "+
|
||||||
t.Errorf("report = %q, want a and b and none of "+
|
|
||||||
"the database's files", report)
|
"the database's files", report)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -2497,9 +2016,8 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
|
|||||||
|
|
||||||
// assertScanSummary scans operands and fails the test unless the
|
// assertScanSummary scans operands and fails the test unless the
|
||||||
// summary line gives counts with nothing skipped, and the line naming
|
// summary line gives counts with nothing skipped, and the line naming
|
||||||
// the database is the only one that names one of its files. It returns
|
// the database is the only one that names one of its files.
|
||||||
// everything the scan printed to stderr.
|
func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
||||||
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
stderr := scanOK(t, operands...)
|
stderr := scanOK(t, operands...)
|
||||||
@@ -2513,6 +2031,4 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
|||||||
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
||||||
"database to name one of its files", operands, stderr)
|
"database to name one of its files", operands, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return stderr
|
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package sfdupes
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -22,10 +22,6 @@ type treeNode struct {
|
|||||||
digest [sha256.Size]byte
|
digest [sha256.Size]byte
|
||||||
fileCount int64
|
fileCount int64
|
||||||
totalSize int64
|
totalSize int64
|
||||||
// diskSize is the bytes the directory's files take on disk. Two
|
|
||||||
// copies of one tree have the same totalSize, but their diskSize
|
|
||||||
// can differ.
|
|
||||||
diskSize int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// treesPhase names one phase of trees with its place among all of them,
|
// treesPhase names one phase of trees with its place among all of them,
|
||||||
@@ -38,18 +34,17 @@ func treesPhase(name string) string {
|
|||||||
// runTrees implements the trees subcommand: it reads every record from
|
// runTrees implements the trees subcommand: it reads every record from
|
||||||
// the database in path order, reconstructs the directory hierarchy from
|
// the database in path order, reconstructs the directory hierarchy from
|
||||||
// the record paths, computes a Merkle-style digest per directory, and
|
// the record paths, computes a Merkle-style digest per directory, and
|
||||||
// prints maximal duplicate-tree groups as TSV on stdout. Of the scanned
|
// prints maximal duplicate-tree groups as TSV on stdout. It never
|
||||||
// filesystem it only checks, with pathExists, that the root of each
|
// touches the scanned filesystem; its only I/O is the database, stdout,
|
||||||
// duplicate tree is still there, never the files below it. It names the
|
// and stderr. It names the database on stderr before it opens it. Any
|
||||||
// database on stderr before it opens it. Any database problem,
|
// database problem, including a missing database, is fatal.
|
||||||
// including a missing database, is fatal.
|
|
||||||
//
|
//
|
||||||
// The work runs in four phases, each under a progress display of its
|
// The work runs in four phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database), read (read the number of
|
// own on stderr: open (open the database), read (count the records,
|
||||||
// records from the totals table, then read the records into the
|
// then read them into the hierarchy), group (group the directories by
|
||||||
// hierarchy), group (group the directories by digest) and write (write
|
// digest) and write (write the groups to stdout). Each display ends
|
||||||
// the groups to stdout). Each display ends before the next phase
|
// before the next phase starts, and before an error is returned to be
|
||||||
// starts, and before an error is returned to be printed.
|
// printed.
|
||||||
func runTrees(ctx context.Context, stdout io.Writer) error {
|
func runTrees(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -87,13 +82,13 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
|||||||
return fmt.Errorf("database %s: %w", dbPath, err)
|
return fmt.Errorf("database %s: %w", dbPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Grouping works in memory, then checks the root of each duplicate
|
// Grouping works in memory, so it waits on nothing but the
|
||||||
// tree on disk, so it waits on the processor or the disk.
|
// processor.
|
||||||
prog = newProgress(progressParams{
|
prog = newProgress(progressParams{
|
||||||
label: treesPhase("group"), unit: "directories",
|
label: treesPhase("group"), unit: "directories",
|
||||||
total: int64(len(allDirs)), stalledOn: "cpu/disk",
|
total: int64(len(allDirs)), stalledOn: "cpu",
|
||||||
})
|
})
|
||||||
dupes := collectTreeGroups(allDirs, super, pathExists, prog)
|
dupes := collectTreeGroups(allDirs, super, prog)
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
@@ -144,16 +139,15 @@ func readTree(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
||||||
// a row for every member of each group but its first, counting each
|
// a row for every member of each group but its first, counting each
|
||||||
// group on prog once its rows are written. The disk_size column is the
|
// group on prog once its rows are written. stdout may be the terminal
|
||||||
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
|
// prog is drawn on, so the lines go through prog's writer. It returns
|
||||||
// go through prog's writer. It returns the number of rows (dupe trees)
|
// the number of rows (dupe trees) and the bytes they could reclaim.
|
||||||
// and the bytes they could reclaim.
|
|
||||||
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
) (int, int64, error) {
|
) (int, int64, error) {
|
||||||
out := prog.writer(stdout)
|
out := prog.writer(stdout)
|
||||||
|
|
||||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
|
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -165,9 +159,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
|||||||
for _, g := range dupes {
|
for _, g := range dupes {
|
||||||
first := g[0]
|
first := g[0]
|
||||||
for _, n := range g[1:] {
|
for _, n := range g[1:] {
|
||||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
|
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||||
escapePath(first.path), escapePath(n.path),
|
escapePath(first.path), escapePath(n.path),
|
||||||
first.fileCount, first.totalSize, n.diskSize)
|
first.fileCount, first.totalSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -238,7 +232,6 @@ func (b *treeBuilder) add(r scanRec) {
|
|||||||
dir.entries = append(dir.entries, fileEntry(name, r))
|
dir.entries = append(dir.entries, fileEntry(name, r))
|
||||||
dir.fileCount++
|
dir.fileCount++
|
||||||
dir.totalSize += r.size
|
dir.totalSize += r.size
|
||||||
dir.diskSize += r.diskSize
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// openDir starts the directory called name inside the innermost open
|
// openDir starts the directory called name inside the innermost open
|
||||||
@@ -277,7 +270,6 @@ func (b *treeBuilder) closeTo(n int) {
|
|||||||
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
||||||
dir.parent.fileCount += dir.fileCount
|
dir.parent.fileCount += dir.fileCount
|
||||||
dir.parent.totalSize += dir.totalSize
|
dir.parent.totalSize += dir.totalSize
|
||||||
dir.parent.diskSize += dir.diskSize
|
|
||||||
b.dirs = append(b.dirs, dir)
|
b.dirs = append(b.dirs, dir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -295,20 +287,15 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
|
|||||||
// excluded.
|
// excluded.
|
||||||
func fileEntry(name string, r scanRec) string {
|
func fileEntry(name string, r scanRec) string {
|
||||||
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
|
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
|
||||||
"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
|
"\x00" + r.head + "\x00" + r.tail + "\x00" + r.content
|
||||||
"\x00" + string(r.content[:])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// collectTreeGroups groups directories by digest, counting each one on
|
// collectTreeGroups groups directories by digest, counting each one on
|
||||||
// prog as it is grouped, and returns every maximal group with two or
|
// prog as it is grouped, and returns every maximal group with two or
|
||||||
// more members for which exists returns true, each group's members
|
// more members, each group's members sorted by path, groups ordered by
|
||||||
// sorted by path, groups ordered by tree size descending then by first
|
// tree size descending then by first path ascending.
|
||||||
// path ascending. A group whose trees hold 0 bytes is left out, since
|
|
||||||
// removing one frees nothing. The groups it implies hold 0 bytes too,
|
|
||||||
// so leaving it out hides nothing. exists is called only on the members
|
|
||||||
// of maximal groups.
|
|
||||||
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
||||||
exists func(path string) bool, prog *progress,
|
prog *progress,
|
||||||
) [][]*treeNode {
|
) [][]*treeNode {
|
||||||
groups := make(map[[sha256.Size]byte][]*treeNode)
|
groups := make(map[[sha256.Size]byte][]*treeNode)
|
||||||
for _, d := range allDirs {
|
for _, d := range allDirs {
|
||||||
@@ -320,19 +307,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
|||||||
var dupes [][]*treeNode
|
var dupes [][]*treeNode
|
||||||
|
|
||||||
for _, g := range groups {
|
for _, g := range groups {
|
||||||
// The members share a digest, so they share a total size.
|
if len(g) < minGroupSize || suppressed(g, super) {
|
||||||
if len(g) < minGroupSize || g[0].totalSize == 0 ||
|
|
||||||
suppressed(g, super) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// A member that is gone leaves the group. A suppressed group is
|
|
||||||
// still implied: its members still on disk have parents still
|
|
||||||
// on disk, and those parents' group is checked the same way.
|
|
||||||
g = slices.DeleteFunc(g, func(n *treeNode) bool {
|
|
||||||
return !exists(n.path)
|
|
||||||
})
|
|
||||||
if len(g) < minGroupSize {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,8 +337,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
|||||||
// digest is the SHA-256 of its child entries — files serialized with
|
// digest is the SHA-256 of its child entries — files serialized with
|
||||||
// name and signature, subdirectories with name and recursive digest —
|
// name and signature, subdirectories with name and recursive digest —
|
||||||
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
|
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
|
||||||
// and every hash and digest is 32 bytes, so NUL delimiters are
|
// so NUL delimiters are unambiguous.
|
||||||
// unambiguous.
|
|
||||||
func (n *treeNode) computeDigest() {
|
func (n *treeNode) computeDigest() {
|
||||||
slices.Sort(n.entries)
|
slices.Sort(n.entries)
|
||||||
|
|
||||||
@@ -1,42 +1,41 @@
|
|||||||
package sfdupes //nolint:testpackage // checks the unexported treeNode results
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Signature hashes shared by the smoke-test records.
|
||||||
|
const (
|
||||||
|
f1Head = "f1h"
|
||||||
|
f1Tail = "f1t"
|
||||||
|
f1Content = "f1c"
|
||||||
|
f2Head = "f2h"
|
||||||
|
f2Tail = "f2t"
|
||||||
|
f2Content = "f2c"
|
||||||
|
)
|
||||||
|
|
||||||
// treesOpenLine is the plain line of trees' open phase when it opens
|
// treesOpenLine is the plain line of trees' open phase when it opens
|
||||||
// the database at once; it prints it as the phase starts and as it ends.
|
// the database at once; it prints it as the phase starts and as it ends.
|
||||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// smokeTreeRecs mirrors the tree of the test under README "Definition
|
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||||
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
|
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||||
// by one filename. Each file takes one 4 KiB block on disk.
|
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
||||||
content: f1Content, path: "/d/t1/f1"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
|
||||||
content: f2Content, path: "/d/t1/sub/f2"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
|
||||||
content: f1Content, path: "/d/t2/f1"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
path: "/d/t3/sub/f2renamed"},
|
||||||
content: f2Content, path: "/d/t2/sub/f2"},
|
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
|
||||||
content: f1Content, path: "/d/t3/f1"},
|
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
|
||||||
content: f2Content, path: "/d/t3/sub/f2renamed"},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -154,14 +153,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
|
|||||||
|
|
||||||
for _, top := range []string{"/a", "/c"} {
|
for _, top := range []string{"/a", "/c"} {
|
||||||
for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
|
for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
|
||||||
content := testHash("c")
|
content := "c"
|
||||||
if p == "/b-x/f" {
|
if p == "/b-x/f" {
|
||||||
content = testHash("other")
|
content = "other"
|
||||||
}
|
}
|
||||||
|
|
||||||
recs = append(recs, scanRec{
|
recs = append(recs, scanRec{
|
||||||
size: 1, head: testHash("h"), tail: testHash("t"),
|
size: 1, head: "h", tail: "t", content: content, path: top + p,
|
||||||
content: content, path: top + p,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -180,7 +178,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
|
|||||||
t.Fatalf("directory paths = %q, want %q", got, want)
|
t.Fatalf("directory paths = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := collectTreeGroups(dirs, super, everyPathExists, nil)
|
groups := collectTreeGroups(dirs, super, nil)
|
||||||
|
|
||||||
gotGroups := groupPaths(groups)
|
gotGroups := groupPaths(groups)
|
||||||
wantGroups := [][]string{{"/a", "/c"}}
|
wantGroups := [][]string{{"/a", "/c"}}
|
||||||
@@ -196,8 +194,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunTreesEscapesPaths(t *testing.T) {
|
func TestRunTreesEscapesPaths(t *testing.T) {
|
||||||
dir, recs := recsOnDisk(t, awkwardPairRecs())
|
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
|
||||||
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -207,45 +204,22 @@ func TestRunTreesEscapesPaths(t *testing.T) {
|
|||||||
code, exitOK, stderr.String())
|
code, exitOK, stderr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want := "first\tdupe\tfiles\tsize\n" +
|
||||||
dir + `/d/\tone\ntwo\rthree\\four` + "\t" + dir + "/d/A\t1\t5\t4096\n"
|
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
|
|
||||||
// take different space on disk. The row's disk_size is the sum over the
|
|
||||||
// dupe copy's files, those in its subdirectory included.
|
|
||||||
func TestRunTreesDiskSize(t *testing.T) {
|
|
||||||
f, g := testHash("f"), testHash("g")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
|
|
||||||
{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
|
|
||||||
{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
|
|
||||||
{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
|
|
||||||
}
|
|
||||||
|
|
||||||
dir, recs := recsOnDisk(t, recs)
|
|
||||||
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
|
||||||
|
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
|
||||||
dir + "/a\t" + dir + "/b\t2\t30\t1536\n"
|
|
||||||
if got := runStdout(t, cmdTrees); got != want {
|
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
|
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
|
||||||
// records with stderr not a terminal, and checks every line it prints
|
// records with stderr not a terminal, and checks every line it prints
|
||||||
// there. After the line naming the database, each phase prints a line
|
// there. After the line naming the database, each phase prints a line
|
||||||
// naming it with its place among the four as it starts and another as
|
// naming it with its place among the four as it starts and another as
|
||||||
// it ends. read counts the 6 records before it reads them, group counts
|
// it ends. read counts the 6 records before it reads them, group counts
|
||||||
// the directories, and write the 1 group. The summary comes last, and
|
// the 8 directories, and write the 1 group. The summary comes last, and
|
||||||
// stdout holds the report alone.
|
// stdout holds the report alone.
|
||||||
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
||||||
dir, recs := recsOnDisk(t, smokeTreeRecs())
|
path := seedDatabase(t, smokeTreeRecs())
|
||||||
path := seedDatabase(t, recs)
|
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
@@ -258,27 +232,21 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
|||||||
code, exitOK, stderr())
|
code, exitOK, stderr())
|
||||||
}
|
}
|
||||||
|
|
||||||
report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
|
||||||
dir + "/d/t1\t" + dir + "/d/t2\t2\t3100\t8192\n"
|
|
||||||
if got := stdout.String(); got != report {
|
if got := stdout.String(); got != report {
|
||||||
t.Errorf("stdout = %q, want %q", got, report)
|
t.Errorf("stdout = %q, want %q", got, report)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The smoke-test records name 8 directories, / included. Moved under
|
|
||||||
// dir, they name one more for each slash in dir.
|
|
||||||
dirs := strconv.Itoa(8 + strings.Count(dir, "/"))
|
|
||||||
|
|
||||||
// How each line starts; the rate at the end of a phase varies.
|
// How each line starts; the rate at the end of a phase varies.
|
||||||
want := []string{
|
want := []string{
|
||||||
"trees: database " + path + " (from " + databaseEnv + ")",
|
"trees: database " + path + " (from " + databaseEnv + ")",
|
||||||
treesOpenLine,
|
treesOpenLine,
|
||||||
treesOpenLine,
|
treesOpenLine,
|
||||||
"[2/4] read: counting the records, elapsed 0s, remain unknown",
|
"[2/4] read: counting the records, elapsed 0s, eta unknown",
|
||||||
"[2/4] read: [6/6] 100%, ",
|
"[2/4] read: [6/6] 100%, ",
|
||||||
"[3/4] group: [0/" + dirs + "] 0%, 0 directories/s, elapsed 0s, " +
|
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, eta unknown",
|
||||||
"remain unknown",
|
"[3/4] group: [8/8] 100%, ",
|
||||||
"[3/4] group: [" + dirs + "/" + dirs + "] 100%, ",
|
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, eta unknown",
|
||||||
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
|
|
||||||
"[4/4] write: [1/1] 100%, ",
|
"[4/4] write: [1/1] 100%, ",
|
||||||
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
|
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
|
||||||
}
|
}
|
||||||
@@ -306,7 +274,7 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
|||||||
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
super, dirs := treeOf(t, smokeTreeRecs())
|
super, dirs := treeOf(t, smokeTreeRecs())
|
||||||
dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
|
dupes := collectTreeGroups(dirs, super, nil)
|
||||||
|
|
||||||
// No timed draws: the display is drawn as it starts and as it ends.
|
// No timed draws: the display is drawn as it starts and as it ends.
|
||||||
p := startProgress(progressParams{
|
p := startProgress(progressParams{
|
||||||
@@ -329,9 +297,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
|||||||
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
|
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
|
||||||
}
|
}
|
||||||
|
|
||||||
want := []string{
|
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
|
||||||
"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
|
|
||||||
}
|
|
||||||
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
|
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
|
||||||
t.Errorf("terminal shows lines %q, want %q first", shown, want)
|
t.Errorf("terminal shows lines %q, want %q first", shown, want)
|
||||||
}
|
}
|
||||||
@@ -366,10 +332,11 @@ func TestTreeDigests(t *testing.T) {
|
|||||||
func TestTreeDigestContentSensitivity(t *testing.T) {
|
func TestTreeDigestContentSensitivity(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
same, c := testHash("same"), testHash("c")
|
const sharedTail = "same"
|
||||||
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
|
{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
|
||||||
{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
|
{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, dirs := treeOf(t, recs)
|
_, dirs := treeOf(t, recs)
|
||||||
@@ -387,7 +354,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
|
|||||||
|
|
||||||
super, dirs := treeOf(t, smokeTreeRecs())
|
super, dirs := treeOf(t, smokeTreeRecs())
|
||||||
|
|
||||||
groups := collectTreeGroups(dirs, super, everyPathExists, nil)
|
groups := collectTreeGroups(dirs, super, nil)
|
||||||
|
|
||||||
want := [][]string{{"/d/t1", "/d/t2"}}
|
want := [][]string{{"/d/t1", "/d/t2"}}
|
||||||
if got := groupPaths(groups); !slices.EqualFunc(got, want,
|
if got := groupPaths(groups); !slices.EqualFunc(got, want,
|
||||||
@@ -411,16 +378,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
|
|||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
super, dirs := treeOf(t, recs)
|
||||||
|
|
||||||
forward := groupPaths(collectTreeGroups(dirs, super, everyPathExists,
|
forward := groupPaths(collectTreeGroups(dirs, super, nil))
|
||||||
nil))
|
|
||||||
|
|
||||||
reversed := slices.Clone(recs)
|
reversed := slices.Clone(recs)
|
||||||
slices.Reverse(reversed)
|
slices.Reverse(reversed)
|
||||||
|
|
||||||
superR, dirsR := treeOf(t, reversed)
|
superR, dirsR := treeOf(t, reversed)
|
||||||
|
|
||||||
backward := groupPaths(collectTreeGroups(dirsR, superR, everyPathExists,
|
backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
|
||||||
nil))
|
|
||||||
if !slices.EqualFunc(forward, backward, slices.Equal) {
|
if !slices.EqualFunc(forward, backward, slices.Equal) {
|
||||||
t.Fatalf("output depends on record order: %v vs %v",
|
t.Fatalf("output depends on record order: %v vs %v",
|
||||||
forward, backward)
|
forward, backward)
|
||||||
@@ -432,44 +397,14 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
|
|||||||
|
|
||||||
// Identical sibling dirs share a parent, so their group cannot be
|
// Identical sibling dirs share a parent, so their group cannot be
|
||||||
// implied by a parent group and must be reported.
|
// implied by a parent group and must be reported.
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
super, dirs := treeOf(t, recs)
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
|
got := groupPaths(collectTreeGroups(dirs, super, nil))
|
||||||
|
|
||||||
want := [][]string{{"/p/x1", "/p/x2"}}
|
|
||||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
|
||||||
t.Fatalf("groups = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
|
|
||||||
// holding only empty files form no group, and that an empty file still
|
|
||||||
// counts toward the digest of a tree that holds other files.
|
|
||||||
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
|
|
||||||
sha256.Sum256(nil)
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
|
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
|
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
|
|
||||||
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
|
|
||||||
// Without the empty file, /p/x3 is no copy of /p/x1.
|
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
|
|
||||||
}
|
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
|
|
||||||
|
|
||||||
want := [][]string{{"/p/x1", "/p/x2"}}
|
want := [][]string{{"/p/x1", "/p/x2"}}
|
||||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
if !slices.EqualFunc(got, want, slices.Equal) {
|
||||||
@@ -483,107 +418,18 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
|||||||
// /p/a and /q/b contain identical x subtrees, but /p/a has an
|
// /p/a and /q/b contain identical x subtrees, but /p/a has an
|
||||||
// extra file, so the parents' digests differ and the x group must
|
// extra file, so the parents' digests differ and the x group must
|
||||||
// be reported.
|
// be reported.
|
||||||
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
|
||||||
{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
|
{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
super, dirs := treeOf(t, recs)
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
|
got := groupPaths(collectTreeGroups(dirs, super, nil))
|
||||||
|
|
||||||
want := [][]string{{"/p/a/x", "/q/b/x"}}
|
want := [][]string{{"/p/a/x", "/q/b/x"}}
|
||||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
if !slices.EqualFunc(got, want, slices.Equal) {
|
||||||
t.Fatalf("groups = %v, want %v", got, want)
|
t.Fatalf("groups = %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCollectTreeGroupsChecksOnlyReportedCopies checks that trees asks
|
|
||||||
// whether a directory is on disk only for the members of the groups it
|
|
||||||
// returns. It never asks about /a/proj/sub, which lies inside a copy
|
|
||||||
// and whose group the group of /a/proj implies.
|
|
||||||
func TestCollectTreeGroupsChecksOnlyReportedCopies(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, g, u := testHash("f"), testHash("g"), testHash("u")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/a/proj/f"},
|
|
||||||
{size: 20, head: g, tail: g, content: g, path: "/a/proj/sub/g"},
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/b/proj/f"},
|
|
||||||
{size: 20, head: g, tail: g, content: g, path: "/b/proj/sub/g"},
|
|
||||||
// /a lacks this file, so /a and /b are no copies of each other.
|
|
||||||
{size: 30, head: u, tail: u, content: u, path: "/b/lonely"},
|
|
||||||
}
|
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
|
||||||
|
|
||||||
var checked []string
|
|
||||||
|
|
||||||
exists := func(path string) bool {
|
|
||||||
checked = append(checked, path)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
|
|
||||||
|
|
||||||
want := [][]string{{"/a/proj", "/b/proj"}}
|
|
||||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
|
||||||
t.Fatalf("groups = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.Sort(checked)
|
|
||||||
|
|
||||||
if !slices.Equal(checked, want[0]) {
|
|
||||||
t.Errorf("directories checked on disk = %q, want %q",
|
|
||||||
checked, want[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCollectTreeGroupsDropsGroupLeftWithOneCopy checks that a group of
|
|
||||||
// two copies, one of them gone, is not returned. If kept, it would
|
|
||||||
// write no row, but the summary line would count it.
|
|
||||||
func TestCollectTreeGroupsDropsGroupLeftWithOneCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := testHash("f")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/c/kept/f"},
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/c/gone/f"},
|
|
||||||
}
|
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
|
||||||
|
|
||||||
exists := func(path string) bool { return path != "/c/gone" }
|
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
|
|
||||||
if len(got) != 0 {
|
|
||||||
t.Errorf("groups = %v, want none", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies checks that a group
|
|
||||||
// of three copies, one of them gone, is returned with the other two.
|
|
||||||
func TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := testHash("f")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/d/kept1/f"},
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/d/gone/f"},
|
|
||||||
{size: 10, head: f, tail: f, content: f, path: "/d/kept2/f"},
|
|
||||||
}
|
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
|
||||||
|
|
||||||
exists := func(path string) bool { return path != "/d/gone" }
|
|
||||||
|
|
||||||
got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
|
|
||||||
|
|
||||||
want := [][]string{{"/d/kept1", "/d/kept2"}}
|
|
||||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
|
||||||
t.Errorf("groups = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user