Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af0cef6f1f | ||
|
|
202f083cc0 | ||
|
|
1017332d68 | ||
|
|
5c7284c351 | ||
|
|
7cb10ed303 | ||
|
|
da75b713aa | ||
|
|
24b0fc486d | ||
|
|
f52bbe5d95 | ||
|
|
30dae35c51 | ||
|
|
4ecf8dc484 | ||
|
|
2322f8bb02 | ||
|
|
eebf579f11 | ||
|
|
8a4616039b | ||
|
|
c2804d088b |
No files matched your search
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
|
|||||||
|
|
||||||
BINARY := sfdupes
|
BINARY := sfdupes
|
||||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||||
LDFLAGS := -X main.Version=$(VERSION)
|
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
||||||
|
|
||||||
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ LDFLAGS := -X main.Version=$(VERSION)
|
|||||||
# Default target: build the binary. Phony so go build (which has its
|
# Default target: build the binary. Phony so go build (which has its
|
||||||
# own build cache) always decides what to recompile.
|
# own build cache) always decides what to recompile.
|
||||||
sfdupes:
|
sfdupes:
|
||||||
go build -ldflags "$(LDFLAGS)" -o $(BINARY)
|
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes
|
||||||
|
|
||||||
build: sfdupes
|
build: sfdupes
|
||||||
|
|
||||||
|
|||||||
@@ -1,573 +1,32 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
- take an issue from the `1.0.0` milestone on the tracker; work not yet on the
|
- Take an issue from the `1.0.0` milestone on the tracker. Work that is not on
|
||||||
tracker gets filed as an issue first
|
the tracker yet is filed as an issue first.
|
||||||
- branch from `next`
|
- Branch from `next`.
|
||||||
- do the work, with tests, in small focused commits
|
- Do the work, with tests, in small focused commits.
|
||||||
- record it at the top of Completed Steps (`TODO.md` changes in the same commit
|
- Push the branch and open a PR against `next`. Its title ends with
|
||||||
as the work)
|
` (closes #N)`.
|
||||||
- push the branch and open a PR against `next` whose title ends with
|
- An independent review gates each merge to `next`. Every finding is addressed
|
||||||
` (closes #N)`
|
or rebutted on the PR.
|
||||||
- an independent review gates each merge to `next`; every finding is addressed
|
- Only the owner merges `next` into `main`.
|
||||||
or explicitly rebutted on the PR
|
|
||||||
- only the owner merges `next` to `main`
|
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
- pre-1.0
|
- Pre-1.0.
|
||||||
- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues
|
- The tracker is authoritative for the pre-1.0 backlog. The open issues under
|
||||||
under the `1.0.0` milestone are what remains before the tag, and this file
|
the `1.0.0` milestone are what remains before the tag.
|
||||||
records history and process, not the queue
|
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
- take the next issue from the `1.0.0` milestone on the tracker:
|
- Take the next issue from the `1.0.0` milestone,
|
||||||
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source
|
https://git.eeqj.de/sneak/sfdupes/milestone/17. The issues are not restated
|
||||||
of truth for what is left before 1.0.0. Individual issues are deliberately not
|
here, because a copy drifts as soon as the tracker moves.
|
||||||
restated here; a copy in this file drifts out of date the moment the tracker
|
|
||||||
moves
|
|
||||||
|
|
||||||
# Completed Steps
|
|
||||||
|
|
||||||
- each record holds `disk_size`, the bytes the file takes on disk: its allocated
|
|
||||||
blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
|
|
||||||
prints a `disk_size` column after `size`, the sum over the `dupe` copy's
|
|
||||||
files; on a compressed or sparse file it differs from `size`. A rescan updates
|
|
||||||
the `disk_size` of an unchanged file whose size on disk changed, without
|
|
||||||
reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147)
|
|
||||||
|
|
||||||
- the database keeps the number of records and the sum of their sizes in the one
|
|
||||||
row of a `totals` table, which three triggers on `files` keep current. `info`
|
|
||||||
reads that row, and so do `scan`'s `load` phase and the `read` phases of
|
|
||||||
`report` and `trees` for their totals; no count reads every record any more.
|
|
||||||
The triggers make each write to `files` slower. `scan` keeps SQLite's
|
|
||||||
temporary data in memory, which wins back part of that time (2026-10-27,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/146)
|
|
||||||
|
|
||||||
- an operand at or inside a directory named `.zfs` is never scanned, whether
|
|
||||||
given alone or under another operand. `scan` warns about it, counts it as
|
|
||||||
skipped and does not walk it (2026-10-26,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/142)
|
|
||||||
|
|
||||||
- a file or directory removed while the scan runs is not an error, in the walk
|
|
||||||
or in the hash phase. The scan prints no line for it and does not count it as
|
|
||||||
skipped. When the first of several hard links to a file is gone, the hash
|
|
||||||
phase reads the next one instead. On a terminal every progress display starts
|
|
||||||
with the spinner, the hash phase's included. A phase with an exact total ends
|
|
||||||
its line with its percent and a 10-column bar at the terminal's right edge,
|
|
||||||
which no longer moves as the line's length changes. A line too wide for the
|
|
||||||
terminal now loses its rate before its bar. The time remaining is labelled
|
|
||||||
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
|
|
||||||
|
|
||||||
- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
|
|
||||||
bytes of a SHA-256, half the size of the hex text they held. The hashes stay
|
|
||||||
raw from the hash phase through the database to `report` and `trees`. Reading
|
|
||||||
a record whose hash is not 32 bytes is an error (2026-10-26,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/134)
|
|
||||||
|
|
||||||
- the hash phase gives every file the walk records three hashes: `head` and
|
|
||||||
`tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
|
|
||||||
of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
|
|
||||||
The 10 MiB and 50 MiB thresholds, the size census, the records stored without
|
|
||||||
hashes and the content phase are gone, so a scan has four phases and a rescan
|
|
||||||
reads only new and changed files. `info` no longer counts the records with
|
|
||||||
hashes, since every record has them (2026-10-25,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
|
||||||
|
|
||||||
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
|
|
||||||
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
|
|
||||||
stages of the `Dockerfile` the module download, and in `builder` the
|
|
||||||
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
|
|
||||||
until the base image, the install line, `go.mod` or `go.sum` changes, and
|
|
||||||
every step that lints, tests or compiles still comes after it;
|
|
||||||
`golangci-lint run` also keeps golangci-lint's cache. README says the same
|
|
||||||
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
|
|
||||||
|
|
||||||
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
|
|
||||||
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
|
|
||||||
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
|
|
||||||
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
|
|
||||||
early build without a database. When the linker set no version,
|
|
||||||
`sfdupes --version` prints the main module's version that Go recorded in the
|
|
||||||
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
|
|
||||||
when neither is known (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/116)
|
|
||||||
|
|
||||||
- `scan` creates the database's directory where the database path leads: the
|
|
||||||
path with its last element cut off as text, never cleaned, so a `..` after a
|
|
||||||
symlinked directory is followed through the symlink, as when the file is
|
|
||||||
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
|
|
||||||
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
|
|
||||||
used to create `/srv/new` and then fail (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
|
||||||
|
|
||||||
- `scan` never records its own database: once the database and its `.lock` file
|
|
||||||
are open, it notes the device and inode of the database file, its `-wal` and
|
|
||||||
`-shm` files and its `.lock` file, and the walk passes over a file with one of
|
|
||||||
them wherever the operands reach it, through a symlinked directory included,
|
|
||||||
printing no line and counting nothing. A record an earlier scan left for one
|
|
||||||
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
|
||||||
|
|
||||||
- `scan` never records or reads anything that is not on a persistent filesystem
|
|
||||||
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
|
|
||||||
matched by the device ID of each such filesystem, listed as the scan starts
|
|
||||||
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
|
|
||||||
however the walk or the content phase reaches it; no flag turns that off.
|
|
||||||
Below an operand, its walk does not enter the default excludes either, so
|
|
||||||
`sfdupes scan /` skips the kernel's view of the running system and the
|
|
||||||
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
|
|
||||||
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
|
|
||||||
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
|
|
||||||
Windows list, not used yet. An operand at or inside a default exclude is still
|
|
||||||
walked. The scan prints a line for each path it leaves out, without counting
|
|
||||||
it as skipped, and the records there count as outside the scanned operands, as
|
|
||||||
for `-x`. `--no-default-excludes` turns off the default excludes only, and
|
|
||||||
`scan --help` lists them (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
|
||||||
|
|
||||||
- the rate a progress display shows covers the items completed over the last 5
|
|
||||||
seconds, or since the line before when stderr is not a terminal, so it falls
|
|
||||||
to 0 while nothing completes, and a burst after a wait no longer reads as a
|
|
||||||
steady rate of over 1,000 files a second; the ETA still comes from the phase's
|
|
||||||
average. Once no item has completed for 10 seconds, the word `stalled` and
|
|
||||||
where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
|
|
||||||
`db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
|
|
||||||
either, each phase giving its word where it is defined. A line too wide for
|
|
||||||
the terminal loses its bar first, then its rate or those words, so the phase,
|
|
||||||
progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
|
|
||||||
like those of README's example, the content phase's checks and reads lose
|
|
||||||
those words at most times in minutes, and every other phase keeps them
|
|
||||||
whenever the elapsed time and the ETA are both in minutes. The content phase's
|
|
||||||
last partial batch is now written under a display of its own (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/125)
|
|
||||||
|
|
||||||
- new subcommand `info` prints the database's details on stdout, one per line as
|
|
||||||
a name, a tab and the value: its path and where it came from, the sizes of the
|
|
||||||
database file and of its `-wal` file when there is one, when the database file
|
|
||||||
was last changed, the schema version, the number of records, how many have
|
|
||||||
head and tail hashes and how many a content hash, and the sizes of all records
|
|
||||||
added up. It opens the database as `report` does, and shows two phases on
|
|
||||||
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
|
|
||||||
from one read-only transaction, the counts from one query (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/117)
|
|
||||||
|
|
||||||
- `report` shows its work on stderr in two phases, `[1/2] open` and
|
|
||||||
`[2/2] read`, with the progress display `scan` and `trees` use, from the
|
|
||||||
moment it starts until the summary line: `open` says it is opening the
|
|
||||||
database, also while it waits on a busy one; `read` counts the records, then
|
|
||||||
reads them against that count in signature order, which SQLite takes from the
|
|
||||||
`files_signature` index without sorting, so the first record comes at once.
|
|
||||||
`report` forms the duplicate groups itself, holding only those of the size it
|
|
||||||
is reading, and writes them ordered by first path before it reads the next
|
|
||||||
size. stdout and the summary line are unchanged, and SQLite no longer needs a
|
|
||||||
temporary file to sort in (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/114)
|
|
||||||
|
|
||||||
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
|
|
||||||
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
|
|
||||||
from the moment it starts until the summary line: `open` says it is opening
|
|
||||||
the database, also while it waits on a busy one; `read` counts the records,
|
|
||||||
then reads them against that count; `group` counts the directories it groups,
|
|
||||||
and `write` the groups it writes. A display's rate names what its phase
|
|
||||||
counts: files, records, directories or groups. stdout and the summary line are
|
|
||||||
unchanged. When stdout and stderr are the same file, as one terminal is, none
|
|
||||||
of stdout's lines shares a line with the display; when they are not, the
|
|
||||||
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/113)
|
|
||||||
|
|
||||||
- each `scan` phase's progress display names the phase and its place among the
|
|
||||||
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
|
|
||||||
time and ETA: `load` counts the records first, the walk's total is estimated
|
|
||||||
from the records under the operands, and the content phase counts the records
|
|
||||||
it checks from SQLite's grouping alone, then says it is searching the database
|
|
||||||
until the first one is returned. On a terminal a phase with an exact total
|
|
||||||
draws its line followed by the library's bar, and any other a spinner followed
|
|
||||||
by the line; every display is drawn again several times a second. No line
|
|
||||||
reaches the terminal's last column, however narrow, also after the terminal is
|
|
||||||
made narrower during a phase: a line too wide leaves out its rate, then its
|
|
||||||
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
|
|
||||||
only when those alone do not fit, as below about 70 columns late in a large
|
|
||||||
scan. In plain mode the line is printed every 5 seconds whether or not an item
|
|
||||||
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
|
|
||||||
|
|
||||||
- `scan`, `report` and `trees` first print a line on stderr naming the database
|
|
||||||
they use, as an absolute path, and whether that path is the default or comes
|
|
||||||
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
|
|
||||||
"Database" says when `scan` writes each file (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/111)
|
|
||||||
|
|
||||||
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
|
||||||
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
|
||||||
that write no image, and `make test` runs the suite under the race detector,
|
|
||||||
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
|
||||||
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
|
||||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
|
|
||||||
every build, while the steps that compile keep Go's build cache and the tests
|
|
||||||
run with `-count=1`; prettier runs on the host, from the node and yarn
|
|
||||||
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
|
|
||||||
and the build stage installs `git` and `make` itself; a new push cancels the
|
|
||||||
workflow's older run on the same branch, and a run stops after 20 minutes;
|
|
||||||
`.claude/settings.json` is deleted (2026-10-15,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
|
|
||||||
`test` and the build stage, and the install of `git` and `make` in the build
|
|
||||||
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
|
|
||||||
every step that lints, tests or compiles still runs on every build
|
|
||||||
|
|
||||||
- with `-x`, an operand under another operand is walked on its own instead of
|
|
||||||
being dropped when a directory from it (from its directory, for a regular
|
|
||||||
file) up to but not including that operand is on a different filesystem than
|
|
||||||
that operand, whose walk then cannot reach it, and the records of files gone
|
|
||||||
from below it are deleted (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
|
||||||
|
|
||||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
|
||||||
under its operand, and counts their sizes in the size census, instead of
|
|
||||||
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
|
||||||
|
|
||||||
- the hash and content phases open each file without blocking and check that it
|
|
||||||
is still a regular file, so one replaced by a FIFO or device node after it was
|
|
||||||
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
|
||||||
|
|
||||||
- the scan lock file is named from the database's path with symlinks resolved,
|
|
||||||
after `scan` creates the database file if it is missing, so a scan through a
|
|
||||||
symlink to the database fails while another scan holds the lock (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
|
||||||
|
|
||||||
- on a terminal, a warning for the last file of a phase no longer erases the
|
|
||||||
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
|
||||||
|
|
||||||
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
|
||||||
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
|
||||||
within the same second is re-hashed (2026-10-07,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/12)
|
|
||||||
|
|
||||||
- cut the narration from `TODO.md` Completed Steps and from the comments in
|
|
||||||
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
|
|
||||||
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
|
|
||||||
|
|
||||||
- `make test-race` ran the test suite under the race detector in a cgo-enabled
|
|
||||||
container, outside `make check` (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/18). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
|
|
||||||
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
|
|
||||||
`make test-race` is gone
|
|
||||||
|
|
||||||
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
|
|
||||||
rather than serve the gates from cache (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
|
||||||
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
|
||||||
`script/` pass a new `CACHEBUST` value, so theirs always run
|
|
||||||
|
|
||||||
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
|
||||||
it; all Markdown reformatted (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
|
||||||
|
|
||||||
- `script/lint` writes no image, so a run no longer leaves an untagged one
|
|
||||||
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
|
|
||||||
|
|
||||||
- tests cover a missing database, `scan` keeping stdout empty, its skip warning,
|
|
||||||
the `report` and `trees` summary lines, and every subcommand going through
|
|
||||||
`runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
|
|
||||||
|
|
||||||
- `.golangci.yml` replaced with the current canonical copy, which uses
|
|
||||||
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
|
||||||
|
|
||||||
- a test fails when either `hashWorker` cancellation check in `scan.go` is
|
|
||||||
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
|
|
||||||
|
|
||||||
- a database path holding `?`, `#` or `%` opens exactly the file it names
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
|
||||||
|
|
||||||
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
|
||||||
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
|
||||||
|
|
||||||
- a test fails when either walk cancellation check in `scan.go` is removed
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
|
||||||
|
|
||||||
- test that `scan` refuses a database with another schema version (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/64)
|
|
||||||
|
|
||||||
- correct four inaccurate comments in `cancel_test.go` and rename
|
|
||||||
`walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
|
||||||
|
|
||||||
- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
|
||||||
|
|
||||||
- `scan` creates the schema in one transaction; a version-0 database with a
|
|
||||||
`files` table is refused with a clear schema-version error (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
|
||||||
|
|
||||||
- README documents install, Docker, a daily cron scan and how to read and check
|
|
||||||
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
|
|
||||||
|
|
||||||
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home
|
|
||||||
and copied the sources with `--chown`, so no `chown -R` walked them
|
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
|
|
||||||
nothing changes owner: the build stage only compiles, as root, and the tests
|
|
||||||
run as `nobody` in the `test` phase
|
|
||||||
|
|
||||||
- `--version` prints `sfdupes VERSION` to stdout; README documents it and
|
|
||||||
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
|
|
||||||
|
|
||||||
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed,
|
|
||||||
deletes nothing more, exits 1 (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/5)
|
|
||||||
|
|
||||||
- `report` and `trees` stream the records instead of holding them all in memory;
|
|
||||||
the schema gains the `files_signature` index (2026-10-04,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/14)
|
|
||||||
|
|
||||||
- progress prints at once on a non-terminal, uses a real terminal test, and
|
|
||||||
prints warnings through a spinner instead of racing its redraw (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/13)
|
|
||||||
|
|
||||||
- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping
|
|
||||||
the records beneath them (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/9)
|
|
||||||
|
|
||||||
- `scan` holds a lock on a lock file beside the database for its whole run, so a
|
|
||||||
second `scan` fails at once with exit 1 (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/53)
|
|
||||||
|
|
||||||
- test stdout write failures in `report` and `trees`; README states that
|
|
||||||
`| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null`
|
|
||||||
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
|
|
||||||
|
|
||||||
- `report` and `trees` open the database read-only, and `scan` leaves it out of
|
|
||||||
WAL mode, so reading needs only read access (2026-10-03, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/8)
|
|
||||||
|
|
||||||
- escape tabs, newlines, carriage returns and backslashes in report, trees and
|
|
||||||
warning paths; the root directory's path is `/` (2026-10-03,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/7)
|
|
||||||
|
|
||||||
- stamp the git tag or short commit in a plain `docker build .` instead of `dev`
|
|
||||||
(2026-10-02, branch `next`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git`
|
|
||||||
without `.git/config`; the build stage stamps the `VERSION` build argument,
|
|
||||||
else `git describe --tags --always`, and fails if the context carries `.git`
|
|
||||||
and the version is still empty, `dev` or `unknown`. CI checks out the full
|
|
||||||
history (`fetch-depth: 0`) so it stamps the same value as `make build`.
|
|
||||||
|
|
||||||
- replace the 1 KiB end-window sampling with the head/tail plus content-hash
|
|
||||||
ladder (2026-09-22, branch `next`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection"
|
|
||||||
documents every rung. A file under 10 MiB is hashed in full, and its `head`,
|
|
||||||
`tail` and `content` all hold that hash. A larger file gets only its 64 KiB
|
|
||||||
`head` and `tail` in the hash phase; the content phase, after the update
|
|
||||||
phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1
|
|
||||||
MiB samples at or above) only when its size, `head` and `tail` match another
|
|
||||||
record's from this scan or an earlier one, and never reads a file gone or
|
|
||||||
changed since its record was written. `report` and `trees` leave out any
|
|
||||||
record without a `content` hash. The `content` column is part of the version 1
|
|
||||||
schema.
|
|
||||||
|
|
||||||
- remove the dead `files.dat` references from `Makefile`, `.gitignore` and
|
|
||||||
`.dockerignore` (2026-09-21, branch `next`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/22)
|
|
||||||
|
|
||||||
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and
|
|
||||||
`Dockerfile.lint` (2026-08-10, branch `next`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy
|
|
||||||
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
|
|
||||||
without the false `(Debian-based)` note or the tag. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
|
|
||||||
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and
|
|
||||||
`script/verify-lint-image-pin`, which compared the two pins, are gone.
|
|
||||||
|
|
||||||
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
|
|
||||||
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
|
|
||||||
owner ruling the linter is never installed on a host, and
|
|
||||||
`golangci-lint config verify` runs before `golangci-lint run`.
|
|
||||||
`script/bootstrap` stopped installing or pinning the linter, and
|
|
||||||
`script/verify-linter-pin` was retired. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
|
|
||||||
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
|
|
||||||
stage depends on through `COPY --from=lint /src/go.sum /dev/null`;
|
|
||||||
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
|
|
||||||
an image build may run docker, so the phase calls `golangci-lint` directly.
|
|
||||||
|
|
||||||
- install the Docker build stage's prerequisites by running `script/bootstrap`
|
|
||||||
instead of `apk add --no-cache make` inline (2026-08-09, branch
|
|
||||||
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42),
|
|
||||||
with `script/verify-linter-pin` failing the build unless the linter copied
|
|
||||||
from the lint stage was the version `script/bootstrap` pinned. Builds then
|
|
||||||
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/43. Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
|
|
||||||
`make` with `apk add --no-cache` and only compiles; the `lint` and `test`
|
|
||||||
phases are the gates
|
|
||||||
- bust the Docker layer cache for the gate steps, so `script/cibuild` and
|
|
||||||
`script/docker` cannot report a green they did not earn (2026-08-09, branch
|
|
||||||
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
|
|
||||||
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
|
|
||||||
served them from cache and the build exited 0 having run nothing. The fix was
|
|
||||||
a `CHECK_EPOCH` build argument; since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
|
||||||
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
|
||||||
declares before every step that lints, tests or compiles. Run as root, the
|
|
||||||
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
|
|
||||||
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
|
|
||||||
- check the installed golangci-lint version in `script/bootstrap` instead of
|
|
||||||
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
|
|
||||||
`GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and
|
|
||||||
any installed version that is not the pin — older, newer, absent or
|
|
||||||
unparseable — is reinstalled. `go install` writes into `GOBIN` (or
|
|
||||||
`GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so
|
|
||||||
bootstrap re-reads the effective version after installing and, on a mismatch,
|
|
||||||
prints both paths and both versions and exits non-zero; it does not reorder
|
|
||||||
`PATH` or delete anyone's binary. The `--version` call keeps its stderr and is
|
|
||||||
bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep
|
|
||||||
presence-only checks. Verified by bootstrapping this host from v2.10.1 to
|
|
||||||
v2.12.2 and again to a no-op, and by stub runs of the script under `dash`
|
|
||||||
covering a thirteen-input version-parse matrix, a shadowed install that must
|
|
||||||
exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged
|
|
||||||
binary that must hit the timeout; `make check` and `make lint` are clean at
|
|
||||||
v2.12.2, so v2.10.1 was not hiding any findings on `main`
|
|
||||||
- unwind the hash worker pool on the error path (2026-08-09, branch
|
|
||||||
`hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the
|
|
||||||
pool is now an owned, context-aware `hashPool`: every blocking send in the
|
|
||||||
feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path
|
|
||||||
out, and `hashPhase` defers `pool.stop()`, which cancels and then drains
|
|
||||||
`results` until the last goroutine has exited — draining is what frees a
|
|
||||||
worker already parked on a send. `ctx` is threaded from `cmd.Context()`
|
|
||||||
through `runScan`, `syncScan`, both worker pools and the whole database layer,
|
|
||||||
as the first parameter everywhere. The walk pool gets the same treatment plus
|
|
||||||
a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size
|
|
||||||
census, and every file it never reached looks vanished to the update phase.
|
|
||||||
That phase's own `BeginTx` also fails on the cancelled context before deleting
|
|
||||||
anything, but the guard is the barrier that still holds once an interrupted
|
|
||||||
scan may commit what it has. Tests drive `run(scan)` against a database whose
|
|
||||||
insert trigger aborts and assert that the scan fails instead of hanging and
|
|
||||||
that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others
|
|
||||||
cancel a scan part-way through the walk, deterministically, by counting its
|
|
||||||
own consultations of `ctx.Done()`, and assert that it stops at the guard
|
|
||||||
holding a partial census and a still-populated record index, with every record
|
|
||||||
intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`,
|
|
||||||
`feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation
|
|
||||||
branches of both pools
|
|
||||||
- guarantee the database is closed on every fatal exit path (2026-08-09, branch
|
|
||||||
`db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4):
|
|
||||||
`fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and
|
|
||||||
with it the SQLite WAL checkpoint — now actually runs when a subcommand fails;
|
|
||||||
`runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return
|
|
||||||
errors instead. The single exit point is `run` in `main.go`: it maps a
|
|
||||||
`fatalError` (anything a subcommand returned) to exit 1 and cobra's own
|
|
||||||
argument and flag errors to exit 2, which keeps a runtime failure from being
|
|
||||||
reported as a usage error or printing the usage text. New `main_test.go`
|
|
||||||
drives the CLI in-process and asserts the exit codes from README §Error
|
|
||||||
handling plus the stdout/stderr split, including that a fatal error raised
|
|
||||||
after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`,
|
|
||||||
`report` or `trees`
|
|
||||||
- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch
|
|
||||||
`golangci-v2.12.2`, merged as `38a01bd`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the
|
|
||||||
`Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and
|
|
||||||
replaced `.golangci.yml` with the canonical file — the linter settings (`lll`,
|
|
||||||
`funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per
|
|
||||||
the v2 schema, so they are actually applied; no new lint findings surfaced
|
|
||||||
- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints
|
|
||||||
like the other managed repos (2026-07-26, commit `3abeacf`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints
|
|
||||||
exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
|
|
||||||
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`)
|
|
||||||
and every Makefile target is now a thin shim over them
|
|
||||||
- make the binary the default Make target (2026-07-24, branch
|
|
||||||
`make-default-target`): plain `make` now builds `sfdupes` (previously it ran
|
|
||||||
`check` plus `build`); `make build` remains as an alias
|
|
||||||
- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch
|
|
||||||
`scan-wide-phases`): all operands seed the shared walk pool and every pass
|
|
||||||
runs once over the whole scan, so totals and ETAs are scan-global; the
|
|
||||||
per-operand walk/hash/update cycles and their stderr announcements are gone;
|
|
||||||
the update pass commits in batched transactions — the filesystem is
|
|
||||||
authoritative and the database an eventually-consistent reflection, so
|
|
||||||
scan-level atomicity is not required
|
|
||||||
- split the stat pass back out of the walk (2026-07-24, branch
|
|
||||||
`parallel-phases`): phases are strictly sequential again — walk, stat, hash,
|
|
||||||
update per operand — with parallelism only inside each phase; the walk
|
|
||||||
enumerates paths with per-directory workers and the stat pass lstats them with
|
|
||||||
per-file workers, restoring the exact total/ETA stat bar
|
|
||||||
- announce each operand on stderr before its passes (2026-07-24, branch
|
|
||||||
`scan-operand-progress`): with per-operand walk/hash/update cycles, a
|
|
||||||
multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the
|
|
||||||
whole run's
|
|
||||||
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single
|
|
||||||
goroutine and took hours at ~20M files on a busy pool (observed: 22M files in
|
|
||||||
4h on a ZFS server); it is now a per-directory worker-pool traversal that
|
|
||||||
records size/mtime during the walk (folding away the separate stat pass,
|
|
||||||
halving metadata I/O), and each `PATH` operand commits in its own transaction
|
|
||||||
so an interrupted scan keeps completed operands
|
|
||||||
|
|
||||||
- persistent scan database (2026-07-24, branch `persistent-database`): `scan`
|
|
||||||
now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays
|
|
||||||
disabled) keyed by absolute path that survives between runs — a rescan hashes
|
|
||||||
only new or changed files (by mtime/size), deletes records for files vanished
|
|
||||||
from under the scanned operands, and leaves records outside them untouched, so
|
|
||||||
`scan` can be cronned daily; `report` and `trees` read the database (no
|
|
||||||
positional arguments) instead of a scan stream. Database at
|
|
||||||
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL
|
|
||||||
journaling plus a single-transaction update keep a report run during a scan
|
|
||||||
safe
|
|
||||||
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`,
|
|
||||||
and push `main` plus tags (2026-07-23)
|
|
||||||
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
|
|
||||||
`PATH...` operands via cobra flags replacing the `/srv` `-root` default; new
|
|
||||||
`-x`/`--one-file-system` flag (GNU convention) to stop at filesystem
|
|
||||||
boundaries, which are crossed by default
|
|
||||||
- bring the repo into full policy compliance (2026-07-23, branch
|
|
||||||
`repo-policy-compliance`; checklist below)
|
|
||||||
- `git init` with README-only first commit; code baseline committed on `main`
|
|
||||||
(2026-07-22)
|
|
||||||
- implement `scan`, `report`, and `trees` subcommands (pre-git history)
|
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- possible later features (explicitly out of scope per README): full-content
|
- Possible later features, out of scope per README, are a byte-for-byte check of
|
||||||
verification of candidates, removal-script helpers
|
candidates and helper scripts for removal.
|
||||||
|
|
||||||
# Repo Policy Compliance
|
# Accepted Divergences
|
||||||
|
|
||||||
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo
|
None.
|
||||||
checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone
|
|
||||||
formatting commit is needed.
|
|
||||||
|
|
||||||
- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit
|
|
||||||
untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns
|
|
||||||
- [x] `.editorconfig` missing
|
|
||||||
- [x] `LICENSE` missing and README has no License section (MIT assumed from
|
|
||||||
house convention — user to confirm)
|
|
||||||
- [x] `REPO_POLICIES.md` missing from repo root
|
|
||||||
- [x] `.golangci.yml` missing (install canonical copy); code must then pass
|
|
||||||
`make lint` (150 findings fixed; `make lint` is clean)
|
|
||||||
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`,
|
|
||||||
`docker`, `hooks`; `check` currently depends on `build`, which writes the
|
|
||||||
binary (`make check` must not modify files)
|
|
||||||
- [x] no tests — `go test ./...` has nothing to run; policy requires real tests
|
|
||||||
with a 30-second timeout and the conditional `-v` rerun pattern (suite
|
|
||||||
covers parsing, grouping, digests, suppression, hashing, and the scan
|
|
||||||
pipeline; 64% coverage)
|
|
||||||
- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast
|
|
||||||
lint stage, build stage running `make check`
|
|
||||||
- [x] `.dockerignore` missing
|
|
||||||
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout
|
|
||||||
action pinned by commit SHA)
|
|
||||||
- [x] README lacks required sections: Description first line
|
|
||||||
(name/purpose/category/license/author), Getting Started, Rationale, TODO,
|
|
||||||
License, Author
|
|
||||||
- [x] README non-goal "no git repository setup and no CI" is stale now that the
|
|
||||||
repo is under git with CI
|
|
||||||
- [x] pre-commit hook not installed (`make hooks` once the target exists)
|
|
||||||
|
|
||||||
Accepted divergences (no action):
|
|
||||||
|
|
||||||
- flat single-package layout with `.go` files in the repo root — fine for a
|
|
||||||
small single-binary tool per the Go styleguide; the tracker audit agrees
|
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
// Command sfdupes quickly identifies candidate duplicate files across
|
||||||
|
// very large filesystems without reading every byte of every file.
|
||||||
|
// Files are considered duplicates when their sizes are equal and they
|
||||||
|
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
||||||
|
// KiB, and of the content. The content hash covers the whole file under
|
||||||
|
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||||
|
// file is never read in full. scan maintains a persistent SQLite
|
||||||
|
// database of file signatures (SFDUPES_DATABASE, default
|
||||||
|
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||||
|
// sfdupes report > dupes.tsv
|
||||||
|
// sfdupes trees > dupetrees.tsv
|
||||||
|
// sfdupes info
|
||||||
|
// sfdupes dwim
|
||||||
|
// sfdupes --version
|
||||||
|
//
|
||||||
|
// See README.md for the complete specification.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"sneak.berlin/go/sfdupes/internal/sfdupes"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() { os.Exit(sfdupes.Main()) }
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,23 +1,6 @@
|
|||||||
// Command sfdupes quickly identifies candidate duplicate files across
|
// Package sfdupes implements the sfdupes command and the database its
|
||||||
// very large filesystems without reading every byte of every file.
|
// subcommands share.
|
||||||
// Files are considered duplicates when their sizes are equal and they
|
package sfdupes
|
||||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
|
||||||
// KiB, and of the content. The content hash covers the whole file under
|
|
||||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
|
||||||
// file is never read in full. scan maintains a persistent SQLite
|
|
||||||
// database of file signatures (SFDUPES_DATABASE, default
|
|
||||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
|
||||||
//
|
|
||||||
// Usage:
|
|
||||||
//
|
|
||||||
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
|
||||||
// sfdupes report > dupes.tsv
|
|
||||||
// sfdupes trees > dupetrees.tsv
|
|
||||||
// sfdupes info
|
|
||||||
// sfdupes --version
|
|
||||||
//
|
|
||||||
// See README.md for the complete specification.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -45,6 +28,7 @@ const (
|
|||||||
cmdReport = "report"
|
cmdReport = "report"
|
||||||
cmdTrees = "trees"
|
cmdTrees = "trees"
|
||||||
cmdInfo = "info"
|
cmdInfo = "info"
|
||||||
|
cmdDwim = "dwim"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errNoSubcommand is returned by the root command when it is invoked
|
// errNoSubcommand is returned by the root command when it is invoked
|
||||||
@@ -84,20 +68,22 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
|
|||||||
return info.Main.Version
|
return info.Main.Version
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
// Main runs the command line in os.Args and returns the process exit
|
||||||
|
// code.
|
||||||
|
func Main() int {
|
||||||
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
||||||
// head", the Go runtime ends the process with SIGPIPE on the next
|
// head", the Go runtime ends the process with SIGPIPE on the next
|
||||||
// write instead of returning an error (README "Error handling").
|
// write instead of returning an error (README "Error handling").
|
||||||
// Registering for SIGPIPE with os/signal would change that.
|
// Registering for SIGPIPE with os/signal would change that.
|
||||||
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
return run(os.Args[1:], os.Stdout, os.Stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// run executes args against the command tree and returns the process
|
// run executes args against the command tree and returns the process
|
||||||
// exit code. It is the program's single exit point: the subcommands
|
// exit code. It is the program's single exit point: the subcommands
|
||||||
// return their errors instead of exiting, so every deferred cleanup —
|
// return their errors instead of exiting, so every deferred cleanup —
|
||||||
// above all closing the database, which checkpoints the SQLite WAL —
|
// above all closing the database, which checkpoints the SQLite WAL —
|
||||||
// runs before the process ends. The report, trees and info subcommands
|
// runs before the process ends. The report, trees, info and dwim
|
||||||
// write their data to stdout.
|
// subcommands write their data to stdout.
|
||||||
func run(args []string, stdout, stderr io.Writer) int {
|
func run(args []string, stdout, stderr io.Writer) int {
|
||||||
// A nil slice makes cobra fall back to os.Args, which would let a
|
// A nil slice makes cobra fall back to os.Args, which would let a
|
||||||
// test binary's own flags reach the command tree.
|
// test binary's own flags reach the command tree.
|
||||||
@@ -201,7 +187,16 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
|
dwimCmd := &cobra.Command{
|
||||||
|
Use: cmdDwim,
|
||||||
|
Short: "Scan / with the default excludes, then print the duplicate-tree report",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: runE(func(ctx context.Context, _ []string) error {
|
||||||
|
return runDwim(ctx, "/", stdout)
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
||||||
|
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported run
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -14,6 +14,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
// usageMarker is the first line of cobra's usage text, which a usage
|
// usageMarker is the first line of cobra's usage text, which a usage
|
||||||
@@ -394,7 +396,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
||||||
// README §Database: the path names the file exactly. With link a
|
// README §Settings: the path names the file exactly. With link a
|
||||||
// symlink to the directory real/sub, link/../db.sqlite names
|
// symlink to the directory real/sub, link/../db.sqlite names
|
||||||
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
||||||
// use that file, relative or absolute, and its first line must name
|
// use that file, relative or absolute, and its first line must name
|
||||||
@@ -620,6 +622,7 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
||||||
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
||||||
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
||||||
|
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
|
||||||
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
||||||
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
||||||
}
|
}
|
||||||
@@ -649,7 +652,7 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||||
// README §scan mode: --workers below 1 is a usage error reported in
|
// README §Settings: --workers below 1 is a usage error reported in
|
||||||
// one line on stderr, before the scan opens the database.
|
// one line on stderr, before the scan opens the database.
|
||||||
for _, workers := range []string{"0", "-1"} {
|
for _, workers := range []string{"0", "-1"} {
|
||||||
t.Run(workers, func(t *testing.T) {
|
t.Run(workers, func(t *testing.T) {
|
||||||
@@ -687,8 +690,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
func TestRunHelp(t *testing.T) {
|
func TestRunHelp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Subcommands: help goes to stderr, exits 0, and leaves
|
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
||||||
// stdout empty.
|
// empty.
|
||||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||||
|
|
||||||
for _, args := range cases {
|
for _, args := range cases {
|
||||||
@@ -714,8 +717,8 @@ func TestRunHelp(t *testing.T) {
|
|||||||
func TestRunVersion(t *testing.T) {
|
func TestRunVersion(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Subcommands: the version is one line on stdout, with
|
// README §Settings: the version is one line on stdout, with nothing
|
||||||
// nothing on stderr, and exits 0.
|
// on stderr, and exits 0.
|
||||||
info, _ := debug.ReadBuildInfo()
|
info, _ := debug.ReadBuildInfo()
|
||||||
|
|
||||||
for _, arg := range []string{"--version", "-v"} {
|
for _, arg := range []string{"--version", "-v"} {
|
||||||
@@ -774,8 +777,8 @@ func TestBuildVersion(t *testing.T) {
|
|||||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Error handling: a stdout write failure exits 1, reported
|
// README §Error handling: a failed write to stdout is a fatal error,
|
||||||
// in one line on stderr.
|
// exit 1, reported on stderr starting with "sfdupes:".
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
||||||
@@ -797,7 +800,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
|||||||
func scanFixture(t *testing.T) []string {
|
func scanFixture(t *testing.T) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
|
|
||||||
dupes := []string{
|
dupes := []string{
|
||||||
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
||||||
@@ -867,7 +870,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "a.bin", pattern(1, 300))
|
writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||||
|
|
||||||
// Same size as a.bin, so the scan reads it, and the read fails.
|
// Same size as a.bin, so the scan reads it, and the read fails.
|
||||||
@@ -895,58 +898,278 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
assertNoSidecars(t, path)
|
assertNoSidecars(t, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
|
// realTempDir returns a new temporary directory by its real path, so
|
||||||
path := testDBPath(t)
|
// tests hold where the temporary directory is reached through a
|
||||||
t.Setenv(databaseEnv, path)
|
// symlink.
|
||||||
|
func realTempDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scanning a directory through the symlink stores a record beneath
|
return dir
|
||||||
// the symlink's own path for a file beneath its target.
|
|
||||||
scanOK(t, filepath.Join(link, "sub"))
|
|
||||||
|
|
||||||
assertOperandSkipped(t, path, link, "symlink",
|
|
||||||
filepath.Join(link, "sub", "f"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// symlink creates link pointing at target.
|
||||||
|
func symlink(t *testing.T, target, link string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.Symlink(target, link)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
|
||||||
|
// The operand is scanned at its real path, and its file is recorded
|
||||||
|
// there.
|
||||||
|
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, filepath.Join(dir, "target"), link)
|
||||||
|
|
||||||
|
stderr := scanOK(t, link)
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
|
||||||
|
// with link a symlink to target. Both resolve under target, so
|
||||||
|
// target/sub is walked once, as part of target.
|
||||||
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, filepath.Join(dir, "target"), link)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
stderr := scanOK(t, link, filepath.Join(link, "sub"))
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
|
||||||
|
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
|
||||||
|
// with dir/link a symlink to a directory outside dir. The walk of dir
|
||||||
|
// does not follow link, so dir/link/sub is walked on its own at its
|
||||||
|
// real path, and the record an earlier scan stored there survives.
|
||||||
|
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
a := writeFile(t, dir, "a", pattern(1, 10))
|
||||||
|
|
||||||
|
outside := realTempDir(t)
|
||||||
|
f := writeFile(t, outside, "sub/f", pattern(2, 10))
|
||||||
|
|
||||||
|
symlink(t, outside, filepath.Join(dir, "link"))
|
||||||
|
|
||||||
|
scanOK(t, filepath.Join(outside, "sub"))
|
||||||
|
|
||||||
|
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
|
||||||
|
|
||||||
|
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"1 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{a, f}
|
||||||
|
slices.Sort(want)
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||||
|
t.Errorf("records = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
|
||||||
|
// link, outside dir, a symlink to dir/real. link resolves inside dir,
|
||||||
|
// so it is dropped as covered by dir, and its file is recorded once.
|
||||||
|
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "real/f", pattern(1, 10))
|
||||||
|
|
||||||
|
link := filepath.Join(realTempDir(t), "link")
|
||||||
|
symlink(t, filepath.Join(dir, "real"), link)
|
||||||
|
|
||||||
|
stderr := scanOK(t, dir, link)
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
|
||||||
|
// a symlink whose target does not exist. The scan fails with exit 1 and
|
||||||
|
// an error naming the symlink as given, before it walks anything, so no
|
||||||
|
// database is created.
|
||||||
|
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
writeFile(t, dir, "f", pattern(1, 10))
|
||||||
|
|
||||||
|
dangling := filepath.Join(realTempDir(t), "dangling")
|
||||||
|
symlink(t, filepath.Join(dir, "missing"), dangling)
|
||||||
|
|
||||||
|
stdout := captureStdout(t)
|
||||||
|
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
|
||||||
|
if code != exitFatal {
|
||||||
|
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
|
||||||
|
exitFatal)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertFatalOutput(t, stderr.String(), stdout())
|
||||||
|
|
||||||
|
want := "sfdupes: resolve " + dangling + ": "
|
||||||
|
if !strings.Contains(stderr.String(), want) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
if !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
t.Errorf("database: stat error %v, want it missing", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
|
||||||
|
// after a cd through home/proj, a symlink to data/proj. The file is
|
||||||
|
// recorded under data/proj, its real path.
|
||||||
|
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||||
|
|
||||||
|
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// link is dropped as a symlink, but link/sub must still be scanned,
|
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||||
// not dropped as lying under link.
|
filepath.Join(dir, "home", "proj"))
|
||||||
scanOK(t, link, filepath.Join(link, "sub"))
|
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
// t.Chdir also sets PWD to the path through the symlink, as a
|
||||||
|
// shell does after cd.
|
||||||
|
t.Chdir(filepath.Join(dir, "home", "proj"))
|
||||||
|
|
||||||
|
scanOK(t, ".")
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
|
||||||
|
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
|
||||||
|
// a cd through home/proj. As with realpath, the ".." counts from
|
||||||
|
// data/proj, so data is scanned and home is not.
|
||||||
|
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ wd, operand string }{
|
||||||
|
{"", "home/proj/.."},
|
||||||
|
{"home/proj", ".."},
|
||||||
|
} {
|
||||||
|
t.Run(tc.operand, func(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||||
|
g := writeFile(t, dir, "data/g", pattern(2, 10))
|
||||||
|
writeFile(t, dir, "home/h", pattern(3, 10))
|
||||||
|
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||||
|
filepath.Join(dir, "home", "proj"))
|
||||||
|
|
||||||
|
// t.Chdir also sets PWD to the path given, as a shell does
|
||||||
|
// after cd.
|
||||||
|
t.Chdir(filepath.Join(dir, tc.wd))
|
||||||
|
|
||||||
|
scanOK(t, tc.operand)
|
||||||
|
|
||||||
|
want := []string{f, g}
|
||||||
|
slices.Sort(want)
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||||
|
t.Errorf("records = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
|
||||||
|
// The scan warns about the FIFO by its real path, counts it as skipped,
|
||||||
|
// and exits 0.
|
||||||
|
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
fifo := filepath.Join(dir, "fifo")
|
||||||
|
|
||||||
|
err := unix.Mkfifo(fifo, 0o600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, fifo, link)
|
||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
|
stderr := scanOK(t, link)
|
||||||
|
|
||||||
|
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
||||||
|
if !strings.Contains(stderr, warning) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 1 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||||
zfs := filepath.Join(t.TempDir(), ".zfs")
|
zfs := filepath.Join(realTempDir(t), ".zfs")
|
||||||
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
||||||
|
|
||||||
// No scan records a file inside a .zfs directory, so the record
|
// No scan records a file inside a .zfs directory, so the record
|
||||||
@@ -997,7 +1220,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
||||||
writeFile(t, hourly, "f", pattern(1, 10))
|
writeFile(t, hourly, "f", pattern(1, 10))
|
||||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
f := writeFile(t, dir, "f", pattern(2, 10))
|
||||||
@@ -1041,23 +1264,19 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|||||||
|
|
||||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
||||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
||||||
// by the path given, counted as skipped, and not walked.
|
// by its real path, counted as skipped, and not walked.
|
||||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
||||||
|
|
||||||
snaps := filepath.Join(dir, "snaps")
|
snaps := filepath.Join(dir, "snaps")
|
||||||
|
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(snaps, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
|
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
||||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||||
"0 unchanged), 1 skipped\n"
|
"0 unchanged), 1 skipped\n"
|
||||||
@@ -1069,24 +1288,19 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
||||||
// a symlink to a directory outside any .zfs directory. The operand is
|
// a symlink to pool/data. The operand is scanned as usual, and its file
|
||||||
// scanned as usual, and its file is recorded by the path given.
|
// is recorded by its real path.
|
||||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||||
|
|
||||||
data := filepath.Join(dir, "data")
|
data := filepath.Join(dir, "data")
|
||||||
|
symlink(t, filepath.Join(dir, "pool", "data"), data)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
|
stderr := scanOK(t, filepath.Join(data, "daily"))
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(data, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
"0 unchanged), 0 skipped\n"
|
"0 unchanged), 0 skipped\n"
|
||||||
@@ -1094,14 +1308,9 @@ func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
|||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
}
|
}
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
if err != nil {
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunReportSucceeds(t *testing.T) {
|
func TestRunReportSucceeds(t *testing.T) {
|
||||||
@@ -1257,7 +1466,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
||||||
// README §Database: the path names the database file exactly. In
|
// README §Settings: the path names the database file exactly. In
|
||||||
// SQLite's connection string an unescaped ? or # would end the file
|
// SQLite's connection string an unescaped ? or # would end the file
|
||||||
// name and % would start an escape, and a path starting with //
|
// name and % would start an escape, and a path starting with //
|
||||||
// could be read as a host name. %25 is a valid escape, so unescaped
|
// could be read as a host name. %25 is a valid escape, so unescaped
|
||||||
@@ -1302,9 +1511,8 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: while one scan holds the lock, a second scan
|
// README §Database: a second scan that finds the lock held fails
|
||||||
// fails at once, naming the lock file, without opening the
|
// at once, naming the lock file, without opening the database.
|
||||||
// database.
|
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
@@ -1363,7 +1571,7 @@ func assertSecondScanFails(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||||
// README §Database: every path to one database names the same lock
|
// README §Database: a path through a symlink names the same lock
|
||||||
// file, so while a scan holds it through one path, a scan through
|
// file, so while a scan holds it through one path, a scan through
|
||||||
// another fails, naming the lock file beside the database's real
|
// another fails, naming the lock file beside the database's real
|
||||||
// path.
|
// path.
|
||||||
@@ -1468,8 +1676,8 @@ func TestRunReportsDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
||||||
// README §Error handling: a stdout write failure exits 1, reported
|
// README §Error handling: a failed write to stdout is a fatal error,
|
||||||
// in one line on stderr.
|
// exit 1, reported on stderr starting with "sfdupes:".
|
||||||
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -146,33 +146,44 @@ var errHashSize = errors.New("hash is not 32 bytes")
|
|||||||
|
|
||||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||||
// path, with the working directory, its symlinks resolved, put in front
|
// path, made absolute by absolutePath, and where the path came from,
|
||||||
// of it when it is relative, and where the path came from, "default" or
|
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
||||||
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
|
// prints first on stderr.
|
||||||
// stderr.
|
|
||||||
func databasePath() (string, string, error) {
|
func databasePath() (string, string, error) {
|
||||||
path, source := defaultDatabasePath, "default"
|
path, source := defaultDatabasePath, "default"
|
||||||
if p := os.Getenv(databaseEnv); p != "" {
|
if p := os.Getenv(databaseEnv); p != "" {
|
||||||
path, source = p, "from "+databaseEnv
|
path, source = p, "from "+databaseEnv
|
||||||
}
|
}
|
||||||
|
|
||||||
if filepath.IsAbs(path) {
|
abs, err := absolutePath(path)
|
||||||
return path, source, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The working directory as the kernel reports it, with no symlink
|
|
||||||
// in it, as README "Database" says. os.Getwd may return PWD
|
|
||||||
// instead, which can name it through a symlink.
|
|
||||||
wd, err := unix.Getwd()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Not filepath.Abs or filepath.Join: they clean the path, which
|
return abs, source, nil
|
||||||
// drops each ".." together with the directory name before it. When
|
}
|
||||||
// that directory is a symlink, the cleaned path names another file.
|
|
||||||
|
// absolutePath returns path unchanged when it is absolute, and
|
||||||
|
// otherwise the working directory, a "/" and path. The working
|
||||||
|
// directory is the one the kernel reports, with no symlink in it;
|
||||||
|
// os.Getwd may return PWD instead, which can name it through a symlink.
|
||||||
|
// The result is not cleaned, as filepath.Abs and filepath.Join would
|
||||||
|
// clean it. Cleaning drops each ".." together with the directory name
|
||||||
|
// before it. When that name is a symlink, the cleaned path names
|
||||||
|
// another file than the one the system reaches by following the symlink
|
||||||
|
// first.
|
||||||
|
func absolutePath(path string) (string, error) {
|
||||||
|
if filepath.IsAbs(path) {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
wd, err := unix.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
// The working directory ends in "/" only when it is the root.
|
// The working directory ends in "/" only when it is the root.
|
||||||
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
return strings.TrimSuffix(wd, "/") + "/" + path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createDatabaseDirectory creates the directory the database at path
|
// createDatabaseDirectory creates the directory the database at path
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // calls the unexported database openers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package sfdupes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
||||||
|
// root" runs with no flags, then what "sfdupes trees" runs, and ends
|
||||||
|
// with a line on stderr saying how to print the report again. The dwim
|
||||||
|
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
|
||||||
|
// does, but only during the scan. An error from the scan, an interrupt
|
||||||
|
// included, ends it before the report. During the report, a signal
|
||||||
|
// ends it at once, as it ends trees.
|
||||||
|
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
|
||||||
|
scanCtx, stop := interruptContext(ctx)
|
||||||
|
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false)
|
||||||
|
|
||||||
|
stop()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = runTrees(ctx, stdout)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(os.Stderr,
|
||||||
|
`dwim: run "sfdupes trees" to print this report again`)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package sfdupes //nolint:testpackage // calls the unexported runDwim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
||||||
|
// dwim scans its root, then prints the trees report, the larger of
|
||||||
|
// two duplicate-tree pairs first, and ends by saying how to print
|
||||||
|
// the report again.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
root := realTempDir(t)
|
||||||
|
|
||||||
|
small := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
||||||
|
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
||||||
|
}
|
||||||
|
large := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
|
||||||
|
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), root, &stdout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
||||||
|
if len(lines) != 3 {
|
||||||
|
t.Fatalf("stdout = %q, want a header and two groups",
|
||||||
|
stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
|
||||||
|
if !strings.HasPrefix(lines[1], wantFirst) {
|
||||||
|
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
|
||||||
|
if !strings.HasPrefix(lines[2], wantSecond) {
|
||||||
|
t.Errorf("second group = %q, want it to start %q",
|
||||||
|
lines[2], wantSecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
|
||||||
|
if got := stderr(); !strings.HasSuffix(got, last) {
|
||||||
|
t.Errorf("stderr = %q, want it to end %q", got, last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
||||||
|
// A fatal scan error ends dwim before the report: a root that does
|
||||||
|
// not exist fails the scan as it fails scan itself.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
missing := filepath.Join(realTempDir(t), "missing")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), missing, &stdout)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stdout.String(); got != "" {
|
||||||
|
t.Errorf("stdout = %q, want no report", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stderr(); strings.Contains(got, "trees:") ||
|
||||||
|
strings.Contains(got, "dwim:") {
|
||||||
|
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
// This file holds two rules for the walk (README "Rules for the walk"):
|
// This file holds two rules for the walk (README "Rules for the walk"):
|
||||||
// the default excludes, the paths a scan's walk does not enter below an
|
// the default excludes, the paths a scan's walk does not enter below an
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"slices"
|
"slices"
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -17,7 +17,9 @@ import (
|
|||||||
func TestDefaultExcludes(t *testing.T) {
|
func TestDefaultExcludes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README "Rules for the walk": each system's list.
|
// README "Rules for the walk": the Linux and macOS lists, and none
|
||||||
|
// on any other system. The Windows list exists in the code only: the
|
||||||
|
// binary does not build for Windows, so the README leaves it out.
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
@@ -284,7 +286,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
||||||
t.Skip(dir + " is not inside a default exclude")
|
t.Skip(dir + " is not inside a default exclude")
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported startProgress
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
@@ -209,29 +209,30 @@ func interrupted(walked int) error {
|
|||||||
return errInterrupted
|
return errInterrupted
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveRoots converts each PATH operand to an absolute, lexically
|
// resolveRoots converts each PATH operand to its real path: absolute,
|
||||||
// cleaned path (symlinks are not resolved) and verifies that it
|
// with every symlink in it followed. Database records are keyed by
|
||||||
// exists. Database records are keyed by absolute path, so scan results
|
// that path, so scan results depend neither on the working directory
|
||||||
// must not depend on the working directory.
|
// nor on the symlinks an operand was given through. An operand that
|
||||||
|
// cannot be resolved, because it or a symlink's target does not exist
|
||||||
|
// or cannot be read, is a fatal error before any scanning.
|
||||||
func resolveRoots(roots []string) ([]string, error) {
|
func resolveRoots(roots []string) ([]string, error) {
|
||||||
abs := make([]string, 0, len(roots))
|
resolved := make([]string, 0, len(roots))
|
||||||
|
|
||||||
for _, root := range roots {
|
for _, root := range roots {
|
||||||
a, err := filepath.Abs(root)
|
a, err := absolutePath(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A nonexistent operand is a fatal error before any scanning.
|
r, err := filepath.EvalSymlinks(a)
|
||||||
_, err = os.Lstat(a)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
abs = append(abs, a)
|
resolved = append(resolved, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
return abs, nil
|
return resolved, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneRoots drops operands already covered by another operand:
|
// pruneRoots drops operands already covered by another operand:
|
||||||
@@ -417,14 +418,16 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
|
|
||||||
// walkableRoots returns the operands the walk can start from: regular
|
// walkableRoots returns the operands the walk can start from: regular
|
||||||
// files, and directories not named .zfs, none of them inside a
|
// files, and directories not named .zfs, none of them inside a
|
||||||
// directory named .zfs. Every other operand is warned about, counted as
|
// directory named .zfs. The operands are real paths from resolveRoots,
|
||||||
// skipped, and dropped. A dropped operand is no longer a root, so the
|
// so these checks apply to what a symlink operand points at. Every
|
||||||
// records stored beneath it are treated as if it had not been given:
|
// other operand is warned about, counted as skipped, and dropped. A
|
||||||
// those under no other root count as outside the roots and are not
|
// dropped operand is no longer a root, so the records stored beneath it
|
||||||
// deleted as unverified. An operand on a filesystem in rules that is not
|
// are treated as if it had not been given: those under no other root
|
||||||
// persistent is dropped the same way, with a line saying so that is not
|
// count as outside the roots and are not deleted as unverified. An
|
||||||
// a warning, and is not counted as skipped. An operand that fails lstat
|
// operand on a filesystem in rules that is not persistent is dropped
|
||||||
// here is kept, and the walk warns about it unless it no longer exists.
|
// the same way, with a line saying so that is not a warning, and is not
|
||||||
|
// counted as skipped. An operand that fails lstat here is kept, and the
|
||||||
|
// walk warns about it unless it no longer exists.
|
||||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||||
kept := make([]string, 0, len(roots))
|
kept := make([]string, 0, len(roots))
|
||||||
|
|
||||||
@@ -554,7 +557,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
// The space a file takes on disk can change while its size and
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
||||||
// only seconds after the write.
|
// up to seconds after the write.
|
||||||
if old.diskSize != ev.rec.diskSize {
|
if old.diskSize != ev.rec.diskSize {
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
s.diskSizes = append(s.diskSizes, ev.rec)
|
||||||
}
|
}
|
||||||
@@ -894,19 +897,13 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
|||||||
// operandWarning returns the one-line warning for an operand the walk
|
// operandWarning returns the one-line warning for an operand the walk
|
||||||
// does not start from, naming the path and what it is, or "" for one it
|
// does not start from, naming the path and what it is, or "" for one it
|
||||||
// does: a regular file, or a directory not named .zfs, that is not
|
// does: a regular file, or a directory not named .zfs, that is not
|
||||||
// inside a directory named .zfs, neither as given nor with its symlinks
|
// inside a directory named .zfs. resolveRoots has already followed the
|
||||||
// resolved. Symlinks are never followed, including as operands.
|
// operand's symlinks, so a symlink here is an operand replaced by one
|
||||||
|
// after it was resolved, and is not followed.
|
||||||
func operandWarning(root string, fi fs.FileInfo) string {
|
func operandWarning(root string, fi fs.FileInfo) string {
|
||||||
// The walk never enters a .zfs directory, so it starts from nothing
|
// The walk never enters a .zfs directory, so it starts from nothing
|
||||||
// inside one either, not even through a symlink. An operand whose
|
// inside one either.
|
||||||
// symlinks cannot be resolved is checked as given.
|
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
||||||
resolved, err := filepath.EvalSymlinks(root)
|
|
||||||
if err != nil {
|
|
||||||
resolved = root
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
|
|
||||||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
|
|
||||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
||||||
root)
|
root)
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported syncTree
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -43,10 +43,16 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
||||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
|
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's
|
||||||
// blocks only seconds after the write, and fsync does not hasten that.
|
// blocks only when it commits the write, up to seconds later, and one
|
||||||
// A test therefore calls unix.Sync, which waits for it, before the scan
|
// commit can take the file through several growing sizes. So this value
|
||||||
// whose disk_size it compares with this.
|
// can change at any moment. Neither fsync nor unix.Sync hastens that,
|
||||||
|
// but a unix.Sync call often takes long enough that the blocks are
|
||||||
|
// counted when it returns. A test that needs the value to change waits
|
||||||
|
// for the change. A test that compares it with a scan's disk_size reads
|
||||||
|
// it before and after the scan. The rewrite test rescans until the two
|
||||||
|
// reads match, then compares exactly. The other two tests fail only on
|
||||||
|
// a disk_size outside the two reads.
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
func diskSizeOf(t *testing.T, path string) int64 {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1082,12 +1088,13 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
// buildSmokeTree recreates the tree of the test under README
|
||||||
// with deterministic content and returns the tree root.
|
// "Definition of done" with deterministic content and returns the tree
|
||||||
|
// root.
|
||||||
func buildSmokeTree(t *testing.T) string {
|
func buildSmokeTree(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
one := pattern(10, 2000)
|
one := pattern(10, 2000)
|
||||||
f1 := pattern(30, 3000)
|
f1 := pattern(30, 3000)
|
||||||
f2 := pattern(40, 100)
|
f2 := pattern(40, 100)
|
||||||
@@ -1447,20 +1454,26 @@ func TestScanStoresDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
unix.Sync() // see diskSizeOf
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
size := diskSizeOf(t, p)
|
||||||
if want == 0 || want >= length {
|
if size == 0 || size >= length {
|
||||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
||||||
"and fewer than its length %d", want, length)
|
"and fewer than its length %d", size, length)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, operand := range []string{filepath.Dir(p), p} {
|
for _, operand := range []string{filepath.Dir(p), p} {
|
||||||
db := openTestDB(t)
|
db := openTestDB(t)
|
||||||
|
|
||||||
|
// ZFS may count the blocks during the scan; see diskSizeOf.
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
syncTree(t, db, operand)
|
syncTree(t, db, operand)
|
||||||
|
|
||||||
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
if got != want {
|
if got < before || got > after {
|
||||||
t.Errorf("scan of %s: disk_size = %d, want %d",
|
t.Errorf("scan of %s: disk_size = %d, want %d to %d",
|
||||||
operand, got, want)
|
operand, got, before, after)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1478,7 +1491,7 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
syncTree(t, db, dir)
|
syncTree(t, db, dir)
|
||||||
|
|
||||||
before := recordByPath(t, dbRecords(t, db), p).diskSize
|
recorded := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
|
|
||||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
||||||
|
|
||||||
@@ -1491,22 +1504,53 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
// Wait for ZFS to count the write; see diskSizeOf.
|
||||||
|
deadline := time.Now().Add(30 * time.Second)
|
||||||
want := diskSizeOf(t, p)
|
for diskSizeOf(t, p) == recorded {
|
||||||
if want == before {
|
if time.Now().After(deadline) {
|
||||||
t.Fatalf("the file still takes %d bytes on disk after the write "+
|
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
|
||||||
"into its hole", want)
|
"after the write into its hole", recorded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unix.Sync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZFS may still be counting, so the size may change during a rescan.
|
||||||
|
// Only a rescan with the same size on both sides of it can be compared
|
||||||
|
// exactly. A rescan that keeps the old size still fails, because the
|
||||||
|
// wait made before differ from it.
|
||||||
|
for {
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
st := syncTree(t, db, dir)
|
||||||
|
|
||||||
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
if st != (scanStats{walked: 1, updated: 1}) {
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if before == after {
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
if got != want {
|
if got != after {
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
t.Errorf("disk_size = %d, want %d", got, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("the size on disk was still changing 30 seconds after "+
|
||||||
|
"the write into its hole: %d, then %d", before, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later mtime makes the next rescan read the file again.
|
||||||
|
later = later.Add(time.Second)
|
||||||
|
|
||||||
|
err = os.Chtimes(p, later, later)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1532,15 +1576,21 @@ func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
unix.Sync() // see diskSizeOf
|
||||||
|
|
||||||
|
// ZFS may count the blocks during the rescan; see diskSizeOf.
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
// A file read again would count as updated.
|
// A file read again would count as updated.
|
||||||
st := syncTree(t, db, dir)
|
st := syncTree(t, db, dir)
|
||||||
|
|
||||||
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
if st != (scanStats{walked: 1, unchanged: 1}) {
|
||||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
if want := diskSizeOf(t, p); got != want {
|
if got < before || got > after {
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2260,33 +2310,24 @@ func TestUnderRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
||||||
// database, reached directly and through a symlinked directory. The
|
// database, and in one case also a hard link to the database file. The
|
||||||
// walk passes over the database file, its -wal and -shm files and its
|
// walk knows the database's files by device ID and inode number, so it
|
||||||
// .lock file with no line and no count, and the record an earlier scan
|
// passes over the database file, its -wal and -shm files, its .lock
|
||||||
// left for the .lock file is deleted. A rescan of the unchanged tree
|
// file and the hard link with no line and no count. The record an
|
||||||
// updates nothing, and report lists none of the database's files.
|
// earlier scan left for the .lock file is deleted. A rescan of the
|
||||||
|
// unchanged tree updates nothing, and report lists none of the
|
||||||
|
// database's files.
|
||||||
func TestRunScanPassesOverDatabase(t *testing.T) {
|
func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||||
for _, viaSymlink := range []bool{false, true} {
|
for _, viaHardLink := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
|
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
|
||||||
base := t.TempDir()
|
tree := realTempDir(t)
|
||||||
tree := filepath.Join(base, "real", "tree")
|
hardLink := filepath.Join(tree, "hardlink")
|
||||||
operand := tree
|
|
||||||
|
|
||||||
writeFile(t, tree, "a", pattern(1, 100))
|
writeFile(t, tree, "a", pattern(1, 100))
|
||||||
writeFile(t, tree, "b", pattern(1, 100))
|
writeFile(t, tree, "b", pattern(1, 100))
|
||||||
writeFile(t, tree, "empty1", nil)
|
writeFile(t, tree, "empty1", nil)
|
||||||
writeFile(t, tree, "empty2", nil)
|
writeFile(t, tree, "empty2", nil)
|
||||||
|
|
||||||
if viaSymlink {
|
|
||||||
err := os.Symlink(filepath.Join(base, "real"),
|
|
||||||
filepath.Join(base, "link"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
operand = filepath.Join(base, "link", "tree")
|
|
||||||
}
|
|
||||||
|
|
||||||
dbPath := filepath.Join(tree, "db.sqlite")
|
dbPath := filepath.Join(tree, "db.sqlite")
|
||||||
t.Setenv(databaseEnv, dbPath)
|
t.Setenv(databaseEnv, dbPath)
|
||||||
|
|
||||||
@@ -2296,7 +2337,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = applyChanges(t.Context(), db, []scanRec{
|
err = applyChanges(t.Context(), db, []scanRec{
|
||||||
{path: filepath.Join(operand, "db.sqlite.lock")},
|
{path: filepath.Join(tree, "db.sqlite.lock")},
|
||||||
}, nil, nil)
|
}, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -2304,16 +2345,29 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
|
|
||||||
closeScanDatabase(t.Context(), db, dbPath)
|
closeScanDatabase(t.Context(), db, dbPath)
|
||||||
|
|
||||||
assertScanSummary(t,
|
if viaHardLink {
|
||||||
|
err = os.Link(dbPath, hardLink)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stderr := assertScanSummary(t,
|
||||||
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
||||||
operand)
|
tree)
|
||||||
assertScanSummary(t,
|
stderr += assertScanSummary(t,
|
||||||
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
||||||
operand)
|
tree)
|
||||||
|
|
||||||
|
if strings.Contains(stderr, hardLink) {
|
||||||
|
t.Errorf("scan stderr = %q, want no line naming %s",
|
||||||
|
stderr, hardLink)
|
||||||
|
}
|
||||||
|
|
||||||
report := runStdout(t, cmdReport)
|
report := runStdout(t, cmdReport)
|
||||||
if strings.Contains(report, "db.sqlite") ||
|
if strings.Contains(report, "db.sqlite") ||
|
||||||
!strings.Contains(report, filepath.Join(operand, "empty2")) {
|
strings.Contains(report, hardLink) ||
|
||||||
|
!strings.Contains(report, filepath.Join(tree, "empty2")) {
|
||||||
t.Errorf("report = %q, want the empty files and none of "+
|
t.Errorf("report = %q, want the empty files and none of "+
|
||||||
"the database's files", report)
|
"the database's files", report)
|
||||||
}
|
}
|
||||||
@@ -2338,8 +2392,9 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
|
|||||||
|
|
||||||
// assertScanSummary scans operands and fails the test unless the
|
// assertScanSummary scans operands and fails the test unless the
|
||||||
// summary line gives counts with nothing skipped, and the line naming
|
// summary line gives counts with nothing skipped, and the line naming
|
||||||
// the database is the only one that names one of its files.
|
// the database is the only one that names one of its files. It returns
|
||||||
func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
// everything the scan printed to stderr.
|
||||||
|
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
stderr := scanOK(t, operands...)
|
stderr := scanOK(t, operands...)
|
||||||
@@ -2353,4 +2408,6 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
|||||||
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
||||||
"database to name one of its files", operands, stderr)
|
"database to name one of its files", operands, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return stderr
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported treeNode results
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -15,9 +15,9 @@ import (
|
|||||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"remain unknown"
|
||||||
|
|
||||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
// smokeTreeRecs mirrors the tree of the test under README "Definition
|
||||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
|
||||||
// Each file takes one 4 KiB block on disk.
|
// by one filename. Each file takes one 4 KiB block on disk.
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||||
Reference in new issue
Block a user