1 Commits
Author SHA1 Message Date
clawbot b2f599f35f Re-vendor the canonical files from sneak/prompts at dd4027b (closes #95)
check / check (push) Canceled after 0s
The shared files are the copies at sneak/prompts commit dd4027b, with this
repository's own entries kept after them. script/lint, script/test and
REPO_POLICIES.md come from its next at c55a0cb, so the lint and test
builds write no image. golangci-lint is v2.14.0 and raises no findings.
Lint and test are phases of the Dockerfile; the tests run under the race
detector as nobody, so Dockerfile.lint, script/verify-lint-image-pin and
make test-race are gone. Every docker build in script/ passes --no-cache.
Formatting runs on the host: script/bootstrap installs the pinned node and
yarn, and the prettier and markdown stages are gone. .claude/settings.json
is deleted.

Deviation: the workflow keeps fetch-depth: 0.
Deviation: .gitignore keeps the scan database patterns.
Over the cap: make test takes 82 to 100 seconds on this host.

Model: opus-5-5
2026-10-08 00:32:31 +00:00
39 changed files with 1026 additions and 6675 deletions
-3
View File
@@ -13,6 +13,3 @@ indent_style = tab
[*.go] [*.go]
indent_style = tab indent_style = tab
# This repository's own sections, such as one for another language it
# uses, go below this comment, and a re-vendor keeps them.
-9
View File
@@ -1,20 +1,11 @@
name: check name: check
on: [push] on: [push]
# Free the shared runner: a new push cancels only the same branch's older run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Free the shared runner from a hung build.
timeout-minutes: 20
steps: steps:
# actions/checkout v4.2.2, 2026-02-22 # actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
# script/cibuild needs no token, so none is left in .git/config.
with: with:
persist-credentials: false
# All history and tags, so git describe finds the version tag.
fetch-depth: 0 fetch-depth: 0
- run: script/cibuild - run: script/cibuild
+3 -5
View File
@@ -27,7 +27,7 @@ node_modules/
# Environment files. `*.env` covers bare `.env` and the `prod.env` # Environment files. `*.env` covers bare `.env` and the `prod.env`
# convention. Only the templates `example.env` and `sample.env` are # convention. Only the templates `example.env` and `sample.env` are
# re-included below. A repository that commits any other template adds # re-included below. A repository that commits any other template adds
# its own negation at the end of this file, for example `!.env.example`. # its own negation after these lines, for example `!.env.example`.
*.[eE][nN][vV] *.[eE][nN][vV]
.[eE][nN][vV].* .[eE][nN][vV].*
.[eE][nN][vV][rR][cC] .[eE][nN][vV][rR][cC]
@@ -46,13 +46,11 @@ node_modules/
[iI][dD]_[eE][dD]25519 [iI][dD]_[eE][dD]25519
[iI][dD]_[eE][dD]25519_[sS][kK] [iI][dD]_[eE][dD]25519_[sS][kK]
# This repository's own entries, such as its build outputs, go below # This repository's own entries, kept after the canonical content above.
# this comment, and a re-vendor keeps them. Anchor a binary built at the
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
/sfdupes
*.log *.log
*.out *.out
*.test *.test
/sfdupes
# A scan database lists every path it scanned. # A scan database lists every path it scanned.
*.sqlite *.sqlite
-2
View File
@@ -25,8 +25,6 @@ linters:
# silenced by disabling that name, not by enabling the successor. # silenced by disabling that name, not by enabling the successor.
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- gomodguard # Deprecated, replaced by gomodguard_v2 - gomodguard # Deprecated, replaced by gomodguard_v2
# Misses findings at random in v2.14.0; back once a pinned release fixes it
- canonicalheader
settings: settings:
lll: lll:
line-length: 88 line-length: 88
+11 -34
View File
@@ -1,14 +1,3 @@
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build. The
# test phase runs as nobody and so mounts Go's build cache at
# /tmp/.cache/go-build; its own comment says why.
# Lint phase, built alone by script/lint. The tools are invoked directly # Lint phase, built alone by script/lint. The tools are invoked directly
# rather than through `make lint`, which runs docker itself and so cannot # rather than through `make lint`, which runs docker itself and so cannot
# run inside a build step. # run inside a build step.
@@ -17,7 +6,6 @@ FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a924
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a # The gofmt half of `make fmt-check`. gofmt's output is assigned to a
@@ -30,20 +18,14 @@ RUN files="$(gofmt -s -l .)" && \
# Validates .golangci.yml against the schema the pinned binary embeds. # Validates .golangci.yml against the schema the pinned binary embeds.
RUN golangci-lint config verify --config .golangci.yml RUN golangci-lint config verify --config .golangci.yml
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN golangci-lint run --config .golangci.yml ./...
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
# Test phase, built alone by script/test. -race needs cgo and so a C # Test phase, built alone by script/test. -race needs cgo and so a C
# compiler, which the Debian Go image ships and the alpine one does not. # compiler, which the Debian Go image ships and the alpine one does not.
# -count=1 because the mounted cache also holds test results, which go
# test would otherwise replay.
# #
# The tests run as nobody: several of them make a file unreadable and # The tests run as nobody: several of them make a file unreadable and
# expect reading it to fail, and root reads it anyway. nobody has no home # expect reading it to fail, and root reads it anyway. nobody has no home
# directory, so HOME is /tmp, and Go keeps its build cache in # directory, so HOME is /tmp, where Go puts its build cache.
# /tmp/.cache/go-build. That is where the cache is mounted, owned by
# nobody (65534) so the tests can write to it.
# golang:1.25-trixie, 2026-10-04 # golang:1.25-trixie, 2026-10-04
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
USER nobody USER nobody
@@ -51,27 +33,24 @@ ENV HOME=/tmp
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \ RUN go test -timeout 90s -race -cover ./... || \
go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies are # Build stage. Nothing is wanted from either phase above; the copies are
# what make BuildKit build them first, so nothing after them can run # what make BuildKit build them first, so this stage cannot run unless
# unless lint and test passed. # lint and test passed.
# golang:1.25-alpine, 2026-07-23 # golang:1.25-alpine, 2026-07-23
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git make RUN apk add --no-cache git make
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
COPY . . COPY . .
# The version stamped into the binary: the VERSION build argument when # The version stamped into the binary: the VERSION build argument when
@@ -81,8 +60,7 @@ COPY . .
# "dev". `make build` rather than `go build`, so the image and a host # "dev". `make build` rather than `go build`, so the image and a host
# build share one compile recipe, cgo disabled included. # build share one compile recipe, cgo disabled included.
ARG VERSION ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
[ "$version" = unknown ]; }; then \ [ "$version" = unknown ]; }; then \
echo "no version could be derived although the build context carries .git" >&2; \ echo "no version could be derived although the build context carries .git" >&2; \
@@ -94,7 +72,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
# stage's chain and nothing else. # stage's chain and nothing else.
# alpine:3.22, 2026-07-23 # alpine:3.22, 2026-07-23
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
ARG CACHEBUST
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
+150 -493
View File
@@ -38,9 +38,8 @@ database persists between runs; a rescan only hashes files that are new or
changed, or that may have gained a duplicate since the last scan, and removes changed, or that may have gained a duplicate since the last scan, and removes
records for files that no longer exist. `report` reads the database and prints records for files that no longer exist. `report` reads the database and prints
the file-level duplicates report. `trees` reads the same database and prints the the file-level duplicates report. `trees` reads the same database and prints the
duplicate-tree report. `info` prints the database's details: its path, the sizes duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation
of its files and the number of its records. A missing/invalid subcommand — or a with no `PATH` operand — prints a usage message and exits 2.
`scan` invocation with no `PATH` operand — prints a usage message and exits 2.
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
@@ -49,40 +48,15 @@ results are as fresh as the last completed scan.
### Install ### Install
With Go installed, this builds and installs the newest release: With Go installed, this builds and installs the current `main` branch:
```sh
go install sneak.berlin/go/sfdupes@latest
```
Until 1.0, releases are `v0.0.N` tags on the tip of the `main` branch: each time
`main` moves and its tip passes `script/cibuild`, that tip is tagged with the
next `v0.0.N`. So `@latest` installs `main`'s tip once it has passed the build.
`@main` installs the tip before it is tagged:
```sh ```sh
go install sneak.berlin/go/sfdupes@main go install sneak.berlin/go/sfdupes@main
``` ```
`v0.0.1` is an early build without a database. It does not work as this README The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. A
describes, and `go.mod` retracts it. binary installed this way reports its version as `dev`; one built from a clone
or into the Docker image carries the git tag or commit it was built from.
The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. To
see which version an installed binary is:
```sh
sfdupes --version
go version -m "$(command -v sfdupes)"
```
`sfdupes --version` prints `sfdupes VERSION`. For a binary from `go install`,
`VERSION` is the version Go recorded in it: the release tag, such as `v0.0.3`,
or for `@main` a pseudo-version ending in the first 12 characters of the
commit's hash. A binary built from a clone or into the Docker image carries the
git tag or commit it was built from instead, and `VERSION` is `dev` only when
the binary carries no version at all. `go version -m` prints what Go recorded in
the binary; its `mod` line names the module's version. Binaries from `v0.0.2`
and earlier print `dev`, so check those with `go version -m`.
From a clone, `make build` writes the binary to `./sfdupes`: From a clone, `make build` writes the binary to `./sfdupes`:
@@ -122,13 +96,9 @@ As a file `/etc/cron.d/sfdupes`:
the first scan. To keep it elsewhere, set the first scan. To keep it elsewhere, set
`SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line. `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line.
- `scan` writes nothing to stdout. Its stderr, appended here to - `scan` writes nothing to stdout. Its stderr, appended here to
`/var/log/sfdupes.log`, holds a first line naming the database (see `/var/log/sfdupes.log`, holds a plain progress line as each phase starts and
"Database"), plain progress lines naming each phase and its place among the then at most every 5 seconds, a warning for each path it skips, and the
five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it summary line (see "Progress" and "`scan` mode"). The log grows with every
runs, and one as it ends), a warning for each path it skips, a line for each
default exclude its walk does not enter and for each path it leaves out
because it is not on a persistent filesystem (see "Rules for the walk"), and
the summary line (see "Progress" and "`scan` mode"). The log grows with every
scan; rotate it like any other. scan; rotate it like any other.
- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A - Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A
scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the
@@ -137,9 +107,9 @@ As a file `/etc/cron.d/sfdupes`:
- A scan still running when the next one starts carries on. The new one fails at - A scan still running when the next one starts carries on. The new one fails at
once, and the lines cron mails include once, and the lines cron mails include
`sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`.
- `report`, `trees` and `info` need only read access to the database (see - `report` and `trees` need only read access to the database (see "Database").
"Database"). Under the usual umask of `022` the first scan creates it readable Under the usual umask of `022` the first scan creates it readable by every
by every user, so an unprivileged user can run them against root's database. user, so an unprivileged user can run them against root's database.
### Reading the reports ### Reading the reports
@@ -227,13 +197,12 @@ Goals, in order:
under resilver). Holding one small record (path, size, mtime) per file in under resilver). Holding one small record (path, size, mtime) per file in
memory during a scan is acceptable; holding every file's hashes is not (they memory during a scan is acceptable; holding every file's hashes is not (they
stay in the database). The reporting commands do not hold every file's stay in the database). The reporting commands do not hold every file's
hashes either: `report` reads the records in signature order and holds the hashes either: `report` lets SQLite group and order the records and writes
duplicate paths of one size at a time, so its memory grows with the each row as it reads it, so its memory does not grow with the database, and
duplicates of the size that has the most of them, and `trees` reads the `trees` reads the records in path order and keeps each directory's path,
records in path order and keeps each directory's path, digest and totals, digest and totals, plus the hashes of only the files in the directories
plus the hashes of only the files in the directories holding the record holding the record being read, so its memory grows with the number of
being read, so its memory grows with the number of directories and with the directories and with the size of the largest directory.
size of the largest directory.
3. **Scan incrementally, analyze offline.** The expensive filesystem scan 3. **Scan incrementally, analyze offline.** The expensive filesystem scan
maintains a persistent database; an unchanged file is never read again on a maintains a persistent database; an unchanged file is never read again on a
rescan, except to compute its content hash once a file of 10 MiB or more rescan, except to compute its content hash once a file of 10 MiB or more
@@ -263,7 +232,7 @@ Goals, in order:
### Subcommands ### Subcommands
Four subcommands, all implemented: Three subcommands, all implemented:
1. `scan` — walk the filesystem and synchronize the database: one signature 1. `scan` — walk the filesystem and synchronize the database: one signature
record per regular file. record per regular file.
@@ -271,76 +240,47 @@ Four subcommands, all implemented:
3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy 3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy
from the database records, compute a Merkle-style digest per directory, and from the database records, compute a Merkle-style digest per directory, and
report maximal groups of identical trees. report maximal groups of identical trees.
4. `info` — the database's details: its path, the sizes of its files, its
schema version and the counts of its records.
``` ```
sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... sfdupes scan [--workers N] [-x] PATH...
sfdupes report > dupes.tsv sfdupes report > dupes.tsv
sfdupes trees > dupetrees.tsv sfdupes trees > dupetrees.tsv
sfdupes info
sfdupes --version sfdupes --version
sfdupes [command] --help sfdupes [command] --help
``` ```
`--workers N` sets the size of each `scan` worker pool (default: the number of `--workers N` sets the size of each `scan` worker pool (default: the number of
CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that
operand's filesystem, and `--no-default-excludes` lets the walk enter the operand's filesystem; both are described under "`scan` mode".
default excludes, a list of paths; all three are described under "`scan` mode". `sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and
`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION` (see exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a
"Install"), to stdout and exits 0, writing nothing to stderr. `-h` or `--help`, subcommand, prints the help text to stderr and exits 0, writing nothing to
alone or after a subcommand, prints the help text to stderr and exits 0, writing stdout.
nothing to stdout.
### Database ### Database
All four subcommands operate on a single SQLite database file: All three subcommands operate on a single SQLite database file:
- Location: the value of the `SFDUPES_DATABASE` environment variable when set - Location: the value of the `SFDUPES_DATABASE` environment variable when set
and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no
command-line flag. The path names the file exactly, whatever characters it command-line flag. The path names the file exactly, whatever characters it
holds (`?`, `#` and `%` included); a relative path is relative to the working holds (`?`, `#` and `%` included); a relative path is relative to the working
directory. directory.
- Each subcommand first prints one line on stderr naming the database it uses, - `scan` creates the database (and its parent directory) on first use. `report`
before anything else it prints (progress displays included), whether or not and `trees` require an existing database; a missing database file is a fatal
stderr is a terminal. The line gives the path with the working directory put error (exit 1) telling the user to run `scan` first.
in front of it when it is relative, and otherwise as set (a `..` in it stays,
so the line names the same file), escaped like the paths in a report (see
"Report output format"), and says where the path came from:
```
scan: new database /var/lib/sfdupes/db.sqlite (default)
trees: database /srv/tmp/sfdupes/db.db (from SFDUPES_DATABASE)
```
The working directory put in front is the one `pwd -P` prints, with its
symlinks resolved. After a `cd` through a symlink to a directory, a path
starting `../` therefore names a file beside the directory the symlink leads
to, as it does for any other program. `scan` says `new database` when no
file is at that path as it starts. `report`, `trees` and `info` print the
line before they open the database, so the error for a missing database
comes after it.
- `scan` creates the database (and its parent directory) on first use. `report`,
`trees` and `info` require an existing database; a missing database file is a
fatal error (exit 1) telling the user to run `scan` first.
- Only one `scan` runs against a database at a time. For its whole run, `scan` - Only one `scan` runs against a database at a time. For its whole run, `scan`
holds an exclusive `flock(2)` lock on a lock file beside the database, named holds an exclusive `flock(2)` lock on a lock file beside the database, named
by appending `.lock` to the database's path with symlinks resolved by appending `.lock` to the database path (`/var/lib/sfdupes/db.sqlite.lock`
(`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the by default), taken before it walks the filesystem or opens the database. A
filesystem or opens the database. Every path to one database, through a second `scan` against the same database does not wait: it fails at once with a
symlink to the file or to a directory above it, names the same lock file. Just one-line error naming the lock file and exits 1, without walking anything or
before it takes the lock, `scan` creates the database file, empty, if it is opening the database, and the running scan carries on. The lock file is
missing (through a symlink at the database path, it creates the symlink's created on first use, open to its owner only, and left in place: a leftover
target), so the lock file is named the same way before the first scan sets up file blocks nothing, because the lock ends with the process holding it however
the database as after. A second `scan` against the same database does not it ends, a fatal error or an interrupt included, and deleting the file while a
wait: it fails at once with a one-line error naming the lock file and exits 1, scan runs would let a second scan start. `report` and `trees` never take the
without walking anything or opening the database, and the running scan carries lock, so they run during a scan.
on. The lock file is created on first use, open to its owner only, and left in
place: a leftover file blocks nothing, because the lock ends with the process
holding it however it ends, a fatal error or an interrupt included, and
deleting the file while a scan runs would let a second scan start. `report`,
`trees` and `info` never take the lock, so they run during a scan.
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so - While `scan` runs, the database is in WAL journal mode with a busy timeout, so
running a report while a cron `scan` is in progress is safe. The filesystem is running a report while a cron `scan` is in progress is safe. The filesystem is
authoritative; the database is an eventually-consistent reflection of it. authoritative; the database is an eventually-consistent reflection of it.
@@ -356,27 +296,13 @@ All four subcommands operate on a single SQLite database file:
needs the database to itself: a `scan` that starts while a report is still needs the database to itself: a `scan` that starts while a report is still
reading waits for it up to the 10-second busy timeout, then fails; a `scan` reading waits for it up to the 10-second busy timeout, then fails; a `scan`
that ends while a report has the database open warns and leaves the database that ends while a report has the database open warns and leaves the database
in WAL mode until the next scan. `report` writes its rows while it reads the in WAL mode until the next scan. `report` writes each row as it reads it, so
records, so it is still reading while its output is paused (a pager, a stalled it is still reading while its output is paused (a pager, a stalled pipe), and
pipe), and a `scan` started then fails after the busy timeout. a `scan` started then fails after the busy timeout.
- A `scan` writes to the database and the files beside it at these points (its - `report` and `trees` open the database read-only and need only read access to
phases are described under "`scan` mode"): the database file, and no write access to its directory. While the database is
- As it starts, before the walk: the directory, an empty database file and in WAL mode they also read the `-wal` and `-shm` files beside it, which SQLite
the `.lock` file, each only if missing. creates with the database file's permissions.
- Then, still before the walk: the schema, in a new database, and the `-wal`
and `-shm` files, which stay beside the database while the scan runs.
- While it loads the existing records (`load`) and walks: no records. A
first scan's database holds none, however long its walk has run.
- During the hash phase: the first records, committed 10,000 at a time.
- In the update phase: the rest of the hashed records, the records for
size-unique files, and the deletions.
- In the content phase: content hashes, again committed 10,000 at a time.
- At the end, as it closes the database: the `-wal` and `-shm` files go
away, unless a report still has the database open (see above).
- `report`, `trees` and `info` open the database read-only and need only read
access to the database file, and no write access to its directory. While the
database is in WAL mode they also read the `-wal` and `-shm` files beside it,
which SQLite creates with the database file's permissions.
- Schema (`PRAGMA user_version` is the schema version, currently 1; a database - Schema (`PRAGMA user_version` is the schema version, currently 1; a database
with any other version is a fatal error. `scan` creates the schema and sets with any other version is a fatal error. `scan` creates the schema and sets
the version in one transaction, so a first scan stopped while doing so leaves the version in one transaction, so a first scan stopped while doing so leaves
@@ -386,34 +312,29 @@ All four subcommands operate on a single SQLite database file:
```sql ```sql
CREATE TABLE files ( CREATE TABLE files (
path BLOB PRIMARY KEY, -- absolute path, raw bytes path BLOB PRIMARY KEY, -- absolute path, raw bytes
size INTEGER NOT NULL, -- bytes, from lstat size INTEGER NOT NULL, -- bytes, from lstat
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat mtime INTEGER NOT NULL, -- Unix seconds, from lstat
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB
head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB
tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples
content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples
) WITHOUT ROWID; ) WITHOUT ROWID;
CREATE INDEX files_signature ON files (size, head, tail, content); CREATE INDEX files_signature ON files (size, head, tail, content);
``` ```
Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
UTF-8. `mtime` and `mtime_nsec` hold the file's mtime to the nanosecond: UTF-8. `mtime` is used only for change detection; it is not part of the
`mtime` the whole Unix seconds, rounded down, and `mtime_nsec` the duplicate key. For a file under 10 MiB `head`, `tail`, and `content` all
nanoseconds past that second. Split this way they hold any mtime a hold the whole-file hash (that range is hashed in full, with no end
filesystem can record, one before 1678 or after 2262 included, which a windows); for a larger file `head` and `tail` hold the first- and last-64
single 64-bit count of nanoseconds cannot. They are used only for change KiB hashes and `content` the whole-file or sampled hash. All three are empty
detection and are not part of the duplicate key. For a file under 10 MiB strings when the file has never been hashed because its size was unique as
`head`, `tail`, and `content` all hold the whole-file hash (that range is of the last scan that covered it. For a file of 10 MiB or more, `content`
hashed in full, with no end windows); for a larger file `head` and `tail` stays empty until the content phase of a scan (see "`scan` mode" below) has
hold the first- and last-64 KiB hashes and `content` the whole-file or read the file. A record with an empty `content` is never part of a duplicate
sampled hash. All three are empty strings when the file has never been group, though it still defines the file for tree reconstruction. The
hashed because its size was unique as of the last scan that covered it. For `files_signature` index lets SQLite group the records by signature for
a file of 10 MiB or more, `content` stays empty until the content phase of a `report` without sorting the whole table.
scan (see "`scan` mode" below) has read the file. A record with an empty
`content` is never part of a duplicate group, though it still defines the
file for tree reconstruction. The `files_signature` index lets `report` read
the records in signature order without sorting the table.
### Duplicate detection ### Duplicate detection
@@ -467,12 +388,7 @@ results do not depend on the working directory. All operands belong to a single
scan and are enumerated concurrently: every operand seeds the shared walk worker scan and are enumerated concurrently: every operand seeds the shared walk worker
pool. Overlapping operands are harmless — an operand that duplicates another or pool. Overlapping operands are harmless — an operand that duplicates another or
lies under another is dropped before walking, so every file is reached exactly lies under another is dropped before walking, so every file is reached exactly
once and produces one database record. An operand under another one is kept, once and produces one database record.
though, and walked on its own when any directory from it (from its directory,
for a regular file) up to but not including the operand covering it is a default
exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with
`-x`, is on a different filesystem than that operand, because that operand's
walk does not enter such a directory and so does not reach it.
An operand that is a symlink (never followed, not even as an operand), socket, An operand that is a symlink (never followed, not even as an operand), socket,
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
@@ -481,11 +397,9 @@ drops it from the scanned operands before reading the database. Another operand
beneath it is still scanned. The records stored beneath it are not deleted: they beneath it is still scanned. The records stored beneath it are not deleted: they
are treated like any other record outside the scanned operands, including the are treated like any other record outside the scanned operands, including the
content-phase exception below. If it lies under another operand, they are under content-phase exception below. If it lies under another operand, they are under
that operand instead, and are treated like any other record there. This is not that operand instead, and are deleted like any other record there that this scan
an error: a scan whose every operand is dropped walks nothing and exits 0. An did not verify. This is not an error: a scan whose every operand is dropped
operand that is not on a persistent filesystem (see "Rules for the walk") is walks nothing and exits 0.
dropped the same way, except that the line `scan` prints for it is not a
warning, and it is not counted as skipped.
`scan` synchronizes the database with the filesystem state under the scanned `scan` synchronizes the database with the filesystem state under the scanned
operands: operands:
@@ -507,9 +421,6 @@ operands:
once its size, `head`, and `tail` match another record's. once its size, `head`, and `tail` match another record's.
- A file whose mtime is newer than recorded, or whose size differs, is processed - A file whose mtime is newer than recorded, or whose size differs, is processed
as if new: re-hashed, or recorded without hashes, per the shared-size rule. as if new: re-hashed, or recorded without hashes, per the shared-size rule.
Change detection compares the mtime to the nanosecond, as finely as the
filesystem records it, so a same-size rewrite counts as a change whenever the
filesystem gives it a later mtime than recorded, even within the same second.
- A database record whose path lies under one of the scanned operands but was - A database record whose path lies under one of the scanned operands but was
not successfully processed this run is deleted. This removes records for not successfully processed this run is deleted. This removes records for
deleted files. It also removes records for paths that failed to stat or hash deleted files. It also removes records for paths that failed to stat or hash
@@ -521,26 +432,13 @@ operands:
can be scanned on different schedules into the same database. The one can be scanned on different schedules into the same database. The one
exception is the content phase below: a stored file of 10 MiB or more without exception is the content phase below: a stored file of 10 MiB or more without
a `content` hash is read for one, wherever it lies, once its size, `head`, and a `content` hash is read for one, wherever it lies, once its size, `head`, and
`tail` match another record's. If that file is gone, has changed since its `tail` match another record's. If that file is gone or has changed since its
record was written, or is not on a persistent filesystem, it is not read and record was written, the record is left as it is.
the record is left as it is.
- A record at or below a path the walk left out, because that path is a default
exclude, is not on a persistent filesystem, or, with `-x`, is a directory on
another filesystem than its operand, counts as outside the scanned operands:
it is not deleted, and its size joins the size census. A filesystem mounted
under an operand can therefore be scanned on its own schedule into the same
database. Records below an operand at or inside such a path are not kept this
way, since that operand's own walk verifies them.
`scan` runs **five sequential phases over the whole scan**. Parallelism lives `scan` runs **four sequential phases over the whole scan**. Parallelism lives
inside the walk, hash and content phases; batched database writes begin during inside each phase; batched database writes begin during the hash phase:
the hash phase:
1. **load** — read the existing records. Each record under the operands is kept 1. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
in memory, by path, with its size, its mtime and whether it has hashes, for
the walk to compare each file with; the size of every other record joins the
size census.
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
with the walk worker pool: every operand seeds the shared queue, and each with the walk worker pool: every operand seeds the shared queue, and each
worker reads one directory at a time, handing discovered subdirectories back worker reads one directory at a time, handing discovered subdirectories back
to the queue and running `lstat` on each regular file as it is discovered to the queue and running `lstat` on each regular file as it is discovered
@@ -550,7 +448,7 @@ the hash phase:
or busy pools. The walk builds the size census and resolves unchanged or busy pools. The walk builds the size census and resolves unchanged
already-hashed files on the fly; every other file is carried to the hash already-hashed files on the fly; every other file is carried to the hash
phase as a (path, size, mtime) record. phase as a (path, size, mtime) record.
3. **hash** — with the census complete, each carried file's size decides its 2. **hash** — with the census complete, each carried file's size decides its
fate. Size-unique files are never read: new or changed ones are recorded fate. Size-unique files are never read: new or changed ones are recorded
without hashes in the update phase, unchanged unhashed ones simply keep without hashes in the update phase, unchanged unhashed ones simply keep
their records. Every file with a shared size is hashed by the worker pool as their records. Every file with a shared size is hashed by the worker pool as
@@ -566,30 +464,29 @@ the hash phase:
**while hashing runs**, so a scan interrupted after hours keeps everything **while hashing runs**, so a scan interrupted after hours keeps everything
hashed so far and the next scan resumes cheaply, skipping records already hashed so far and the next scan resumes cheaply, skipping records already
written. written.
4. **update** — commit the final partial batch, the hash-less records for 3. **update** — commit the final partial batch, the hash-less records for
size-unique new and changed files, and the deletions for records the scan size-unique new and changed files, and the deletions for records the scan
did not verify (vanished files, plus paths that failed to stat or hash). did not verify (vanished files, plus paths that failed to stat or hash).
5. **content** — find every record of 10 MiB or more without a `content` hash 4. **content** — find every record of 10 MiB or more without a `content` hash
whose size, `head`, and `tail` equal another record's, anywhere in the whose size, `head`, and `tail` equal another record's, anywhere in the
database: records from this scan and records stored by earlier scans, inside database: records from this scan and records stored by earlier scans, inside
or outside the scanned operands. SQLite finds them, so only the records to or outside the scanned operands. SQLite finds them, so only the records to
be read are kept in memory, never every file's hashes. Every record sharing be read are kept in memory, never every file's hashes. Every record sharing
their size, `head`, and `tail`, including one that already has a `content` their size, `head`, and `tail`, including one that already has a `content`
hash, has its file checked with `lstat` first. A file that is gone, is no hash, has its file checked with `lstat` first. A file that is gone, is no
longer a regular file, has changed (a different size, or an mtime newer than longer a regular file, or has changed (a different size, or an mtime newer
recorded), or is not on a persistent filesystem (see "Rules for the walk") than recorded) keeps its record as it is and does not count as a match for
keeps its record as it is and does not count as a match for the others. Any the others. Any other `lstat` error is warned about and counted as skipped,
other `lstat` error is warned about and counted as skipped, with the same with the same result. If such a record has no `content` hash, it stays out
result. If such a record has no `content` hash, it stays out of duplicate of duplicate groups; if it has one, it is still reported until a scan
groups; if it has one, it is still reported until a scan covering its own covering its own tree updates or removes it. The files that pass and have no
tree updates or removes it. The files that pass and have no `content` hash `content` hash are read only if at least two of those records pass, so a
are read only if at least two of those records pass, so a file whose only file whose only matches are stale costs no read; a file that already has a
matches are stale costs no read; a file that already has a `content` hash is `content` hash is never read again. They are read by a worker pool as in the
never read again. They are read by a worker pool as in the hash phase, in hash phase, in inode order and once per inode, and their content hashes are
inode order and once per inode, and their content hashes are committed in committed in batches. A failed read is warned about and counted as skipped;
batches. A failed read is warned about and counted as skipped; its record its record keeps an empty `content`, so it is not a duplicate, and a later
keeps an empty `content`, so it is not a duplicate, and a later scan tries scan tries again.
again.
Rules for the walk: Rules for the walk:
@@ -600,99 +497,18 @@ Rules for the walk:
- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking - Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking
them would list every file once per snapshot), not even when it is an operand; them would list every file once per snapshot), not even when it is an operand;
such an operand is dropped the same way. such an operand is dropped the same way.
- Never record the scan's own database. Once the database and its `.lock` file
are open, `scan` notes the device and inode of the database file, its `-wal`
and `-shm` files and its `.lock` file, and the walk passes over a file with
one of them, however it reaches it: as an operand, below one, or through a
symlinked directory. It prints no line for such a file and does not count it
as skipped. A record that an earlier scan left for one of them is deleted,
like the record of any file the scan no longer verifies. There is no
`-journal` file during the walk, because the database is in WAL mode then (see
"Database").
- Filesystem boundaries are crossed by default. With `-x` (long form - Filesystem boundaries are crossed by default. With `-x` (long form
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend `--one-file-system`, following the GNU `du`/`rsync` convention), never descend
into a directory on a different filesystem than its `PATH` operand; each into a directory on a different filesystem than its `PATH` operand; each
operand is bounded by its own filesystem. The records below such a directory, operand is bounded by its own filesystem.
unless they lie below an operand at or inside that directory, count as outside
the scanned operands, as described in "`scan` mode" above: they are not
deleted, and their sizes join the size census.
- Never record anything that is not on a persistent filesystem. A filesystem is
not persistent when the kernel makes it up or keeps it in memory: one of type
`proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`,
`debugfs`, `tracefs`, `securityfs`, `pstore`, `bpf`, `configfs`, `fusectl`,
`mqueue`, `hugetlbfs`, `efivarfs`, `binfmt_misc`, `autofs`, `nsfs`,
`rpc_pipefs`, `selinuxfs` or `devfs` (the macOS filesystem of `/dev`). After
it names the database and before it takes the lock, `scan` lists the mounted
filesystems, from `/proc/self/mountinfo` on Linux and from the system's list
of mounted filesystems on macOS (on any other system it lists none), and notes
the device ID of each one of those types. On Linux that is the device ID the
list gives, so a filesystem that a later mount hides still gives its own. On
macOS it is read with `lstat` on the mount point, and where one filesystem is
mounted over another on the same mount point, the one mounted last counts. If
`scan` cannot list them, it fails (exit 1). The walk then never enters a
directory, and never records a file, on one of those devices, however it
reaches it: as an operand, below one, or through a bind mount, which keeps the
device ID of what it binds. The content phase never reads a file on one of
them either. No flag turns this off. For each operand, directory or file it
leaves out this way, `scan` prints a line on stderr naming it and the type of
its filesystem, as it prints a warning, so it never lands inside a progress
display:
```
scan: skipped /dev/shm (tmpfs, not a persistent filesystem)
```
That line is not a warning, and the path is not counted as skipped. Records
that an earlier scan left at or below it, unless they lie below an operand
at or inside it, count as outside the scanned operands, as described in
"`scan` mode" above: they are not deleted, and their sizes join the size
census.
- Below an operand, never descend into a default exclude: a path that holds the
kernel's view of the running system, or temporary files, so that
`sfdupes scan /` walks only the files worth comparing. The default excludes
depend on the system, and `sfdupes scan --help` lists those of the system it
runs on:
- On Linux: `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`.
- On macOS: `/dev`, `/private/tmp`, `/private/var/tmp`, `/private/var/vm`,
`/System/Volumes/VM` and `/System/Volumes/Data`. A walk of `/` reaches
everything in `/System/Volumes/Data` through the firmlinks, so walking it
as well would record each of those files twice.
- On Windows, relative to the root of the drive an operand lies on: the
page, swap and hibernation files `pagefile.sys`, `swapfile.sys` and
`hiberfil.sys`, and the directories `System Volume Information`,
`$Recycle.Bin`, `Windows\Temp` and `Users\*\AppData\Local\Temp`, where `*`
matches one path component, as it does in any default exclude. sfdupes
does not build for Windows yet, so this list is not used.
- On any other system there are none.
`--no-default-excludes` turns them all off; what is not on a persistent
filesystem stays left out. An operand at or inside a default exclude is
still walked, so `sfdupes scan /tmp/x` works, unless it is not on a
persistent filesystem. For each default exclude its walk does not enter,
`scan` prints a line on stderr naming it, as it prints a warning:
```
scan: skipped /var/tmp (default exclude)
```
That line is not a warning, and the directory is not counted as skipped. A
default exclude that is not on a persistent filesystem, as `/proc` is, gets
the line for its filesystem instead, and one that is itself an operand gets
no line, since it is walked as that operand. The records below a default
exclude the walk does not enter, unless they lie below an operand at or
inside it, count as outside the scanned operands, as described in "`scan`
mode" above: they are not deleted, and their sizes join the size census.
- On any per-path error (permission denied, file vanished between passes, - On any per-path error (permission denied, file vanished between passes,
unreadable): print a one-line warning to stderr, skip the path, and continue. unreadable): print a one-line warning to stderr, skip the path, and continue.
Per-file errors never abort the run; the final summary reports how many were Per-file errors never abort the run; the final summary reports how many were
skipped. As specified above, a skipped path that has a database record from an skipped. As specified above, a skipped path that has a database record from an
earlier scan loses that record, unless it failed only in the content phase, or earlier scan loses that record, unless it failed only in the content phase, or
is an operand dropped before the database was read whose record counts as is an operand dropped before the database was read that lies under no other
outside the scanned operands; an unreadable directory subtree likewise loses operand; an unreadable directory subtree likewise loses its records (accepted:
its records (accepted: the database mirrors what the latest scan could the database mirrors what the latest scan could actually verify).
actually verify).
Concurrency: the walk phase (which also stats files), the hash phase, and the Concurrency: the walk phase (which also stats files), the hash phase, and the
content phase each use a worker pool of `--workers` workers (default content phase each use a worker pool of `--workers` workers (default
@@ -701,9 +517,8 @@ files. `--workers` must be at least 1: a smaller value is a usage error,
reported in one line on stderr with exit 2 before anything is scanned. All three reported in one line on stderr with exit 2 before anything is scanned. All three
phases are seek-bound on spinning disks, so raising `--workers` well past the phases are seek-bound on spinning disks, so raising `--workers` well past the
core count can help on pools with many spindles. The main goroutine owns core count can help on pools with many spindles. The main goroutine owns
partitioning, database writes, and the progress counts, which each progress partitioning, database writes, and progress rendering; progress display must
display draws from a goroutine of its own; progress display must never block the never block the workers.
workers.
`scan` writes nothing to stdout. The summary line on stderr reports the files `scan` writes nothing to stdout. The summary line on stderr reports the files
seen this run broken down by disposition, plus skips: seen this run broken down by disposition, plus skips:
@@ -721,8 +536,11 @@ total.)
**`report` must never touch the filesystem being analyzed.** It does not stat, **`report` must never touch the filesystem being analyzed.** It does not stat,
open, or otherwise access any path that appears in the records; its only I/O is open, or otherwise access any path that appears in the records; its only I/O is
reading the database and writing stdout/stderr. `report` must produce identical reading the database, writing stdout/stderr, and the temporary file SQLite sorts
output whether or not the scanned filesystem is still mounted. in when the duplicate rows do not fit in memory. SQLite puts that file in
`$SQLITE_TMPDIR` or `$TMPDIR` when set, otherwise in `/var/tmp` (or `/tmp`), and
deletes it as soon as it has opened it. `report` must produce identical output
whether or not the scanned filesystem is still mounted.
Processing: Processing:
@@ -736,16 +554,6 @@ Processing:
by `first` path ascending. Output must be fully deterministic for a given by `first` path ascending. Output must be fully deterministic for a given
database state. database state.
`report` runs in two phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `read` counts the records, then reads
them in signature order, largest size first. That is the order of the
`files_signature` index read backwards, so SQLite sorts nothing and returns the
first record at once. The records sharing a signature come together, and so do
the records of one size, so `report` holds only the duplicate groups of the size
it is reading, and writes them, ordered by `first` path, once it reaches a
smaller size. Its memory holds at most the duplicate paths of one size at a
time, never every record.
#### Report output format #### Report output format
TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a
@@ -819,12 +627,6 @@ Processing:
- Order groups by total tree size descending, tie-broken by `first` path - Order groups by total tree size descending, tie-broken by `first` path
ascending. Output must be fully deterministic for a given input. ascending. Output must be fully deterministic for a given input.
`trees` runs in four phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `read` counts the records, then reads
them in path order, building the hierarchy and computing the digests as it goes;
`group` groups the directories by digest and picks the maximal groups; `write`
writes the groups to stdout.
#### Trees output format #### Trees output format
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
@@ -843,192 +645,57 @@ Summary to stderr: records read, number of duplicate-tree groups, number of dupe
trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human
units. units.
### `info` mode
`info` takes no positional arguments and prints the database's details on
stdout, one per line: a name, a tab, and the value. It opens the database the
way `report` does, read-only and without the scan lock, so it runs during a scan
and needs only read access (see "Database"). Like `report`, it never touches the
scanned filesystem.
```
database /var/lib/sfdupes/db.sqlite (default)
database file size 2147483648 (2.0 GiB)
-wal file size 4124152 (3.9 MiB)
modified 2026-10-16T12:02:50Z
schema version 1
records 12345678
records with head and tail hashes 1234567
records with content hash 123456
size of recorded files 150000000000000 (136.4 TiB)
```
- `database` is the path and where it came from, as on the line every subcommand
prints first on stderr (see "Database").
- `database file size` and `-wal file size` are in bytes, then in human units.
The `-wal` line is there only when a `-wal` file is beside the database, as
while a scan runs.
- `modified` is the time the database file was last changed, in UTC. While the
database is in WAL mode, the newest changes may be in the `-wal` file alone.
- `schema version` is the database's `PRAGMA user_version`. A database with
another version than this build's is refused, as by `report`.
- `records` is the number of records. `records with head and tail hashes` and
`records with content hash` count the records that have those hashes (see
"Database"), and `size of recorded files` adds up the sizes of all records, in
bytes, then in human units.
`info` runs in two phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `count` reads the schema version and
counts the records with one query, a single pass over the table, both in one
read-only transaction, so the numbers agree with each other while a scan is
committing. `info` writes its lines once the `count` display has ended, so on a
terminal they come after the displays.
### Progress ### Progress
Use the progress-bar library for all progress of `scan`, `report`, `trees` and Use the progress-bar library for all scan progress; rendering in the style of
`info`; rendering in the style of `pv` is the model. All progress goes to `pv` is the model. All progress goes to stderr.
stderr.
`scan`, `report`, `trees` and `info` must always show that they are working and Each phase gets its own display, rendered the moment the phase starts — a scan
not hung. Each phase gets its own display from the moment it starts, naming the must never look hung. Loading the existing-record index (`load`) and the walk
phase and its place among the command's phases and showing its progress, its have no known totals while running: show a live count, rate, and elapsed time
elapsed time and its estimated time remaining (ETA). (spinner-style, no percentage or ETA). The content phase's display (`content`)
starts the same way, counting the records checked while SQLite finds the files
to read and `lstat` checks them, then shows a bar once reading starts. The hash
and update phases, and the content phase's reads, have exact totals — only files
that actually need hashing appear in the hash and content totals, so their ETAs
are meaningful. Required elements for the bars with known totals:
A scan has five phases (`[1/5] load`, `[2/5] walk`, `[3/5] hash`,
`[4/5] update`, `[5/5] content`):
- `load` counts the records first, then reads them against that count. While the
count runs, the display says it is counting the records.
- `walk` has no exact total. Its estimate is the number of records the last scan
left under the operands, known once `load` is done and shown with a `~`. Once
the walk passes the estimate, or when there are no such records, as on a first
scan, the ETA is unknown.
- `hash` and `update` have exact totals. Only files that actually need hashing
appear in the hash total, so its ETA is meaningful.
- `content` first counts the records whose files it checks with `lstat`, which
SQLite does by adding up the number of records with each matching size,
`head`, and `tail`, without fetching the records. It then says it is searching
the database until SQLite returns the first record; the checks, and their rate
and ETA, start there. The files to read then get a display of their own, with
their count as its total, and so does the writing of the last of their records
to the database, with the count of those records as its total.
`trees` has four phases (`[1/4] open`, `[2/4] read`, `[3/4] group`,
`[4/4] write`, see "`trees` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `read` counts the records first, then reads them against that count. While the
count runs, the display says it is counting the records.
- `group` has the directories `read` found as its total, and `write` the
duplicate-tree groups.
`report` has two phases (`[1/2] open`, `[2/2] read`, see "`report` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `read` counts the records first, then reads them against that count, writing
the rows to stdout as it goes. While the count runs, the display says it is
counting the records.
`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `count` has no total either, because one query counts everything: the display
says it is counting the records until the query returns, and its ETA is
unknown.
Each display is one line holding:
- the phase and its place among the phases
- a `[m/n] x%` display (items processed / total items, percent); before a phase
has a total, the items processed, or what the phase is waiting for
- current rate (items/s), naming the items the phase counts: files, records,
directories or groups. It covers the items completed over the last 5 seconds,
or, when stderr is not a terminal, since the line before, so it falls to 0
while nothing completes, and a burst of items after a wait shows as a burst
rather than as a steady rate. Once no item has completed for 10 seconds, the
word `stalled` and where the phase waits take the rate's place (see below).
- elapsed time - elapsed time
- estimated time remaining, or `unknown`. It comes from the phase's average rate - estimated time remaining
since it started counting, so that it does not swing with each burst. - a `[m/n] x%` display (items processed / total items, percent)
- current rate (items/s)
Example shape (exact layout is flexible, content is not): Example shape (exact layout is flexible, content is not):
``` ```
[3/5] hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54 hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54
[2/5] walk: [1234567/~1200000] 103%, 5120 files/s, elapsed 4m1s, eta unknown
[3/5] hash: [12345/45678] 27%, stalled disk/db, elapsed 4m30s, eta 12m9s
``` ```
When no item has completed for 10 seconds, as while one large file is read, a
disk is slow or the database is busy, the line says `stalled` and where the
phase waits in place of the rate, which would read 0: `disk` for files, `db` for
the database, `out` for stdout, and two of them, as `disk/db`, for a phase that
may wait on either. `walk` says `disk`. `hash` and the content phase's reads say
`disk/db`, as they read files and write their records to the database between
them, and so do the content phase's checks, which `lstat` each record's file as
the database search returns it. `load`, `update`, the content phase's last write
and the `read` phase of `trees` say `db`. `trees`' `write` says `out`, and its
`group`, which works in memory, says `cpu`. The `read` phase of `report` writes
the report while it reads the records, so it says `db/out`. A phase still
waiting for its total already says what it waits for, and says nothing more.
Additional requirements: Additional requirements:
- On a terminal, a phase with an exact total (every phase but `scan`'s `walk`,
the `open` phase of `report` and `trees`, and both phases of `info`, once it
has counted what it will do) is drawn as its line followed by the progress-bar
library's bar, 10 columns wide, which shows the percent in place of the line's
own. Any other phase, and a phase whose bar has no room, is drawn as the
library's spinner followed by its line. Each display is drawn again several
times a second, so its elapsed time keeps moving while a phase waits for its
next item.
- On a terminal no line reaches the terminal's last column, however narrow the
terminal is. A line too wide for the terminal first loses its bar, which gives
way to the spinner, then its rate, the words of a stalled phase, or what the
phase is waiting for. The phase and its place, the progress, the elapsed time
and the ETA are kept whole. On an 80-column terminal, at counts like those of
the stalled line in the example above, a stalled line keeps its words whenever
the elapsed time and the ETA are both in minutes, except in the content
phase's checks and reads. Their lines are the widest, and at most times in
minutes they lose the words. Where the words and the ETA do not both fit, as
at millions of files and times in hours, the words give way. Only when those
four alone are too wide, as on a terminal narrower than about 70 columns hours
into a scan of millions of files, is the line cut short at the terminal's
edge, so the end of the ETA is lost first. A terminal made narrower while a
phase runs gets the same: the next draw erases the line within the new width
and fits the line to it, and the display goes on redrawing in place.
- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI - When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI
redraws: print the line the moment each phase starts, every 5 seconds while it redraws: print a plain one-line progress update the moment each phase starts,
runs (whether or not an item has completed, and never more often), and once then no more often than every 5 seconds.
more as the phase ends. - Progress updates are driven from the main goroutine and must be non-blocking
- Progress must be non-blocking with respect to the worker pool: the main with respect to the worker pool. On a terminal the spinner-style displays also
goroutine counts the items, and each display draws them from a goroutine of redraw on their own several times a second, so their count and elapsed time
its own. stay current while a phase waits for its next item.
- A warning printed during a phase always lands on a line of its own, never - A warning printed during a phase always lands on a line of its own, never
inside the progress display. So does every line written to stdout when stdout inside the progress display.
and stderr are the same file (the same device and inode): one terminal, as in - A bar whose phase stops short of its total, as an interrupted one does, is
a plain `sfdupes report` or `sfdupes trees`, or one pipe or file that both are left as last drawn rather than filled up.
sent to. When they are not, stdout is written without waiting on the display, - `report` and `trees` modes need no progress display, only their stderr
and the display keeps being drawn while a slow reader of stdout holds up a summaries.
write.
- A phase that stops short of its total, as an interrupted one does, is left
showing the count it stopped at rather than filled up.
### Error handling and exit codes ### Error handling and exit codes
- `0`: success, even if individual files were skipped with warnings. - `0`: success, even if individual files were skipped with warnings.
- `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is - `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is
already running against the same database, the database cannot be already running against the same database, the database cannot be
created/opened/read/written, a missing database for `report`/`trees`/`info`, created/opened/read/written, a missing database for `report`/`trees`, stdout
stdout write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below).
below).
- `2`: usage error (including `scan` with no `PATH` operand, `scan` with - `2`: usage error (including `scan` with no `PATH` operand, `scan` with
`--workers` below 1, and `report`/`trees`/`info` with any positional `--workers` below 1, and `report`/`trees` with any positional argument).
argument).
A stdout write failure, such as a full disk, is reported in one line on stderr A stdout write failure, such as a full disk, is reported in one line on stderr
and exits 1. Two cases never reach sfdupes as a failed write: and exits 1. Two cases never reach sfdupes as a failed write:
@@ -1071,16 +738,13 @@ entrypoints are:
- `script/bootstrap` — install everything needed to build and develop this - `script/bootstrap` — install everything needed to build and develop this
repository, idempotently, assuming nothing is present. `git`, `make`, and `go` repository, idempotently, assuming nothing is present. `git`, `make`, and `go`
come from the first of nix, apt, brew, or apk found on the host, and are come from the first of nix, apt, brew, or apk found on the host, and are
presence-checked only. An installed node is used when its major version is 22; presence-checked only. An installed node is used as it is; otherwise node
otherwise node 22.17.0 is installed through nvm, which comes from a release 22.17.0 is installed through nvm, which comes from a release archive whose
archive whose sha256 the script checks, and yarn 1.22.22 is activated under it sha256 the script checks. yarn 1.22.22 comes through corepack, and
through corepack. With an installed node 22, a yarn already on `PATH` is used `yarn install --frozen-lockfile` installs the prettier that `package.json` and
as it is; otherwise yarn 1.22.22 is activated through corepack, or installed `yarn.lock` pin. `golangci-lint` is never installed: it runs in Docker (see
with `npm` when there is no corepack. `yarn install --frozen-lockfile` then `script/lint`). `docker` is not installed either; testing, linting and the
installs the prettier that `package.json` and `yarn.lock` pin. `golangci-lint` image build need it. Ends with `go mod download`.
is never installed: it runs in Docker (see `script/lint`). `docker` is not
installed either; testing, linting and the image build need it. Ends with
`go mod download`.
- `script/setup` — make a fresh clone ready for development: runs - `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit`. `script/bootstrap`, then `script/install-precommit`.
- `script/projectname` — print this project's name (`sfdupes`). Scripts that - `script/projectname` — print this project's name (`sfdupes`). Scripts that
@@ -1101,8 +765,8 @@ entrypoints are:
- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every - `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every
Markdown file with prettier, at the settings in `.prettierrc` (4-space Markdown file with prettier, at the settings in `.prettierrc` (4-space
indents, prose wrapped at 80 columns). Both run on the host, prettier through indents, prose wrapped at 80 columns). Both run on the host, prettier through
yarn at the version `yarn.lock` pins. When the node on `PATH` is not node 22, yarn at the version `yarn.lock` pins. When yarn is not on `PATH`, this loads
this loads the node 22.17.0 that `script/bootstrap` installed through nvm. the node 22.17.0 that `script/bootstrap` installed through nvm.
- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any - `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any
unformatted file and exits non-zero instead of writing. gofmt and prettier unformatted file and exits non-zero instead of writing. gofmt and prettier
both run every time, and each names itself when it fails. The `Dockerfile`'s both run every time, and each names itself when it fails. The `Dockerfile`'s
@@ -1122,25 +786,14 @@ entrypoints are:
- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a - `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a
no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then
`script/check`. `script/check`.
- `script/install-precommit` — install the git pre-commit hook, as - `script/install-precommit` — install the git pre-commit hook that runs
`.git/hooks/pre-commit`, that runs `script/precommit`. `script/precommit`. The hook is written to the common git directory, so the
main checkout and every worktree share it.
Every `docker build` in `script/` passes a new `CACHEBUST` build argument. In Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps Docker would serve the gate steps from its cache, and the build would pass
that install system packages and download the Go modules, or right after `FROM` having run no test and no linter. Every run therefore downloads the Go modules
in a stage with neither, and every step that lints, tests or compiles comes again and needs the network.
after it, so every `RUN` step after it runs again each time. On an unchanged
tree Docker would otherwise serve the gate steps from its cache, and the build
would pass having run no test and no linter. The layers above that line stay in
Docker's build cache until the base image, their own lines, `go.mod` or `go.sum`
change, or until that cache is cleared; only then are the packages installed and
the modules downloaded again. Go's build cache is kept between builds on the
steps that compile, so only what changed is compiled again; the tests run with
`-count=1`, so none reports a stored result. `golangci-lint run` also keeps
golangci-lint's cache, and takes a package's findings from it only when the
package's files and those of the packages it imports, `go.mod`, the linters and
their settings in `.golangci.yml`, and the golangci-lint version are all
unchanged.
## Build ## Build
@@ -1204,6 +857,10 @@ All of the following, run in this directory, must pass:
./sfdupes report ./sfdupes report
``` ```
(The database lives in a temp directory of its own: inside `$d`, the scan
would record it, and its empty lock file would join the `empty1`/`empty2`
group.)
Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
group (two dupe rows, `first` is the lexicographically smallest path); group (two dupe rows, `first` is the lexicographically smallest path);
`t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/ `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
+80 -157
View File
@@ -1,6 +1,6 @@
--- ---
title: Repository Policies title: Repository Policies
last_modified: 2026-10-18 last_modified: 2026-10-07
--- ---
This document covers repository structure, tooling, and workflow standards. Code This document covers repository structure, tooling, and workflow standards. Code
@@ -54,39 +54,34 @@ style conventions are in separate documents:
`cibuild`. `script/bootstrap` installs all dependencies idempotently and `cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the (detected in that order; apt runs noninteractive). For node it uses the
installed node only when its major version is the pinned one; otherwise (node installed node if present; otherwise it installs a PINNED node version via
missing, or another major, such as the node 24 the CI runner image ships) it nvm, first installing nvm itself if missing — from a hash-verified GitHub
installs the PINNED node version via nvm and installs yarn and the packages release archive (never `curl | sh`), with bash installed as an explicit
under it. nvm itself is installed first if missing, from a hash-verified prerequisite since nvm requires bash. yarn is then pinned via
GitHub release archive (never `curl | sh`), with bash installed as an explicit `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
prerequisite since nvm requires bash. Only the major version is compared always exact versions. `script/cibuild` runs the CI build: it changes to the
because the `Dockerfile` stages start from a node image whose exact version is repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned with the version; the Gitea workflow calls it. **`script/cibuild` runs
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs `script/bootstrap` first**, because the workflow checks out the repo and runs
nothing else, while `script/fmt-check` runs the formatter on the host: on a nothing else, while `script/fmt-check` runs the formatter on the host: on a
pristine checkout with nothing installed the run dies there, after the pristine checkout with nothing installed the run dies there, after the
containerised gates have passed. **The bootstrap alone is not enough**: when containerised gates have passed. **The bootstrap alone is not enough**:
`script/bootstrap` installs node and yarn under nvm it leaves neither on the `script/bootstrap` installs node and yarn under nvm and leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
under another node. The host entrypoints that need yarn — `script/fmt` and entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when source nvm for the pinned node version before invoking it, exactly as
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, `script/bootstrap`'s own install step does. A runner carrying nothing but
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the docker and git then gets through `script/check`. Four further scripts are our
pinned node. A runner carrying nothing but docker and git then gets through own extensions to the standard: `script/check` runs `script/test`,
`script/check`. Four further scripts are our own extensions to the standard: `script/lint` and `script/fmt-check`; `script/precommit` is what the git
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`; pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
`script/precommit` is what the git pre-commit hook runs, and it calls installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/check`; `script/install-precommit` installs the git pre-commit hook `script/projectname` (literally that filename) simply outputs the project's
(the `make hooks` target shims to it); and `script/projectname` (literally name. Scripts that need the name call `script/projectname` — e.g.
that filename) simply outputs the project's name. Scripts that need the name `script/docker` assembles its image tag from it — so those scripts stay
call `script/projectname` — e.g. `script/docker` assembles its image tag from byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
it — so those scripts stay byte-identical across all repos. Repo-type-specific `go mod tidy` verification in Go repos) belong in `script/precommit`, not in
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in the hook itself. Model scripts are at
`script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the must document the provided scripts in an **Entrypoints** section (see the
README requirements below). README requirements below).
@@ -123,10 +118,8 @@ style conventions are in separate documents:
and nothing else: and nothing else:
```sh ```sh
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build --no-cache --target lint --output type=cacheonly .
--target lint --output type=cacheonly . docker build --no-cache --target test --output type=cacheonly .
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target test --output type=cacheonly .
``` ```
**A stage that is not the last one in the file is built only when the final **A stage that is not the last one in the file is built only when the final
@@ -139,12 +132,12 @@ style conventions are in separate documents:
**The gate builds write no image.** With `--output type=cacheonly` the phase **The gate builds write no image.** With `--output type=cacheonly` the phase
runs and a failing step fails the build, but the result is not exported. runs and a failing step fails the build, but the result is not exported.
Nothing uses those images, and writing one out is slow: a Go test phase's Nothing uses those images, and writing one out is slow: a Go test phase's
image holds the toolchain. A build given neither `--output` nor `-t` writes image holds the toolchain and every compiled package. A build given neither
an untagged image and leaves it dangling, on every developer host and every `--output` nor `-t` writes an untagged image and leaves it dangling, on
CI runner. `script/cibuild` and `script/docker` build the image that ships every developer host and every CI runner. `script/cibuild` and
and tag it, so each build replaces the previous image; each assigns the tag `script/docker` build the image that ships and tag it, so each build
on its own line before the build, so `set -e` stops it where replaces the previous image; each assigns the tag on its own line before the
`script/projectname` fails. build, so `set -e` stops it where `script/projectname` fails.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are `eslint`, `prettier` — never through `make lint` or `script/test`, which are
@@ -159,42 +152,22 @@ style conventions are in separate documents:
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own `TMPDIR`, so the lock is org, in both directions. A container has its own cache, its own `TMPDIR` and
out of reach. The lint phase does keep golangci-lint's cache between check a digest-pinned binary, so neither is reachable.
builds (below), but it lints every checkout at `/src`, and a finding is
taken from that cache only for the same files, settings and golangci-lint
version.
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker - **Any build that runs checks is built with `--no-cache`.** Docker invalidates
invalidates a `COPY` layer only when the copied content changes, so on an a `COPY` layer only when the copied content changes, so on an unchanged tree
unchanged tree the check `RUN` is served from cache, nothing executes, and the the check `RUN` is served from cache, nothing executes, and the build still
build still exits 0. Every `docker build` in `script/` therefore passes exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
id, so no two builds share a value, not even the three in one four, and there is no fifth — `script/check` runs the two gate phases and
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and `script/fmt-check`, and builds no image of its own. A bare `docker build .` is
`script/docker` are the four, and there is no fifth — `script/check` runs the not evidence that anything ran: a sub-second build reporting success is a
two gate phases and `script/fmt-check`, and builds no image of its own. In cache hit, not a result. Never invalidate by pruning — `docker builder prune`
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps and friends destroy a build cache shared with every other build on the host.
that install system packages and download Go's modules, or right after `FROM` When a check is added or changed, prove it works by planting a defect it must
in a stage with neither, and every step that lints, tests or compiles comes catch and watching the run fail on it, then revert the defect. A green run
after it. A step that installs anything else as well, such as a alone shows neither that the check ran nor that it covers what it should.
`script/bootstrap` that also installs node packages, comes after it too. A new
value reruns every `RUN` step after that line; `COPY` steps are compared by
content and need nothing. Docker keeps the layers above the line until the
base image, their own lines or the files they copy change. A stage without the
`ARG` line is served from cache on an unchanged tree, so a repository
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
not pass `--no-cache`: it reruns every step too, but it also builds the layers
above the line again and empties every cache mount, so Go's build cache and
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
survive between check builds, would start empty every time. A bare
`docker build .` is not evidence that anything ran: a sub-second build
reporting success is a cache hit, not a result. Never invalidate by pruning —
`docker builder prune` and friends destroy a build cache shared with every
other build on the host. When a check is added or changed, prove it works by
planting a defect it must catch and watching the run fail on it, then revert
the defect. A green run alone shows neither that the check ran nor that it
covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.** - **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by The lint phase is based on the `golangci/golangci-lint` image (pinned by
@@ -203,56 +176,40 @@ style conventions are in separate documents:
`Dockerfile`: `Dockerfile`:
```dockerfile ```dockerfile
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build.
# Lint phase # Lint phase
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN golangci-lint run --config .golangci.yml ./...
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Test phase. -race needs cgo and so a C compiler, which the Debian Go
# image ships and the alpine one does not. -count=1 because the mounted # image ships and the alpine one does not.
# cache also holds test results, which go test would otherwise replay.
# golang:1.x, YYYY-MM-DD # golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test FROM golang@sha256:... AS test
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN go test -timeout 90s -race -cover ./... || \
go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies # Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so nothing after them can # are what make BuildKit build them first, so this stage cannot run
# run unless lint and test passed. # unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD # golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder FROM golang@sha256:... AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git RUN apk add --no-cache git
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
COPY . . COPY . .
# The VERSION build arg when one is given, otherwise # The VERSION build arg when one is given, otherwise
@@ -260,8 +217,7 @@ style conventions are in separate documents:
# .git present, a version that is still empty, dev or unknown fails the # .git present, a version that is still empty, dev or unknown fails the
# build: git is missing or could not read the checkout. # build: git is missing or could not read the checkout.
ARG VERSION ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \ if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \ case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \ echo "version is '$VERSION' although .git is present" >&2; \
@@ -274,7 +230,6 @@ style conventions are in separate documents:
# Runtime stage, and the last one # Runtime stage, and the last one
FROM alpine@sha256:... FROM alpine@sha256:...
ARG CACHEBUST
COPY --from=builder /app /usr/local/bin/app COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"] ENTRYPOINT ["app"]
``` ```
@@ -289,26 +244,6 @@ style conventions are in separate documents:
- Keep the runtime stage last, and if you add a stage after it, give it the - Keep the runtime stage last, and if you add a stage after it, give it the
same two copies. A plain `docker build .` builds the last stage's chain same two copies. A plain `docker build .` builds the last stage's chain
and nothing else. and nothing else.
- Check builds keep these, and none of them can hand back a stale result:
- The layers above `ARG CACHEBUST`: the system packages and Go's
modules. Docker installs the packages again when the base image or the
install line changes, and downloads the modules again when `go.mod` or
`go.sum` changes. Go checks every module against `go.sum` as it
downloads it, and no later build can change a kept layer. The modules
are kept in a layer rather than a cache mount because a cache mount is
shared by every build on the host, and Go builds from whatever files
it holds without checking them again (only `go mod verify` does), so a
file changed there would go unnoticed.
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
root), mounted on the steps that compile: `golangci-lint run`, both
`go test` runs and `go build`. Go reuses a compiled package only when
all its inputs are unchanged. The cache also holds test results, which
is why both `go test` runs carry `-count=1`.
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
a package's findings only when the package's files and those of every
package it imports, `go.mod`, the linters and their settings in
`.golangci.yml`, and the golangci-lint version are all unchanged.
- If the project uses `//go:embed` directives that reference build artifacts - If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must (e.g. a web frontend compiled in a separate stage), the lint phase must
create placeholder files so the embed directives resolve. Example: create placeholder files so the embed directives resolve. Example:
@@ -317,20 +252,12 @@ style conventions are in separate documents:
in the lint phase. The `golangci/golangci-lint` image is Debian-based and in the lint phase. The `golangci/golangci-lint` image is Debian-based and
has no `apk`, so install with `apt-get` under the Debian package name has no `apk`, so install with `apt-get` under the Debian package name
(`libvips-dev`, where alpine says `vips-dev`), and delete the package (`libvips-dev`, where alpine says `vips-dev`), and delete the package
lists in the same `RUN`, so the layer does not keep them. The test phase, lists in the same `RUN`, so the layer does not keep them:
on the Debian Go image, installs the libraries its tests need the same
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
Docker keeps it until the base image or the install line changes:
```dockerfile ```dockerfile
FROM golangci/golangci-lint@sha256:... AS lint
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \ && apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
``` ```
- `.dockerignore` lets `.git` into the build context. It keeps out every git - `.dockerignore` lets `.git` into the build context. It keeps out every git
@@ -377,18 +304,18 @@ style conventions are in separate documents:
`script/cibuild` bootstraps, runs the gate phases, and then builds the image, `script/cibuild` bootstraps, runs the gate phases, and then builds the image,
so a successful run means every check passed; a bare `docker build .` does not so a successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The carry the same guarantee, because its gate phases may come from the cache. The
image build gets a `CACHEBUST` value of its own and so runs the gate phases a image build is uncached and so runs the gate phases a second time. That is the
second time. That is the price of the rule above, and it is worth paying: the price of the rule above, and it is worth paying: the image that ships is built
image that ships is built from a run of its own gates rather than from a cache from a run of its own gates rather than from a cache entry. The `check` job
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
shared runner after 20 minutes. That allows for the three Docker builds minutes. That allows for the three Docker builds described above (the test
described above (the test phase, the lint phase, then the image), each held to phase, the lint phase, then the image), each held to the 5-minute Docker build
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow limit below, plus the bootstrap. A separate workflow limited to `main` by a
limited to `main` by a `branches` list under `on: push` cannot be checked by `branches` list under `on: push` cannot be checked by review: to try a change
review: to try a change to it, add the feature branch to that list and push, to it, add the feature branch to that list and push, then remove the branch
then remove the branch from the list again before merging. Keep any job in it from the list again before merging. Keep any job in it that publishes behind
that publishes behind `if: github.ref_name == 'main'`, so the run from the `if: github.ref_name == 'main'`, so the run from the feature branch publishes
feature branch publishes nothing. nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for - Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -448,11 +375,12 @@ style conventions are in separate documents:
That cache is Go's own, separate from Docker's layer cache. Go stores a That cache is Go's own, separate from Docker's layer cache. Go stores a
passing result in its cache directory (`GOCACHE`), and when the same tests passing result in its cache directory (`GOCACHE`), and when the same tests
run again on unchanged code it prints that result, marked `(cached)`, run again on unchanged code it prints that result, marked `(cached)`,
without running them. The directory lasts from one run to the next on a without running them. That matters on a developer's machine, where this
developer's machine, where this target runs, and in the `test` phase of the target runs and the directory lasts from one run to the next. The `test`
`Dockerfile`, which mounts it to keep compiled packages between check phase of the `Dockerfile` needs no `-count=1`: its base image holds no
builds, so the phase's `go test` runs carry `-count=1` too. A new result for this repo's tests and nothing before its `go test` step runs a
`CACHEBUST` value (above) is what makes that step run on an unchanged tree. test, so there is nothing to replay. `--no-cache` (above) is what makes that
step run on an unchanged tree.
Python example: Python example:
@@ -539,7 +467,7 @@ style conventions are in separate documents:
version="$(git describe --tags --always --dirty 2>/dev/null || true)" version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown" [ -n "$version" ] || version="unknown"
tag="$(script/projectname)" tag="$(script/projectname)"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build --no-cache \
--build-arg VERSION="$version" \ --build-arg VERSION="$version" \
-t "$tag" . -t "$tag" .
``` ```
@@ -621,11 +549,6 @@ style conventions are in separate documents:
function defined and never invoked has the same exit status and the same function defined and never invoked has the same exit status and the same
empty output as one that worked. empty output as one that worked.
Node is the exception to the whole-token comparison: the canonical
`script/bootstrap` compares only its major version, because the node 22
alpine image the `Dockerfile` stages start from is not the exact pin, and
nvm cannot install a prebuilt node on alpine.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to A Go tool a repo needs on the host is installed with `go install` pinned to
+81 -211
View File
@@ -29,194 +29,31 @@
# Completed Steps # Completed Steps
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from - re-vendor the canonical files from `sneak/prompts` commit `dd4027b`, with
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder` `script/lint`, `script/test` and `REPO_POLICIES.md` from its `next` at
stages of the `Dockerfile` the module download, and in `builder` the `c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
until the base image, the install line, `go.mod` or `go.sum` changes, and
every step that lints, tests or compiles still comes after it;
`golangci-lint run` also keeps golangci-lint's cache. README says the same
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
early build without a database. When the linker set no version,
`sfdupes --version` prints the main module's version that Go recorded in the
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
when neither is known (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/116)
- `scan` creates the database's directory where the database path leads: the
path with its last element cut off as text, never cleaned, so a `..` after a
symlinked directory is followed through the symlink, as when the file is
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
used to create `/srv/new` and then fail (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/119)
- `scan` never records its own database: once the database and its `.lock` file
are open, it notes the device and inode of the database file, its `-wal` and
`-shm` files and its `.lock` file, and the walk passes over a file with one of
them wherever the operands reach it, through a symlinked directory included,
printing no line and counting nothing. A record an earlier scan left for one
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
- `scan` never records or reads anything that is not on a persistent filesystem
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
matched by the device ID of each such filesystem, listed as the scan starts
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
however the walk or the content phase reaches it; no flag turns that off.
Below an operand, its walk does not enter the default excludes either, so
`sfdupes scan /` skips the kernel's view of the running system and the
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
Windows list, not used yet. An operand at or inside a default exclude is still
walked. The scan prints a line for each path it leaves out, without counting
it as skipped, and the records there count as outside the scanned operands, as
for `-x`. `--no-default-excludes` turns off the default excludes only, and
`scan --help` lists them (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/124)
- the rate a progress display shows covers the items completed over the last 5
seconds, or since the line before when stderr is not a terminal, so it falls
to 0 while nothing completes, and a burst after a wait no longer reads as a
steady rate of over 1,000 files a second; the ETA still comes from the phase's
average. Once no item has completed for 10 seconds, the word `stalled` and
where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
`db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
either, each phase giving its word where it is defined. A line too wide for
the terminal loses its bar first, then its rate or those words, so the phase,
progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
like those of README's example, the content phase's checks and reads lose
those words at most times in minutes, and every other phase keeps them
whenever the elapsed time and the ETA are both in minutes. The content phase's
last partial batch is now written under a display of its own (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/125)
- new subcommand `info` prints the database's details on stdout, one per line as
a name, a tab and the value: its path and where it came from, the sizes of the
database file and of its `-wal` file when there is one, when the database file
was last changed, the schema version, the number of records, how many have
head and tail hashes and how many a content hash, and the sizes of all records
added up. It opens the database as `report` does, and shows two phases on
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
from one read-only transaction, the counts from one query (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/117)
- `report` shows its work on stderr in two phases, `[1/2] open` and
`[2/2] read`, with the progress display `scan` and `trees` use, from the
moment it starts until the summary line: `open` says it is opening the
database, also while it waits on a busy one; `read` counts the records, then
reads them against that count in signature order, which SQLite takes from the
`files_signature` index without sorting, so the first record comes at once.
`report` forms the duplicate groups itself, holding only those of the size it
is reading, and writes them ordered by first path before it reads the next
size. stdout and the summary line are unchanged, and SQLite no longer needs a
temporary file to sort in (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/114)
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
from the moment it starts until the summary line: `open` says it is opening
the database, also while it waits on a busy one; `read` counts the records,
then reads them against that count; `group` counts the directories it groups,
and `write` the groups it writes. A display's rate names what its phase
counts: files, records, directories or groups. stdout and the summary line are
unchanged. When stdout and stderr are the same file, as one terminal is, none
of stdout's lines shares a line with the display; when they are not, the
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/113)
- each `scan` phase's progress display names the phase and its place among the
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
time and ETA: `load` counts the records first, the walk's total is estimated
from the records under the operands, and the content phase counts the records
it checks from SQLite's grouping alone, then says it is searching the database
until the first one is returned. On a terminal a phase with an exact total
draws its line followed by the library's bar, and any other a spinner followed
by the line; every display is drawn again several times a second. No line
reaches the terminal's last column, however narrow, also after the terminal is
made narrower during a phase: a line too wide leaves out its rate, then its
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
only when those alone do not fit, as below about 70 columns late in a large
scan. In plain mode the line is printed every 5 seconds whether or not an item
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
- `scan`, `report` and `trees` first print a line on stderr naming the database
they use, as an absolute path, and whether that path is the default or comes
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
"Database" says when `scan` writes each file (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/111)
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
that write no image, and `make test` runs the suite under the race detector, that write no image, and `make test` runs the suite under the race detector,
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
every stage of the `Dockerfile` declares first, so every `RUN` step runs on the host, from the node and yarn `script/bootstrap` installs, so the
every build, while the steps that compile keep Go's build cache and the tests `prettier` and `markdown` stages are gone and the build stage installs `git`
run with `-count=1`; prettier runs on the host, from the node and yarn and `make` itself; `.claude/settings.json` is deleted (2026-10-08,
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone https://git.eeqj.de/sneak/sfdupes/issues/95)
and the build stage installs `git` and `make` itself; a new push cancels the
workflow's older run on the same branch, and a run stops after 20 minutes;
`.claude/settings.json` is deleted (2026-10-15,
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
`test` and the build stage, and the install of `git` and `make` in the build
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
every step that lints, tests or compiles still runs on every build
- with `-x`, an operand under another operand is walked on its own instead of
being dropped when a directory from it (from its directory, for a regular
file) up to but not including that operand is on a different filesystem than
that operand, whose walk then cannot reach it, and the records of files gone
from below it are deleted (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/106)
- with `-x`, `scan` keeps the records below a directory on another filesystem
under its operand, and counts their sizes in the size census, instead of
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
- the hash and content phases open each file without blocking and check that it
is still a regular file, so one replaced by a FIFO or device node after it was
statted is warned about and skipped instead of hanging the scan (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/100)
- the scan lock file is named from the database's path with symlinks resolved,
after `scan` creates the database file if it is missing, so a scan through a
symlink to the database fails while another scan holds the lock (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/101)
- on a terminal, a warning for the last file of a phase no longer erases the
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/102)
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
within the same second is re-hashed (2026-10-07,
https://git.eeqj.de/sneak/sfdupes/issues/12)
- cut the narration from `TODO.md` Completed Steps and from the comments in - cut the narration from `TODO.md` Completed Steps and from the comments in
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since `script/` and both Dockerfiles; §Workflow now branches from and merges to
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
- `make test-race` ran the test suite under the race detector in a cgo-enabled - `make test-race` runs the test suite under the race detector in a cgo-enabled
container, outside `make check` (2026-10-04, container, outside `make check` (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/18). Since https://git.eeqj.de/sneak/sfdupes/issues/18)
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
`make test-race` is gone
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`, - a bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
rather than serve the gates from cache (2026-10-04, rather than serve the gates from cache (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/39). Since https://git.eeqj.de/sneak/sfdupes/issues/39). Since
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in `REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
`script/` pass a new `CACHEBUST` value, so theirs always run `script/` pass `--no-cache`, so theirs always run
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks - `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
it; all Markdown reformatted (2026-10-04, it; all Markdown reformatted (2026-10-04,
@@ -262,12 +99,9 @@
- README documents install, Docker, a daily cron scan and how to read and check - README documents install, Docker, a daily cron scan and how to read and check
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home - the `Dockerfile` build stage keeps the Go module cache out of `builder`'s home
and copied the sources with `--chown`, so no `chown -R` walked them and copies the sources with `--chown`, so no `chown -R` walks them
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43)
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
nothing changes owner: the build stage only compiles, as root, and the tests
run as `nobody` in the `test` phase
- `--version` prints `sfdupes VERSION` to stdout; README documents it and - `--version` prints `sfdupes VERSION` to stdout; README documents it and
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
@@ -331,46 +165,82 @@
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and - fix the lint-image pin comments and `FROM` form in `Dockerfile` and
`Dockerfile.lint` (2026-08-10, branch `next`, closes `Dockerfile.lint` (2026-08-10, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`, `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
without the false `(Debian-based)` note or the tag. Since without the false `(Debian-based)` note or the tag; digest unchanged.
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase `script/verify-lint-image-pin` still matches the tagless form, and a tag on
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and one side only is caught as a plain mismatch.
`script/verify-lint-image-pin`, which compared the two pins, are gone.
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, - run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
owner ruling the linter is never installed on a host, and owner ruling the linter is never installed on a host. `Dockerfile.lint` copies
`golangci-lint config verify` runs before `golangci-lint run`. the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and
`script/bootstrap` stopped installing or pinning the linter, and runs `golangci-lint config verify` and `golangci-lint run` as build steps;
`script/verify-linter-pin` was retired. Since `script/lint` builds it. `script/bootstrap` no longer installs or pins the
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the linter, and warns rather than fails when `docker` is absent;
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build `ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`.
stage depends on through `COPY --from=lint /src/go.sum /dev/null`; `script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside in both files, compares their two `FROM` lines and restates neither pin.
an image build may run docker, so the phase calls `golangci-lint` directly. Traps: nothing inside an image build may shell out to docker, so the
`Dockerfile` lint stage calls `golangci-lint` directly and the build stage
runs `make test` and `make fmt-check` instead of `make check`, through `make`
because the Makefile's `export CGO_ENABLED = 0` only reaches what it invokes.
`COPY --from=lint /src/go.sum /dev/null` replaces the copied linter binary as
the only edge making the build stage wait for lint; dropping it would end
fail-fast linting under a still-green build. `golangci-lint config verify`,
included per the ruling, validates from an embedded schema with no network
call, but `go mod download` above the gates still needs the network on a cold
cache. Verified: `make lint` green with no `golangci-lint` on `PATH`; two
back-to-back `script/lint` runs on an untouched tree both ran the linter
(27.7s and 28.7s in the lint step, `COPY . .` `CACHED` above); a planted
unused variable failed `script/lint`, and failed `make docker` at `[lint 9/9]`
with the build stage stopped at `[builder 3/12]`; the drift guard fails on a
tag-only, a digest-only and an unreadable reference, naming both sides; under
`--network none` config verify passes a valid config and rejects an invalid
one; `make docker` green in 5m35s with all six gates run (lint 37.6s, test
25.2s reporting `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not
`(cached)`); in the builder image with the Go test cache off, `--user 0:0`
still fails `TestScanHardlinkRunFailsTogether` where the unprivileged user
passes. Noted for follow-up, not fixed here: `golangci-lint` warns that
`gomodguard` is deprecated since v2.12.0 in favour of `gomodguard_v2`.
- install the Docker build stage's prerequisites by running `script/bootstrap` - install the Docker build stage's prerequisites by running `script/bootstrap`
instead of `apk add --no-cache make` inline (2026-08-09, branch instead of `apk add --no-cache make` inline (2026-08-09, branch
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42), `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42):
with `script/verify-linter-pin` failing the build unless the linter copied the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
from the lint stage was the version `script/bootstrap` pinned. Builds then which ends in `go mod download`, so the separate call to it is gone.
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as `COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap
https://git.eeqj.de/sneak/sfdupes/issues/43. Since layer: it is the only edge making this stage depend on the lint stage, so
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and deleting it would end fail-fast linting silently. A new
`make` with `apk add --no-cache` and only compiles; the `lint` and `test` `script/verify-linter-pin`, run in the build stage before bootstrap, fails the
phases are the gates build naming both versions unless that copied binary is the version
`script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip.
`$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. The `chown`
and `USER builder` still precede `make check`. Verified: the guard fails the
build with both versions named when the lint stage's linter is faked to
another version, and passes an unmodified build; bootstrap runs clean under
Alpine's `sh` and `apk`, finding the copied linter already at the pin; a
second build served the bootstrap and dependency layers `CACHED` while both
gates ran; a planted `unused` finding failed the build at the lint gate in
48.9s with the build stage's `make check` never starting; and the suite run in
the image as `--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the
drop to the unprivileged user is still needed. That last check needs the Go
test cache off: as root it first reported `ok ... (cached)`, reusing the
build-time result. Build times on a noisy shared host: 2m13s on an unchanged
tree, 2m17s and 4m29s after a source change, 5m14s cold, which breaches the
policy ceiling; `chown -R builder:builder /src /home/builder` walks the module
cache and alone varied from 77s to 210s across those builds, and `main`
measured 5m03s cold with a 209s `chown`. Filed as
https://git.eeqj.de/sneak/sfdupes/issues/43
- bust the Docker layer cache for the gate steps, so `script/cibuild` and - bust the Docker layer cache for the gate steps, so `script/cibuild` and
`script/docker` cannot report a green they did not earn (2026-08-09, branch `script/docker` cannot report a green they did not earn (2026-08-09, branch
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the `cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker `Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
served them from cache and the build exited 0 having run nothing. The fix was served them from cache and the build exited 0 having run nothing. Every
a `CHECK_EPOCH` build argument; since `docker build` in `script/` now passes `--no-cache` instead
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/` (https://git.eeqj.de/sneak/sfdupes/issues/95). Run as root, the tests fail
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile` `TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
declares before every step that lints, tests or compiles. Run as root, the the test relies on, so they run as an unprivileged user
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
- check the installed golangci-lint version in `script/bootstrap` instead of - check the installed golangci-lint version in `script/bootstrap` instead of
only its presence (2026-08-09, branch `bootstrap-version-check`, closes only its presence (2026-08-09, branch `bootstrap-version-check`, closes
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
+14 -21
View File
@@ -196,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
base := baselineGoroutines(t) base := baselineGoroutines(t)
st, err := syncScan(newWalkClock(walkCancelAtDone), db, st, err := syncScan(newWalkClock(walkCancelAtDone), db,
[]string{dir}, walkCancelWorkers, false, walkRules{}) []string{dir}, walkCancelWorkers, false)
assertWalkGuardAborted(t, st, err) assertWalkGuardAborted(t, st, err)
assertRecordsIntact(t, db, before) assertRecordsIntact(t, db, before)
@@ -268,8 +268,7 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context()) ctx, cancel := context.WithCancel(t.Context())
cancel() cancel()
st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false, st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false)
walkRules{})
if !errors.Is(err, context.Canceled) { if !errors.Is(err, context.Canceled) {
t.Fatalf("syncScan on a cancelled context = %v, want %v", t.Fatalf("syncScan on a cancelled context = %v, want %v",
err, context.Canceled) err, context.Canceled)
@@ -302,7 +301,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) {
db := openTestDB(t) db := openTestDB(t)
st, err := syncScan(newWalkClock(hashCancelAtDone), db, st, err := syncScan(newWalkClock(hashCancelAtDone), db,
[]string{dir}, walkCancelWorkers, false, walkRules{}) []string{dir}, walkCancelWorkers, false)
if !errors.Is(err, context.Canceled) { if !errors.Is(err, context.Canceled) {
t.Fatalf("syncScan cancelled mid-hash = %v, want %v", t.Fatalf("syncScan cancelled mid-hash = %v, want %v",
err, context.Canceled) err, context.Canceled)
@@ -345,10 +344,9 @@ func storedPaths(t *testing.T, path string) []string {
// TestRunScanInterrupted calls the scan entrypoint with a context that // TestRunScanInterrupted calls the scan entrypoint with a context that
// is already cancelled, as when a signal arrives at once. It must return // is already cancelled, as when a signal arrives at once. It must return
// errInterrupted promptly, with its line on stderr after the one naming // errInterrupted promptly with its one line on stderr and nothing on
// the database and nothing on stdout, leave the database valid and as // stdout, leave the database valid and as it was, and leave nothing in
// it was, and leave nothing in the way of the next scan, which must // the way of the next scan, which must bring the database up to date.
// bring the database up to date.
func TestRunScanInterrupted(t *testing.T) { func TestRunScanInterrupted(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
@@ -357,8 +355,7 @@ func TestRunScanInterrupted(t *testing.T) {
stderr := captureStderr(t) stderr := captureStderr(t)
dir := buildSmokeTree(t) dir := buildSmokeTree(t)
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false, err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -382,7 +379,7 @@ func TestRunScanInterrupted(t *testing.T) {
defer close(done) defer close(done)
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
false, false) false)
}() }()
awaitReturn(t, done, "runScan") awaitReturn(t, done, "runScan")
@@ -392,8 +389,7 @@ func TestRunScanInterrupted(t *testing.T) {
err, errInterrupted) err, errInterrupted)
} }
want := "scan: database " + path + " (from " + databaseEnv + ")\n" + want := "scan: interrupted after 0 files\n"
"scan: interrupted after 0 files\n"
if got := stderr()[shown:]; got != want { if got := stderr()[shown:]; got != want {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
@@ -409,8 +405,7 @@ func TestRunScanInterrupted(t *testing.T) {
got, before) got, before)
} }
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false, err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -436,7 +431,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
dir := buildWalkCancelTree(t) dir := buildWalkCancelTree(t)
err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
walkCancelWorkers, false, false) walkCancelWorkers, false)
if !errors.Is(err, errInterrupted) { if !errors.Is(err, errInterrupted) {
t.Fatalf("runScan interrupted mid-hash = %v, want %v", t.Fatalf("runScan interrupted mid-hash = %v, want %v",
err, errInterrupted) err, errInterrupted)
@@ -620,8 +615,7 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) {
// Unbuffered and unread: on a cancelled scan every send gives up. // Unbuffered and unread: on a cancelled scan every send gives up.
events := make(chan walkEvent) events := make(chan walkEvent)
subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events)
walkRules{}, events)
if len(subs) != 0 { if len(subs) != 0 {
t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ t.Errorf("cancelled walkOneDir returned %+v to descend into, "+
"want none", subs) "want none", subs)
@@ -641,8 +635,7 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) {
missing := filepath.Join(t.TempDir(), "missing") missing := filepath.Join(t.TempDir(), "missing")
jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false, jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false)
walkRules{})
for range 64 { for range 64 {
jobs <- dirJob{path: missing} jobs <- dirJob{path: missing}
@@ -673,7 +666,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context()) ctx, cancel := context.WithCancel(t.Context())
defer cancel() defer cancel()
jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{}) jobs, subdirs, events := startWalkWorkers(ctx, 1, false)
// Fill the hand-back channel to its capacity — one slot per worker // Fill the hand-back channel to its capacity — one slot per worker
// — so the worker's own hand-back is certain to block. // — so the worker's own hand-back is certain to block.
+125 -339
View File
@@ -11,8 +11,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"strconv" "strconv"
"strings"
"time"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
@@ -36,33 +34,26 @@ const schemaVersion = 1
// scan. // scan.
const dbDirPerm = 0o755 const dbDirPerm = 0o755
// dbFilePerm is the mode for a database file created by scan, the mode
// SQLite gives a database file it creates.
const dbFilePerm = 0o644
// lockFilePerm is the mode for the scan lock file. Anyone who can open // lockFilePerm is the mode for the scan lock file. Anyone who can open
// the file can hold the lock and keep every scan from running, so it // the file can hold the lock and keep every scan from running, so it
// is open to its owner only. // is open to its owner only.
const lockFilePerm = 0o600 const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are // createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second.
const createTableSQL = ` const createTableSQL = `
CREATE TABLE files ( CREATE TABLE files (
path BLOB PRIMARY KEY, path BLOB PRIMARY KEY,
size INTEGER NOT NULL, size INTEGER NOT NULL,
mtime INTEGER NOT NULL, mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL, head TEXT NOT NULL,
head TEXT NOT NULL, tail TEXT NOT NULL,
tail TEXT NOT NULL, content TEXT NOT NULL
content TEXT NOT NULL
) WITHOUT ROWID ) WITHOUT ROWID
` `
// createIndexSQL indexes the records by signature, so report can read // createIndexSQL indexes the records by signature, so report can have
// them in signature order without sorting the table. // SQLite group them without sorting the whole table.
const createIndexSQL = ` const createIndexSQL = `
CREATE INDEX files_signature ON files (size, head, tail, content) CREATE INDEX files_signature ON files (size, head, tail, content)
` `
@@ -70,17 +61,15 @@ CREATE INDEX files_signature ON files (size, head, tail, content)
// upsertSQL inserts one file record, replacing any existing record for // upsertSQL inserts one file record, replacing any existing record for
// the same path. // the same path.
const upsertSQL = ` const upsertSQL = `
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content) INSERT INTO files (path, size, mtime, head, tail, content)
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (path) DO UPDATE SET ON CONFLICT (path) DO UPDATE SET
size = excluded.size, mtime = excluded.mtime, size = excluded.size, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
head = excluded.head, tail = excluded.tail, head = excluded.head, tail = excluded.tail,
content = excluded.content content = excluded.content
` `
// errNoDatabase reports a missing database file for report, trees and // errNoDatabase reports a missing database file for report/trees.
// info.
var errNoDatabase = errors.New( var errNoDatabase = errors.New(
"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")") "no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
@@ -93,52 +82,13 @@ var errSchemaVersion = errors.New("unsupported database schema version")
var errScanRunning = errors.New("another scan is running") var errScanRunning = errors.New("another scan is running")
// databasePath resolves the database location: SFDUPES_DATABASE when // databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the // set and non-empty, the compiled-in default otherwise.
// path, with the working directory, its symlinks resolved, put in front func databasePath() string {
// of it when it is relative, and where the path came from, "default" or
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
// stderr.
func databasePath() (string, string, error) {
path, source := defaultDatabasePath, "default"
if p := os.Getenv(databaseEnv); p != "" { if p := os.Getenv(databaseEnv); p != "" {
path, source = p, "from "+databaseEnv return p
} }
if filepath.IsAbs(path) { return defaultDatabasePath
return path, source, nil
}
// The working directory as the kernel reports it, with no symlink
// in it, as README "Database" says. os.Getwd may return PWD
// instead, which can name it through a symlink.
wd, err := unix.Getwd()
if err != nil {
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
}
// Not filepath.Abs or filepath.Join: they clean the path, which
// drops each ".." together with the directory name before it. When
// that directory is a symlink, the cleaned path names another file.
// The working directory ends in "/" only when it is the root.
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
}
// createDatabaseDirectory creates the directory the database at path
// goes in, and any directory above it, if missing. path is absolute, as
// databasePath returns it. The directory is path with its last element
// cut off as text, by filepath.Split. filepath.Dir would clean it,
// dropping each ".." together with the directory name before it; when
// that name is a symlink, the cleaned path names another directory than
// the one the system reaches when it opens path.
func createDatabaseDirectory(path string) error {
dir, _ := filepath.Split(path)
err := os.MkdirAll(dir, dbDirPerm)
if err != nil {
return fmt.Errorf("create database directory: %w", err)
}
return nil
} }
// scanParams are the connection parameters for scan: read-write, with // scanParams are the connection parameters for scan: read-write, with
@@ -148,9 +98,9 @@ const scanParams = "_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(WAL)" + "&_pragma=journal_mode(WAL)" +
"&_pragma=synchronous(NORMAL)" "&_pragma=synchronous(NORMAL)"
// reportParams are the connection parameters for report, trees and // reportParams are the connection parameters for report and trees:
// info: read-only, with the same busy timeout. They set no journal // read-only, with the same busy timeout. They set no journal mode,
// mode, because setting one is a write. // because setting one is a write.
const reportParams = "mode=ro" + const reportParams = "mode=ro" +
"&_pragma=busy_timeout(10000)" + "&_pragma=busy_timeout(10000)" +
"&_pragma=query_only(1)" "&_pragma=query_only(1)"
@@ -185,37 +135,19 @@ func openDB(path, params string) (*sql.DB, error) {
// lockScanDatabase takes the lock that keeps a second scan off the // lockScanDatabase takes the lock that keeps a second scan off the
// database at path: an exclusive flock(2) on the file beside it named // database at path: an exclusive flock(2) on the file beside it named
// by the database's path with symlinks resolved and ".lock" appended, // path with ".lock" appended, created along with the database's parent
// so every path to one database names the same lock file. If missing, // directory if missing. A lock held by another scan fails at once
// the database's parent directory and an empty database file are // instead of waiting. The lock lasts until the returned file is closed
// created first, so the lock file is named the same way before the // or the process ends. The file is never deleted: a scan that deleted
// database is set up as after. A lock held by another scan fails at // it would let the next scan lock a new file while another still holds
// once instead of waiting. The lock lasts until the returned file is // the old one.
// closed or the process ends. The lock file is never deleted: a scan
// that deleted it would let the next scan lock a new file while
// another still holds the old one.
func lockScanDatabase(path string) (*os.File, error) { func lockScanDatabase(path string) (*os.File, error) {
err := createDatabaseDirectory(path) err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("create database directory: %w", err)
} }
// Like SQLite, this follows a symlink at path and creates its lockPath := path + ".lock"
// target. SQLite opens an empty file as a new database.
//nolint:gosec // the operator chooses the database path
dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
if err != nil {
return nil, fmt.Errorf("create database: %w", err)
}
_ = dbFile.Close()
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return nil, fmt.Errorf("resolve database path: %w", err)
}
lockPath := resolved + ".lock"
//nolint:gosec // the operator chooses the database path //nolint:gosec // the operator chooses the database path
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm) f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
@@ -241,9 +173,9 @@ func lockScanDatabase(path string) (*os.File, error) {
// openScanDatabase opens the database for the scan subcommand, creating // openScanDatabase opens the database for the scan subcommand, creating
// the file, its parent directory, and the schema as needed. // the file, its parent directory, and the schema as needed.
func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
err := createDatabaseDirectory(path) err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("create database directory: %w", err)
} }
db, err := openDB(path, scanParams) db, err := openDB(path, scanParams)
@@ -261,42 +193,6 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
return db, nil return db, nil
} }
// databaseFiles returns, by device and inode, the files of the database
// at path that the scan keeps open: the database file, its -wal and
// -shm files and its .lock file, each named from the database's path
// with symlinks resolved, as SQLite and lockScanDatabase name them. The
// scan calls it once the database and the lock file are open, so that
// its walk passes over them however it reaches them, through a
// symlinked directory included. A file that does not exist then is left
// out, since no walk can reach it. There is no -journal file while the
// database is in WAL mode.
func databaseFiles(path string) ([]fileRec, error) {
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return nil, fmt.Errorf("resolve database path: %w", err)
}
var files []fileRec
for _, p := range []string{
resolved, resolved + "-wal", resolved + "-shm", resolved + ".lock",
} {
fi, err := os.Lstat(p)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return nil, fmt.Errorf("database: %w", err)
}
dev, ino := inodeOfInfo(fi)
files = append(files, fileRec{path: p, dev: dev, ino: ino})
}
return files, nil
}
// closeScanDatabase switches the database at path from WAL back to // closeScanDatabase switches the database at path from WAL back to
// rollback-journal mode and closes it. Out of WAL mode the database // rollback-journal mode and closes it. Out of WAL mode the database
// file alone holds the whole database, so a reader needs no -wal or // file alone holds the whole database, so a reader needs no -wal or
@@ -315,9 +211,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) {
_ = db.Close() _ = db.Close()
} }
// openReportDatabase opens an existing database for the report, trees // openReportDatabase opens an existing database for the report and
// and info subcommands. A missing database file is an error directing // trees subcommands. A missing database file is an error directing the
// the user to run scan first; the schema version must match exactly. // user to run scan first; the schema version must match exactly.
func openReportDatabase(ctx context.Context, func openReportDatabase(ctx context.Context,
path string, path string,
) (*sql.DB, error) { ) (*sql.DB, error) {
@@ -456,103 +352,12 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
return v, nil return v, nil
} }
// loadFileRows gives total the number of records, then streams every // loadFileRows streams every record to fn in path order: byte order,
// record to fn in path order: byte order, which is the order of the // which is the order of the primary key, so SQLite does not sort.
// primary key, so SQLite does not sort. The count and the records are func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
// read in one transaction, so they agree while a scan is committing. rows, err := db.QueryContext(ctx,
func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64), "SELECT path, size, mtime, head, tail, content FROM files "+
fn func(r scanRec), "ORDER BY path")
) error {
// Everything goes through tx: db has one connection, so a query on
// db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = tx.Rollback() }()
var n int64
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
total(n)
rows, err := tx.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
"FROM files ORDER BY path")
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
path []byte
sec, nsec int64
r scanRec
)
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
&r.content)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
r.mtime = time.Unix(sec, nsec)
fn(r)
}
err = rows.Err()
if err != nil {
return fmt.Errorf("read records: %w", err)
}
return nil
}
// signatureRowsSQL selects every record's signature and path in
// signature order, largest size first. That is the order of the
// files_signature index read backwards, so SQLite reads the index alone
// and returns the first row at once, with nothing to sort.
const signatureRowsSQL = `
SELECT size, head, tail, content, path
FROM files
ORDER BY size DESC, head DESC, tail DESC, content DESC
`
// loadSignatureRows gives total the number of records, then streams
// every record's signature and path to fn, without its mtime, in the
// order of signatureRowsSQL. The count and the records are read in one
// transaction, so they agree while a scan is committing. An error from
// fn stops the reading and is returned as it is.
func loadSignatureRows(ctx context.Context, db *sql.DB,
total func(n int64), fn func(r scanRec) error,
) error {
// Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = tx.Rollback() }()
var n int64
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
total(n)
rows, err := tx.QueryContext(ctx, signatureRowsSQL)
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return fmt.Errorf("read records: %w", err)
} }
@@ -565,17 +370,14 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
r scanRec r scanRec
) )
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path) err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail,
&r.content)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
r.path = string(path) r.path = string(path)
fn(r)
err = fn(r)
if err != nil {
return err
}
} }
err = rows.Err() err = rows.Err()
@@ -586,70 +388,77 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
return nil return nil
} }
// countRecords returns the number of records in the database. // dupeRowsSQL selects every record in a duplicate group, with the
func countRecords(ctx context.Context, db *sql.DB) (int64, error) { // group's first path. A group is the records with a content hash that
var n int64 // share a size, head, tail, and content, when there are two or more of
// them. The rows come in report order: groups by size descending, then
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n) // by first path, and each group's paths ascending.
if err != nil { const dupeRowsSQL = `
return 0, fmt.Errorf("count records: %w", err) SELECT g.first, f.path, f.size
} FROM files AS f
JOIN (
return n, nil SELECT size, head, tail, content, MIN(path) AS first
} FROM files
WHERE content <> ''
// databaseCountsSQL counts the records, those with head and tail GROUP BY size, head, tail, content
// hashes and those with a content hash, and adds up their sizes, in one HAVING COUNT(*) > 1
// pass over the table. A SUM over no rows is NULL, so on an empty table ) AS g USING (size, head, tail, content)
// each SUM falls back to 0. ORDER BY f.size DESC, g.first, f.path
const databaseCountsSQL = `
SELECT COUNT(*),
COALESCE(SUM(head <> '' AND tail <> ''), 0),
COALESCE(SUM(content <> ''), 0),
COALESCE(SUM(size), 0)
FROM files
` `
// databaseCounts is what info reads from the database: its schema // loadDupeRows streams the rows of dupeRowsSQL to fn and returns the
// version and the counts of databaseCountsSQL. // number of records in the database. The count and the rows are read
type databaseCounts struct { // in one transaction, so they agree while a scan is committing. An
schemaVersion int // error from fn stops the reading and is returned as it is.
records int64 // every record func loadDupeRows(ctx context.Context, db *sql.DB,
headTail int64 // records with head and tail hashes fn func(first, path string, size int64) error,
content int64 // records with a content hash ) (int, error) {
size int64 // the sizes of all records added up
}
// readDatabaseCounts reads the schema version, then the counts of
// databaseCountsSQL, in one read-only transaction, so both describe the
// database at one moment while a scan is committing.
func readDatabaseCounts(ctx context.Context,
db *sql.DB,
) (databaseCounts, error) {
var c databaseCounts
// Everything goes through tx: the report connection is the only // Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever. // one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil { if err != nil {
return c, fmt.Errorf("count records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
defer func() { _ = tx.Rollback() }() defer func() { _ = tx.Rollback() }()
err = tx.QueryRowContext(ctx, "PRAGMA user_version"). var records int
Scan(&c.schemaVersion)
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records)
if err != nil { if err != nil {
return c, fmt.Errorf("read schema version: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
err = tx.QueryRowContext(ctx, databaseCountsSQL). rows, err := tx.QueryContext(ctx, dupeRowsSQL)
Scan(&c.records, &c.headTail, &c.content, &c.size)
if err != nil { if err != nil {
return c, fmt.Errorf("count records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
return c, nil defer func() { _ = rows.Close() }()
for rows.Next() {
var (
first, path []byte
size int64
)
err = rows.Scan(&first, &path, &size)
if err != nil {
return 0, fmt.Errorf("read record: %w", err)
}
err = fn(string(first), string(path), size)
if err != nil {
return 0, err
}
}
err = rows.Err()
if err != nil {
return 0, fmt.Errorf("read records: %w", err)
}
return records, nil
} }
// loadFileMeta streams every record's path, size, mtime, and whether // loadFileMeta streams every record's path, size, mtime, and whether
@@ -657,10 +466,10 @@ func readDatabaseCounts(ctx context.Context,
// values, and skipping the hash columns keeps the scan's in-memory // values, and skipping the hash columns keeps the scan's in-memory
// index small on multi-million-file databases. // index small on multi-million-file databases.
func loadFileMeta(ctx context.Context, db *sql.DB, func loadFileMeta(ctx context.Context, db *sql.DB,
fn func(path string, size int64, mtime time.Time, hashed bool), fn func(path string, size, mtime int64, hashed bool),
) error { ) error {
rows, err := db.QueryContext(ctx, rows, err := db.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files") "SELECT path, size, mtime, head <> '' FROM files")
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return fmt.Errorf("read records: %w", err)
} }
@@ -669,17 +478,17 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
for rows.Next() { for rows.Next() {
var ( var (
path []byte path []byte
size, sec, nsec int64 size, mtime int64
hashed int64 hashed int64
) )
err = rows.Scan(&path, &size, &sec, &nsec, &hashed) err = rows.Scan(&path, &size, &mtime, &hashed)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
fn(string(path), size, time.Unix(sec, nsec), hashed != 0) fn(string(path), size, mtime, hashed != 0)
} }
err = rows.Err() err = rows.Err()
@@ -690,46 +499,26 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
return nil return nil
} }
// contentGroupsSQL selects each group (the records sharing a size, // contentCandidatesSQL selects every record of at least headTailMin
// head, and tail) of records of at least headTailMin bytes that has more // bytes whose size, head, and tail equal another record's, in each
// than one record and at least one record without a content hash, with // group (the records sharing a size, head, and tail) where at least one
// the number of its records. // record has no content hash, with whether each record has one. SQLite
const contentGroupsSQL = ` // does the grouping, so no other record's hashes are loaded into
SELECT size, head, tail, COUNT(*) AS records // memory; the rows come ordered by size, head, and tail, so each
FROM files // group's rows arrive together.
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
`
// contentCandidatesSQL selects every record of the groups
// contentGroupsSQL selects, with whether each record has a content
// hash. SQLite does the grouping, so no other record's hashes are
// loaded into memory; the rows come ordered by size, head, and tail, so
// each group's rows arrive together.
const contentCandidatesSQL = ` const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> '' SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
FROM files AS f FROM files AS f
JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail) JOIN (
SELECT size, head, tail
FROM files
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
) AS g USING (size, head, tail)
ORDER BY size, head, tail ORDER BY size, head, tail
` `
// countContentCandidates returns the number of rows of
// contentCandidatesSQL, adding up the records of the groups
// contentGroupsSQL selects, so it neither joins nor sorts.
func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
err := db.QueryRowContext(ctx,
"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
headTailMin).Scan(&n)
if err != nil {
return 0, fmt.Errorf("count content candidates: %w", err)
}
return n, nil
}
// loadContentCandidates streams the rows of contentCandidatesSQL to fn: // loadContentCandidates streams the rows of contentCandidatesSQL to fn:
// each record, without its content hash, and whether it has one. // each record, without its content hash, and whether it has one.
func loadContentCandidates(ctx context.Context, db *sql.DB, func loadContentCandidates(ctx context.Context, db *sql.DB,
@@ -744,20 +533,17 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
for rows.Next() { for rows.Next() {
var ( var (
path []byte path []byte
sec, nsec int64 r scanRec
r scanRec hashed int64
hashed int64
) )
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
&hashed)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
r.path = string(path) r.path = string(path)
r.mtime = time.Unix(sec, nsec)
fn(r, hashed != 0) fn(r, hashed != 0)
} }
@@ -841,8 +627,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
defer func() { _ = st.Close() }() defer func() { _ = st.Close() }()
for _, r := range upserts { for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size, _, err = st.ExecContext(ctx,
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content) []byte(r.path), r.size, r.mtime, r.head, r.tail, r.content)
if err != nil { if err != nil {
return fmt.Errorf("upsert %s: %w", r.path, err) return fmt.Errorf("upsert %s: %w", r.path, err)
} }
+22 -355
View File
@@ -5,28 +5,18 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"strings" "strings"
"testing" "testing"
"time"
) )
// testDBPath returns a database path inside a fresh temp dir, with // testDBPath returns a database path inside a fresh temp dir.
// symlinks resolved as the scan lock resolves them, so the lock file
// is this path plus ".lock" even where the temp dir is reached through
// a symlink.
func testDBPath(t *testing.T) string { func testDBPath(t *testing.T) string {
t.Helper() t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir()) return filepath.Join(t.TempDir(), "db.sqlite")
if err != nil {
t.Fatal(err)
}
return filepath.Join(dir, "db.sqlite")
} }
// openTestDB creates a fresh scan database in a temp dir. // openTestDB creates a fresh scan database in a temp dir.
@@ -44,47 +34,16 @@ func openTestDB(t *testing.T) *sql.DB {
} }
func TestDatabasePath(t *testing.T) { func TestDatabasePath(t *testing.T) {
// README §Database: SFDUPES_DATABASE when set and non-empty, the t.Setenv(databaseEnv, "")
// default otherwise, with the working directory, its symlinks
// resolved, put in front of a relative path. Nothing is cleaned, so if got := databasePath(); got != defaultDatabasePath {
// a ".." stays. t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
} }
t.Chdir(dir) t.Setenv(databaseEnv, "/custom/place.sqlite")
fromEnv := "from " + databaseEnv if got := databasePath(); got != "/custom/place.sqlite" {
cases := []struct { t.Errorf("databasePath() = %q, want the env override", got)
env, path, source string
}{
{"", defaultDatabasePath, "default"},
{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
fromEnv},
{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
}
for _, tc := range cases {
t.Setenv(databaseEnv, tc.env)
path, source, err := databasePath()
if err != nil || path != tc.path || source != tc.source {
t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
databaseEnv, tc.env, path, source, err, tc.path, tc.source)
}
}
// In the root directory the path gets one slash in front, not two.
t.Chdir("/")
t.Setenv(databaseEnv, "place.sqlite")
path, _, err := databasePath()
if err != nil || path != "/place.sqlite" {
t.Errorf("databasePath() in / = %q, %v; want %q",
path, err, "/place.sqlite")
} }
} }
@@ -119,116 +78,6 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
} }
} }
// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
// through a symlink to a directory, then "..", to a missing directory.
// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
// places scan creates the database's directory, must each create the
// database there, and nothing at srv/new, where the path leads once
// cleaned as text.
func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
t.Parallel()
creates := map[string]func(ctx context.Context, path string) error{
"lockScanDatabase": func(_ context.Context, path string) error {
lock, err := lockScanDatabase(path)
if err == nil {
_ = lock.Close()
}
return err
},
"openScanDatabase": func(ctx context.Context, path string) error {
db, err := openScanDatabase(ctx, path)
if err == nil {
_ = db.Close()
}
return err
},
}
for name, create := range creates {
t.Run(name, func(t *testing.T) {
t.Parallel()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "data", "x"),
filepath.Join(dir, "srv", "link"))
if err != nil {
t.Fatal(err)
}
err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
if err != nil {
t.Fatalf("%s: %v", name, err)
}
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
if err != nil {
t.Errorf("database the path leads to: %v", err)
}
_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("srv/new: stat error %v, want it missing", err)
}
})
}
}
// TestDatabaseFiles opens a database through a symlink to its file:
// databaseFiles finds the database file and its -wal and -shm files
// beside the symlink's target, where SQLite keeps them, and leaves out
// the .lock file, which only a scan creates.
func TestDatabaseFiles(t *testing.T) {
t.Parallel()
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
db, err := openScanDatabase(t.Context(), link)
if err != nil {
t.Fatal(err)
}
defer func() { _ = db.Close() }()
files, err := databaseFiles(link)
if err != nil {
t.Fatal(err)
}
got := make([]string, 0, len(files))
for _, f := range files {
got = append(got, f.path)
}
want := []string{path, path + "-wal", path + "-shm"}
if !slices.Equal(got, want) {
t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
}
}
func TestOpenDatabaseUnversionedForeign(t *testing.T) { func TestOpenDatabaseUnversionedForeign(t *testing.T) {
t.Parallel() t.Parallel()
@@ -395,128 +244,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
defer func() { _ = reportDB.Close() }() defer func() { _ = reportDB.Close() }()
err = loadFileRows(t.Context(), reportDB, func(int64) {}, err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
func(scanRec) {})
if err != nil { if err != nil {
t.Fatalf("loadFileRows: %v", err) t.Fatalf("loadFileRows: %v", err)
} }
} }
// TestLoadRowsCountAgreeDuringScan has a scan commit records after
// loadFileRows, as trees calls it, or loadSignatureRows, as report
// calls it, has counted the records and before it reads them, as when
// trees or report runs during a scan: each must read exactly as many
// records as it counted.
func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
t.Parallel()
loads := map[string]func(context.Context, *sql.DB, func(int64),
func(scanRec)) error{
"loadFileRows": loadFileRows,
"loadSignatureRows": func(ctx context.Context, db *sql.DB,
total func(int64), fn func(scanRec),
) error {
return loadSignatureRows(ctx, db, total, func(r scanRec) error {
fn(r)
return nil
})
},
}
for name, load := range loads {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := testDBPath(t)
scanDB, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = scanDB.Close() }()
recs := smokeTreeRecs()
err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
if err != nil {
t.Fatal(err)
}
reportDB, err := openReportDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = reportDB.Close() }()
var counted, read int64
err = load(t.Context(), reportDB, func(n int64) {
counted = n
err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
if err != nil {
t.Error(err)
}
}, func(scanRec) { read++ })
if err != nil {
t.Fatal(err)
}
if counted != 3 || read != counted {
t.Errorf("counted %d records and read %d, want 3 and 3",
counted, read)
}
})
}
}
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
t.Parallel()
db := openTestDB(t)
rows, err := db.QueryContext(t.Context(),
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
if err != nil {
t.Fatal(err)
}
defer func() { _ = rows.Close() }()
var plan []string
for rows.Next() {
var (
id, parent, unused int
detail string
)
err = rows.Scan(&id, &parent, &unused, &detail)
if err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
want := []string{"SCAN files USING COVERING INDEX files_signature"}
if !slices.Equal(plan, want) {
t.Errorf("query plan = %q, want %q", plan, want)
}
}
func TestApplyChangesRoundTrip(t *testing.T) { func TestApplyChangesRoundTrip(t *testing.T) {
t.Parallel() t.Parallel()
@@ -527,19 +260,14 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// written. // written.
recs := []scanRec{ recs := []scanRec{
{ {
size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2", size: 2, mtime: 20, head: "h2", tail: "t2", content: "c2",
content: "c2", path: "/a/tab\tnew\nline", path: "/a/tab\tnew\nline",
},
{
size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
content: "c1", path: "/a/x",
}, },
{size: 1, mtime: 10, head: "h1", tail: "t1", content: "c1", path: "/a/x"},
} }
prog := newProgress(progressParams{label: "update", total: 4}) err := applyChanges(t.Context(), db, recs, nil,
defer prog.finish() newProgress("update", 2))
err := applyChanges(t.Context(), db, recs, nil, prog)
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -553,12 +281,11 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// An upsert for an existing path updates in place; a delete // An upsert for an existing path updates in place; a delete
// removes exactly its path. // removes exactly its path.
upd := scanRec{ upd := scanRec{
size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3", size: 3, mtime: 30, head: "h3", tail: "t3", content: "c3", path: "/a/x",
path: "/a/x",
} }
err = applyChanges(t.Context(), db, []scanRec{upd}, err = applyChanges(t.Context(), db, []scanRec{upd},
[]string{"/a/tab\tnew\nline"}, prog) []string{"/a/tab\tnew\nline"}, newProgress("update", 2))
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -581,15 +308,13 @@ func TestApplyChangesBatching(t *testing.T) {
recs := make([]scanRec, 0, n) recs := make([]scanRec, 0, n)
for i := range n { for i := range n {
recs = append(recs, scanRec{ recs = append(recs, scanRec{
size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t", size: int64(i), mtime: 1, head: "h", tail: "t",
path: fmt.Sprintf("/batch/%07d", i), path: fmt.Sprintf("/batch/%07d", i),
}) })
} }
prog := newProgress(progressParams{label: "update", total: int64(2 * n)}) err := applyChanges(t.Context(), db, recs, nil,
defer prog.finish() newProgress("update", int64(n)))
err := applyChanges(t.Context(), db, recs, nil, prog)
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -603,7 +328,8 @@ func TestApplyChangesBatching(t *testing.T) {
deletes = append(deletes, r.path) deletes = append(deletes, r.path)
} }
err = applyChanges(t.Context(), db, nil, deletes, prog) err = applyChanges(t.Context(), db, nil, deletes,
newProgress("update", int64(n)))
if err != nil { if err != nil {
t.Fatalf("applyChanges deletes: %v", err) t.Fatalf("applyChanges deletes: %v", err)
} }
@@ -612,62 +338,3 @@ func TestApplyChangesBatching(t *testing.T) {
t.Fatalf("records = %d, want 0", len(got)) t.Fatalf("records = %d, want 0", len(got))
} }
} }
// TestCountContentCandidates checks that countContentCandidates counts
// the records the content phase's search returns: those of each group
// of two or more records of headTailMin or more with a head, where at
// least one has no content hash.
func TestCountContentCandidates(t *testing.T) {
t.Parallel()
db := openTestDB(t)
// Each group's records: size, head, and content, one per record.
groups := []struct {
size int64
head string
contents []string
}{
{headTailMin, "h", []string{"", "c", "c"}}, // returned
{headTailMin + 1, "h", []string{"", ""}}, // returned
{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
{headTailMin + 3, "h", []string{""}}, // one record
{headTailMin - 1, "h", []string{"", ""}}, // too small
{headTailMin + 4, "", []string{"", ""}}, // no head
}
var recs []scanRec
for _, g := range groups {
for _, content := range g.contents {
recs = append(recs, scanRec{
size: g.size, mtime: time.Unix(1, 0), head: g.head,
tail: "t", content: content,
path: fmt.Sprintf("/r/%d", len(recs)),
})
}
}
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
got, err := countContentCandidates(t.Context(), db)
if err != nil {
t.Fatalf("countContentCandidates: %v", err)
}
var returned int64
err = loadContentCandidates(t.Context(), db,
func(scanRec, bool) { returned++ })
if err != nil {
t.Fatalf("loadContentCandidates: %v", err)
}
if got != 5 || returned != 5 {
t.Errorf("count = %d, records returned = %d, want 5 and 5",
got, returned)
}
}
-195
View File
@@ -1,195 +0,0 @@
package main
// This file holds two rules for the walk (README "Rules for the walk"):
// the default excludes, the paths a scan's walk does not enter below an
// operand unless scan is given --no-default-excludes, and the
// filesystems that are not persistent, on which the scan never records
// or reads anything.
import (
"bufio"
"errors"
"fmt"
"io"
"io/fs"
"path/filepath"
"slices"
"strings"
"golang.org/x/sys/unix"
)
// The systems that have default excludes, as runtime.GOOS names them.
const (
goosLinux = "linux"
goosDarwin = "darwin"
goosWindows = "windows"
)
// The fields of a line of mountInfoPath that readMountInfo reads,
// counted from 0 (see proc(5)): the device ID, as major:minor, is the
// third field, and after the sixth come optional fields, then a field
// holding "-", then the filesystem type.
const (
deviceField = 2
optionalFieldsStart = 6
)
// typesPerHelpLine is how many filesystem types the scan command's help
// lists on each line.
const typesPerHelpLine = 6
// errMountInfoLine reports a line of mountInfoPath without the fields
// readMountInfo reads.
var errMountInfoLine = errors.New("unexpected line")
// walkRules holds the rules for the walk that a scan sets up as it
// starts. excludes are the default excludes, or none with
// --no-default-excludes. nonPersistent maps the device ID of every
// mounted filesystem that is not persistent to its type; the scan
// records and reads nothing on one of those, whatever the flags.
// databaseFiles are the scan's own database files, which the walk
// passes over (see databaseFiles in db.go).
type walkRules struct {
excludes []string
nonPersistent map[uint64]string
databaseFiles []fileRec
}
// defaultExcludes returns the default excludes of the system goos
// names, as runtime.GOOS does: a fixed list on Linux, macOS and
// Windows, and none on any other system.
func defaultExcludes(goos string) []string {
switch goos {
case goosLinux:
return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
case goosDarwin:
// A walk of / reaches everything in /System/Volumes/Data through
// the firmlinks, so walking it as well would record each of
// those files twice.
return []string{
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
"/System/Volumes/VM", "/System/Volumes/Data",
}
case goosWindows:
// Relative to the root of the drive an operand lies on. The
// first three are the page, swap and hibernation files.
return []string{
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
`Users\*\AppData\Local\Temp`,
}
default:
return nil
}
}
// isDefaultExclude reports whether path is one of excludes, in which a
// * matches one path component.
func isDefaultExclude(path string, excludes []string) bool {
for _, pattern := range excludes {
// The patterns are fixed, so none is malformed.
if ok, _ := filepath.Match(pattern, path); ok {
return true
}
}
return false
}
// nonPersistentFilesystemTypes returns the filesystem types that are
// not persistent: filesystems that the kernel makes up or that are kept
// in memory. devfs is the macOS filesystem of /dev; the others are
// Linux names.
func nonPersistentFilesystemTypes() []string {
return []string{
"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs",
"cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore",
"bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs",
"binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs",
}
}
// nonPersistentLine returns the line the scan prints for path when fi,
// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
// when it is on any other.
func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
dev, ok := deviceOfInfo(fi)
if !ok {
return ""
}
fsType, found := rules.nonPersistent[dev]
if !found {
return ""
}
return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)",
path, fsType)
}
// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
// returns the device ID of every mounted filesystem that is not
// persistent, with its type. The device ID is the one each line gives,
// not that of the file now at its mount point, so a mount that a later
// mount hides never adds the device of another filesystem.
func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
types := nonPersistentFilesystemTypes()
devices := make(map[uint64]string)
lines := bufio.NewScanner(mountinfo)
for lines.Scan() {
fields := strings.Fields(lines.Text())
sep := slices.Index(fields, "-")
if sep < optionalFieldsStart || sep+1 >= len(fields) {
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
}
var major, minor uint32
_, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor)
if err != nil {
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
}
fsType := fields[sep+1]
if slices.Contains(types, fsType) {
devices[unix.Mkdev(major, minor)] = fsType
}
}
return devices, lines.Err()
}
// defaultExcludesHelp describes, for the scan command's help, the
// default excludes of the system goos names and, where the scan lists
// the mounted filesystems, the types it never records anything on.
func defaultExcludesHelp(goos string) string {
var help strings.Builder
dirs := defaultExcludes(goos)
if len(dirs) == 0 {
help.WriteString("This system has no default excludes.\n")
} else {
help.WriteString("Below an operand, the walk does not enter these " +
"default excludes, unless\n--no-default-excludes is given:\n\n")
for _, dir := range dirs {
fmt.Fprintf(&help, " %s\n", dir)
}
}
if goos == goosLinux || goos == goosDarwin {
help.WriteString("\nWith or without --no-default-excludes, the scan " +
"records nothing on a\nfilesystem that is not persistent, " +
"one of these types:\n\n")
types := nonPersistentFilesystemTypes()
for line := range slices.Chunk(types, typesPerHelpLine) {
fmt.Fprintf(&help, " %s\n", strings.Join(line, ", "))
}
}
return help.String()
}
-52
View File
@@ -1,52 +0,0 @@
//go:build darwin
package main
import (
"slices"
"golang.org/x/sys/unix"
)
// nonPersistentFilesystems returns the device ID of every mounted
// filesystem that is not persistent, with its type, from the system's
// list of mounted filesystems. The device ID is read with lstat on the
// mount point. Where one filesystem is mounted over another on the same
// mount point, the later one in the list counts. A mount point that
// lstat cannot reach is left out.
func nonPersistentFilesystems() (map[uint64]string, error) {
// The first call counts the filesystems, the second lists them.
n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
if err != nil {
return nil, err
}
list := make([]unix.Statfs_t, n)
n, err = unix.Getfsstat(list, unix.MNT_NOWAIT)
if err != nil {
return nil, err
}
mounts := make(map[string]string, n)
for _, st := range list[:n] {
mounts[unix.ByteSliceToString(st.Mntonname[:])] =
unix.ByteSliceToString(st.Fstypename[:])
}
types := nonPersistentFilesystemTypes()
devices := make(map[uint64]string)
for mountPoint, fsType := range mounts {
if !slices.Contains(types, fsType) {
continue
}
if dev, ok := deviceOfPath(mountPoint); ok {
devices[dev] = fsType
}
}
return devices, nil
}
-36
View File
@@ -1,36 +0,0 @@
//go:build !darwin
package main
import (
"fmt"
"os"
"runtime"
)
// mountInfoPath is the file in which Linux lists the mounts the reading
// process sees.
const mountInfoPath = "/proc/self/mountinfo"
// nonPersistentFilesystems returns the device ID of every mounted
// filesystem that is not persistent, with its type: on Linux read from
// mountInfoPath (see readMountInfo), and on any other system none.
func nonPersistentFilesystems() (map[uint64]string, error) {
if runtime.GOOS != goosLinux {
return map[uint64]string{}, nil
}
f, err := os.Open(mountInfoPath)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
devices, err := readMountInfo(f)
if err != nil {
return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
}
return devices, nil
}
-583
View File
@@ -1,583 +0,0 @@
package main
import (
"bytes"
"errors"
"maps"
"os"
"path/filepath"
"runtime"
"slices"
"strings"
"testing"
"golang.org/x/sys/unix"
)
func TestDefaultExcludes(t *testing.T) {
t.Parallel()
// README "Rules for the walk": each system's list.
cases := map[string][]string{
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
goosDarwin: {
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
"/System/Volumes/VM", "/System/Volumes/Data",
},
goosWindows: {
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
`Users\*\AppData\Local\Temp`,
},
"freebsd": nil,
}
for goos, want := range cases {
if got := defaultExcludes(goos); !slices.Equal(got, want) {
t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
}
}
}
func TestIsDefaultExclude(t *testing.T) {
t.Parallel()
// A * matches one path component, no more and no fewer.
excludes := []string{"/scratch", "/home/*/cache"}
cases := map[string]bool{
"/scratch": true,
"/scratch/x": false,
"/home/u/cache": true,
"/home/u/v/cache": false,
"/home/cache": false,
}
for path, want := range cases {
if got := isDefaultExclude(path, excludes); got != want {
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
}
}
}
func TestReadMountInfo(t *testing.T) {
t.Parallel()
// The second-to-last line has no optional fields. The tmpfs on
// /data/a/b is hidden by the root filesystem's /srv, mounted over
// /data/a after it: the tmpfs still gives its own device, and the
// root filesystem's device is not listed.
mountinfo := strings.Join([]string{
"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
"25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw",
"26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw",
"27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
}, "\n")
got, err := readMountInfo(strings.NewReader(mountinfo))
if err != nil {
t.Fatal(err)
}
want := map[uint64]string{
unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs",
unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs",
}
if !maps.Equal(got, want) {
t.Errorf("readMountInfo = %v, want %v", got, want)
}
for _, line := range []string{
"22 1 8:1 / / rw,relatime\n",
"22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n",
} {
_, err = readMountInfo(strings.NewReader(line))
if !errors.Is(err, errMountInfoLine) {
t.Errorf("readMountInfo(%q) = %v, want %v", line, err,
errMountInfoLine)
}
}
}
func TestNonPersistentFilesystems(t *testing.T) {
t.Parallel()
if runtime.GOOS != goosLinux {
t.Skip("the test knows a mount only on Linux: /proc")
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfPath("/proc")
if !ok {
t.Fatal("no device for /proc")
}
if got := nonPersistent[dev]; got != "proc" {
t.Errorf("filesystem of /proc = %q, want %q", got, "proc")
}
}
func TestDefaultExcludesHelp(t *testing.T) {
t.Parallel()
for _, goos := range []string{goosLinux, goosDarwin, goosWindows} {
help := defaultExcludesHelp(goos)
for _, dir := range defaultExcludes(goos) {
if !strings.Contains(help, "\n "+dir+"\n") {
t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
}
}
// Only Linux and macOS list their mounted filesystems.
for _, fsType := range nonPersistentFilesystemTypes() {
listed := strings.Contains(help, " "+fsType+",") ||
strings.Contains(help, " "+fsType+"\n")
if listed != (goos != goosWindows) {
t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
}
}
}
want := "This system has no default excludes.\n"
if got := defaultExcludesHelp("freebsd"); got != want {
t.Errorf("help for freebsd = %q, want %q", got, want)
}
// scan's help shows those of the system it runs on.
var stdout, stderr bytes.Buffer
run([]string{cmdScan, "-h"}, &stdout, &stderr)
help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
if !strings.Contains(stderr.String(), help) {
t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
}
}
// TestScanDoesNotEnterDefaultExclude checks that the walk does not
// enter a default exclude below an operand. It says so on a line of its
// own and does not count it as skipped. The records below it count as
// outside the operands: the record of a file gone from it is kept, and
// its size joins the size census, while a file added to it is not
// recorded.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
excluded := filepath.Join(dir, "tmp")
// A scan with no default excludes records the file in excluded.
gone := writeFile(t, excluded, "gone", pattern(1, 100))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
unseen := writeFile(t, excluded, "new", pattern(2, 10))
// The same size as gone, so it is hashed only if the census holds the
// size of gone's record.
added := writeFile(t, dir, "added", pattern(3, 100))
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.skipped != 0 {
t.Errorf("skipped = %d, want 0", st.skipped)
}
recs := dbRecords(t, db)
recordByPath(t, recs, gone)
if recordByPath(t, recs, added).head == "" {
t.Errorf("%s not hashed: the census lacks the size of %s",
added, gone)
}
if slices.Contains(recordPaths(recs), unseen) {
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
}
line := "\nscan: skipped " + excluded + " (default exclude)\n"
if !strings.Contains(stderr(), line) {
t.Errorf("stderr = %q, want %q", stderr(), line)
}
}
// TestScanWalksOperandInDefaultExclude checks that an operand at or
// inside a default exclude is walked, also when it lies under another
// operand, whose walk does not enter the default exclude.
func TestScanWalksOperandInDefaultExclude(t *testing.T) {
t.Parallel()
dir := t.TempDir()
excluded := filepath.Join(dir, "tmp")
f := writeFile(t, excluded, "sub/f", pattern(1, 10))
sub := filepath.Dir(f)
cases := map[string][]string{
"at it": {excluded},
"inside it": {sub},
"inside it under an operand": {dir, sub},
}
for name, roots := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
db := openTestDB(t)
st, err := syncScan(t.Context(), db, roots, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.added != 1 {
t.Errorf("added = %d, want %s recorded once", st.added, f)
}
recordByPath(t, dbRecords(t, db), f)
})
}
}
// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default
// exclude given as an operand under another operand, which the walk of
// that other operand does not enter, gets no line saying it was
// skipped: it is walked as an operand of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
excluded := filepath.Join(dir, "tmp")
f := writeFile(t, excluded, "f", pattern(1, 10))
st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.added != 1 {
t.Errorf("added = %d, want %s recorded once", st.added, f)
}
recordByPath(t, dbRecords(t, db), f)
if strings.Contains(stderr(), "scan: skipped") {
t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
}
}
// TestRunScanTempDirOperand scans a tree in the tests' temporary
// directory, which on Linux lies inside the default exclude /tmp unless
// TMPDIR names another place, with the system's own default excludes:
// the scan walks the whole tree and reports no default exclude.
func TestRunScanTempDirOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := t.TempDir()
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
t.Skip(dir + " is not inside a default exclude")
}
f := writeFile(t, dir, "sub/f", pattern(1, 10))
stderr := scanOK(t, dir)
if strings.Contains(stderr, "scan: skipped") {
t.Errorf("stderr = %q, want no line saying a path was skipped",
stderr)
}
assertRecorded(t, path, f, true)
}
// TestRunScanNoDefaultExcludes scans /var through the command. Its
// /var/tmp, on a persistent filesystem, is a default exclude on Linux:
// without --no-default-excludes the walk does not enter it and says so,
// and with that flag the walk records a file the test puts there.
func TestRunScanNoDefaultExcludes(t *testing.T) {
if runtime.GOOS != goosLinux {
t.Skip("only Linux has /var/tmp among its default excludes")
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
rules := walkRules{nonPersistent: nonPersistent}
for _, dir := range []string{"/var", "/var/tmp"} {
fi, lstatErr := os.Lstat(dir)
if lstatErr != nil || !fi.IsDir() ||
rules.nonPersistentLine(dir, fi) != "" {
t.Skip(dir + " is not a directory on a persistent filesystem here")
}
}
//nolint:usetesting // the directory must be inside /var/tmp
tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-")
if err != nil {
t.Skip("cannot write to /var/tmp here: " + err.Error())
}
t.Cleanup(func() { _ = os.RemoveAll(tmp) })
f := writeFile(t, tmp, "f", pattern(1, 10))
path := testDBPath(t)
t.Setenv(databaseEnv, path)
line := "scan: skipped /var/tmp (default exclude)\n"
if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
}
assertRecorded(t, path, f, false)
stderr := scanOK(t, "--no-default-excludes", "/var")
if strings.Contains(stderr, line) {
t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q",
stderr, line)
}
assertRecorded(t, path, f, true)
}
// TestRunScanSkipsNonPersistentOperand scans, through the command, a
// directory on a filesystem that is not persistent, with and without
// --no-default-excludes: either way it is skipped, with a line saying
// so, and nothing in it is recorded.
func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
shm, fsType, _ := shmTempDir(t)
writeFile(t, shm, "f", pattern(1, 10))
t.Setenv(databaseEnv, testDBPath(t))
line := "scan: skipped " + shm + " (" + fsType +
", not a persistent filesystem)\n"
summary := "scan: 0 files seen"
for _, flags := range [][]string{nil, {"--no-default-excludes"}} {
stderr := scanOK(t, append(flags, shm)...)
if !strings.Contains(stderr, line) ||
!strings.Contains(stderr, summary) {
t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr,
line, summary)
}
}
}
// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
// that is not persistent beside one on the tests' temporary directory.
// The records an earlier scan left in it count as outside the operands:
// the record of a file gone from it is kept, and its size joins the size
// census, while a file added to it is not recorded, and its size does
// not join the census.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
shm, fsType, rules := shmTempDir(t)
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
// A scan that, without the list of filesystems, records gone.
gone := writeFile(t, shm, "gone", pattern(1, 100))
syncTree(t, db, shm)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
unseen := writeFile(t, shm, "new", pattern(2, 10))
// added has the size of gone, and lonely that of unseen: each is
// hashed only if the census holds that size.
added := writeFile(t, dir, "added", pattern(3, 100))
lonely := writeFile(t, dir, "lonely", pattern(4, 10))
st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
if err != nil {
t.Fatal(err)
}
if st.skipped != 0 {
t.Errorf("skipped = %d, want 0", st.skipped)
}
recs := dbRecords(t, db)
recordByPath(t, recs, gone)
if recordByPath(t, recs, added).head == "" {
t.Errorf("%s not hashed: the census lacks the size of %s",
added, gone)
}
if recordByPath(t, recs, lonely).head != "" {
t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen)
}
if slices.Contains(recordPaths(recs), unseen) {
t.Errorf("%s recorded", unseen)
}
line := "scan: skipped " + shm + " (" + fsType +
", not a persistent filesystem)\n"
if !strings.Contains(stderr(), line) {
t.Errorf("stderr = %q, want %q", stderr(), line)
}
}
// TestScanContentSkipsNonPersistent checks that the content phase does
// not read a file on a filesystem that is not persistent, though an
// earlier scan left a record for it: it is treated as a changed file,
// so its record is left as it is and its identical copy in another
// tree, having no other match, is not read either.
func TestScanContentSkipsNonPersistent(t *testing.T) {
t.Parallel()
shm, _, rules := shmTempDir(t)
db := openTestDB(t)
// A scan that, without the list of filesystems, records a with a
// head and tail but no content hash.
a := sparseFileWithoutMatch(t, shm, "a", headTailMin)
syncTree(t, db, shm)
before := recordByPath(t, dbRecords(t, db), a)
if before.head == "" || before.content != "" {
t.Fatalf("after scanning %s: %+v, want head and tail only", shm, before)
}
b := sparseFile(t, t.TempDir(), "b", headTailMin)
st, err := syncScan(t.Context(), db, []string{filepath.Dir(b)}, 1, false,
rules)
if err != nil {
t.Fatal(err)
}
if st.skipped != 0 {
t.Errorf("skipped = %d, want 0", st.skipped)
}
recs := dbRecords(t, db)
if r := recordByPath(t, recs, a); r != before {
t.Errorf("record = %+v, want it left as %+v", r, before)
}
if r := recordByPath(t, recs, b); r.content != "" {
t.Errorf("%s: content = %q, want none: its only match is on %s",
b, r.content, shm)
}
}
// TestWalkLeavesOutNonPersistent starts the walk itself in a directory
// on a filesystem that is not persistent, which a scan never does: it
// drops such an operand first. Below a directory on a persistent
// filesystem, one on another takes a mount, which a test cannot make.
// The walk enters no directory and records no file there, and names
// each.
func TestWalkLeavesOutNonPersistent(t *testing.T) {
t.Parallel()
shm, fsType, rules := shmTempDir(t)
f := writeFile(t, shm, "f", pattern(1, 10))
sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10)))
var leftOut []string
for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) {
if ev.rec.path != "" {
t.Errorf("walk recorded %s", ev.rec.path)
}
if ev.leftOut == "" {
continue
}
leftOut = append(leftOut, ev.leftOut)
want := "scan: skipped " + ev.leftOut + " (" + fsType +
", not a persistent filesystem)"
if ev.warn != want {
t.Errorf("line = %q, want %q", ev.warn, want)
}
}
slices.Sort(leftOut)
if want := []string{f, sub}; !slices.Equal(leftOut, want) {
t.Errorf("walk left out %q, want %q", leftOut, want)
}
}
// shmTempDir returns a new directory in /dev/shm, removed when the test
// ends, the type of its filesystem, and the rules holding the real
// filesystems that are not persistent. It skips the test where /dev/shm
// is on a persistent filesystem, or cannot be written to.
func shmTempDir(t *testing.T) (string, string, walkRules) {
t.Helper()
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfPath("/dev/shm")
fsType, found := nonPersistent[dev]
if !ok || !found {
t.Skip("/dev/shm is not a memory-backed filesystem here")
}
//nolint:usetesting // the directory must be inside /dev/shm
dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-")
if err != nil {
t.Skip("cannot write to /dev/shm here: " + err.Error())
}
t.Cleanup(func() { _ = os.RemoveAll(dir) })
return dir, fsType, walkRules{nonPersistent: nonPersistent}
}
// assertRecorded checks that the database at dbPath holds a record for
// path when want is set, and none when it is not.
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
t.Helper()
db, err := openDB(dbPath, reportParams)
if err != nil {
t.Fatal(err)
}
defer func() { _ = db.Close() }()
if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want {
t.Errorf("%s recorded: %v, want %v", path, got, want)
}
}
-2
View File
@@ -24,5 +24,3 @@ require (
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect modernc.org/memory v1.11.0 // indirect
) )
retract v0.0.1 // early build without a database; does not work as the README describes
-128
View File
@@ -1,128 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
// infoPhase names one phase of info with its place among all of them,
// as "[2/2] count", for the phase's progress display. The list here is
// the one place the phases are counted.
func infoPhase(name string) string {
return phaseLabel([]string{openPhase, "count"}, name)
}
// runInfo implements the info subcommand: it prints the database's
// details on stdout (see writeInfo). It opens the database as report
// does, read-only and without the scan lock, and never touches the
// scanned filesystem. It names the database on stderr before it opens
// it. Any database problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and count (count the records,
// with one query). Each display ends before the next phase starts, and
// before an error is returned to be printed. The details are written
// once the last display has ended, so no display is drawn while they
// are written.
func runInfo(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "info: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: infoPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil {
return err
}
defer func() { _ = db.Close() }()
// One query counts everything, so the phase never has a total.
prog = newProgress(progressParams{
label: infoPhase("count"), total: -1,
waitingFor: countingRecords,
})
counts, err := readDatabaseCounts(ctx, db)
prog.finish()
if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err)
}
return writeInfo(stdout, dbPath, source, counts)
}
// writeInfo writes the details of the database at dbPath to stdout,
// one per line, as a name, a tab and the value (README "`info` mode"):
// the path and source as databasePath returns them, the sizes of the
// database file and of its -wal file when there is one, when the
// database file was last changed, and the schema version and record
// counts that readDatabaseCounts read.
func writeInfo(stdout io.Writer, dbPath, source string,
counts databaseCounts,
) error {
// SQLite keeps the -wal file beside the database file, named from
// its path with symlinks resolved.
file, err := filepath.EvalSymlinks(dbPath)
if err != nil {
return fmt.Errorf("database: %w", err)
}
dbFile, err := os.Stat(file)
if err != nil {
return fmt.Errorf("database: %w", err)
}
walFile, err := os.Stat(file + "-wal")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("database: %w", err)
}
var b strings.Builder
fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source)
fmt.Fprintf(&b, "database file size\t%d (%s)\n",
dbFile.Size(), humanBytes(dbFile.Size()))
if walFile != nil {
fmt.Fprintf(&b, "-wal file size\t%d (%s)\n",
walFile.Size(), humanBytes(walFile.Size()))
}
fmt.Fprintf(&b, "modified\t%s\n",
dbFile.ModTime().UTC().Format(time.RFC3339))
fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
fmt.Fprintf(&b, "records\t%d\n", counts.records)
fmt.Fprintf(&b, "records with head and tail hashes\t%d\n",
counts.headTail)
fmt.Fprintf(&b, "records with content hash\t%d\n", counts.content)
fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
counts.size, humanBytes(counts.size))
_, err = io.WriteString(stdout, b.String())
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
return nil
}
-168
View File
@@ -1,168 +0,0 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// infoOpenLine is the plain line of info's open phase when it opens the
// database at once; it prints it as the phase starts and as it ends.
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"eta unknown"
// infoCountLine is the plain line of info's count phase when the count
// takes under a second; it prints it as the phase starts and as it
// ends.
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
"eta unknown"
// infoRecs are four records: a duplicate pair with every hash, a file
// of 20 MiB with head and tail hashes but no content hash yet, and a
// file never hashed.
func infoRecs() []scanRec {
return []scanRec{
{size: 5, head: "h", tail: "t", content: "c", path: "/d/a"},
{size: 5, head: "h", tail: "t", content: "c", path: "/d/b"},
{size: 20 << 20, head: "bh", tail: "bt", path: "/d/big"},
{size: 7, path: "/d/unique"},
}
}
// TestRunInfo runs info on an empty and on a filled database, with
// stdout and stderr the same file that is not a terminal, and checks
// everything written there: the line naming the database, each phase's
// line as it starts and as it ends, and then the details, each on a
// line of its own. No -wal file is beside a database scan has closed.
func TestRunInfo(t *testing.T) {
cases := map[string]struct {
recs []scanRec
want string // the details after the database file's time
}{
"empty": {
want: "schema version\t1\n" +
"records\t0\n" +
"records with head and tail hashes\t0\n" +
"records with content hash\t0\n" +
"size of recorded files\t0 (0 B)\n",
},
"filled": {
recs: infoRecs(),
want: "schema version\t1\n" +
"records\t4\n" +
"records with head and tail hashes\t3\n" +
"records with content hash\t2\n" +
"size of recorded files\t20971537 (20.0 MiB)\n",
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
path := seedDatabase(t, tc.recs)
t.Setenv(databaseEnv, path)
file, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
stderr := captureStderr(t)
code := run([]string{cmdInfo}, os.Stderr, os.Stderr)
if code != exitOK {
t.Fatalf("run(info) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
named := path + " (from " + databaseEnv + ")\n"
want := "info: database " + named +
infoOpenLine + "\n" + infoOpenLine + "\n" +
infoCountLine + "\n" + infoCountLine + "\n" +
"database\t" + named +
fmt.Sprintf("database file size\t%d (%s)\n",
file.Size(), humanBytes(file.Size())) +
"modified\t" + file.ModTime().UTC().Format(time.RFC3339) +
"\n" + tc.want
if got := stderr(); got != want {
t.Errorf("stdout and stderr = %q, want %q", got, want)
}
})
}
}
// TestRunInfoDuringScan runs info while a scan holds the lock and has
// the database open in WAL mode, with records committed to the -wal
// file only. info must count those records and show the -wal file's
// size, also when the database path is a symlink: SQLite keeps the
// -wal file beside the symlink's target.
func TestRunInfoDuringScan(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
db, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
err = applyChanges(t.Context(), db, infoRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
wal, err := os.Stat(path + "-wal")
if err != nil {
t.Fatal(err)
}
for _, named := range []string{path, link} {
t.Run(filepath.Base(named), func(t *testing.T) {
t.Setenv(databaseEnv, named)
stdout := runStdout(t, cmdInfo)
for _, want := range []string{
"database\t" + named + " (from " + databaseEnv + ")\n",
fmt.Sprintf("\n-wal file size\t%d (%s)\n",
wal.Size(), humanBytes(wal.Size())),
"\nrecords\t4\n",
} {
if !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
}
})
}
}
func TestRunInfoNeedsOnlyReadAccess(t *testing.T) {
// README "`info` mode": like report, info needs only read access to
// the database file and none to its directory.
path := testDBPath(t)
t.Setenv(databaseEnv, path)
scanFixture(t)
makeReadOnly(t, path)
stdout := runStdout(t, cmdInfo)
if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
if strings.Contains(stdout, "-wal") {
t.Errorf("stdout = %q, want no -wal file between scans", stdout)
}
}
+32 -84
View File
@@ -8,14 +8,13 @@
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB // under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
// or larger. scan maintains a persistent SQLite database of file // or larger. scan maintains a persistent SQLite database of file
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) // signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
// that the other subcommands read. // that the reporting subcommands read.
// //
// Usage: // Usage:
// //
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... // sfdupes scan [--workers N] [-x] PATH...
// sfdupes report > dupes.tsv // sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv // sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes --version // sfdupes --version
// //
// See README.md for the complete specification. // See README.md for the complete specification.
@@ -28,7 +27,6 @@ import (
"io" "io"
"os" "os"
"runtime" "runtime"
"runtime/debug"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@@ -46,7 +44,6 @@ const (
cmdScan = "scan" cmdScan = "scan"
cmdReport = "report" cmdReport = "report"
cmdTrees = "trees" cmdTrees = "trees"
cmdInfo = "info"
) )
// errNoSubcommand is returned by the root command when it is invoked // errNoSubcommand is returned by the root command when it is invoked
@@ -59,32 +56,10 @@ var errNoSubcommand = errors.New("no subcommand")
var errWorkersBelowOne = errors.New("--workers must be at least 1") var errWorkersBelowOne = errors.New("--workers must be at least 1")
// Version is the build version, injected at link time via -ldflags // Version is the build version, injected at link time via -ldflags
// (see the Makefile); empty when the linker set none, as for a binary // (see the Makefile); "dev" for a plain go build.
// built by go install. buildVersion turns it into what --version prints.
// //
//nolint:gochecknoglobals // written only by the linker //nolint:gochecknoglobals // written only by the linker
var Version string var Version = "dev"
// buildVersion returns the version that sfdupes --version prints
// (README "Install"): the version the linker set, when it set one;
// otherwise the main module's version that Go recorded in the binary,
// a tag such as v0.0.2 or the pseudo-version of an @main build; and
// "dev" when neither is known. info is what debug.ReadBuildInfo
// returns, nil when the binary carries none. Go records "(devel)" for
// a build that has no version, such as one inside a copy of the source
// without git.
func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
if linkerVersion != "" {
return linkerVersion
}
if info == nil || info.Main.Version == "" ||
info.Main.Version == "(devel)" {
return "dev"
}
return info.Main.Version
}
func main() { func main() {
// Once the reader of a stdout pipe has gone, as in "sfdupes report | // Once the reader of a stdout pipe has gone, as in "sfdupes report |
@@ -98,8 +73,8 @@ func main() {
// exit code. It is the program's single exit point: the subcommands // exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup — // return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL — // above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees and info subcommands // runs before the process ends. The report and trees subcommands write
// write their data to stdout. // their data to stdout.
func run(args []string, stdout, stderr io.Writer) int { func run(args []string, stdout, stderr io.Writer) int {
// A nil slice makes cobra fall back to os.Args, which would let a // A nil slice makes cobra fall back to os.Args, which would let a
// test binary's own flags reach the command tree. // test binary's own flags reach the command tree.
@@ -139,10 +114,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
var showVersion bool var showVersion bool
printVersion := runE(func(context.Context, []string) error { printVersion := runE(func(context.Context, []string) error {
info, _ := debug.ReadBuildInfo() _, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version)
_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
buildVersion(Version, info))
if err != nil { if err != nil {
return fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
@@ -176,6 +148,30 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
root.Flags().BoolVarP(&showVersion, "version", "v", false, root.Flags().BoolVarP(&showVersion, "version", "v", false,
"print the version to stdout") "print the version to stdout")
var (
scanWorkers int
scanOneFS bool
)
scanCmd := &cobra.Command{
Use: cmdScan + " [--workers N] [-x] PATH...",
Short: "Walk trees and synchronize the scan database",
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, _ []string) error {
return checkScanWorkers(cmd, scanWorkers)
},
RunE: runE(func(ctx context.Context, args []string) error {
ctx, stop := interruptContext(ctx)
defer stop()
return runScan(ctx, args, scanWorkers, scanOneFS)
}),
}
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
"concurrent workers for the walk, hash, and content phases")
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
"do not cross filesystem boundaries")
reportCmd := &cobra.Command{ reportCmd := &cobra.Command{
Use: cmdReport, Use: cmdReport,
Short: "Read the scan database and print the file-level duplicates report", Short: "Read the scan database and print the file-level duplicates report",
@@ -194,59 +190,11 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
}), }),
} }
infoCmd := &cobra.Command{ root.AddCommand(scanCmd, reportCmd, treesCmd)
Use: cmdInfo,
Short: "Read the scan database and print its details",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runInfo(ctx, stdout)
}),
}
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
return root return root
} }
// newScanCommand builds the scan subcommand, with its --workers, -x and
// --no-default-excludes flags. Its help lists the default excludes of
// the system it runs on, and the filesystem types it never records
// anything on.
func newScanCommand() *cobra.Command {
var (
scanWorkers int
scanOneFS bool
scanNoDefaultExcludes bool
)
short := "Walk trees and synchronize the scan database"
scanCmd := &cobra.Command{
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
Short: short,
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, _ []string) error {
return checkScanWorkers(cmd, scanWorkers)
},
RunE: runE(func(ctx context.Context, args []string) error {
ctx, stop := interruptContext(ctx)
defer stop()
return runScan(ctx, args, scanWorkers, scanOneFS,
scanNoDefaultExcludes)
}),
}
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
"concurrent workers for the walk, hash, and content phases")
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
"do not cross filesystem boundaries")
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
false, "enter the default excludes, the paths listed above")
return scanCmd
}
// checkScanWorkers rejects a scan --workers value below 1. That is a // checkScanWorkers rejects a scan --workers value below 1. That is a
// usage error reported in one line: cobra prints the returned message // usage error reported in one line: cobra prints the returned message
// without the usage text, and run exits 2. // without the usage text, and run exits 2.
+20 -456
View File
@@ -4,12 +4,10 @@ import (
"bytes" "bytes"
"context" "context"
"errors" "errors"
"fmt"
"io" "io"
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"runtime/debug"
"slices" "slices"
"strconv" "strconv"
"strings" "strings"
@@ -20,36 +18,6 @@ import (
// error prints and a runtime failure must not. // error prints and a runtime failure must not.
const usageMarker = "Usage:" const usageMarker = "Usage:"
// TestMain stops the tests at once, saying why, when their temporary
// directory is not on a persistent filesystem: scan records nothing
// there (README "Rules for the walk"), so most scan tests would fail
// without saying why.
func TestMain(m *testing.M) {
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err)
os.Exit(1)
}
dir := os.TempDir()
fi, err := os.Stat(dir)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
dev, ok := deviceOfInfo(fi)
if fsType, found := nonPersistent[dev]; ok && found {
fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+
"and scan records nothing there: set TMPDIR to a directory on a "+
"persistent filesystem\n", dir, fsType)
os.Exit(1)
}
os.Exit(m.Run())
}
// walSuffixes are the SQLite sidecar files a WAL-mode database keeps // walSuffixes are the SQLite sidecar files a WAL-mode database keeps
// while it is open. A clean close checkpoints the WAL and removes // while it is open. A clean close checkpoints the WAL and removes
// both; finding either afterwards means the database was never closed. // both; finding either afterwards means the database was never closed.
@@ -316,38 +284,23 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) {
} }
func TestRunMissingDatabaseIsFatal(t *testing.T) { func TestRunMissingDatabaseIsFatal(t *testing.T) {
// README §Database: report, trees and info need an existing // README §Database: report and trees need an existing database; a
// database; a missing one exits 1 with a message telling the user to // missing one exits 1 with a message telling the user to run scan.
// run scan, after the line naming the database. for _, name := range []string{cmdReport, cmdTrees} {
openLines := map[string]string{
cmdReport: reportOpenLine,
cmdTrees: treesOpenLine,
cmdInfo: infoOpenLine,
}
for name, openLine := range openLines {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
var stdout bytes.Buffer var stdout, stderr bytes.Buffer
// run gets os.Stderr, where the subcommand prints its line, code := run([]string{name}, &stdout, &stderr)
// so the capture shows the two in the order printed.
stderr := captureStderr(t)
code := run([]string{name}, &stdout, os.Stderr)
if code != exitFatal { if code != exitFatal {
t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
} }
// Each shows its open phase, as it starts and as it ends, want := "sfdupes: " + path + ": no database (run \"sfdupes " +
// before the error.
want := name + ": database " + path + " (from " + databaseEnv +
")\n" + openLine + "\n" + openLine + "\n" +
"sfdupes: " + path + ": no database (run \"sfdupes " +
"scan\" first, or set " + databaseEnv + ")\n" "scan\" first, or set " + databaseEnv + ")\n"
if got := stderr(); got != want { if got := stderr.String(); got != want {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
@@ -358,256 +311,6 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) {
} }
} }
func TestRunNamesRelativeDatabase(t *testing.T) {
// README §Database: each subcommand's first line on stderr names the
// database, a relative SFDUPES_DATABASE made absolute against the
// working directory. scan calls it new until a scan has created it.
path := testDBPath(t)
t.Chdir(filepath.Dir(path))
t.Setenv(databaseEnv, filepath.Base(path))
dir := t.TempDir()
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("first scan stderr = %q, want it to start %q", got, want)
}
want = "scan: database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("second scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + path + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
// README §Database: the path names the file exactly. With link a
// symlink to the directory real/sub, link/../db.sqlite names
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
// use that file, relative or absolute, and its first line must name
// the path as set, with the working directory put in front of a
// relative one.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Chdir(dir)
err = os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "real", "sub"),
filepath.Join(dir, "link"))
if err != nil {
t.Fatal(err)
}
named := dir + "/link/../db.sqlite"
from := " (from " + databaseEnv + ")\n"
tree := t.TempDir()
t.Setenv(databaseEnv, "link/../db.sqlite")
want := "scan: new database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("relative scan stderr = %q, want it to start %q", got, want)
}
// The absolute path names the database the relative one created.
t.Setenv(databaseEnv, named)
want = "scan: database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("absolute scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + named + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
_, err = os.Stat(filepath.Join(dir, "real", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
// With no file at the cleaned path, report and trees succeeded on
// the database in real.
_, err = os.Stat(filepath.Join(dir, "db.sqlite"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("database at the cleaned path: stat error %v, want it "+
"missing", err)
}
}
func TestRunScanFromSymlinkedWorkingDirectory(t *testing.T) {
// README §Database: a relative path gets the working directory, its
// symlinks resolved, put in front of it. With home/proj a symlink
// to data/proj, entered as the working directory through the
// symlink, ../new/db.sqlite names data/new/db.sqlite. scan must
// create that directory and file, and nothing beside home/proj.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Join(dir, "data", "proj"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(dir, "home"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
if err != nil {
t.Fatal(err)
}
// t.Chdir also sets PWD to the path through the symlink, as a
// shell does after cd.
t.Chdir(filepath.Join(dir, "home", "proj"))
t.Setenv(databaseEnv, "../new/db.sqlite")
want := "scan: new database " + dir + "/data/proj/../new/db.sqlite" +
" (from " + databaseEnv + ")\n"
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
_, err = os.Lstat(filepath.Join(dir, "home", "new"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("home/new: stat error %v, want it missing", err)
}
}
func TestRunEscapesDatabasePath(t *testing.T) {
// README §Database: each subcommand's first line escapes the
// database path like the paths in a report, so a tab, newline or
// backslash in it keeps the line one line.
dir := t.TempDir()
t.Setenv(databaseEnv, dir+"/a\tb\nc\\d.sqlite")
escaped := dir + `/a\tb\nc\\d.sqlite`
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + escaped + from
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + escaped + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunNamesDefaultDatabase(t *testing.T) {
// README §Database: with SFDUPES_DATABASE empty, each subcommand's
// first line on stderr names the default database. A database may
// exist at the default path on the machine running the test, so
// nothing here may create or read one: scan gets an operand that
// does not exist, which stops it before it takes the lock, and
// report, trees and info a cancelled context, which stops them
// before they read anything. They show their open phase after the
// line.
t.Setenv(databaseEnv, "")
// Whether scan calls the database new depends on that machine too.
scanWant := "scan: database " + defaultDatabasePath + " (default)\n"
_, err := os.Stat(defaultDatabasePath)
if errors.Is(err, fs.ErrNotExist) {
scanWant = "scan: new database " + defaultDatabasePath +
" (default)\n"
}
missing := filepath.Join(t.TempDir(), "nope")
cases := map[string]struct {
run func() error
want string
}{
cmdScan: {
run: func() error {
return runScan(t.Context(), []string{missing}, 1, false, false)
},
want: scanWant,
},
cmdReport: {
run: func() error {
return runReport(cancelledContext(t), io.Discard)
},
want: "report: database " + defaultDatabasePath + " (default)\n" +
reportOpenLine + "\n" + reportOpenLine + "\n",
},
cmdTrees: {
run: func() error {
return runTrees(cancelledContext(t), io.Discard)
},
want: "trees: database " + defaultDatabasePath + " (default)\n" +
treesOpenLine + "\n" + treesOpenLine + "\n",
},
cmdInfo: {
run: func() error {
return runInfo(cancelledContext(t), io.Discard)
},
want: "info: database " + defaultDatabasePath + " (default)\n" +
infoOpenLine + "\n" + infoOpenLine + "\n",
},
}
for name, tc := range cases {
stderr := captureStderr(t)
err = tc.run()
if err == nil {
t.Errorf("%s: error = nil, want it stopped", name)
}
if got := stderr(); got != tc.want {
t.Errorf("%s stderr = %q, want %q", name, got, tc.want)
}
}
}
func TestRunUsageErrors(t *testing.T) { func TestRunUsageErrors(t *testing.T) {
// Usage errors keep exiting 2 with cobra's own report on stderr. // Usage errors keep exiting 2 with cobra's own report on stderr.
cases := map[string]struct { cases := map[string]struct {
@@ -618,7 +321,6 @@ func TestRunUsageErrors(t *testing.T) {
"scan without paths": {[]string{cmdScan}, usageMarker}, "scan without paths": {[]string{cmdScan}, usageMarker},
"report with args": {[]string{cmdReport, "x"}, usageMarker}, "report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker}, "trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker}, "unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
"unknown subcommand": {[]string{"nope"}, "unknown command"}, "unknown subcommand": {[]string{"nope"}, "unknown command"},
} }
@@ -715,8 +417,6 @@ func TestRunVersion(t *testing.T) {
// README §Subcommands: the version is one line on stdout, with // README §Subcommands: the version is one line on stdout, with
// nothing on stderr, and exits 0. // nothing on stderr, and exits 0.
info, _ := debug.ReadBuildInfo()
for _, arg := range []string{"--version", "-v"} { for _, arg := range []string{"--version", "-v"} {
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
@@ -725,7 +425,7 @@ func TestRunVersion(t *testing.T) {
t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
} }
want := "sfdupes " + buildVersion(Version, info) + "\n" want := "sfdupes " + Version + "\n"
if got := stdout.String(); got != want { if got := stdout.String(); got != want {
t.Errorf("run(%s) stdout = %q, want %q", arg, got, want) t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
} }
@@ -736,40 +436,6 @@ func TestRunVersion(t *testing.T) {
} }
} }
func TestBuildVersion(t *testing.T) {
t.Parallel()
// README "Install": the linker's version wins; otherwise the main
// module's version that Go recorded; "dev" when neither is known.
tagged := &debug.BuildInfo{Main: debug.Module{Version: "v0.0.2"}}
pseudo := &debug.BuildInfo{Main: debug.Module{
Version: "v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1",
}}
devel := &debug.BuildInfo{Main: debug.Module{Version: "(devel)"}}
cases := []struct {
name string
linker string
info *debug.BuildInfo
want string
}{
{"linker set", "v0.0.2-3-g4e4a94d", tagged, "v0.0.2-3-g4e4a94d"},
{"linker set, no build info", "4e4a94d", nil, "4e4a94d"},
{"build info with a tag", "", tagged, "v0.0.2"},
{"build info with a pseudo-version", "", pseudo,
"v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1"},
{"(devel)", "", devel, "dev"},
{"no build info", "", nil, "dev"},
}
for _, c := range cases {
if got := buildVersion(c.linker, c.info); got != c.want {
t.Errorf("%s: buildVersion(%q, ...) = %q, want %q",
c.name, c.linker, got, c.want)
}
}
}
func TestRunVersionWriteFailureIsFatal(t *testing.T) { func TestRunVersionWriteFailureIsFatal(t *testing.T) {
t.Parallel() t.Parallel()
@@ -995,16 +661,10 @@ func TestRunReportSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want) t.Errorf("stdout = %q, want %q", got, want)
} }
// Its phases come between the two lines; see want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " +
// TestRunReportProgressNamesEachPhase.
first := "report: database " + path + " (from " + databaseEnv + ")\n"
last := "\nreport: 2 records read, 1 duplicate groups, 1 dupe files, " +
"300 B reclaimable\n" "300 B reclaimable\n"
if got := stderr(); got != want {
if got := stderr(); !strings.HasPrefix(got, first) || t.Errorf("stderr = %q, want %q", got, want)
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
} }
assertNoSidecars(t, path) assertNoSidecars(t, path)
@@ -1034,16 +694,10 @@ func TestRunTreesSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want) t.Errorf("stdout = %q, want %q", got, want)
} }
// Its phases come between the two lines; see want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " +
// TestRunTreesProgressNamesEachPhase. "300 B reclaimable\n"
first := "trees: database " + path + " (from " + databaseEnv + ")\n" if got := stderr(); got != want {
last := "\ntrees: 2 records read, 1 duplicate tree groups, " + t.Errorf("stderr = %q, want %q", got, want)
"1 dupe trees, 300 B reclaimable\n"
if got := stderr(); !strings.HasPrefix(got, first) ||
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
} }
assertNoSidecars(t, path) assertNoSidecars(t, path)
@@ -1168,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
func TestRunSecondScanFails(t *testing.T) { func TestRunSecondScanFails(t *testing.T) {
// README §Database: while one scan holds the lock, a second scan // README §Database: while one scan holds the lock, a second scan
// fails at once, naming the lock file, without opening the // fails at once, naming the lock file, without creating the
// database. // database.
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
@@ -1194,99 +848,10 @@ func TestRunSecondScanFails(t *testing.T) {
t.Errorf("stdout = %q, want nothing (data only)", got) t.Errorf("stdout = %q, want nothing (data only)", got)
} }
// Taking the lock created the database file, empty; opening it as _, err := os.Stat(path)
// a database would have written to it. if !errors.Is(err, fs.ErrNotExist) {
info, err := os.Stat(path) t.Errorf("stat %s = %v, want the database not created", path, err)
if err != nil {
t.Fatal(err)
} }
if info.Size() != 0 {
t.Errorf("database size = %d, want 0 (not opened)", info.Size())
}
}
// assertSecondScanFails runs a scan against the database
// SFDUPES_DATABASE names while the test holds the lock on it. It fails
// the test unless that scan fails at once, naming the lock file beside
// path, the database's real path.
func assertSecondScanFails(t *testing.T, path string) {
t.Helper()
var stderr bytes.Buffer
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
if code != exitFatal {
t.Errorf("run(scan) = %d, want %d", code, exitFatal)
}
want := "sfdupes: another scan is running (lock held on " +
path + ".lock)\n"
if got := stderr.String(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
}
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
// README §Database: every path to one database names the same lock
// file, so while a scan holds it through one path, a scan through
// another fails, naming the lock file beside the database's real
// path.
t.Run("symlink to the database", func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
scanFixture(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
t.Setenv(databaseEnv, link)
assertSecondScanFails(t, path)
})
t.Run("symlink to its directory, no database yet", func(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link")
err := os.Symlink(filepath.Dir(path), link)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
assertSecondScanFails(t, path)
})
t.Run("symlink whose target does not exist yet", func(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
// The first scan takes the lock through the link before the
// database exists, then creates the database through it.
holdScanLock(t, link)
db, err := openScanDatabase(t.Context(), link)
if err != nil {
t.Fatal(err)
}
closeScanDatabase(t.Context(), db, link)
t.Setenv(databaseEnv, link)
assertSecondScanFails(t, path)
})
} }
func TestRunScanReleasesLock(t *testing.T) { func TestRunScanReleasesLock(t *testing.T) {
@@ -1335,7 +900,7 @@ func TestRunReportsDuringScan(t *testing.T) {
func TestRunStdoutClosedIsFatal(t *testing.T) { func TestRunStdoutClosedIsFatal(t *testing.T) {
// README §Error handling: a stdout write failure exits 1, reported // README §Error handling: a stdout write failure exits 1, reported
// in one line on stderr. // in one line on stderr.
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} { for _, name := range []string{cmdReport, cmdTrees} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t)) t.Setenv(databaseEnv, testDBPath(t))
@@ -1385,7 +950,6 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
cases := map[string]func(context.Context, io.Writer) error{ cases := map[string]func(context.Context, io.Writer) error{
cmdReport: runReport, cmdReport: runReport,
cmdTrees: runTrees, cmdTrees: runTrees,
cmdInfo: runInfo,
} }
for name, fn := range cases { for name, fn := range cases {
+111 -462
View File
@@ -2,333 +2,122 @@ package main
import ( import (
"fmt" "fmt"
"io"
"math"
"os" "os"
"slices"
"strings"
"sync"
"time" "time"
"github.com/schollz/progressbar/v3" "github.com/schollz/progressbar/v3"
"golang.org/x/term" "golang.org/x/term"
) )
// plainInterval is the time between progress lines while a phase runs, // plainInterval is the minimum time between progress lines when stderr
// when stderr is not a TTY. // is not a TTY.
const plainInterval = 5 * time.Second const plainInterval = 5 * time.Second
// redrawInterval is the time between redraws of the progress display on // barThrottle caps how often the TTY progress bar redraws.
// a TTY. const barThrottle = 100 * time.Millisecond
const redrawInterval = 100 * time.Millisecond
// rateWindow is the time the rate a display shows covers: the items // walkSpinnerType selects the progressbar library's spinner style used
// completed over the last rateWindow, which is the time since the line // for the walk pass, which has no known total.
// before when stderr is not a TTY. const walkSpinnerType = 14
const rateWindow = 5 * time.Second
// stallTime is how long a phase goes with no item completed before its
// line says it has stalled, and where it waits, in place of the rate.
const stallTime = 10 * time.Second
// spinnerType selects the progressbar library's spinner style.
const spinnerType = 14
// barWidth is the width of the progressbar library's bar between its
// two ends, as in the example line in README "Progress".
const barWidth = 10
// percentScale converts a fraction to a percentage. // percentScale converts a fraction to a percentage.
const percentScale = 100 const percentScale = 100
// fallbackWidth is the width taken for a terminal whose width cannot be
// read, as the progressbar library takes it.
const fallbackWidth = 80
// spinnerMargin is the columns of a spinner's line besides its text:
// the spinner and the three spaces the library draws around the text,
// and the terminal's last column, which every line stays clear of so
// that no terminal wraps it.
const spinnerMargin = 5
// barMargin is the columns of a bar's line besides its text: the
// percent and the bar the library draws after the text, the spaces
// around them, and the terminal's last column.
const barMargin = barWidth + 11
// countingRecords is what the display of a phase that reads the records
// says while it counts them, before it has its total.
const countingRecords = "counting the records"
// openPhase is the name of the phase in which report, trees and info
// open the database, and openingDatabase what its display says while
// it does.
const (
openPhase = "open"
openingDatabase = "opening the database"
)
// stderrIsTTY reports whether stderr is attached to a terminal. // stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool { func stderrIsTTY() bool {
return term.IsTerminal(int(os.Stderr.Fd())) return term.IsTerminal(int(os.Stderr.Fd()))
} }
// terminalWidth returns the width of the terminal on stderr, or // progress renders one scan pass's progress on stderr. On a TTY it
// fallbackWidth, as the progressbar library does, when it cannot read // delegates to the progressbar library (spinner style when the total is
// one. // unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
func terminalWidth() int { // stderr is not a TTY it emits no ANSI redraws: it prints a plain
width, _, err := term.GetSize(int(os.Stderr.Fd())) // one-line update as the pass starts, then no more often than every
if err != nil || width <= 0 { // plainInterval.
return fallbackWidth
}
return width
}
// phaseLabel names phase with its place among phases, as "[2/5] walk".
func phaseLabel(phases []string, phase string) string {
return fmt.Sprintf("[%d/%d] %s",
slices.Index(phases, phase)+1, len(phases), phase)
}
// progressParams describe one phase's progress display as it starts.
type progressParams struct {
label string // the phase and its place, from phaseLabel
unit string // what the phase counts, as "files"
total int64 // the items the phase will complete; -1 when not known
estimate bool // total is an estimate, which the count may pass
waitingFor string // what the phase waits for until setTotal is called
stalledOn string // where the phase waits, as "disk/db"; see stallTime
}
// drawnCount is the count of items completed as of one draw, and when
// that draw was.
type drawnCount struct {
at time.Time
count int64
}
// progress renders one phase's progress on stderr. On a TTY the
// progressbar library draws line, fitted to the terminal's width: after
// the library's spinner, or, for a phase with an exact total, before the
// library's bar when the terminal has room for the bar beside it. It is
// drawn again every redrawInterval, so its elapsed time keeps moving
// while the phase waits for its next item. When stderr is not a TTY it
// emits no ANSI redraws: it prints line as the phase starts, every
// plainInterval while it runs, and as it ends.
// //
// The draws come from a goroutine of their own, so mu guards the fields // All methods must be called from the main goroutine only. On a TTY
// below it. Every method must be called from the main goroutine. A nil // the library also redraws a spinner from its own goroutine, several
// *progress is a valid no-display receiver: every method is a no-op, so // times a second, so its count and elapsed time stay current while a
// batched database flushes during the hash phase can reuse the update // pass waits for its next item. A nil *progress is a valid
// phase's helpers without rendering anything, and its writer writes // no-display receiver: every method is a no-op, so batched database
// with no display to clear. // flushes during the streaming pass can reuse the update-pass helpers
// without rendering anything.
type progress struct { type progress struct {
now func() time.Time // reads the time label string
width func() int // reads the terminal's width; nil when not a TTY total int64 // -1 when unknown (walk pass)
stop chan struct{} // closed by finish to end the draws bar *progressbar.ProgressBar
done chan struct{} // closed once the draws have ended count int64
start time.Time
mu sync.Mutex last time.Time
bar *progressbar.ProgressBar // what the library draws; nil until then
withBar bool // bar is the library's bar, not its spinner
widest int // the widest line bar has drawn, in columns
label string
unit string
total int64
estimate bool
waitingFor string
stalledOn string
count int64
start time.Time // when the phase started, for the elapsed time
rateStart time.Time // when the count started, for the ETA
lastItem time.Time // when the last item was completed, or the count started
// drawn holds the count at each draw over the last rateWindow, oldest
// first; the rate counts from the first. It keeps the draw before the
// latest even when that is older, as the line before is when lines
// are plainInterval apart and a tick comes late.
drawn []drawnCount
} }
// newProgress starts the display of one phase on stderr. func newProgress(label string, total int64) *progress {
func newProgress(params progressParams) *progress { p := &progress{label: label, total: total, start: time.Now()}
if stderrIsTTY() { if stderrIsTTY() {
return startProgress(params, terminalWidth, redrawInterval, time.Now) p.bar = newBar(label, total)
return p
} }
return startProgress(params, nil, plainInterval, time.Now) // Print the zero state at once: the first item may take minutes,
} // and a pass must never look hung.
p.last = p.start
// startProgress shows the phase params describes at once, and again fmt.Fprintln(os.Stderr, p.plainLine())
// every interval until finish: with the progressbar library on a
// terminal, whose width each draw reads by calling width, or as a plain
// line when width is nil. The display reads the time by calling now.
func startProgress(params progressParams, width func() int,
interval time.Duration, now func() time.Time,
) *progress {
start := now()
p := &progress{
now: now, width: width,
stop: make(chan struct{}), done: make(chan struct{}),
label: params.label, unit: params.unit, total: params.total,
estimate: params.estimate, waitingFor: params.waitingFor,
stalledOn: params.stalledOn, start: start, rateStart: start,
lastItem: start, drawn: []drawnCount{{at: start}},
}
// Show the zero state at once: the first item may take minutes,
// and a phase must never look hung.
p.draw()
go func() {
defer close(p.done)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-p.stop:
return
case <-ticker.C:
p.draw()
}
}
}()
return p return p
} }
// draw shows the phase's progress: on a TTY in place of the line // newBar builds the TTY display for newProgress.
// before, and otherwise as a plain line. It first adds the count to func newBar(label string, total int64) *progressbar.ProgressBar {
// drawn, then drops from drawn the draws older than rateWindow, except opts := []progressbar.Option{
// the one before this draw.
func (p *progress) draw() {
p.mu.Lock()
defer p.mu.Unlock()
now := p.now()
p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
p.drawn = p.drawn[1:]
}
if p.width == nil {
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, true))
return
}
width := p.width()
// A phase with an exact total gets the library's bar when its whole
// line, without the percent the bar shows, fits beside the bar: the
// bar gives way before any part of the line. A total of 0 leaves the
// bar nothing to fill.
text, withBar := "", false
if p.total > 0 && !p.estimate {
text = p.line(now, math.MaxInt, false)
withBar = len(text) <= width-barMargin
}
margin := barMargin
if !withBar {
margin = spinnerMargin
text = p.line(now, width-margin, true)
}
// The library makes a bar or a spinner, and before each draw erases
// its line with as many spaces as the widest line it has drawn. A
// display gets a new one when it turns from one to the other, as its
// line grows or the terminal is resized, and when the terminal has
// been made too narrow for that erase, which would wrap and leave a
// line behind at every draw.
if p.bar == nil || p.withBar != withBar || p.widest >= width {
p.newBar(withBar, width)
}
if withBar {
_ = p.bar.Set64(p.count)
}
p.bar.Describe(text)
// The library drew the text and its margin, less the terminal's last
// column, which the margin keeps clear.
p.widest = max(p.widest, len(text)+margin-1)
}
// newBar has the library make what draw draws: a bar for the phase's
// total when withBar is set, and a spinner when not. Either shows the
// text draw gives it, and nothing of its own besides the spinner, or
// the bar and its percent. p.mu must be held.
func (p *progress) newBar(withBar bool, width int) {
if p.bar != nil {
// Erase what was drawn before, short of the last column of a
// terminal that may have been made narrower since: the library's
// own erase is as wide as the widest line it drew.
blank := strings.Repeat(" ", min(p.widest, width-1))
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
}
total := int64(-1) // the library's spinner
if withBar {
total = p.total
}
p.bar = progressbar.NewOptions64(total,
progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetWidth(barWidth), progressbar.OptionSetDescription(label),
progressbar.OptionSpinnerType(spinnerType), progressbar.OptionShowCount(),
// The spinner turns at each draw, not on a timer of the progressbar.OptionShowIts(),
// library's own. progressbar.OptionSetItsString("files"),
progressbar.OptionSetSpinnerChangeInterval(0), progressbar.OptionSetElapsedTime(true),
// The text carries the elapsed time and the ETA. progressbar.OptionThrottle(barThrottle),
progressbar.OptionSetElapsedTime(false), // Render at zero immediately: a phase must be visible the
progressbar.OptionSetPredictTime(false), // moment it starts, even before its first item completes.
) progressbar.OptionSetRenderBlankState(true),
p.withBar, p.widest = withBar, 0 }
} if total >= 0 {
opts = append(opts,
// setTotal gives a phase that started without a total its total, once progressbar.OptionSetPredictTime(true),
// it is known. The rate, the ETA and the time with no item completed progressbar.OptionShowElapsedTimeOnFinish(),
// count from then on: the time spent waiting for the total says nothing )
// about how fast items go. } else {
func (p *progress) setTotal(total int64) { opts = append(opts,
if p == nil { progressbar.OptionSetPredictTime(false),
return progressbar.OptionSpinnerType(walkSpinnerType),
)
} }
now := p.now() return progressbar.NewOptions64(total, opts...)
p.mu.Lock()
defer p.mu.Unlock()
p.total = total
p.rateStart, p.lastItem = now, now
p.drawn = []drawnCount{{at: now, count: p.count}}
} }
// increment records one completed item, which the display shows at its // increment records one completed item and refreshes the display.
// next draw.
func (p *progress) increment() { func (p *progress) increment() {
if p == nil { if p == nil {
return return
} }
now := p.now()
p.mu.Lock()
defer p.mu.Unlock()
p.count++ p.count++
p.lastItem = now if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
}
} }
// warnf prints a one-line warning to stderr without corrupting the // warnf prints a one-line warning to stderr without corrupting the bar.
// display. The whole message is escaped like a report's path columns, // The whole message is escaped like a report's path columns, so a path
// so a path holding a newline cannot split the warning. // holding a newline cannot split the warning.
func (p *progress) warnf(format string, args ...any) { func (p *progress) warnf(format string, args ...any) {
if p == nil { if p == nil {
return return
@@ -336,186 +125,51 @@ func (p *progress) warnf(format string, args ...any) {
msg := escapePath(fmt.Sprintf(format, args...)) msg := escapePath(fmt.Sprintf(format, args...))
p.mu.Lock() if p.bar != nil && p.total < 0 {
defer p.mu.Unlock() // The library also redraws a spinner from its own goroutine, so
// a direct write could land inside a redraw. The bar prints the
// warning itself, just before its next redraw.
_, _ = progressbar.Bprintln(p.bar, msg)
return
}
if p.bar != nil { if p.bar != nil {
// The warning takes the display's line; the next draw shows
// the display again, on the line below.
_ = p.bar.Clear() _ = p.bar.Clear()
} }
fmt.Fprintln(os.Stderr, msg) fmt.Fprintln(os.Stderr, msg)
} }
// progressWriter writes a phase's output to out, which may be the file // finish terminates the pass's display. A bar whose pass stopped short
// the phase's display is drawn on, as stdout is in a plain "sfdupes // of its total, as an interrupted one does, is left as last drawn; the
// trees" on a terminal. It holds what is written to it and writes that // library's Finish would fill it up.
// to out in pieces of about ioBufSize bytes. When out is the file
// stderr is, each piece is written under the display's lock with the
// display cleared first, as warnf does for a warning: no draw lands in
// the middle of a piece, and the next draw shows the display again, on
// the line below. Every write to it must end with a newline, as a row
// written with one fmt.Fprintf does, so that each piece ends a line.
// Flush writes what it holds once the last row is written.
type progressWriter struct {
// p is the display to clear, nil unless out is the file stderr is.
p *progress
out io.Writer
buf []byte
}
// writer returns a progressWriter writing to out around p's display.
// It clears the display and takes its lock only when out is the file
// stderr is: the same device and inode, as os.SameFile compares them,
// which is one terminal, or one pipe or file that both are sent to.
// Otherwise it writes to out without the lock, so the display keeps
// being drawn while a write to out waits for a slow reader.
func (p *progress) writer(out io.Writer) *progressWriter {
w := &progressWriter{out: out}
if f, ok := out.(*os.File); ok {
outInfo, outErr := f.Stat()
errInfo, errErr := os.Stderr.Stat()
if outErr == nil && errErr == nil && os.SameFile(outInfo, errInfo) {
w.p = p
}
}
return w
}
// Write holds b, and writes what it holds to out once that comes to
// ioBufSize bytes.
func (w *progressWriter) Write(b []byte) (int, error) {
w.buf = append(w.buf, b...)
if len(w.buf) < ioBufSize {
return len(b), nil
}
return len(b), w.Flush()
}
// Flush writes what w holds to out. When out is the file stderr is, the
// display is cleared first and not drawn again until it is written.
func (w *progressWriter) Flush() error {
if w.p != nil {
w.p.mu.Lock()
defer w.p.mu.Unlock()
if w.p.bar != nil {
_ = w.p.bar.Clear()
}
}
_, err := w.out.Write(w.buf)
w.buf = w.buf[:0]
return err
}
// finish ends the phase's display, drawn once more as the phase ends.
// A phase stopped short of its total, as an interrupted one is, shows
// the count it stopped at rather than a full bar.
func (p *progress) finish() { func (p *progress) finish() {
if p == nil { if p == nil {
return return
} }
close(p.stop) if p.bar != nil {
<-p.done if p.total >= 0 && p.count < p.total {
_ = p.bar.Exit()
p.draw() } else {
_ = p.bar.Finish()
if p.width != nil {
// The display stays on the TTY as drawn; what follows starts
// on a line of its own.
fmt.Fprintln(os.Stderr)
}
}
// line formats the phase's progress as of now in at most room columns,
// as
//
// [3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
//
// where "files" is the phase's unit, without the percent when percent
// is false, as before the library's bar, which shows its own. A phase
// with no total shows the items done instead, or what it waits for. The
// rate is that since the first draw in drawn, and the ETA comes from
// the rate since the count started. Once no item has been completed for
// stallTime, the rate, which would read 0, gives way to the word
// "stalled" and where the phase waits, as "stalled disk/db".
//
// The phase and its place, the progress, the elapsed time and the ETA
// are kept whole. A line wider than room leaves out the rate, the words
// of a stalled phase, or what the phase waits for; only if it is still
// wider is it cut at room. Every part is ASCII, so its length is its
// width in columns. p.mu must be held.
func (p *progress) line(now time.Time, room int, percent bool) string {
elapsed := now.Sub(p.start).Round(time.Second)
// head and tail hold the parts kept whole; extra, between them, is
// the part left out.
var head, extra, tail string
if p.total < 0 && p.waitingFor != "" {
head, extra = p.label+": ", p.waitingFor+", "
tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
} else {
since := p.drawn[0]
rate := perSecond(p.count-since.count, now.Sub(since.at))
average := perSecond(p.count, now.Sub(p.rateStart))
head = p.label + ": " + p.itemsDone(percent) + ", "
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
if now.Sub(p.lastItem) >= stallTime {
extra = "stalled " + p.stalledOn + ", "
} }
fmt.Fprintln(os.Stderr)
return
} }
full, kept := head+extra+tail, head+tail fmt.Fprintln(os.Stderr, p.plainLine())
switch {
case len(full) <= room:
return full
case len(kept) <= room:
return kept
}
return kept[:max(room, 0)]
} }
// perSecond returns count items over d as items a second, or 0 when no // plainLine formats a non-TTY progress line.
// time has passed. func (p *progress) plainLine() string {
func perSecond(count int64, d time.Duration) float64 { elapsed := time.Since(p.start)
if d <= 0 {
return 0
}
return float64(count) / d.Seconds()
}
// itemsDone formats the items done, as "[1234/5000] 25%", with "~"
// before a total that is an estimate and without the percent when
// percent is false, or as "1234 files", in the phase's unit, when the
// phase has no total. p.mu must be held.
func (p *progress) itemsDone(percent bool) string {
if p.total < 0 { if p.total < 0 {
return fmt.Sprintf("%d %s", p.count, p.unit) return fmt.Sprintf("%s: %d files, elapsed %s",
} p.label, p.count, elapsed.Round(time.Second))
approx := ""
if p.estimate {
approx = "~"
}
done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
if !percent {
return done
} }
pct := float64(percentScale) pct := float64(percentScale)
@@ -523,24 +177,19 @@ func (p *progress) itemsDone(percent bool) string {
pct = percentScale * float64(p.count) / float64(p.total) pct = percentScale * float64(p.count) / float64(p.total)
} }
return fmt.Sprintf("%s %.0f%%", done, pct) rate := 0.0
} if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
// eta returns the time the phase has left at rate items a second, or
// "unknown": with no total, before the first item, and once the count
// passes an estimate. p.mu must be held.
func (p *progress) eta(rate float64) string {
switch {
case p.total < 0, p.estimate && p.count > p.total:
return "unknown"
case p.count >= p.total:
return "0s"
case rate <= 0:
return "unknown"
} }
remaining := time.Duration( eta := "?"
float64(p.total-p.count) / rate * float64(time.Second))
return remaining.Round(time.Second).String() if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)
} }
+69 -869
View File
File diff suppressed because it is too large Load Diff
+48 -200
View File
@@ -1,14 +1,12 @@
package main package main
import ( import (
"bufio"
"context" "context"
"database/sql"
"fmt" "fmt"
"io" "io"
"os" "os"
"slices"
"strings" "strings"
"time"
) )
// ioBufSize is the buffer size for the buffered stdout writers. // ioBufSize is the buffer size for the buffered stdout writers.
@@ -23,231 +21,81 @@ const minGroupSize = 2
// only and used by scan for change detection. // only and used by scan for change detection.
type scanRec struct { type scanRec struct {
size int64 size int64
mtime time.Time mtime int64
head string head string
tail string tail string
content string content string
path string path string
} }
// reportPhase names one phase of report with its place among all of
// them, as "[2/2] read", for the phase's progress display. The list
// here is the one place the phases are counted.
func reportPhase(name string) string {
return phaseLabel([]string{openPhase, "read"}, name)
}
// runReport implements the report subcommand: it prints the file-level // runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. It never touches the scanned // duplicates report as TSV on stdout. SQLite groups and orders the
// filesystem; its only I/O is the database, stdout, and stderr. It // records, and each row is written as it is read, so no group is held
// names the database on stderr before it opens it. Any database // in memory. It never touches the scanned filesystem; its only I/O is
// problem, including a missing database, is fatal. // the database (with SQLite's temporary sort file), stdout, and stderr.
// // Any database problem, including a missing database, is fatal.
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (count the records,
// then read them in signature order, writing each size's duplicate
// groups to stdout once that size is read; see readReport). Each
// display ends before the next phase starts, and before an error is
// returned to be printed.
func runReport(ctx context.Context, stdout io.Writer) error { func runReport(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath() dbPath := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: reportPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath) db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
// The read phase writes the report to stdout while it reads the out := bufio.NewWriterSize(stdout, ioBufSize)
// records, so it may wait on either.
prog = newProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db/out",
})
report, err := readReport(ctx, db, dbPath, stdout, prog)
prog.finish() _, err = fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
var (
groups, dupeFiles int
reclaimable int64
writeErr error
)
records, err := loadDupeRows(ctx, db,
func(first, path string, size int64) error {
// A group's first path is its first row; every other
// path is a dupe.
if path == first {
groups++
return nil
}
_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
escapePath(first), escapePath(path), size)
dupeFiles++
reclaimable += size
return writeErr
})
if writeErr != nil {
return fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil { if err != nil {
return err return fmt.Errorf("database %s: %w", dbPath, err)
}
err = out.Flush()
if err != nil {
return fmt.Errorf("write stdout: %w", err)
} }
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
"report: %d records read, %d duplicate groups, %d dupe files, "+ "report: %d records read, %d duplicate groups, %d dupe files, "+
"%s reclaimable\n", "%s reclaimable\n",
report.records, report.dupeGroups, report.dupeFiles, records, groups, dupeFiles, humanBytes(reclaimable))
humanBytes(report.reclaimable))
return nil return nil
} }
// readReport gives prog the number of records in db, the database at
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
stdout io.Writer, prog *progress,
) (*reportWriter, error) {
out := prog.writer(stdout)
report := &reportWriter{out: out}
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
var writeErr error
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
prog.increment()
writeErr = report.add(r)
return writeErr
})
if writeErr != nil {
return nil, fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil {
return nil, fmt.Errorf("database %s: %w", dbPath, err)
}
err = report.finish()
if err == nil {
err = out.Flush()
}
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
return report, nil
}
// reportWriter writes the report's rows to out from the records, added
// in signature order, largest size first, as loadSignatureRows reads
// them. The records sharing a signature come together, and so do the
// records of one size, so it holds only the duplicate groups of the
// size being read, and writes them, ordered by first path, once a
// record of a smaller size comes. It counts what the summary line
// reports.
type reportWriter struct {
out io.Writer
sig scanRec // the signature being read; its path is not used
paths []string // the paths read so far with sig
groups [][]string // the duplicate groups of sig.size, each sorted
records, dupeGroups, dupeFiles int
reclaimable int64
}
// add takes the next record. One with the signature being read joins
// its paths; any other ends them, and one of a smaller size first
// writes the groups of the size before.
func (w *reportWriter) add(r scanRec) error {
w.records++
// A record without a content hash has unknown content (README
// "Database"), so it is never a duplicate.
if r.content == "" {
return nil
}
if r.size == w.sig.size && r.head == w.sig.head &&
r.tail == w.sig.tail && r.content == w.sig.content {
w.paths = append(w.paths, r.path)
return nil
}
w.endGroup()
if r.size != w.sig.size {
err := w.writeGroups()
if err != nil {
return err
}
}
w.sig = r
w.paths = append(w.paths, r.path)
return nil
}
// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
func (w *reportWriter) endGroup() {
if len(w.paths) < minGroupSize {
// A path with no duplicate is dropped, and its slice reused.
w.paths = w.paths[:0]
return
}
slices.Sort(w.paths)
w.groups = append(w.groups, w.paths)
w.paths = nil
}
// writeGroups writes the duplicate groups held, all of size w.sig.size,
// ordered by first path: for each group, a row for every path but the
// first. No two groups share a first path, since no two records share
// a path.
func (w *reportWriter) writeGroups() error {
slices.SortFunc(w.groups, func(a, b []string) int {
return strings.Compare(a[0], b[0])
})
for _, g := range w.groups {
for _, path := range g[1:] {
_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
escapePath(g[0]), escapePath(path), w.sig.size)
if err != nil {
return err
}
w.dupeFiles++
w.reclaimable += w.sig.size
}
w.dupeGroups++
}
w.groups = nil
return nil
}
// finish writes the groups of the last size read.
func (w *reportWriter) finish() error {
w.endGroup()
return w.writeGroups()
}
// escapePath returns a path as it is written in a report column (README // escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return // "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is. // becomes \\, \t, \n or \r, and every other byte is kept as it is.
+26 -198
View File
@@ -9,10 +9,8 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"strconv"
"strings" "strings"
"testing" "testing"
"time"
) )
// awkwardDir is a directory name holding every byte the reports escape. // awkwardDir is a directory name holding every byte the reports escape.
@@ -29,8 +27,7 @@ func awkwardPairRecs() []scanRec {
} }
} }
// seedDatabase writes recs into a fresh database, closes it as scan // seedDatabase writes recs into a fresh database and returns its path.
// does, out of WAL mode, and returns its path.
func seedDatabase(t *testing.T, recs []scanRec) string { func seedDatabase(t *testing.T, recs []scanRec) string {
t.Helper() t.Helper()
@@ -46,7 +43,10 @@ func seedDatabase(t *testing.T, recs []scanRec) string {
t.Fatal(err) t.Fatal(err)
} }
closeScanDatabase(t.Context(), db, path) err = db.Close()
if err != nil {
t.Fatal(err)
}
return path return path
} }
@@ -59,39 +59,25 @@ type dupeGroup struct {
} }
// dupeGroups returns the duplicate groups report reads from db, in // dupeGroups returns the duplicate groups report reads from db, in
// report order, from the rows it writes: a group's rows come together, // report order.
// each naming the group's first path.
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup { func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
t.Helper() t.Helper()
var out bytes.Buffer
_, err := readReport(t.Context(), db, "", &out, nil)
if err != nil {
t.Fatal(err)
}
var groups []dupeGroup var groups []dupeGroup
rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n") _, err := loadDupeRows(t.Context(), db,
for _, row := range rows[1:] { // after the header func(first, path string, size int64) error {
fields := strings.Split(row, "\t") if path == first {
if len(fields) != 3 { groups = append(groups, dupeGroup{size: size})
t.Fatalf("row %q, want first, dupe and size", row) }
}
size, err := strconv.ParseInt(fields[2], 10, 64) g := &groups[len(groups)-1]
if err != nil { g.paths = append(g.paths, path)
t.Fatal(err)
}
if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] { return nil
groups = append(groups, })
dupeGroup{size: size, paths: []string{fields[0]}}) if err != nil {
} t.Fatal(err)
g := &groups[len(groups)-1]
g.paths = append(g.paths, fields[1])
} }
return groups return groups
@@ -131,147 +117,26 @@ func TestRunReportEscapesPaths(t *testing.T) {
} }
func TestReportStdoutFailsWhileReading(t *testing.T) { func TestReportStdoutFailsWhileReading(t *testing.T) {
t.Parallel() // Each row holds two paths longer than dir, so the report is more
// than twice the stdout buffer and stdout fails while rows are
// Every two records are a duplicate pair of a size of their own, and // still being read, not at the final flush.
// a pair's row is written once a smaller size is read. Each row
// holds two paths longer than dir, so the rows come to more than
// twice the stdout buffer, and stdout fails while records are still
// being read, not once every record is.
dir := "/" + strings.Repeat("d", 4096) dir := "/" + strings.Repeat("d", 4096)
recs := make([]scanRec, 2*ioBufSize/len(dir)) recs := make([]scanRec, ioBufSize/len(dir))
for i := range recs { for i := range recs {
recs[i] = scanRec{ recs[i] = scanRec{
size: int64(i/2) + 1, head: "h", tail: "t", content: "c", size: 1, head: "h", tail: "t", content: "c",
path: fmt.Sprintf("%s/%d", dir, i), path: fmt.Sprintf("%s/%d", dir, i),
} }
} }
db := openTestDB(t) t.Setenv(databaseEnv, seedDatabase(t, recs))
err := applyChanges(t.Context(), db, recs, nil, nil) err := runReport(t.Context(), failingWriter{})
if err != nil {
t.Fatal(err)
}
// Counts the records read, and draws nothing.
prog := &progress{now: time.Now}
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
if !errors.Is(err, errWriteFailed) || if !errors.Is(err, errWriteFailed) ||
!strings.HasPrefix(err.Error(), "write stdout: ") { !strings.HasPrefix(err.Error(), "write stdout: ") {
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed) t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
} }
if prog.count >= int64(len(recs)) {
t.Errorf("stdout failed after all %d records were read, want it "+
"to stop the read", prog.count)
}
}
// reportOpenLine is the plain line of report's open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"eta unknown"
// smokeReport is report's stdout for the smoke-test records.
const smokeReport = "first\tdupe\tsize\n" +
"/d/t1/f1\t/d/t2/f1\t3000\n" +
"/d/t1/f1\t/d/t3/f1\t3000\n" +
"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"
// TestRunReportProgressNamesEachPhase runs report on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the two as it starts and another as
// it ends; read counts the 6 records before it reads them. The summary
// comes last, and stdout holds the report alone.
func TestRunReportProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdReport}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(report) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
if got := stdout.String(); got != smokeReport {
t.Errorf("stdout = %q, want %q", got, smokeReport)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"report: database " + path + " (from " + databaseEnv + ")",
reportOpenLine,
reportOpenLine,
"[2/2] read: counting the records, elapsed 0s, eta unknown",
"[2/2] read: [6/6] 100%, ",
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
"6.1 KiB reclaimable",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestReadReportOnTerminal writes the smoke-test report to the terminal
// its read phase's display is drawn on, as a plain "sfdupes report"
// does: the terminal must show the header and every row whole, each on
// a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestReadReportOnTerminal(t *testing.T) {
stderr := captureStderr(t)
db := openTestDB(t)
err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords,
}, terminalWidth, time.Hour, time.Now)
_, err = readReport(t.Context(), db, "", os.Stderr, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
} }
func TestRunReportsIgnoreInsertionOrder(t *testing.T) { func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
@@ -448,14 +313,8 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
// mtime is informational only; records differing only in mtime // mtime is informational only; records differing only in mtime
// still group together. // still group together.
recs := []scanRec{ recs := []scanRec{
{ {size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"},
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", {size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"},
content: "c", path: "/m/1",
},
{
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
content: "c", path: "/m/2",
},
} }
groups := dupeGroupsOf(t, recs) groups := dupeGroupsOf(t, recs)
@@ -488,37 +347,6 @@ func TestDupeGroupsTieBreak(t *testing.T) {
} }
} }
func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
t.Parallel()
// report reads the groups of size 50 in signature order, here /a,
// /z, /k, which is neither the order of their first paths nor its
// reverse, and must write them ordered by first path. The group of
// size 7 comes after them all, though its first path sorts first.
recs := []scanRec{
{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
}
groups := dupeGroupsOf(t, recs)
got := make([]string, 0, len(groups))
for _, g := range groups {
got = append(got, g.paths[0])
}
want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
if !slices.Equal(got, want) {
t.Errorf("groups' first paths = %q, want %q", got, want)
}
}
func TestDupeGroupsDeterministic(t *testing.T) { func TestDupeGroupsDeterministic(t *testing.T) {
t.Parallel() t.Parallel()
+124 -478
View File
@@ -13,12 +13,10 @@ import (
"os" "os"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"runtime"
"slices" "slices"
"strings" "strings"
"sync" "sync"
"syscall" "syscall"
"time"
) )
// The duplicate ladder (see hashSignature and README "Duplicate // The duplicate ladder (see hashSignature and README "Duplicate
@@ -70,7 +68,7 @@ var errInterrupted = errors.New("scan interrupted")
type fileRec struct { type fileRec struct {
path string path string
size int64 size int64
mtime time.Time mtime int64
dev uint64 dev uint64
ino uint64 ino uint64
} }
@@ -81,32 +79,18 @@ type fileRec struct {
// they would dominate the scan's memory. // they would dominate the scan's memory.
type fileMeta struct { type fileMeta struct {
size int64 size int64
mtime time.Time mtime int64
hashed bool hashed bool
} }
// scanUnit is what every scan phase counts, as its progress display // runScan implements the scan subcommand: four sequential phases —
// names it. // walk (which stats each file as it is discovered), hash, update,
const scanUnit = "files" // content — that synchronize the persistent database with the
// filesystem state under the PATH operands. Only files whose size at
// scanPhase names one phase of a scan with its place among all of them, // least one other file shares are ever hashed: a size-unique file
// as "[2/5] walk", for the phase's progress display. The list here is // cannot be a duplicate. A file of headTailMin or more gets its content
// the one place the phases are counted. // hash only when its size, head, and tail match another file's. Flag
func scanPhase(name string) string { // parsing and the at-least-one-operand check are done by cobra. The
return phaseLabel(
[]string{"load", "walk", "hash", "update", "content"}, name)
}
// runScan implements the scan subcommand: five sequential phases —
// load (which reads the existing records), walk (which stats each file
// as it is discovered), hash, update, content — that synchronize the
// persistent database with the filesystem state under the PATH
// operands. Only files whose size at least one other file shares are
// ever hashed: a size-unique file cannot be a duplicate. A file of
// headTailMin or more gets its content hash only when its size, head,
// and tail match another file's. Flag parsing and the
// at-least-one-operand check are done by cobra. Before anything else,
// the scan names the database on stderr, saying whether it is new. The
// scan holds the lock on the database for its whole run, so a second // scan holds the lock on the database for its whole run, so a second
// scan fails before it walks the filesystem or opens the database. // scan fails before it walks the filesystem or opens the database.
// Errors are returned rather than exiting, so that the deferred close — // Errors are returned rather than exiting, so that the deferred close —
@@ -115,44 +99,16 @@ func scanPhase(name string) string {
// SIGTERM that interruptContext catches, the scan keeps what it has // SIGTERM that interruptContext catches, the scan keeps what it has
// hashed (see syncScan), prints how many files its walk reached, and // hashed (see syncScan), prints how many files its walk reached, and
// returns errInterrupted. workers must be at least 1; the scan command // returns errInterrupted. workers must be at least 1; the scan command
// rejects anything less. Before the lock is taken, the scan lists the // rejects anything less.
// filesystems that are not persistent, on which the walk records
// nothing; unless noDefaultExcludes is set, the walk does not enter the
// default excludes either. Once the database is open, the scan notes
// its files, which the walk passes over.
func runScan(ctx context.Context, roots []string, workers int, func runScan(ctx context.Context, roots []string, workers int,
oneFS, noDefaultExcludes bool, oneFS bool,
) error { ) error {
dbPath, source, err := databasePath() roots, err := resolveRoots(roots)
if err != nil { if err != nil {
return err return err
} }
// A database file missing now is created by lockScanDatabase below. dbPath := databasePath()
what := "database"
_, err = os.Stat(dbPath)
if errors.Is(err, fs.ErrNotExist) {
what = "new database"
}
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
source)
roots, err = resolveRoots(roots)
if err != nil {
return err
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
return fmt.Errorf("list the mounted filesystems: %w", err)
}
rules := walkRules{nonPersistent: nonPersistent}
if !noDefaultExcludes {
rules.excludes = defaultExcludes(runtime.GOOS)
}
lock, err := lockScanDatabase(dbPath) lock, err := lockScanDatabase(dbPath)
if err != nil { if err != nil {
@@ -174,12 +130,7 @@ func runScan(ctx context.Context, roots []string, workers int,
defer closeScanDatabase(ctx, db, dbPath) defer closeScanDatabase(ctx, db, dbPath)
rules.databaseFiles, err = databaseFiles(dbPath) st, err := syncScan(ctx, db, roots, workers, oneFS)
if err != nil {
return err
}
st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
if errors.Is(err, context.Canceled) { if errors.Is(err, context.Canceled) {
return interrupted(st.walked) return interrupted(st.walked)
} }
@@ -251,14 +202,10 @@ func resolveRoots(roots []string) ([]string, error) {
} }
// pruneRoots drops operands already covered by another operand: // pruneRoots drops operands already covered by another operand:
// duplicates and any operand lying under another one. An operand under // duplicates and any operand lying under another one. Every remaining
// another one is kept, though, when the walk of the deepest kept operand // file is then reachable through exactly one root, so walked paths are
// above it does not reach it (see walkReaches). Every remaining file is // unique without keeping a scan-wide set of every path seen.
// then reachable through exactly one root, so walked paths are unique func pruneRoots(roots []string) []string {
// without keeping a scan-wide set of every path seen.
func pruneRoots(roots []string, oneFS bool, rules walkRules,
device func(path string) (uint64, bool),
) []string {
// Shorter-first ordering guarantees an ancestor is kept before any // Shorter-first ordering guarantees an ancestor is kept before any
// operand under it is considered. // operand under it is considered.
sorted := slices.Clone(roots) sorted := slices.Clone(roots)
@@ -273,16 +220,7 @@ func pruneRoots(roots []string, oneFS bool, rules walkRules,
kept := make([]string, 0, len(sorted)) kept := make([]string, 0, len(sorted))
for _, r := range sorted { for _, r := range sorted {
above := deepestRoot(r, kept) if !underAnyRoot(r, kept) {
if above == "" {
kept = append(kept, r)
continue
}
// A duplicate is dropped, and so is an operand the walk of above
// reaches.
if above != r && !walkReaches(above, r, oneFS, rules, device) {
kept = append(kept, r) kept = append(kept, r)
} }
} }
@@ -290,58 +228,6 @@ func pruneRoots(roots []string, oneFS bool, rules walkRules,
return kept return kept
} }
// walkReaches reports whether the walk of operand above reaches r, an
// operand under it. That walk does not enter a default exclude in
// rules, nor a directory that device puts on a filesystem in rules that
// is not persistent, nor, with -x, one that device puts on a different
// filesystem than above, when device knows above's. It reaches every
// regular file in a directory it enters, so for a regular-file operand
// the directories checked start at its directory; they end below above.
func walkReaches(above, r string, oneFS bool, rules walkRules,
device func(path string) (uint64, bool),
) bool {
aboveDev, aboveDevOK := device(above)
dir := r
fi, err := os.Lstat(r)
if err == nil && fi.Mode().IsRegular() {
dir = filepath.Dir(r)
}
for ; dir != above; dir = filepath.Dir(dir) {
if isDefaultExclude(dir, rules.excludes) {
return false
}
dev, ok := device(dir)
if !ok {
continue
}
if _, found := rules.nonPersistent[dev]; found {
return false
}
if oneFS && aboveDevOK && dev != aboveDev {
return false
}
}
return true
}
// deviceOfPath returns the filesystem device ID of the file at path,
// when lstat succeeds and the platform exposes one.
func deviceOfPath(path string) (uint64, bool) {
fi, err := os.Lstat(path)
if err != nil {
return 0, false
}
return deviceOfInfo(fi)
}
// scanStats summarizes one scan's database synchronization for the // scanStats summarizes one scan's database synchronization for the
// final stderr summary, or for the line an interrupted scan prints. // final stderr summary, or for the line an interrupted scan prints.
type scanStats struct { type scanStats struct {
@@ -372,11 +258,11 @@ type scanState struct {
// the hashed records still waiting in the batch, starts no other write // the hashed records still waiting in the batch, starts no other write
// or deletion, and returns the cancellation. // or deletion, and returns the cancellation.
func syncScan(ctx context.Context, db *sql.DB, roots []string, func syncScan(ctx context.Context, db *sql.DB, roots []string,
workers int, oneFS bool, rules walkRules, workers int, oneFS bool,
) (scanStats, error) { ) (scanStats, error) {
s := &scanState{db: db} s := &scanState{db: db}
err := s.runPhases(ctx, roots, workers, oneFS, rules) err := s.runPhases(ctx, roots, workers, oneFS)
if err == nil || ctx.Err() == nil { if err == nil || ctx.Err() == nil {
return s.st, err return s.st, err
} }
@@ -391,35 +277,31 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
} }
// runPhases synchronizes the database with the filesystem under roots // runPhases synchronizes the database with the filesystem under roots
// in five sequential phases: load (read the existing records), walk // in four sequential phases: walk (enumerate and stat every file,
// (enumerate and stat every file, building a complete size census), // building a complete size census), hash (read only the new or
// hash (read only the new or changed — or previously unhashed — files // changed — or previously unhashed — files whose size at least one
// whose size at least one other file shares, committing results in // other file shares, committing results in batches as they arrive),
// batches as they arrive), update (record the size-unique files without // update (record the size-unique files without reading them, and
// reading them, and delete the records the scan no longer verifies), // delete the records the scan no longer verifies), and content (fill
// and content (fill in the content hash of every record of headTailMin // in the content hash of every record of headTailMin or more whose
// or more whose size, head, and tail match another record's). Records // size, head, and tail match another record's). Records outside the
// outside the roots are never touched, except that the content phase // roots are never touched, except that the content phase fills in
// fills in their content hash. Operands the walk cannot start from, or // their content hash. Operands the walk cannot start from are dropped
// that are not on a persistent filesystem, are dropped first, so the // first, so the records beneath them count as outside the roots unless
// records beneath them are treated as if those operands had not been // they lie under another root.
// given. Below an operand, the walk follows rules.
func (s *scanState) runPhases(ctx context.Context, roots []string, func (s *scanState) runPhases(ctx context.Context, roots []string,
workers int, oneFS bool, rules walkRules, workers int, oneFS bool,
) error { ) error {
// Types and filesystems are checked before pruning so that an // Types are checked before pruning so that an operand under a
// operand under a dropped one is still scanned, not dropped as lying // dropped one is still scanned, not dropped as lying under it.
// under it. roots = pruneRoots(s.walkableRoots(roots))
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
deviceOfPath)
err := s.loadIndex(ctx, roots) err := s.loadIndex(ctx, roots)
if err != nil { if err != nil {
return err return err
} }
changed, unhashed := s.walkPhase( changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers))
startWalk(ctx, roots, oneFS, rules, workers), roots)
// A cancelled walk stops early, so its size census covers only part // A cancelled walk stops early, so its size census covers only part
// of the roots, and every file it never reached would look vanished // of the roots, and every file it never reached would look vanished
@@ -441,19 +323,17 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
return err return err
} }
return s.contentPhase(ctx, workers, rules) return s.contentPhase(ctx, workers)
} }
// walkableRoots returns the operands the walk can start from: regular // walkableRoots returns the operands the walk can start from: regular
// files, and directories not named .zfs. Every other operand is warned // files, and directories not named .zfs. Every other operand is warned
// about, counted as skipped, and dropped. A dropped operand is no // about, counted as skipped, and dropped. A dropped operand is no
// longer a root, so the records stored beneath it are treated as if it // longer a root, so the records stored beneath it count as outside the
// had not been given: those under no other root count as outside the // roots and are not deleted as unverified, unless it lies under another
// roots and are not deleted as unverified. An operand on a filesystem // root. An operand that fails lstat here is kept, and the walk warns
// in rules that is not persistent is dropped the same way, with a line // about it.
// saying so that is not a warning, and is not counted as skipped. An func (s *scanState) walkableRoots(roots []string) []string {
// operand that fails lstat here is kept, and the walk warns about it.
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
kept := make([]string, 0, len(roots)) kept := make([]string, 0, len(roots))
for _, root := range roots { for _, root := range roots {
@@ -467,13 +347,6 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
continue continue
} }
line := rules.nonPersistentLine(root, fi)
if line != "" {
fmt.Fprintln(os.Stderr, escapePath(line))
continue
}
} }
kept = append(kept, root) kept = append(kept, root)
@@ -489,25 +362,14 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
// separately into the same database. // separately into the same database.
func (s *scanState) loadIndex(ctx context.Context, roots []string) error { func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// Indexing tens of millions of records takes real time; without a // Indexing tens of millions of records takes real time; without a
// display the scan looks hung before the walk begins. Counting them // display the scan looks hung before the walk begins.
// first gives the display its total. prog := newProgress("load", -1)
prog := newProgress(progressParams{
label: scanPhase("load"), unit: scanUnit, total: -1,
waitingFor: countingRecords, stalledOn: "db",
})
defer prog.finish() defer prog.finish()
total, err := countRecords(ctx, s.db)
if err != nil {
return err
}
prog.setTotal(total)
s.existing = make(map[string]fileMeta) s.existing = make(map[string]fileMeta)
return loadFileMeta(ctx, s.db, return loadFileMeta(ctx, s.db,
func(path string, size int64, mtime time.Time, hashed bool) { func(path string, size, mtime int64, hashed bool) {
prog.increment() prog.increment()
if underAnyRoot(path, roots) { if underAnyRoot(path, roots) {
@@ -528,52 +390,15 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// (the content phase may still fill in its content hash). It returns // (the content phase may still fill in its content hash). It returns
// the new-or-changed files and the unchanged files whose records lack // the new-or-changed files and the unchanged files whose records lack
// hashes; both remain candidates until the census decides whether // hashes; both remain candidates until the census decides whether
// their sizes are shared. A record at or below a path the walk left // their sizes are shared.
// out, because -x stopped at it, it is a default exclude, or it is not
// on a persistent filesystem, counts as outside the roots: it is not
// deleted, and its size joins the census. That does not hold below a
// root at or inside that path, since that root's own walk verifies the
// records below it; nor is a line printed for such a root.
func (s *scanState) walkPhase( func (s *scanState) walkPhase(
events <-chan walkEvent, roots []string, events <-chan walkEvent,
) ([]fileRec, []fileRec) { ) ([]fileRec, []fileRec) {
// The records the last scan left under the roots estimate how many prog := newProgress("walk", -1)
// files the walk will find. A first scan has none, and no estimate.
estimate := int64(len(s.existing))
if estimate == 0 {
estimate = -1
}
prog := newProgress(progressParams{ var changed, unhashed []fileRec
label: scanPhase("walk"), unit: scanUnit, total: estimate,
estimate: true, stalledOn: "disk",
})
var (
changed, unhashed []fileRec
leftOut []string
)
for ev := range events { for ev := range events {
if ev.otherFSDir != "" {
leftOut = append(leftOut, ev.otherFSDir)
continue
}
// The line saying so is printed as a warning is, but what the
// walk leaves out is not counted as skipped. A default exclude
// that is itself a root is walked as one, so it gets no line.
if ev.leftOut != "" {
leftOut = append(leftOut, ev.leftOut)
if !slices.Contains(roots, ev.leftOut) {
prog.warnf("%s", ev.warn)
}
continue
}
if ev.fail { if ev.fail {
s.st.skipped++ s.st.skipped++
@@ -590,7 +415,7 @@ func (s *scanState) walkPhase(
old, ok := s.existing[ev.rec.path] old, ok := s.existing[ev.rec.path]
switch { switch {
case !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime): case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime:
changed = append(changed, ev.rec) changed = append(changed, ev.rec)
case old.hashed: case old.hashed:
delete(s.existing, ev.rec.path) delete(s.existing, ev.rec.path)
@@ -603,20 +428,6 @@ func (s *scanState) walkPhase(
prog.finish() prog.finish()
for path, old := range s.existing {
// Kept only when a path the walk left out lies below the deepest
// root above the record, or is the record's own. A root at or
// below that path was walked on its own and did not find the
// file. Both contain the record, so the longer path is the
// deeper.
left := deepestRoot(path, leftOut)
if len(left) > len(deepestRoot(path, roots)) {
delete(s.existing, path)
s.sizes = append(s.sizes, old.size)
}
}
return changed, unhashed return changed, unhashed
} }
@@ -716,15 +527,6 @@ func sameInode(a, b fileRec) bool {
return (a.dev != 0 || a.ino != 0) && a.dev == b.dev && a.ino == b.ino return (a.dev != 0 || a.ino != 0) && a.dev == b.dev && a.ino == b.ino
} }
// isDatabaseFile reports whether f, a file the walk reached, is one of
// the scan's own database files in rules, which the walk passes over
// without a line and without counting it.
func isDatabaseFile(f fileRec, rules walkRules) bool {
return slices.ContainsFunc(rules.databaseFiles, func(d fileRec) bool {
return sameInode(d, f)
})
}
// hashPhase hashes every queued file with hashSignature — the head and // hashPhase hashes every queued file with hashSignature — the head and
// tail of a file of headTailMin or more, the whole file below that — // tail of a file of headTailMin or more, the whole file below that —
// committing completed records to the database in batches as results // committing completed records to the database in batches as results
@@ -741,11 +543,10 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
} }
// readRuns reads runs with the worker pool, one read per inode run, in // readRuns reads runs with the worker pool, one read per inode run, in
// the order given, under the progress display of the phase named label. // the order given, under a progress display named label. The workers
// The workers compute each run's hashes with hash, and each result goes // compute each run's hashes with hash, and each result goes to record;
// to record; a run that fails to read is warned about and counted as // a run that fails to read is warned about and counted as skipped
// skipped instead. The total counts actual reads, so the display shows // instead. The total counts actual reads, so the bar shows a real ETA.
// a real ETA.
// //
// Returning early — a failed database write, or a cancelled scan — must // Returning early — a failed database write, or a cancelled scan — must
// not strand the pool: the feeder would park forever on a full jobs // not strand the pool: the feeder would park forever on a full jobs
@@ -759,12 +560,7 @@ func (s *scanState) readRuns(ctx context.Context, workers int,
pool := startHashPool(ctx, runs, workers, hash) pool := startHashPool(ctx, runs, workers, hash)
defer pool.stop() defer pool.stop()
// Between results the phase may wait on a file being read or on a prog := newProgress(label, int64(len(runs)))
// batch of records being written.
prog := newProgress(progressParams{
label: scanPhase(label), unit: scanUnit, total: int64(len(runs)),
stalledOn: "disk/db",
})
defer prog.finish() defer prog.finish()
for range runs { for range runs {
@@ -851,10 +647,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
s.st.removed = len(deletes) s.st.removed = len(deletes)
total := len(s.batch) + len(s.sentinels) + len(deletes) total := len(s.batch) + len(s.sentinels) + len(deletes)
prog := newProgress(progressParams{ prog := newProgress("update", int64(total))
label: scanPhase("update"), unit: scanUnit, total: int64(total),
stalledOn: "db",
})
defer prog.finish() defer prog.finish()
@@ -893,12 +686,9 @@ func (s *scanState) updatePhase(ctx context.Context) error {
// hash phase's worker pool and their records written back in batches. A // hash phase's worker pool and their records written back in batches. A
// failed read is warned about and counted as skipped; the record keeps // failed read is warned about and counted as skipped; the record keeps
// its empty content, so it is never grouped, and a later scan tries // its empty content, so it is never grouped, and a later scan tries
// again. No file on a filesystem in rules that is not persistent is // again.
// read. func (s *scanState) contentPhase(ctx context.Context, workers int) error {
func (s *scanState) contentPhase(ctx context.Context, workers int, toRead, recs, err := s.contentCandidates(ctx)
rules walkRules,
) error {
toRead, recs, err := s.contentCandidates(ctx, rules)
if err != nil { if err != nil {
return err return err
} }
@@ -919,57 +709,35 @@ func (s *scanState) contentPhase(ctx context.Context, workers int,
return err return err
} }
// The last partial batch is written under a display of its own, as return applyChanges(ctx, s.db, s.batch, nil, nil)
// the update phase writes the hash phase's.
prog := newProgress(progressParams{
label: scanPhase("content"), unit: scanUnit,
total: int64(len(s.batch)), stalledOn: "db",
})
defer prog.finish()
return applyChanges(ctx, s.db, s.batch, nil, prog)
} }
// contentCandidates returns the files the content phase reads, and // contentCandidates returns the files the content phase reads, and
// their records by path. Every record contentCandidatesSQL returns has // their records by path. Every record contentCandidatesSQL returns has
// its file checked with lstat, whether or not it already has a content // its file checked with lstat, whether or not it already has a content
// hash: a file that is gone, is no longer a regular file, has changed // hash: a file that is gone, is no longer a regular file, or has
// by the walk's rule, or is on a filesystem in rules that is not // changed by the walk's rule keeps its record as it is and does not
// persistent keeps its record as it is and does not count as a match // count as a match for the others, and any other lstat error is warned
// for the others, and any other lstat error is warned about and counted // about and counted as skipped, with the same result. If such a record
// as skipped, with the same result. If such a record has no content // has no content hash, it stays out of duplicate groups; if it has one,
// hash, it stays out of duplicate groups; if it has one, it is still // it is still reported until a scan covering its own tree updates or
// reported until a scan covering its own tree updates or removes it. // removes it. The files of a group that pass and have no content hash
// The files of a group that pass and have no content hash are read only // are read only if at least minGroupSize of the group's files pass, so
// if at least minGroupSize of the group's files pass, so a group whose // a group whose other members are all stale costs no reads. Only the
// other members are all stale costs no reads. Only the records to be // records to be read are kept.
// read are kept.
func (s *scanState) contentCandidates( func (s *scanState) contentCandidates(
ctx context.Context, rules walkRules, ctx context.Context,
) ([]fileRec, map[string]scanRec, error) { ) ([]fileRec, map[string]scanRec, error) {
// The search and the checks take real time on a large database; // The query and the checks take real time on a large database;
// without a display the scan looks hung before the reads begin. // without a display the scan looks hung before the reads begin.
// The display says it is searching until SQLite returns the first prog := newProgress("content", -1)
// record. The checks start there, against a count of the records
// the search returns, and so do their rate and ETA. Between records
// they may wait on a file's lstat or on the search.
prog := newProgress(progressParams{
label: scanPhase("content"), unit: scanUnit, total: -1,
waitingFor: "searching the database", stalledOn: "disk/db",
})
defer prog.finish() defer prog.finish()
total, err := countContentCandidates(ctx, s.db)
if err != nil {
return nil, nil, err
}
var ( var (
searching = true // no record has been returned yet toRead []fileRec
toRead []fileRec first scanRec // the current group's first record
first scanRec // the current group's first record passed int // the current group's files that passed the check
passed int // the current group's files that passed the check unread []fileRec // those of them without a content hash
unread []fileRec // those of them without a content hash
) )
recs := make(map[string]scanRec) recs := make(map[string]scanRec)
@@ -988,13 +756,7 @@ func (s *scanState) contentCandidates(
passed, unread = 0, nil passed, unread = 0, nil
} }
err = loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) { err := loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) {
if searching {
searching = false
prog.setTotal(total)
}
prog.increment() prog.increment()
if r.size != first.size || r.head != first.head || r.tail != first.tail { if r.size != first.size || r.head != first.head || r.tail != first.tail {
@@ -1003,7 +765,7 @@ func (s *scanState) contentCandidates(
first = r first = r
} }
f, ok, err := unchangedFile(r, rules) f, ok, err := unchangedFile(r)
if err != nil { if err != nil {
s.st.skipped++ s.st.skipped++
@@ -1025,11 +787,6 @@ func (s *scanState) contentCandidates(
return nil, nil, err return nil, nil, err
} }
// A search that returned no record still ends showing its count.
if searching {
prog.setTotal(total)
}
endGroup() endGroup()
return toRead, recs, nil return toRead, recs, nil
@@ -1037,10 +794,9 @@ func (s *scanState) contentCandidates(
// unchangedFile lstats the file r names and returns it for reading if // unchangedFile lstats the file r names and returns it for reading if
// it is still the regular file r records: the same size, and an mtime // it is still the regular file r records: the same size, and an mtime
// no newer than recorded (the walk's change rule). A file that is gone, // no newer than recorded (the walk's change rule). A file that is gone
// has changed, or is on a filesystem in rules that is not persistent // or has changed reports false; any other lstat error is returned.
// reports false; any other lstat error is returned. func unchangedFile(r scanRec) (fileRec, bool, error) {
func unchangedFile(r scanRec, rules walkRules) (fileRec, bool, error) {
fi, err := os.Lstat(r.path) fi, err := os.Lstat(r.path)
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
return fileRec{}, false, nil return fileRec{}, false, nil
@@ -1051,33 +807,17 @@ func unchangedFile(r scanRec, rules walkRules) (fileRec, bool, error) {
} }
if !fi.Mode().IsRegular() || fi.Size() != r.size || if !fi.Mode().IsRegular() || fi.Size() != r.size ||
mtimeAfter(fi.ModTime(), r.mtime) { fi.ModTime().Unix() > r.mtime {
return fileRec{}, false, nil return fileRec{}, false, nil
} }
dev, ino := inodeOfInfo(fi) dev, ino := inodeOfInfo(fi)
// The scan reads nothing on a filesystem that is not persistent,
// whatever record an earlier scan left for the file.
if _, found := rules.nonPersistent[dev]; found {
return fileRec{}, false, nil
}
return fileRec{ return fileRec{
path: r.path, size: r.size, mtime: r.mtime, dev: dev, ino: ino, path: r.path, size: r.size, mtime: r.mtime, dev: dev, ino: ino,
}, true, nil }, true, nil
} }
// mtimeAfter reports whether mtime a is later than mtime b.
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
func mtimeAfter(a, b time.Time) bool {
if a.Unix() != b.Unix() {
return a.Unix() > b.Unix()
}
return a.Nanosecond() > b.Nanosecond()
}
// underAnyRoot reports whether path is any of the roots or lies under // underAnyRoot reports whether path is any of the roots or lies under
// one of them. // one of them.
func underAnyRoot(path string, roots []string) bool { func underAnyRoot(path string, roots []string) bool {
@@ -1105,20 +845,6 @@ func underRoot(path, root string) bool {
return strings.HasPrefix(path, prefix) return strings.HasPrefix(path, prefix)
} }
// deepestRoot returns the longest of the roots that path is or lies
// under, or "" when it lies under none.
func deepestRoot(path string, roots []string) string {
deepest := ""
for _, root := range roots {
if underRoot(path, root) && len(root) > len(deepest) {
deepest = root
}
}
return deepest
}
// dirJob is one directory awaiting traversal by the walk workers. It // dirJob is one directory awaiting traversal by the walk workers. It
// carries its operand's filesystem device so -x can stop at // carries its operand's filesystem device so -x can stop at
// filesystem boundaries. // filesystem boundaries.
@@ -1129,36 +855,28 @@ type dirJob struct {
} }
// walkEvent is one walk result delivered to the walk phase: a regular // walkEvent is one walk result delivered to the walk phase: a regular
// file's statted record, a warning when fail is set, or a path the walk // file's statted record, or a warning when fail is set.
// left out: when otherFSDir is set, a directory -x did not enter
// because it is on another filesystem than its operand, and when
// leftOut is set, a default exclude or a directory or file on a
// filesystem that is not persistent, with warn holding the line that
// says so.
type walkEvent struct { type walkEvent struct {
rec fileRec rec fileRec
warn string warn string
fail bool fail bool
otherFSDir string
leftOut string
} }
// startWalk seeds every root into the shared walk worker pool and // startWalk seeds every root into the shared walk worker pool and
// returns the event stream: one record per regular file, one warning // returns the event stream: one record per regular file, one warning
// event per per-path error, and one event per path -x or rules kept the // event per per-path error. The channel is closed when the walk
// walk from entering or recording. The channel is closed when the walk
// completes, and also when ctx is cancelled — every goroutine in the // completes, and also when ctx is cancelled — every goroutine in the
// pool abandons its blocking send in that case, so the consumer sees a // pool abandons its blocking send in that case, so the consumer sees a
// truncated but properly terminated stream instead of a stalled one. // truncated but properly terminated stream instead of a stalled one.
func startWalk(ctx context.Context, roots []string, oneFS bool, func startWalk(ctx context.Context, roots []string, oneFS bool,
rules walkRules, workers int, workers int,
) <-chan walkEvent { ) <-chan walkEvent {
jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, rules) jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS)
go func() { go func() {
initial := make([]dirJob, 0, len(roots)) initial := make([]dirJob, 0, len(roots))
for _, root := range roots { for _, root := range roots {
initial = append(initial, seedRoot(ctx, root, rules, events)...) initial = append(initial, seedRoot(ctx, root, events)...)
} }
dispatchDirs(ctx, initial, jobs, subdirs) dispatchDirs(ctx, initial, jobs, subdirs)
@@ -1212,14 +930,12 @@ func operandWarning(root string, fi fs.FileInfo) string {
} }
// seedRoot turns one PATH operand into the walk's starting state: a // seedRoot turns one PATH operand into the walk's starting state: a
// regular-file operand is statted and emitted directly, unless it is // regular-file operand is statted and emitted directly, and a directory
// one of the scan's own database files in rules, and a directory
// operand becomes an initial job. walkableRoots has already dropped // operand becomes an initial job. walkableRoots has already dropped
// every other operand. One that has changed into something else since // every other operand. One that has changed into something else since
// is warned about and skipped here; it is still a root, so the records // is warned about and skipped here; it is still a root, so the records
// stored beneath it, other than those under another root, are deleted // stored beneath it are deleted as unverified.
// as unverified. func seedRoot(ctx context.Context, root string,
func seedRoot(ctx context.Context, root string, rules walkRules,
events chan<- walkEvent, events chan<- walkEvent,
) []dirJob { ) []dirJob {
fi, err := os.Lstat(root) fi, err := os.Lstat(root)
@@ -1246,17 +962,14 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
} }
dev, ino := inodeOfInfo(fi) dev, ino := inodeOfInfo(fi)
rec := fileRec{
sendEvent(ctx, events, walkEvent{rec: fileRec{
path: root, path: root,
size: fi.Size(), size: fi.Size(),
mtime: fi.ModTime(), mtime: fi.ModTime().Unix(),
dev: dev, dev: dev,
ino: ino, ino: ino,
} }})
if !isDatabaseFile(rec, rules) {
sendEvent(ctx, events, walkEvent{rec: rec})
}
return nil return nil
} }
@@ -1268,7 +981,7 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
// workers drop the directories still queued rather than stop reading // workers drop the directories still queued rather than stop reading
// jobs, so the range always runs out and the pool always tears down. // jobs, so the range always runs out and the pool always tears down.
func startWalkWorkers(ctx context.Context, workers int, func startWalkWorkers(ctx context.Context, workers int,
oneFS bool, rules walkRules, oneFS bool,
) (chan dirJob, chan []dirJob, chan walkEvent) { ) (chan dirJob, chan []dirJob, chan walkEvent) {
jobs := make(chan dirJob, workQueueDepth) jobs := make(chan dirJob, workQueueDepth)
subdirs := make(chan []dirJob, workers) subdirs := make(chan []dirJob, workers)
@@ -1284,7 +997,7 @@ func startWalkWorkers(ctx context.Context, workers int,
} }
select { select {
case subdirs <- walkOneDir(ctx, job, oneFS, rules, events): case subdirs <- walkOneDir(ctx, job, oneFS, events):
case <-ctx.Done(): case <-ctx.Done():
} }
} }
@@ -1342,7 +1055,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
// entry and a warning event per unreadable one, and returns the // entry and a warning event per unreadable one, and returns the
// subdirectories to descend into. // subdirectories to descend into.
func walkOneDir(ctx context.Context, job dirJob, oneFS bool, func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
rules walkRules, events chan<- walkEvent, events chan<- walkEvent,
) []dirJob { ) []dirJob {
entries, err := os.ReadDir(job.path) entries, err := os.ReadDir(job.path)
if err != nil { if err != nil {
@@ -1366,8 +1079,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
p := filepath.Join(job.path, e.Name()) p := filepath.Join(job.path, e.Name())
if e.IsDir() { if e.IsDir() {
sub, ok := subdirJob(ctx, p, e, job, oneFS, rules, events) if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok {
if ok {
subs = append(subs, sub) subs = append(subs, sub)
} }
@@ -1379,7 +1091,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
continue continue
} }
emitFile(ctx, p, e, rules, events) emitFile(ctx, p, e, events)
} }
return subs return subs
@@ -1389,12 +1101,9 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
// The lstat happens here in the walk worker, while the directory's // The lstat happens here in the walk worker, while the directory's
// metadata is still hot; a path that fails to stat (or stops being a // metadata is still hot; a path that fails to stat (or stops being a
// regular file) between the directory read and the lstat is warned // regular file) between the directory read and the lstat is warned
// about and skipped. A file on a filesystem in rules that is not // about and skipped.
// persistent, as through a bind mount of one file, is not recorded: it
// gets an event naming it instead. One of the scan's own database files
// in rules gets no event at all.
func emitFile(ctx context.Context, p string, e fs.DirEntry, func emitFile(ctx context.Context, p string, e fs.DirEntry,
rules walkRules, events chan<- walkEvent, events chan<- walkEvent,
) { ) {
info, err := e.Info() info, err := e.Info()
if err != nil { if err != nil {
@@ -1410,43 +1119,33 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
return return
} }
line := rules.nonPersistentLine(p, info)
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
return
}
dev, ino := inodeOfInfo(info) dev, ino := inodeOfInfo(info)
rec := fileRec{
sendEvent(ctx, events, walkEvent{rec: fileRec{
path: p, path: p,
size: info.Size(), size: info.Size(),
mtime: info.ModTime(), mtime: info.ModTime().Unix(),
dev: dev, dev: dev,
ino: ino, ino: ino,
} }})
if !isDatabaseFile(rec, rules) {
sendEvent(ctx, events, walkEvent{rec: rec})
}
} }
// subdirJob applies the descent rules to directory p: never enter // subdirJob applies the descent rules to directory p: never enter
// .zfs (ZFS snapshot pseudo-dirs would list every file once per // .zfs (ZFS snapshot pseudo-dirs would list every file once per
// snapshot), never enter a directory on a filesystem in rules that is // snapshot), and with -x never enter a directory on a different
// not persistent, nor a default exclude in rules, and with -x never // filesystem than its operand.
// enter a directory on a different filesystem than its operand. For the
// last three it sends an event that names the directory, so the scan
// keeps the records below it, except those below an operand at or
// inside it.
func subdirJob(ctx context.Context, p string, e fs.DirEntry, func subdirJob(ctx context.Context, p string, e fs.DirEntry,
parent dirJob, oneFS bool, rules walkRules, parent dirJob, oneFS bool, events chan<- walkEvent,
events chan<- walkEvent,
) (dirJob, bool) { ) (dirJob, bool) {
if e.Name() == ".zfs" { if e.Name() == ".zfs" {
return dirJob{}, false return dirJob{}, false
} }
job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
if !oneFS || !parent.rootDevOK {
return job, true
}
info, err := e.Info() info, err := e.Info()
if err != nil { if err != nil {
sendEvent(ctx, events, walkEvent{ sendEvent(ctx, events, walkEvent{
@@ -1457,30 +1156,7 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
return dirJob{}, false return dirJob{}, false
} }
line := rules.nonPersistentLine(p, info)
if line != "" {
sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
return dirJob{}, false
}
if isDefaultExclude(p, rules.excludes) {
sendEvent(ctx, events, walkEvent{
leftOut: p,
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
})
return dirJob{}, false
}
job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
if !oneFS || !parent.rootDevOK {
return job, true
}
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
sendEvent(ctx, events, walkEvent{otherFSDir: p})
return dirJob{}, false return dirJob{}, false
} }
@@ -1622,38 +1298,6 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
} }
} }
// errNotRegular reports a path that is no longer a regular file when it
// is opened to be read.
var errNotRegular = errors.New("not a regular file")
// openRegularFile opens path for reading and returns it only if it is
// still a regular file, since the path may have been replaced after it
// was statted. O_NONBLOCK keeps the open of a FIFO or a device node
// from waiting forever, and fstat then rejects it. Reads of a regular
// file ignore O_NONBLOCK.
func openRegularFile(path string) (*os.File, error) {
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
fi, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, err
}
if !fi.Mode().IsRegular() {
_ = f.Close()
return nil, errNotRegular
}
return f, nil
}
// emptyHash is the lowercase-hex SHA-256 of the empty input: the head, // emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
// tail, and content hash of every zero-length file. // tail, and content hash of every zero-length file.
const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
@@ -1676,7 +1320,8 @@ func hashSignature(path string, size int64) (string, string, string, error) {
return emptyHash, emptyHash, emptyHash, nil return emptyHash, emptyHash, emptyHash, nil
} }
f, err := openRegularFile(path) //nolint:gosec // hashing operator-supplied paths is the tool's purpose
f, err := os.Open(path)
if err != nil { if err != nil {
return "", "", "", err return "", "", "", err
} }
@@ -1707,7 +1352,8 @@ func hashSignature(path string, size int64) (string, string, string, error) {
// tail are returned empty, because the content phase keeps the ones its // tail are returned empty, because the content phase keeps the ones its
// records already hold. // records already hold.
func hashContentOnly(path string, size int64) (string, string, string, error) { func hashContentOnly(path string, size int64) (string, string, string, error) {
f, err := openRegularFile(path) //nolint:gosec // hashing operator-supplied paths is the tool's purpose
f, err := os.Open(path)
if err != nil { if err != nil {
return "", "", "", err return "", "", "", err
} }
+17 -791
View File
@@ -20,8 +20,6 @@ import (
"syscall" "syscall"
"testing" "testing"
"time" "time"
"golang.org/x/sys/unix"
) )
// writeFile creates a file with the given content and returns its path. // writeFile creates a file with the given content and returns its path.
@@ -583,65 +581,6 @@ func TestScanContentHashedStalePartners(t *testing.T) {
} }
} }
// TestScanContentSameSecondRewrite is TestScanContentStalePartners for
// a stored file rewritten in place at the same size with an mtime later
// in the same second than recorded: the file counts as changed, so
// neither it nor its match inside the operand is read.
func TestScanContentSameSecondRewrite(t *testing.T) {
t.Parallel()
db := openTestDB(t)
dirA := t.TempDir()
changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin)
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(changed, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dirA)
before := dbRecords(t, db)
// Rewrite one byte in place, keeping the size.
pokeAt(t, changed, headTailMin/2, []byte{1})
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(changed, later, later)
if err != nil {
t.Fatal(err)
}
dirB := t.TempDir()
sparseFile(t, dirB, "changed-copy", headTailMin)
st := syncTree(t, db, dirB)
if st != (scanStats{walked: 1, added: 1}) {
t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
}
recs := dbRecords(t, db)
for _, r := range recs {
if r.content != "" {
t.Errorf("%s: content = %q, want none: its only match is stale",
r.path, r.content)
}
}
for _, old := range before {
if r := recordByPath(t, recs, old.path); r != old {
t.Errorf("record = %+v, want it left as %+v", r, old)
}
}
if groups := dupeGroups(t, db); len(groups) != 0 {
t.Errorf("groups = %+v, want none", groups)
}
}
// TestScanContentReadFailure checks that a failed content read is // TestScanContentReadFailure checks that a failed content read is
// counted as skipped and leaves the record without a content hash, and // counted as skipped and leaves the record without a content hash, and
// that a later scan tries the read again. // that a later scan tries the read again.
@@ -782,8 +721,7 @@ func collectWalk(t *testing.T, roots []string, oneFS bool,
errs int errs int
) )
for ev := range startWalk(t.Context(), roots, oneFS, walkRules{}, for ev := range startWalk(t.Context(), roots, oneFS, workers) {
workers) {
if ev.fail { if ev.fail {
errs++ errs++
@@ -845,8 +783,8 @@ func TestWalk(t *testing.T) {
t.Errorf("%s: size = %d, want 1..3", r.path, r.size) t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
} }
if r.mtime.Unix() <= 0 { if r.mtime <= 0 {
t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime) t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
} }
} }
} }
@@ -998,15 +936,14 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
} }
// callSubdirJob runs subdirJob against e under parent, collecting any // callSubdirJob runs subdirJob against e under parent, collecting any
// events it emits (subdirJob emits at most one). // warning events it emits (subdirJob emits at most one).
func callSubdirJob(t *testing.T, p string, e fs.DirEntry, func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
parent dirJob, oneFS bool, parent dirJob, oneFS bool,
) (dirJob, bool, []walkEvent) { ) (dirJob, bool, []walkEvent) {
t.Helper() t.Helper()
events := make(chan walkEvent, 1) events := make(chan walkEvent, 1)
job, ok := subdirJob(t.Context(), p, e, parent, oneFS, walkRules{}, job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events)
events)
close(events) close(events)
var evs []walkEvent var evs []walkEvent
@@ -1077,15 +1014,8 @@ func TestSubdirJobOneFilesystem(t *testing.T) {
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK) t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
} }
// A refused descent names the directory, so the scan keeps if len(evs) != 0 {
// the records below it. t.Fatalf("unexpected events: %+v", evs)
var wantEvs []walkEvent
if !ok {
wantEvs = []walkEvent{{otherFSDir: sub}}
}
if !slices.Equal(evs, wantEvs) {
t.Fatalf("events = %+v, want %+v", evs, wantEvs)
} }
// The accepted job must carry the operand's device down, or -x // The accepted job must carry the operand's device down, or -x
@@ -1137,296 +1067,6 @@ func TestSubdirJobStatError(t *testing.T) {
} }
} }
// syncOneFilesystem scans roots into db with -x and returns the scan
// stats. Every device pruneRoots looks up, and the device each
// operand's walk compares its subdirectories with, comes from devices
// instead of from lstat, so a test can put a path on a filesystem that
// no directory here is on. Unlike runPhases, it skips walkableRoots,
// the check for a cancelled walk, and the content phase.
func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
devices map[string]uint64,
) scanStats {
t.Helper()
device := func(path string) (uint64, bool) {
dev, ok := devices[path]
return dev, ok
}
roots = pruneRoots(roots, true, walkRules{}, device)
ctx := t.Context()
s := &scanState{db: db}
err := s.loadIndex(ctx, roots)
if err != nil {
t.Fatal(err)
}
jobs, subdirs, events := startWalkWorkers(ctx, 2, true, walkRules{})
// seedRoot emits a regular-file operand itself and returns a job
// only for a directory operand.
var initial []dirJob
for _, root := range roots {
for _, job := range seedRoot(ctx, root, walkRules{}, events) {
job.rootDev = devices[root]
initial = append(initial, job)
}
}
dispatchDirs(ctx, initial, jobs, subdirs)
changed, unhashed := s.walkPhase(events, roots)
s.partition(changed, unhashed)
err = s.hashPhase(ctx, 2)
if err != nil {
t.Fatal(err)
}
err = s.updatePhase(ctx)
if err != nil {
t.Fatal(err)
}
return s.st
}
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
// record below a directory the walk does not enter, because it is on
// another filesystem than its operand, counts as outside the operands:
// it is not deleted, and its size joins the size census. The record of
// a gone file that is not below that directory is still deleted, even
// when its path begins with the directory's name. The operand is given
// a device that no directory here is on, so no second real filesystem
// is needed.
func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
// A scan without -x records the file below mnt, and mnt2, which is
// beside mnt, not below it.
mounted := writeFile(t, dir, "mnt/f", pattern(1, 100))
gone := writeFile(t, dir, "mnt2", pattern(3, 50))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
// The same size as mnt/f, so it is hashed only if mnt/f's size is
// in the census.
added := writeFile(t, dir, "added", pattern(2, 100))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1})
recs := dbRecords(t, db)
recordByPath(t, recs, mounted)
if recordByPath(t, recs, added).head == "" {
t.Errorf("%s not hashed: the census lacks the size of %s",
added, mounted)
}
if slices.Contains(recordPaths(recs), gone) {
t.Errorf("record for %s kept: the file is gone and not below mnt",
gone)
}
}
// TestScanOneFilesystemWalksNestedOperand checks that with -x an
// operand under another one, on a different filesystem than it, is
// walked on its own: a file added below it is recorded, and the record
// of a file gone from below it is deleted. A record below a directory
// that its own walk does not enter is still kept. Each operand is given
// a device that no directory here is on, so neither walk enters a
// subdirectory.
func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
// A scan without -x records both files below mnt.
gone := writeFile(t, dir, "mnt/gone", pattern(1, 10))
kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
added := writeFile(t, dir, "mnt/added", pattern(3, 30))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
mnt := filepath.Join(dir, "mnt")
syncOneFilesystem(t, db, []string{dir, mnt},
map[string]uint64{dir: dev + 1, mnt: dev + 2})
paths := recordPaths(dbRecords(t, db))
if !slices.Contains(paths, added) {
t.Errorf("no record for %s: the walk did not reach %s", added, mnt)
}
if slices.Contains(paths, gone) {
t.Errorf("record for %s kept: the file is gone", gone)
}
if !slices.Contains(paths, kept) {
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
kept, mnt, filepath.Dir(kept))
}
}
// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
// with -x an operand on the same filesystem as the operand covering it
// is walked on its own when a directory between them is on another
// filesystem, since the walk of the covering operand does not enter
// that directory: a file added below the nested operand is recorded,
// and the record of a file gone from below it is deleted. A record
// below that directory but outside the nested operand is still kept.
// Both operands are given one device that no directory here is on,
// while mnt keeps its own.
func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
// A scan without -x records both files below mnt.
gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
mnt := filepath.Join(dir, "mnt")
back := filepath.Join(mnt, "back")
syncOneFilesystem(t, db, []string{dir, back},
map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
paths := recordPaths(dbRecords(t, db))
if !slices.Contains(paths, added) {
t.Errorf("no record for %s: the walk did not reach %s", added, back)
}
if slices.Contains(paths, gone) {
t.Errorf("record for %s kept: the file is gone", gone)
}
if !slices.Contains(paths, kept) {
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
kept, dir, mnt)
}
}
// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
// regular-file operand under another operand is dropped when the file
// reports a different device than that operand but its directory does
// not, as a file bind-mounted into the tree does: the walk of the other
// operand enters the directory and reaches the file, so the file is
// recorded once.
func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
file := writeFile(t, dir, "sub/f", pattern(1, 10))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
dir: dev, filepath.Dir(file): dev, file: dev + 1,
})
if st.added != 1 {
t.Errorf("%s recorded %d times, want once", file, st.added)
}
}
// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a
// regular-file operand under another operand is kept when its directory
// is on a different filesystem than that operand: the walk of the other
// operand does not enter the directory, so the file is recorded once,
// as its own operand. The other operand is given a device that no
// directory here is on.
func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
file := writeFile(t, dir, "sub/f", pattern(1, 10))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
dir: dev + 1, filepath.Dir(file): dev,
})
if st.added != 1 {
t.Errorf("%s recorded %d times, want once", file, st.added)
}
}
// buildSmokeTree recreates the README smoke-test filesystem layout // buildSmokeTree recreates the README smoke-test filesystem layout
// with deterministic content and returns the tree root. // with deterministic content and returns the tree root.
func buildSmokeTree(t *testing.T) string { func buildSmokeTree(t *testing.T) string {
@@ -1465,7 +1105,7 @@ const smokeTreeFiles = 15
func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
t.Helper() t.Helper()
st, err := syncScan(t.Context(), db, roots, 4, false, walkRules{}) st, err := syncScan(t.Context(), db, roots, 4, false)
if err != nil { if err != nil {
t.Fatalf("syncScan: %v", err) t.Fatalf("syncScan: %v", err)
} }
@@ -1480,7 +1120,7 @@ func dbRecords(t *testing.T, db *sql.DB) []scanRec {
var recs []scanRec var recs []scanRec
err := loadFileRows(t.Context(), db, func(int64) {}, func(r scanRec) { err := loadFileRows(t.Context(), db, func(r scanRec) {
recs = append(recs, r) recs = append(recs, r)
}) })
if err != nil { if err != nil {
@@ -1552,7 +1192,7 @@ func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) {
super, dirs := dbTree(t, db) super, dirs := dbTree(t, db)
tg := collectTreeGroups(dirs, super, nil) tg := collectTreeGroups(dirs, super)
if len(tg) != 1 { if len(tg) != 1 {
t.Fatalf("len(tree groups) = %d, want 1", len(tg)) t.Fatalf("len(tree groups) = %d, want 1", len(tg))
} }
@@ -1588,69 +1228,6 @@ func TestScanPipeline(t *testing.T) {
assertSmokeTreeGroups(t, dir, db) assertSmokeTreeGroups(t, dir, db)
} }
// TestScanProgressNamesEachPhase checks the plain progress lines of a
// first scan, and of a rescan of one operand holding two of the
// database's records. Each phase's first line names it with its place
// among the five: load counts the records before reading them, the
// content phase searches the database, and the walk estimates its total
// from the records under its operand, or says its ETA is unknown when
// there are none.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanProgressNamesEachPhase(t *testing.T) {
stderr := captureStderr(t)
dir := buildSmokeTree(t)
db := openTestDB(t)
syncTree(t, db, dir)
first := stderr()
syncTree(t, db, filepath.Join(dir, "a"))
rescan := stderr()[len(first):]
load := "[1/5] load: counting the records, elapsed 0s, eta unknown"
content := "[5/5] content: searching the database, elapsed 0s, eta unknown"
cases := map[string][]string{
first: {
load, "[2/5] walk: 0 files, 0 files/s, elapsed 0s, eta unknown",
"[3/5] hash: [0/", "[4/5] update: [0/", content,
},
rescan: {
load, "[2/5] walk: [0/~2] 0%, 0 files/s, elapsed 0s, eta unknown",
"[3/5] hash: [0/", "[4/5] update: [0/", content,
},
}
for out, want := range cases {
var got []string
for line := range strings.Lines(out) {
label, _, _ := strings.Cut(line, ":")
if len(got) == 0 || !strings.HasPrefix(got[len(got)-1], label) {
got = append(got, line)
}
}
if len(got) != len(want) {
t.Fatalf("first lines of the phases = %q, want %q", got, want)
}
for i := range want {
if !strings.HasPrefix(got[i], want[i]) {
t.Errorf("phase %d starts %q, want %q", i+1, got[i], want[i])
}
}
}
// The load phase's total is the count of the records.
loaded := fmt.Sprintf("\n[1/5] load: [%d/%d] 100%%, ",
smokeTreeFiles, smokeTreeFiles)
if !strings.Contains(rescan, loaded) {
t.Errorf("rescan stderr = %q, want %q", rescan, loaded)
}
}
func TestSyncScanUnchangedReuse(t *testing.T) { func TestSyncScanUnchangedReuse(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1708,172 +1285,11 @@ func TestSyncScanMtimeBump(t *testing.T) {
t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
} }
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) { if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
t.Fatalf("mtime = %v, want %v", r.mtime, future) t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
} }
} }
// assertWholeFileHashed fails unless the record for path holds the
// whole-file hash of data as its head, tail, and content.
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
data []byte,
) {
t.Helper()
r := recordByPath(t, dbRecords(t, db), path)
if want := hexSum(data); r.head != want || r.tail != want ||
r.content != want {
t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
r.head, r.tail, r.content, want)
}
}
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
// size with an mtime later in the same second as the recorded one: the
// next scan must notice the change and re-hash the file.
func TestSyncScanSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite
// for files given to scan as operands, which scan stats without reading
// their directory.
func TestSyncScanOperandSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
b := writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, a, b)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, a, b)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanRewriteAfter2262 runs assertLateRewriteRehashed with an
// mtime after 2262, a time too late to count in nanoseconds in an int64.
func TestSyncScanRewriteAfter2262(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC))
}
// TestSyncScanRewritePastTimeLimit runs assertLateRewriteRehashed with an
// mtime one second past the latest a time.Time holds without wrapping it
// to a time far in the past.
func TestSyncScanRewritePastTimeLimit(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Unix(9223371974719179008, 0))
}
// assertLateRewriteRehashed scans a directory, rewrites a file in it in
// place at the same size, sets its mtime to late, and fails unless the
// next scan re-hashes the file. It skips where late does not fit the
// platform's timespec or the filesystem does not store it.
func assertLateRewriteRehashed(t *testing.T, late time.Time) {
t.Helper()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
// os.Chtimes cannot set such a time: it converts through UnixNano.
ts, err := unix.TimeToTimespec(late)
if err != nil {
t.Skipf("an mtime %d seconds after 1970 does not fit this platform's "+
"timespec: %v", late.Unix(), err)
}
err = unix.UtimesNano(a, []unix.Timespec{ts, ts})
if err != nil {
t.Fatal(err)
}
fi, err := os.Lstat(a)
if err != nil {
t.Fatal(err)
}
if fi.ModTime().Unix() != late.Unix() {
t.Skipf("the filesystem stored the mtime as %d seconds after 1970, "+
"not %d", fi.ModTime().Unix(), late.Unix())
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
func TestSyncScanAddRemove(t *testing.T) { func TestSyncScanAddRemove(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1919,7 +1335,9 @@ func TestSyncScanSizeChange(t *testing.T) {
writeFile(t, dir, "f", pattern(1, 200)) writeFile(t, dir, "f", pattern(1, 200))
err := os.Chtimes(p, old.mtime, old.mtime) mt := time.Unix(old.mtime, 0)
err := os.Chtimes(p, mt, mt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -2094,7 +1512,7 @@ func TestTreesUnhashedNeverEqual(t *testing.T) {
for name, unknown := range cases { for name, unknown := range cases {
super, dirs := treeOf(t, append(slices.Clone(shared), unknown...)) super, dirs := treeOf(t, append(slices.Clone(shared), unknown...))
if tg := collectTreeGroups(dirs, super, nil); len(tg) != 0 { if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
t.Errorf("%s: tree groups = %d, want 0 (the files may differ)", t.Errorf("%s: tree groups = %d, want 0 (the files may differ)",
name, len(tg)) name, len(tg))
} }
@@ -2159,54 +1577,6 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) {
} }
} }
// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file
// with a FIFO before the hash phase reads it. Opening a FIFO that has
// no writer waits for one, so a blocking open would hang the phase. The
// path must be warned about once and skipped, with no record written.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
stderr := captureStderr(t)
dir := t.TempDir()
p := writeFile(t, dir, "a.bin", pattern(1, 300))
walked, _ := collectWalk(t, []string{dir}, false, 1)
err := os.Remove(p)
if err != nil {
t.Fatal(err)
}
err = unix.Mkfifo(p, 0o600)
if err != nil {
t.Fatal(err)
}
s := &scanState{db: openTestDB(t), toHash: walked}
err = s.hashPhase(t.Context(), 1)
if err != nil {
t.Fatalf("hashPhase = %v, want nil", err)
}
if s.st.skipped != 1 || len(s.batch) != 0 {
t.Errorf("skipped = %d, records = %d, want 1 skipped and none",
s.st.skipped, len(s.batch))
}
warning := "hash " + p + ": not a regular file\n"
if got := stderr(); strings.Count(got, warning) != 1 {
t.Errorf("stderr = %q, want %q once", got, warning)
}
// The content phase opens files the same way.
_, _, content, err := hashContentOnly(p, headTailMin)
if content != "" || !errors.Is(err, errNotRegular) {
t.Errorf("hashContentOnly on a FIFO = %q, %v; want no hash and %v",
content, err, errNotRegular)
}
}
// injectedWriteFailure is the message the injected database trigger // injectedWriteFailure is the message the injected database trigger
// aborts with, so the test can recognize its own failure in the error // aborts with, so the test can recognize its own failure in the error
// the scan reports. // the scan reports.
@@ -2400,62 +1770,14 @@ func TestHashRuns(t *testing.T) {
func TestPruneRoots(t *testing.T) { func TestPruneRoots(t *testing.T) {
t.Parallel() t.Parallel()
const pqRoot = "/p/q"
// /p/q is on another filesystem than /p; /p/q/r is on the same one as
// /p/q, /p/s as /p, and /p/t has no known device.
devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1}
device := func(path string) (uint64, bool) {
dev, ok := devices[path]
return dev, ok
}
// Duplicates and operands under other operands are dropped; /cc is // Duplicates and operands under other operands are dropped; /cc is
// not under /c (sibling with a shared prefix). // not under /c (sibling with a shared prefix).
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
false, walkRules{}, device)
want := []string{"/a", "/c", "/cc"} want := []string{"/a", "/c", "/cc"}
if !slices.Equal(got, want) { if !slices.Equal(got, want) {
t.Fatalf("pruneRoots = %q, want %q", got, want) t.Fatalf("pruneRoots = %q, want %q", got, want)
} }
nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"}
got = pruneRoots(nested, false, walkRules{}, device)
want = []string{"/p"}
if !slices.Equal(got, want) {
t.Fatalf("pruneRoots without -x = %q, want %q", got, want)
}
// With -x, /p/q, on another filesystem than /p, is kept as well.
got = pruneRoots(nested, true, walkRules{}, device)
want = []string{"/p", pqRoot}
if !slices.Equal(got, want) {
t.Fatalf("pruneRoots with -x = %q, want %q", got, want)
}
// The walk of /p does not enter /p/q when it is a default exclude, so
// /p/q is kept, without -x too; /p/q/r lies under it and is dropped.
got = pruneRoots(nested, false, walkRules{excludes: []string{pqRoot}},
device)
if !slices.Equal(got, want) {
t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want)
}
// Nor does it enter /p/q when the filesystem of /p/q is not
// persistent, so /p/q/r/m, of no known device, is kept.
got = pruneRoots([]string{"/p", "/p/q/r/m"}, false,
walkRules{nonPersistent: map[uint64]string{2: "tmpfs"}}, device)
want = []string{"/p", "/p/q/r/m"}
if !slices.Equal(got, want) {
t.Fatalf("pruneRoots with %s not persistent = %q, want %q",
pqRoot, got, want)
}
} }
func TestReportsNeverTouchFilesystem(t *testing.T) { func TestReportsNeverTouchFilesystem(t *testing.T) {
@@ -2501,99 +1823,3 @@ func TestUnderRoot(t *testing.T) {
} }
} }
} }
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
// database, reached directly and through a symlinked directory. The
// walk passes over the database file, its -wal and -shm files and its
// .lock file with no line and no count, and the record an earlier scan
// left for the .lock file is deleted. A rescan of the unchanged tree
// updates nothing, and report lists none of the database's files.
func TestRunScanPassesOverDatabase(t *testing.T) {
for _, viaSymlink := range []bool{false, true} {
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
base := t.TempDir()
tree := filepath.Join(base, "real", "tree")
operand := tree
writeFile(t, tree, "a", pattern(1, 100))
writeFile(t, tree, "b", pattern(1, 100))
writeFile(t, tree, "empty1", nil)
writeFile(t, tree, "empty2", nil)
if viaSymlink {
err := os.Symlink(filepath.Join(base, "real"),
filepath.Join(base, "link"))
if err != nil {
t.Fatal(err)
}
operand = filepath.Join(base, "link", "tree")
}
dbPath := filepath.Join(tree, "db.sqlite")
t.Setenv(databaseEnv, dbPath)
db, err := openScanDatabase(t.Context(), dbPath)
if err != nil {
t.Fatal(err)
}
err = applyChanges(t.Context(), db, []scanRec{
{path: filepath.Join(operand, "db.sqlite.lock")},
}, nil, nil)
if err != nil {
t.Fatal(err)
}
closeScanDatabase(t.Context(), db, dbPath)
assertScanSummary(t,
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
operand)
assertScanSummary(t,
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
operand)
report := runStdout(t, cmdReport)
if strings.Contains(report, "db.sqlite") ||
!strings.Contains(report, filepath.Join(operand, "empty2")) {
t.Errorf("report = %q, want the empty files and none of "+
"the database's files", report)
}
})
}
}
// TestRunScanPassesOverDatabaseOperands scans the database file and its
// .lock file given as operands themselves: neither is recorded or
// counted.
func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
dbPath := testDBPath(t)
t.Setenv(databaseEnv, dbPath)
// The first scan creates both files.
scanOK(t, t.TempDir())
assertScanSummary(t,
"0 files seen (0 added, 0 updated, 0 removed, 0 unchanged)",
dbPath, dbPath+".lock")
}
// assertScanSummary scans operands and fails the test unless the
// summary line gives counts with nothing skipped, and the line naming
// the database is the only one that names one of its files.
func assertScanSummary(t *testing.T, counts string, operands ...string) {
t.Helper()
stderr := scanOK(t, operands...)
want := "scan: " + counts + ", 0 skipped\n"
if !strings.Contains(stderr, want) {
t.Errorf("scan %q: stderr = %q, want %q", operands, stderr, want)
}
if strings.Count(stderr, "db.sqlite") != 1 {
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
"database to name one of its files", operands, stderr)
}
}
+11 -34
View File
@@ -2,12 +2,10 @@
# script/bootstrap: install all dependencies needed to build and develop # script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already # this repo. Idempotent: every install is guarded by a check so already
# installed tools are skipped. Base tooling comes from nix, apt, brew, # installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes nothing is present. The # or apk (detected in that order); assumes nothing is present. Node is
# installed node is used only when its major version is the pinned one; # used directly if installed; otherwise it is installed at a pinned
# otherwise (node missing, or another major) the pinned version is # version via nvm (installing nvm itself first, from a hash-verified
# installed via nvm (installing nvm itself first, from a hash-verified # release archive, never curl | sh).
# release archive, never curl | sh), and yarn and the packages are
# installed under it.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -102,52 +100,31 @@ ensure_nvm() {
rm -rf "$tmp" rm -rf "$tmp"
} }
# True when the node on PATH has the pinned major version. Only the
# major is compared: the Dockerfile stages start from a node 22 alpine
# image whose exact version is not the pin, and nvm cannot install a
# prebuilt node on alpine. Another major is not used: the pinned yarn 1
# prints a deprecation warning under node 24. script/fmt and
# script/fmt-check carry this function unchanged, to run the yarn
# installed here.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
ensure_node() { ensure_node() {
if node_is_pinned_major; then if ! missing node; then return 0; fi
echo "bootstrap: using node $(node --version)"
return 0
fi
ensure_nvm ensure_nvm
nvm_sh "nvm install $NODE_VERSION" nvm_sh "nvm install $NODE_VERSION"
nvm_sh "nvm use $NODE_VERSION >/dev/null && \
echo \"bootstrap: using node \$(node --version)\""
} }
ensure_yarn() { ensure_yarn() {
if ! node_is_pinned_major; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
return 0
fi
if ! missing yarn; then return 0; fi if ! missing yarn; then return 0; fi
if ! missing corepack; then if ! missing corepack; then
corepack enable corepack enable
corepack prepare "yarn@$YARN_VERSION" --activate corepack prepare "yarn@$YARN_VERSION" --activate
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
else else
npm install -g "yarn@$YARN_VERSION" npm install -g "yarn@$YARN_VERSION"
fi fi
} }
install_js_deps() { install_js_deps() {
if node_is_pinned_major; then if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
yarn install --frozen-lockfile
else
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
yarn install --frozen-lockfile" yarn install --frozen-lockfile"
else
yarn install --frozen-lockfile
fi fi
} }
+1 -3
View File
@@ -1,8 +1,6 @@
#!/bin/sh #!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own # script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. test and lint are Docker # extension to scripts-to-rule-them-all. Must not modify any files.
# phases; fmt-check is native, because a formatter writes the working
# tree. Must not modify any files.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+9 -14
View File
@@ -2,12 +2,9 @@
# script/cibuild: run the CI build. It bootstraps first: a CI runner # script/cibuild: run the CI build. It bootstraps first: a CI runner
# checks out and runs this and nothing else, and script/fmt-check runs # checks out and runs this and nothing else, and script/fmt-check runs
# the formatter on the host, which a pristine checkout cannot do. # the formatter on the host, which a pristine checkout cannot do.
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST # --no-cache for the same reason as script/docker: the gate phases the
# line, since a cached gate phase is a check that did not run; unlike # final stage depends on are RUN steps, and a cached one is a check that
# --no-cache, it keeps the layers above that line and the cache mounts. # did not run.
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
# before every step that lints, tests or compiles; a stage without that
# line is served from cache on an unchanged tree, with no warning.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -17,17 +14,15 @@ main() {
cd "$ROOT" cd "$ROOT"
"$SCRIPT_DIR/bootstrap" "$SCRIPT_DIR/bootstrap"
"$SCRIPT_DIR/check" "$SCRIPT_DIR/check"
# The version and the tag each get their own line: a failing # Own line: a failing command substitution inside an argument does
# command substitution inside an argument does not trip `set -e`, # not trip `set -e`, so the inline form degrades silently to an
# so the inline form degrades silently to an empty constant. The # empty constant. The VERSION build argument takes precedence over
# VERSION build argument takes precedence over the version a build # the version a build stage derives from the .git in the context.
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)" version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown" [ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")" docker build --no-cache \
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \ --build-arg VERSION="$version" \
-t "$tag" . -t "$("$SCRIPT_DIR/projectname")" .
} }
main "$@" main "$@"
+8 -14
View File
@@ -1,12 +1,8 @@
#!/bin/sh #!/bin/sh
# script/docker: build the Docker image tagged with the project name. # script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname. # Identical in all repos; the tag comes from script/projectname.
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST # --no-cache because the gate phases the final stage depends on are RUN
# line, since a cached gate phase is a check that did not run; unlike # steps, and a cached one is a check that did not run.
# --no-cache, it keeps the layers above that line and the cache mounts.
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
# before every step that lints, tests or compiles; a stage without that
# line is served from cache on an unchanged tree, with no warning.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -14,17 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
# The version and the tag each get their own line: a failing # Own line: a failing command substitution inside an argument does
# command substitution inside an argument does not trip `set -e`, # not trip `set -e`, so the inline form degrades silently to an
# so the inline form degrades silently to an empty constant. The # empty constant. The VERSION build argument takes precedence over
# VERSION build argument takes precedence over the version a build # the version a build stage derives from the .git in the context.
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)" version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown" [ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")" docker build --no-cache \
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \ --build-arg VERSION="$version" \
-t "$tag" . -t "$("$SCRIPT_DIR/projectname")" .
} }
main "$@" main "$@"
+6 -16
View File
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap. # Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0" NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same # script/bootstrap installs node and yarn under nvm and leaves neither
# function as in script/bootstrap, which uses it to decide where it # on the PATH of the shell that called it, so resolve the pinned
# installs yarn. # toolchain here the way bootstrap's own install step does. nvm is a
node_is_pinned_major() { # bash script, hence the subshell.
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
run_yarn() { run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then if command -v yarn >/dev/null 2>&1; then
exec yarn "$@" exec yarn "$@"
fi fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt: no yarn; run script/bootstrap first" >&2 echo "fmt: no yarn; run script/bootstrap first" >&2
exit 1 exit 1
fi fi
+6 -16
View File
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap. # Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0" NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same # script/bootstrap installs node and yarn under nvm and leaves neither
# function as in script/bootstrap, which uses it to decide where it # on the PATH of the shell that called it, so resolve the pinned
# installs yarn. # toolchain here the way bootstrap's own install step does. nvm is a
node_is_pinned_major() { # bash script, hence the subshell.
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
run_yarn() { run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then if command -v yarn >/dev/null 2>&1; then
exec yarn "$@" exec yarn "$@"
fi fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt-check: no yarn; run script/bootstrap first" >&2 echo "fmt-check: no yarn; run script/bootstrap first" >&2
exit 1 exit 1
fi fi
+7 -3
View File
@@ -1,15 +1,19 @@
#!/bin/sh #!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs # script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all. # script/precommit. Our own extension to scripts-to-rule-them-all.
# Hooks are shared between the main checkout and all worktrees, so
# resolve the common git dir instead of assuming .git is a directory.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
hook=".git/hooks/pre-commit" hooks_dir="$(git rev-parse --git-common-dir)/hooks"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit mkdir -p "$hooks_dir"
chmod +x .git/hooks/pre-commit hook="$hooks_dir/pre-commit"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
chmod +x "$hook"
echo "pre-commit hook installed: runs script/precommit" echo "pre-commit hook installed: runs script/precommit"
} }
+4 -9
View File
@@ -5,14 +5,9 @@
# make its answer untrustworthy. # make its answer untrustworthy.
# #
# The phase is not the last stage in the file, so it is built only when # The phase is not the last stage in the file, so it is built only when
# --target names it. A new CACHEBUST value reruns every RUN step after # --target names it. --no-cache because a cached lint layer is a lint
# the ARG CACHEBUST line, since a cached lint layer is a lint that did # that did not run. --output type=cacheonly writes no image, since
# not run; unlike --no-cache, it keeps the layers above that line and # nothing uses one.
# the cache mounts. This relies on every stage of the Dockerfile
# declaring ARG CACHEBUST before every step that lints, tests or
# compiles; a stage without that line is served from cache on an
# unchanged tree, with no warning.
# --output type=cacheonly writes no image, since nothing uses one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -20,7 +15,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build --no-cache \
--target lint \ --target lint \
--output type=cacheonly . --output type=cacheonly .
} }
+2 -2
View File
@@ -1,13 +1,13 @@
#!/bin/sh #!/bin/sh
# script/precommit: run by the git pre-commit hook; fails the commit if # script/precommit: run by the git pre-commit hook; fails the commit if
# checks fail. Our own extension to scripts-to-rule-them-all. # checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
# go mod tidy must be a no-op before the checks run.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
# Go extra: go mod tidy must be a no-op before the checks run.
cd "$ROOT" cd "$ROOT"
go mod tidy go mod tidy
if ! git diff --exit-code -- go.mod go.sum; then if ! git diff --exit-code -- go.mod go.sum; then
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh #!/bin/sh
# script/setup: set up the repo for development after a fresh clone: # script/setup: set up the repo for development after a fresh clone:
# installs dependencies and the git pre-commit hook. # installs dependencies (script/bootstrap) and the git pre-commit hook.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+3 -8
View File
@@ -2,13 +2,8 @@
# script/test: run the test suite. Testing is a phase of the Dockerfile # script/test: run the test suite. Testing is a phase of the Dockerfile
# and this builds that phase alone, on the same terms as script/lint: # and this builds that phase alone, on the same terms as script/lint:
# --target because a phase that is not the last stage is built only when # --target because a phase that is not the last stage is built only when
# named, and a new CACHEBUST value because a cached test layer is a test # named, and --no-cache because a cached test layer is a test that did
# that did not run; unlike --no-cache, it keeps the layers above the ARG # not run. --output type=cacheonly writes no image, since nothing uses one.
# CACHEBUST line and the cache mounts. This relies on every stage of the
# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
# or compiles; a stage without that line is served from cache on an
# unchanged tree, with no warning. --output type=cacheonly writes no
# image, since nothing uses one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -16,7 +11,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build --no-cache \
--target test \ --target test \
--output type=cacheonly . --output type=cacheonly .
} }
+28 -117
View File
@@ -1,9 +1,9 @@
package main package main
import ( import (
"bufio"
"context" "context"
"crypto/sha256" "crypto/sha256"
"database/sql"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -24,132 +24,44 @@ type treeNode struct {
totalSize int64 totalSize int64
} }
// treesPhase names one phase of trees with its place among all of them,
// as "[2/4] read", for the phase's progress display. The list here is
// the one place the phases are counted.
func treesPhase(name string) string {
return phaseLabel([]string{openPhase, "read", "group", "write"}, name)
}
// runTrees implements the trees subcommand: it reads every record from // runTrees implements the trees subcommand: it reads every record from
// the database in path order, reconstructs the directory hierarchy from // the database in path order, reconstructs the directory hierarchy from
// the record paths, computes a Merkle-style digest per directory, and // the record paths, computes a Merkle-style digest per directory, and
// prints maximal duplicate-tree groups as TSV on stdout. It never // prints maximal duplicate-tree groups as TSV on stdout. It never
// touches the scanned filesystem; its only I/O is the database, stdout, // touches the scanned filesystem; its only I/O is the database, stdout,
// and stderr. It names the database on stderr before it opens it. Any // and stderr. Any database problem, including a missing database, is
// database problem, including a missing database, is fatal. // fatal.
//
// The work runs in four phases, each under a progress display of its
// own on stderr: open (open the database), read (count the records,
// then read them into the hierarchy), group (group the directories by
// digest) and write (write the groups to stdout). Each display ends
// before the next phase starts, and before an error is returned to be
// printed.
func runTrees(ctx context.Context, stdout io.Writer) error { func runTrees(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath() dbPath := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: treesPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath) db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
prog = newProgress(progressParams{ records := 0
label: treesPhase("read"), unit: "records", total: -1, tree := newTreeBuilder()
waitingFor: countingRecords, stalledOn: "db",
err = loadFileRows(ctx, db, func(r scanRec) {
records++
tree.add(r)
}) })
records, super, allDirs, err := readTree(ctx, db, prog)
prog.finish()
if err != nil { if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err) return fmt.Errorf("database %s: %w", dbPath, err)
} }
// Grouping works in memory, so it waits on nothing but the
// processor.
prog = newProgress(progressParams{
label: treesPhase("group"), unit: "directories",
total: int64(len(allDirs)), stalledOn: "cpu",
})
dupes := collectTreeGroups(allDirs, super, prog)
prog.finish()
prog = newProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
stalledOn: "out",
})
dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
prog.finish()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
}
// readTree gives prog the number of records in db as its total, then
// reads every record in path order into the directory hierarchy,
// counting each on prog. It returns the number of records read, the
// super-root and every directory.
func readTree(ctx context.Context, db *sql.DB,
prog *progress,
) (int, *treeNode, []*treeNode, error) {
records := 0
tree := newTreeBuilder()
err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
records++
prog.increment()
tree.add(r)
})
if err != nil {
return 0, nil, nil, err
}
super, allDirs := tree.finish() super, allDirs := tree.finish()
return records, super, allDirs, nil dupes := collectTreeGroups(allDirs, super)
}
// writeTreeGroups writes dupes to stdout as TSV, a header line and then out := bufio.NewWriterSize(stdout, ioBufSize)
// a row for every member of each group but its first, counting each
// group on prog once its rows are written. stdout may be the terminal
// prog is drawn on, so the lines go through prog's writer. It returns
// the number of rows (dupe trees) and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress,
) (int, int64, error) {
out := prog.writer(stdout)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize") _, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
dupeTrees := 0 dupeTrees := 0
@@ -163,22 +75,25 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
escapePath(first.path), escapePath(n.path), escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize) first.fileCount, first.totalSize)
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
dupeTrees++ dupeTrees++
reclaimable += first.totalSize reclaimable += first.totalSize
} }
prog.increment()
} }
err = out.Flush() err = out.Flush()
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
return dupeTrees, reclaimable, nil fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
} }
// treeBuilder reconstructs the directory hierarchy from records added // treeBuilder reconstructs the directory hierarchy from records added
@@ -300,18 +215,14 @@ func fileEntry(name string, r scanRec) string {
"\x00" + r.head + "\x00" + r.tail + "\x00" + content "\x00" + r.head + "\x00" + r.tail + "\x00" + content
} }
// collectTreeGroups groups directories by digest, counting each one on // collectTreeGroups groups directories by digest and returns every
// prog as it is grouped, and returns every maximal group with two or // maximal group with two or more members, each group's members sorted
// more members, each group's members sorted by path, groups ordered by // by path, groups ordered by tree size descending then by first path
// tree size descending then by first path ascending. // ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode, func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
prog *progress,
) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode) groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs { for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d) groups[d.digest] = append(groups[d.digest], d)
prog.increment()
} }
var dupes [][]*treeNode var dupes [][]*treeNode
+7 -107
View File
@@ -3,11 +3,8 @@ package main
import ( import (
"bytes" "bytes"
"database/sql" "database/sql"
"os"
"slices" "slices"
"strings"
"testing" "testing"
"time"
) )
// Signature hashes shared by the smoke-test records. // Signature hashes shared by the smoke-test records.
@@ -20,11 +17,6 @@ const (
f2Content = "f2c" f2Content = "f2c"
) )
// treesOpenLine is the plain line of trees' open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
"eta unknown"
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
// /d/t2 are identical, /d/t3 differs from them only by one filename. // /d/t2 are identical, /d/t3 differs from them only by one filename.
func smokeTreeRecs() []scanRec { func smokeTreeRecs() []scanRec {
@@ -46,7 +38,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
tree := newTreeBuilder() tree := newTreeBuilder()
err := loadFileRows(t.Context(), db, func(int64) {}, tree.add) err := loadFileRows(t.Context(), db, tree.add)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -178,7 +170,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
t.Fatalf("directory paths = %q, want %q", got, want) t.Fatalf("directory paths = %q, want %q", got, want)
} }
groups := collectTreeGroups(dirs, super, nil) groups := collectTreeGroups(dirs, super)
gotGroups := groupPaths(groups) gotGroups := groupPaths(groups)
wantGroups := [][]string{{"/a", "/c"}} wantGroups := [][]string{{"/a", "/c"}}
@@ -211,98 +203,6 @@ func TestRunTreesEscapesPaths(t *testing.T) {
} }
} }
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the four as it starts and another as
// it ends. read counts the 6 records before it reads them, group counts
// the 8 directories, and write the 1 group. The summary comes last, and
// stdout holds the report alone.
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdTrees}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(trees) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
if got := stdout.String(); got != report {
t.Errorf("stdout = %q, want %q", got, report)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"trees: database " + path + " (from " + databaseEnv + ")",
treesOpenLine,
treesOpenLine,
"[2/4] read: counting the records, elapsed 0s, eta unknown",
"[2/4] read: [6/6] 100%, ",
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, eta unknown",
"[3/4] group: [8/8] 100%, ",
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, eta unknown",
"[4/4] write: [1/1] 100%, ",
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
// terminal its write phase's display is drawn on, as a plain "sfdupes
// trees" does: the terminal must show the header and the row whole,
// each on a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
stderr := captureStderr(t)
super, dirs := treeOf(t, smokeTreeRecs())
dupes := collectTreeGroups(dirs, super, nil)
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
}, terminalWidth, time.Hour, time.Now)
_, _, err := writeTreeGroups(os.Stderr, dupes, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
}
func TestTreeDigests(t *testing.T) { func TestTreeDigests(t *testing.T) {
t.Parallel() t.Parallel()
@@ -354,7 +254,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
super, dirs := treeOf(t, smokeTreeRecs()) super, dirs := treeOf(t, smokeTreeRecs())
groups := collectTreeGroups(dirs, super, nil) groups := collectTreeGroups(dirs, super)
want := [][]string{{"/d/t1", "/d/t2"}} want := [][]string{{"/d/t1", "/d/t2"}}
if got := groupPaths(groups); !slices.EqualFunc(got, want, if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -378,14 +278,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
forward := groupPaths(collectTreeGroups(dirs, super, nil)) forward := groupPaths(collectTreeGroups(dirs, super))
reversed := slices.Clone(recs) reversed := slices.Clone(recs)
slices.Reverse(reversed) slices.Reverse(reversed)
superR, dirsR := treeOf(t, reversed) superR, dirsR := treeOf(t, reversed)
backward := groupPaths(collectTreeGroups(dirsR, superR, nil)) backward := groupPaths(collectTreeGroups(dirsR, superR))
if !slices.EqualFunc(forward, backward, slices.Equal) { if !slices.EqualFunc(forward, backward, slices.Equal) {
t.Fatalf("output depends on record order: %v vs %v", t.Fatalf("output depends on record order: %v vs %v",
forward, backward) forward, backward)
@@ -404,7 +304,7 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil)) got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/x1", "/p/x2"}} want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) { if !slices.EqualFunc(got, want, slices.Equal) {
@@ -426,7 +326,7 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil)) got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/a/x", "/q/b/x"}} want := [][]string{{"/p/a/x", "/q/b/x"}}
if !slices.EqualFunc(got, want, slices.Equal) { if !slices.EqualFunc(got, want, slices.Equal) {