sneak/sfdupes
1
0
Fork 0
Code Issues 4 Pull Requests 3 Actions 9 Packages Projects Releases Wiki Activity

Compare commits

base: sneak/sfdupes:issue-135-readme
Branches Tags
sneak/sfdupes:main sneak/sfdupes:issue-135-todo sneak/sfdupes:issue-135-readme sneak/sfdupes:next sneak/sfdupes:issue-142-zfs-symlink-operands sneak/sfdupes:issue-19-markdown-fmt
sneak/sfdupes:v0.0.3 sneak/sfdupes:v0.0.2 sneak/sfdupes:v1.0.0-rc2 sneak/sfdupes:v1.0.0-rc1 sneak/sfdupes:v0.0.1
..
compare: sneak/sfdupes:2acb657a44d7d4cd0c2f094540627db074f08757
Branches Tags
sneak/sfdupes:issue-135-todo sneak/sfdupes:issue-135-readme sneak/sfdupes:next sneak/sfdupes:issue-142-zfs-symlink-operands sneak/sfdupes:main sneak/sfdupes:issue-19-markdown-fmt
sneak/sfdupes:v0.0.3 sneak/sfdupes:v0.0.2 sneak/sfdupes:v1.0.0-rc2 sneak/sfdupes:v1.0.0-rc1 sneak/sfdupes:v0.0.1
1 Commits
issue-135-readme .. 2acb657a44
Author SHA1 Message Date
clawbot 2acb657a44 Re-vendor the canonical files from sneak/prompts at c55a0cb (closes #95)
check / check (push) Canceled after 0s
Details 
Every vendored file, REPO_POLICIES.md and every model script is the
copy at sneak/prompts c55a0cb, with this repository's own entries kept
after the canonical content. Lint and test are phases of the Dockerfile
that write no image, built uncached. make test runs the suite under the
race detector as nobody, because root reads the files the tests make
unreadable. Dockerfile.lint, script/verify-lint-image-pin and
make test-race are gone. Prettier runs on the host, from the node and
yarn that script/bootstrap installs. golangci-lint v2.14.0 raises no
findings. .claude/settings.json is deleted.

Deviation: the set comes from c55a0cb on next rather than dd4027b, as
the instructions on sneak/prompts#78 allow.

Model: opus-5-5
 2026-10-08 01:23:35 +00:00
31 changed files with 2702 additions and 7672 deletions
Expand all files Collapse all files
Dockerfile
+11 -34
View File
@@ -1,14 +1,3 @@
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build. The
# test phase runs as nobody and so mounts Go's build cache at
# /tmp/.cache/go-build; its own comment says why.
# Lint phase, built alone by script/lint. The tools are invoked directly
# rather than through `make lint`, which runs docker itself and so cannot
# run inside a build step.
@@ -17,7 +6,6 @@ FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a924
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY . .
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a
@@ -30,20 +18,14 @@ RUN files="$(gofmt -s -l .)" && \
# Validates .golangci.yml against the schema the pinned binary embeds.
RUN golangci-lint config verify --config .golangci.yml
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
RUN golangci-lint run --config .golangci.yml ./...
# Test phase, built alone by script/test. -race needs cgo and so a C
# compiler, which the Debian Go image ships and the alpine one does not.
# -count=1 because the mounted cache also holds test results, which go
# test would otherwise replay.
#
# The tests run as nobody: several of them make a file unreadable and
# expect reading it to fail, and root reads it anyway. nobody has no home
# directory, so HOME is /tmp, and Go keeps its build cache in
# /tmp/.cache/go-build. That is where the cache is mounted, owned by
# nobody (65534) so the tests can write to it.
# directory, so HOME is /tmp, where Go puts its build cache.
# golang:1.25-trixie, 2026-10-04
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
USER nobody
@@ -51,27 +33,24 @@ ENV HOME=/tmp
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY . .
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
go test -count=1 -timeout 90s -race -cover ./... || \
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies are
# what make BuildKit build them first, so nothing after them can run
# unless lint and test passed.
# what make BuildKit build them first, so this stage cannot run unless
# lint and test passed.
# golang:1.25-alpine, 2026-07-23
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git make
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
COPY . .
# The version stamped into the binary: the VERSION build argument when
@@ -81,8 +60,7 @@ COPY . .
# "dev". `make build` rather than `go build`, so the image and a host
# build share one compile recipe, cgo disabled included.
ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \
version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
[ "$version" = unknown ]; }; then \
echo "no version could be derived although the build context carries .git" >&2; \
@@ -94,7 +72,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
# stage's chain and nothing else.
# alpine:3.22, 2026-07-23
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
ARG CACHEBUST
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
README.md
+679 -1088
View File
File diff suppressed because it is too large Load Diff
REPO_POLICIES.md
+80 -157
View File
@@ -1,6 +1,6 @@
---
title: Repository Policies
last_modified: 2026-10-18
last_modified: 2026-10-07
---
This document covers repository structure, tooling, and workflow standards. Code
@@ -54,39 +54,34 @@ style conventions are in separate documents:
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the
installed node only when its major version is the pinned one; otherwise (node
missing, or another major, such as the node 24 the CI runner image ships) it
installs the PINNED node version via nvm and installs yarn and the packages
under it. nvm itself is installed first if missing, from a hash-verified
GitHub release archive (never `curl | sh`), with bash installed as an explicit
prerequisite since nvm requires bash. Only the major version is compared
because the `Dockerfile` stages start from a node image whose exact version is
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
installed node if present; otherwise it installs a PINNED node version via
nvm, first installing nvm itself if missing — from a hash-verified GitHub
release archive (never `curl | sh`), with bash installed as an explicit
prerequisite since nvm requires bash. yarn is then pinned via
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
always exact versions. `script/cibuild` runs the CI build: it changes to the
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
with the version; the Gitea workflow calls it. **`script/cibuild` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs
nothing else, while `script/fmt-check` runs the formatter on the host: on a
pristine checkout with nothing installed the run dies there, after the
containerised gates have passed. **The bootstrap alone is not enough**: when
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
under another node. The host entrypoints that need yarn — `script/fmt` and
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
pinned node. A runner carrying nothing but docker and git then gets through
`script/check`. Four further scripts are our own extensions to the standard:
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
`script/precommit` is what the git pre-commit hook runs, and it calls
`script/check`; `script/install-precommit` installs the git pre-commit hook
(the `make hooks` target shims to it); and `script/projectname` (literally
that filename) simply outputs the project's name. Scripts that need the name
call `script/projectname` — e.g. `script/docker` assembles its image tag from
it — so those scripts stay byte-identical across all repos. Repo-type-specific
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/precommit`, not in the hook itself. Model scripts are at
containerised gates have passed. **The bootstrap alone is not enough**:
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
source nvm for the pinned node version before invoking it, exactly as
`script/bootstrap`'s own install step does. A runner carrying nothing but
docker and git then gets through `script/check`. Four further scripts are our
own extensions to the standard: `script/check` runs `script/test`,
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/projectname` (literally that filename) simply outputs the project's
name. Scripts that need the name call `script/projectname` — e.g.
`script/docker` assembles its image tag from it — so those scripts stay
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the
README requirements below).
@@ -123,10 +118,8 @@ style conventions are in separate documents:
and nothing else:
```sh
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target lint --output type=cacheonly .
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target test --output type=cacheonly .
docker build --no-cache --target lint --output type=cacheonly .
docker build --no-cache --target test --output type=cacheonly .
```
**A stage that is not the last one in the file is built only when the final
@@ -139,12 +132,12 @@ style conventions are in separate documents:
**The gate builds write no image.** With `--output type=cacheonly` the phase
runs and a failing step fails the build, but the result is not exported.
Nothing uses those images, and writing one out is slow: a Go test phase's
image holds the toolchain. A build given neither `--output` nor `-t` writes
an untagged image and leaves it dangling, on every developer host and every
CI runner. `script/cibuild` and `script/docker` build the image that ships
and tag it, so each build replaces the previous image; each assigns the tag
on its own line before the build, so `set -e` stops it where
`script/projectname` fails.
image holds the toolchain and every compiled package. A build given neither
`--output` nor `-t` writes an untagged image and leaves it dangling, on
every developer host and every CI runner. `script/cibuild` and
`script/docker` build the image that ships and tag it, so each build
replaces the previous image; each assigns the tag on its own line before the
build, so `set -e` stops it where `script/projectname` fails.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
@@ -159,42 +152,22 @@ style conventions are in separate documents:
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own `TMPDIR`, so the lock is
out of reach. The lint phase does keep golangci-lint's cache between check
builds (below), but it lints every checkout at `/src`, and a finding is
taken from that cache only for the same files, settings and golangci-lint
version.
org, in both directions. A container has its own cache, its own `TMPDIR` and
a digest-pinned binary, so neither is reachable.
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
invalidates a `COPY` layer only when the copied content changes, so on an
unchanged tree the check `RUN` is served from cache, nothing executes, and the
build still exits 0. Every `docker build` in `script/` therefore passes
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
id, so no two builds share a value, not even the three in one
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
`script/docker` are the four, and there is no fifth — `script/check` runs the
two gate phases and `script/fmt-check`, and builds no image of its own. In
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
that install system packages and download Go's modules, or right after `FROM`
in a stage with neither, and every step that lints, tests or compiles comes
after it. A step that installs anything else as well, such as a
`script/bootstrap` that also installs node packages, comes after it too. A new
value reruns every `RUN` step after that line; `COPY` steps are compared by
content and need nothing. Docker keeps the layers above the line until the
base image, their own lines or the files they copy change. A stage without the
`ARG` line is served from cache on an unchanged tree, so a repository
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
not pass `--no-cache`: it reruns every step too, but it also builds the layers
above the line again and empties every cache mount, so Go's build cache and
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
survive between check builds, would start empty every time. A bare
`docker build .` is not evidence that anything ran: a sub-second build
reporting success is a cache hit, not a result. Never invalidate by pruning —
`docker builder prune` and friends destroy a build cache shared with every
other build on the host. When a check is added or changed, prove it works by
planting a defect it must catch and watching the run fail on it, then revert
the defect. A green run alone shows neither that the check ran nor that it
covers what it should.
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
a `COPY` layer only when the copied content changes, so on an unchanged tree
the check `RUN` is served from cache, nothing executes, and the build still
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
four, and there is no fifth — `script/check` runs the two gate phases and
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
not evidence that anything ran: a sub-second build reporting success is a
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
and friends destroy a build cache shared with every other build on the host.
When a check is added or changed, prove it works by planting a defect it must
catch and watching the run fail on it, then revert the defect. A green run
alone shows neither that the check ran nor that it covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by
@@ -203,56 +176,40 @@ style conventions are in separate documents:
`Dockerfile`:
```dockerfile
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build.
# Lint phase
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
RUN golangci-lint run --config .golangci.yml ./...
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
# image ships and the alpine one does not. -count=1 because the mounted
# cache also holds test results, which go test would otherwise replay.
# image ships and the alpine one does not.
# golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \
go test -count=1 -timeout 90s -race -cover ./... || \
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so nothing after them can
# run unless lint and test passed.
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
COPY . .
# The VERSION build arg when one is given, otherwise
@@ -260,8 +217,7 @@ style conventions are in separate documents:
# .git present, a version that is still empty, dev or unknown fails the
# build: git is missing or could not read the checkout.
ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \
VERSION="${VERSION:-$(git describe --tags --always)}"; \
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \
@@ -274,7 +230,6 @@ style conventions are in separate documents:
# Runtime stage, and the last one
FROM alpine@sha256:...
ARG CACHEBUST
COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"]
```
@@ -289,26 +244,6 @@ style conventions are in separate documents:
- Keep the runtime stage last, and if you add a stage after it, give it the
same two copies. A plain `docker build .` builds the last stage's chain
and nothing else.
- Check builds keep these, and none of them can hand back a stale result:
- The layers above `ARG CACHEBUST`: the system packages and Go's
modules. Docker installs the packages again when the base image or the
install line changes, and downloads the modules again when `go.mod` or
`go.sum` changes. Go checks every module against `go.sum` as it
downloads it, and no later build can change a kept layer. The modules
are kept in a layer rather than a cache mount because a cache mount is
shared by every build on the host, and Go builds from whatever files
it holds without checking them again (only `go mod verify` does), so a
file changed there would go unnoticed.
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
root), mounted on the steps that compile: `golangci-lint run`, both
`go test` runs and `go build`. Go reuses a compiled package only when
all its inputs are unchanged. The cache also holds test results, which
is why both `go test` runs carry `-count=1`.
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
a package's findings only when the package's files and those of every
package it imports, `go.mod`, the linters and their settings in
`.golangci.yml`, and the golangci-lint version are all unchanged.
- If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must
create placeholder files so the embed directives resolve. Example:
@@ -317,20 +252,12 @@ style conventions are in separate documents:
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
has no `apk`, so install with `apt-get` under the Debian package name
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
lists in the same `RUN`, so the layer does not keep them. The test phase,
on the Debian Go image, installs the libraries its tests need the same
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
Docker keeps it until the base image or the install line changes:
lists in the same `RUN`, so the layer does not keep them:
```dockerfile
FROM golangci/golangci-lint@sha256:... AS lint
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
```
- `.dockerignore` lets `.git` into the build context. It keeps out every git
@@ -377,18 +304,18 @@ style conventions are in separate documents:
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
so a successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
second time. That is the price of the rule above, and it is worth paying: the
image that ships is built from a run of its own gates rather than from a cache
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
shared runner after 20 minutes. That allows for the three Docker builds
described above (the test phase, the lint phase, then the image), each held to
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
limited to `main` by a `branches` list under `on: push` cannot be checked by
review: to try a change to it, add the feature branch to that list and push,
then remove the branch from the list again before merging. Keep any job in it
that publishes behind `if: github.ref_name == 'main'`, so the run from the
feature branch publishes nothing.
image build is uncached and so runs the gate phases a second time. That is the
price of the rule above, and it is worth paying: the image that ships is built
from a run of its own gates rather than from a cache entry. The `check` job
sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
minutes. That allows for the three Docker builds described above (the test
phase, the lint phase, then the image), each held to the 5-minute Docker build
limit below, plus the bootstrap. A separate workflow limited to `main` by a
`branches` list under `on: push` cannot be checked by review: to try a change
to it, add the feature branch to that list and push, then remove the branch
from the list again before merging. Keep any job in it that publishes behind
`if: github.ref_name == 'main'`, so the run from the feature branch publishes
nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -448,11 +375,12 @@ style conventions are in separate documents:
That cache is Go's own, separate from Docker's layer cache. Go stores a
passing result in its cache directory (`GOCACHE`), and when the same tests
run again on unchanged code it prints that result, marked `(cached)`,
without running them. The directory lasts from one run to the next on a
developer's machine, where this target runs, and in the `test` phase of the
`Dockerfile`, which mounts it to keep compiled packages between check
builds, so the phase's `go test` runs carry `-count=1` too. A new
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
without running them. That matters on a developer's machine, where this
target runs and the directory lasts from one run to the next. The `test`
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
result for this repo's tests and nothing before its `go test` step runs a
test, so there is nothing to replay. `--no-cache` (above) is what makes that
step run on an unchanged tree.
Python example:
@@ -539,7 +467,7 @@ style conventions are in separate documents:
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$(script/projectname)"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
```
@@ -621,11 +549,6 @@ style conventions are in separate documents:
function defined and never invoked has the same exit status and the same
empty output as one that worked.
Node is the exception to the whole-token comparison: the canonical
`script/bootstrap` compares only its major version, because the node 22
alpine image the `Dockerfile` stages start from is not the exact pin, and
nvm cannot install a prebuilt node on alpine.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to
TODO.md
+75 -232
View File
@@ -29,208 +29,16 @@
# Completed Steps
- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
it. It leads with what the program is and how to use it. A reference section
covers every setting and every subcommand. Development and testing come last.
Every statement was checked against the code on `next`. Five comments in
`main_test.go` cite "Settings", where their rules now live. One comment each
in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
- an operand at or inside a directory named `.zfs` is never scanned, whether
given alone or under another operand. `scan` warns about it, counts it as
skipped and does not walk it (2026-10-26,
https://git.eeqj.de/sneak/sfdupes/issues/142)
- a file or directory removed while the scan runs is not an error, in the walk
or in the hash phase. The scan prints no line for it and does not count it as
skipped. When the first of several hard links to a file is gone, the hash
phase reads the next one instead. On a terminal every progress display starts
with the spinner, the hash phase's included. A phase with an exact total ends
its line with its percent and a 10-column bar at the terminal's right edge,
which no longer moves as the line's length changes. A line too wide for the
terminal now loses its rate before its bar. The time remaining is labelled
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
bytes of a SHA-256, half the size of the hex text they held. The hashes stay
raw from the hash phase through the database to `report` and `trees`. Reading
a record whose hash is not 32 bytes is an error (2026-10-26,
https://git.eeqj.de/sneak/sfdupes/issues/134)
- the hash phase gives every file the walk records three hashes: `head` and
`tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
The 10 MiB and 50 MiB thresholds, the size census, the records stored without
hashes and the content phase are gone, so a scan has four phases and a rescan
reads only new and changed files. `info` no longer counts the records with
hashes, since every record has them (2026-10-25,
https://git.eeqj.de/sneak/sfdupes/issues/132)
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
stages of the `Dockerfile` the module download, and in `builder` the
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
until the base image, the install line, `go.mod` or `go.sum` changes, and
every step that lints, tests or compiles still comes after it;
`golangci-lint run` also keeps golangci-lint's cache. README says the same
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
early build without a database. When the linker set no version,
`sfdupes --version` prints the main module's version that Go recorded in the
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
when neither is known (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/116)
- `scan` creates the database's directory where the database path leads: the
path with its last element cut off as text, never cleaned, so a `..` after a
symlinked directory is followed through the symlink, as when the file is
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
used to create `/srv/new` and then fail (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/119)
- `scan` never records its own database: once the database and its `.lock` file
are open, it notes the device and inode of the database file, its `-wal` and
`-shm` files and its `.lock` file, and the walk passes over a file with one of
them wherever the operands reach it, through a symlinked directory included,
printing no line and counting nothing. A record an earlier scan left for one
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
- `scan` never records or reads anything that is not on a persistent filesystem
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
matched by the device ID of each such filesystem, listed as the scan starts
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
however the walk or the content phase reaches it; no flag turns that off.
Below an operand, its walk does not enter the default excludes either, so
`sfdupes scan /` skips the kernel's view of the running system and the
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
Windows list, not used yet. An operand at or inside a default exclude is still
walked. The scan prints a line for each path it leaves out, without counting
it as skipped, and the records there count as outside the scanned operands, as
for `-x`. `--no-default-excludes` turns off the default excludes only, and
`scan --help` lists them (2026-10-17,
https://git.eeqj.de/sneak/sfdupes/issues/124)
- the rate a progress display shows covers the items completed over the last 5
seconds, or since the line before when stderr is not a terminal, so it falls
to 0 while nothing completes, and a burst after a wait no longer reads as a
steady rate of over 1,000 files a second; the ETA still comes from the phase's
average. Once no item has completed for 10 seconds, the word `stalled` and
where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
`db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
either, each phase giving its word where it is defined. A line too wide for
the terminal loses its bar first, then its rate or those words, so the phase,
progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
like those of README's example, the content phase's checks and reads lose
those words at most times in minutes, and every other phase keeps them
whenever the elapsed time and the ETA are both in minutes. The content phase's
last partial batch is now written under a display of its own (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/125)
- new subcommand `info` prints the database's details on stdout, one per line as
a name, a tab and the value: its path and where it came from, the sizes of the
database file and of its `-wal` file when there is one, when the database file
was last changed, the schema version, the number of records, how many have
head and tail hashes and how many a content hash, and the sizes of all records
added up. It opens the database as `report` does, and shows two phases on
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
from one read-only transaction, the counts from one query (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/117)
- `report` shows its work on stderr in two phases, `[1/2] open` and
`[2/2] read`, with the progress display `scan` and `trees` use, from the
moment it starts until the summary line: `open` says it is opening the
database, also while it waits on a busy one; `read` counts the records, then
reads them against that count in signature order, which SQLite takes from the
`files_signature` index without sorting, so the first record comes at once.
`report` forms the duplicate groups itself, holding only those of the size it
is reading, and writes them ordered by first path before it reads the next
size. stdout and the summary line are unchanged, and SQLite no longer needs a
temporary file to sort in (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/114)
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
from the moment it starts until the summary line: `open` says it is opening
the database, also while it waits on a busy one; `read` counts the records,
then reads them against that count; `group` counts the directories it groups,
and `write` the groups it writes. A display's rate names what its phase
counts: files, records, directories or groups. stdout and the summary line are
unchanged. When stdout and stderr are the same file, as one terminal is, none
of stdout's lines shares a line with the display; when they are not, the
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/113)
- each `scan` phase's progress display names the phase and its place among the
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
time and ETA: `load` counts the records first, the walk's total is estimated
from the records under the operands, and the content phase counts the records
it checks from SQLite's grouping alone, then says it is searching the database
until the first one is returned. On a terminal a phase with an exact total
draws its line followed by the library's bar, and any other a spinner followed
by the line; every display is drawn again several times a second. No line
reaches the terminal's last column, however narrow, also after the terminal is
made narrower during a phase: a line too wide leaves out its rate, then its
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
only when those alone do not fit, as below about 70 columns late in a large
scan. In plain mode the line is printed every 5 seconds whether or not an item
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
- `scan`, `report` and `trees` first print a line on stderr naming the database
they use, as an absolute path, and whether that path is the default or comes
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
"Database" says when `scan` writes each file (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/111)
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
`c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
that write no image, and `make test` runs the suite under the race detector,
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
every build, while the steps that compile keep Go's build cache and the tests
run with `-count=1`; prettier runs on the host, from the node and yarn
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
and the build stage installs `git` and `make` itself; a new push cancels the
workflow's older run on the same branch, and a run stops after 20 minutes;
`.claude/settings.json` is deleted (2026-10-15,
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
`test` and the build stage, and the install of `git` and `make` in the build
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
every step that lints, tests or compiles still runs on every build
- with `-x`, an operand under another operand is walked on its own instead of
being dropped when a directory from it (from its directory, for a regular
file) up to but not including that operand is on a different filesystem than
that operand, whose walk then cannot reach it, and the records of files gone
from below it are deleted (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/106)
- with `-x`, `scan` keeps the records below a directory on another filesystem
under its operand, and counts their sizes in the size census, instead of
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
- the hash and content phases open each file without blocking and check that it
is still a regular file, so one replaced by a FIFO or device node after it was
statted is warned about and skipped instead of hanging the scan (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/100)
- the scan lock file is named from the database's path with symlinks resolved,
after `scan` creates the database file if it is missing, so a scan through a
symlink to the database fails while another scan holds the lock (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/101)
- on a terminal, a warning for the last file of a phase no longer erases the
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/102)
gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
the host, from the node and yarn `script/bootstrap` installs, so the
`prettier` and `markdown` stages are gone and the build stage installs `git`
and `make` itself; a new push cancels the workflow's older run on the same
branch, and a run stops after 20 minutes; `.claude/settings.json` is deleted
(2026-10-08, https://git.eeqj.de/sneak/sfdupes/issues/95)
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
@@ -238,9 +46,8 @@
https://git.eeqj.de/sneak/sfdupes/issues/12)
- cut the narration from `TODO.md` Completed Steps and from the comments in
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
`script/` and both Dockerfiles; §Workflow now branches from and merges to
`next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
- `make test-race` ran the test suite under the race detector in a cgo-enabled
container, outside `make check` (2026-10-04,
@@ -254,7 +61,7 @@
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
`script/` pass a new `CACHEBUST` value, so theirs always run
`script/` pass `--no-cache`, so theirs always run
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
it; all Markdown reformatted (2026-10-04,
@@ -369,46 +176,82 @@
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and
`Dockerfile.lint` (2026-08-10, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
without the false `(Debian-based)` note or the tag. Since
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and
`script/verify-lint-image-pin`, which compared the two pins, are gone.
without the false `(Debian-based)` note or the tag; digest unchanged.
`script/verify-lint-image-pin` still matches the tagless form, and a tag on
one side only is caught as a plain mismatch.
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
owner ruling the linter is never installed on a host, and
`golangci-lint config verify` runs before `golangci-lint run`.
`script/bootstrap` stopped installing or pinning the linter, and
`script/verify-linter-pin` was retired. Since
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
stage depends on through `COPY --from=lint /src/go.sum /dev/null`;
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
an image build may run docker, so the phase calls `golangci-lint` directly.
owner ruling the linter is never installed on a host. `Dockerfile.lint` copies
the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and
runs `golangci-lint config verify` and `golangci-lint run` as build steps;
`script/lint` builds it. `script/bootstrap` no longer installs or pins the
linter, and warns rather than fails when `docker` is absent;
`ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`.
`script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate
in both files, compares their two `FROM` lines and restates neither pin.
Traps: nothing inside an image build may shell out to docker, so the
`Dockerfile` lint stage calls `golangci-lint` directly and the build stage
runs `make test` and `make fmt-check` instead of `make check`, through `make`
because the Makefile's `export CGO_ENABLED = 0` only reaches what it invokes.
`COPY --from=lint /src/go.sum /dev/null` replaces the copied linter binary as
the only edge making the build stage wait for lint; dropping it would end
fail-fast linting under a still-green build. `golangci-lint config verify`,
included per the ruling, validates from an embedded schema with no network
call, but `go mod download` above the gates still needs the network on a cold
cache. Verified: `make lint` green with no `golangci-lint` on `PATH`; two
back-to-back `script/lint` runs on an untouched tree both ran the linter
(27.7s and 28.7s in the lint step, `COPY . .` `CACHED` above); a planted
unused variable failed `script/lint`, and failed `make docker` at `[lint 9/9]`
with the build stage stopped at `[builder 3/12]`; the drift guard fails on a
tag-only, a digest-only and an unreadable reference, naming both sides; under
`--network none` config verify passes a valid config and rejects an invalid
one; `make docker` green in 5m35s with all six gates run (lint 37.6s, test
25.2s reporting `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not
`(cached)`); in the builder image with the Go test cache off, `--user 0:0`
still fails `TestScanHardlinkRunFailsTogether` where the unprivileged user
passes. Noted for follow-up, not fixed here: `golangci-lint` warns that
`gomodguard` is deprecated since v2.12.0 in favour of `gomodguard_v2`.
- install the Docker build stage's prerequisites by running `script/bootstrap`
instead of `apk add --no-cache make` inline (2026-08-09, branch
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42),
with `script/verify-linter-pin` failing the build unless the linter copied
from the lint stage was the version `script/bootstrap` pinned. Builds then
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as
https://git.eeqj.de/sneak/sfdupes/issues/43. Since
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
`make` with `apk add --no-cache` and only compiles; the `lint` and `test`
phases are the gates
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42):
the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
which ends in `go mod download`, so the separate call to it is gone.
`COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap
layer: it is the only edge making this stage depend on the lint stage, so
deleting it would end fail-fast linting silently. A new
`script/verify-linter-pin`, run in the build stage before bootstrap, fails the
build naming both versions unless that copied binary is the version
`script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip.
`$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. The `chown`
and `USER builder` still precede `make check`. Verified: the guard fails the
build with both versions named when the lint stage's linter is faked to
another version, and passes an unmodified build; bootstrap runs clean under
Alpine's `sh` and `apk`, finding the copied linter already at the pin; a
second build served the bootstrap and dependency layers `CACHED` while both
gates ran; a planted `unused` finding failed the build at the lint gate in
48.9s with the build stage's `make check` never starting; and the suite run in
the image as `--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the
drop to the unprivileged user is still needed. That last check needs the Go
test cache off: as root it first reported `ok ... (cached)`, reusing the
build-time result. Build times on a noisy shared host: 2m13s on an unchanged
tree, 2m17s and 4m29s after a source change, 5m14s cold, which breaches the
policy ceiling; `chown -R builder:builder /src /home/builder` walks the module
cache and alone varied from 77s to 210s across those builds, and `main`
measured 5m03s cold with a 209s `chown`. Filed as
https://git.eeqj.de/sneak/sfdupes/issues/43
- bust the Docker layer cache for the gate steps, so `script/cibuild` and
`script/docker` cannot report a green they did not earn (2026-08-09, branch
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
served them from cache and the build exited 0 having run nothing. The fix was
a `CHECK_EPOCH` build argument; since
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
declares before every step that lints, tests or compiles. Run as root, the
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
served them from cache and the build exited 0 having run nothing. Every
`docker build` in `script/` now passes `--no-cache` instead
(https://git.eeqj.de/sneak/sfdupes/issues/95). Run as root, the tests fail
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
the test relies on, so they run as an unprivileged user
- check the installed golangci-lint version in `script/bootstrap` instead of
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
cancel_test.go
+28 -40
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
@@ -169,8 +168,8 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) {
// TestSyncScanCancelledMidWalkKeepsRecords is the regression net under
// the post-walk guard. The scan is cancelled part-way through the
// walk, so it reaches the guard having walked a genuinely partial tree
// and holding a still-populated index of records the walk never got to.
// walk, so it reaches the guard holding a genuinely partial size
// census and a still-populated index of records the walk never got to.
// Every one of those records would look vanished to the update phase.
// The guard is what stops the scan there, and this test is what
// notices if it stops doing so: deleting the guard, or making it
@@ -197,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
base := baselineGoroutines(t)
st, err := syncScan(newWalkClock(walkCancelAtDone), db,
[]string{dir}, walkCancelWorkers, false, walkRules{})
[]string{dir}, walkCancelWorkers, false)
assertWalkGuardAborted(t, st, err)
assertRecordsIntact(t, db, before)
@@ -209,10 +208,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
}
// assertWalkGuardAborted checks that the scan stopped at the post-walk
// guard: with a walk that reached some files (it really ran) but not
// all of them (it really was cut short), and with no record counted for
// removal. A removal count means the partial walk was carried past
// the guard into the update phase, which is the failure this test
// guard: with a census that is neither empty (the walk really ran)
// nor complete (it really was cut short), and with no record counted
// for removal. A removal count means the partial census was carried
// past the guard into the update phase, which is the failure this test
// exists to catch.
func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
t.Helper()
@@ -223,30 +222,30 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
}
if st.unchanged == 0 {
t.Fatalf("stats = %+v: the walk reached no file, so it never "+
t.Fatalf("stats = %+v: the census is empty, so the walk never "+
"ran and the guard was reached for the wrong reason", st)
}
if st.unchanged >= walkCancelFiles {
t.Fatalf("stats = %+v: the walk reached the whole tree, so it "+
"was not cut short", st)
t.Fatalf("stats = %+v: the census covers the whole tree, so the "+
"walk was not cut short", st)
}
// The workers drop every directory still queued once the scan is
// cancelled, so only the files in the directories already in flight
// can still be reached after the fact. A walk beyond that bound
// can add to the census after the fact. A census beyond that bound
// would mean the cancellation was not observed where it should have
// been.
limit := walkCancelAtDone + walkCancelInFlightFiles
if st.unchanged > limit {
t.Errorf("the walk reached %d files, want at most %d: it kept "+
t.Errorf("census covers %d files, want at most %d: the walk kept "+
"taking directories off the queue after cancellation",
st.unchanged, limit)
}
if st.removed != 0 {
t.Errorf("stats = %+v: the scan counted records for removal from "+
"a partial walk", st)
"a partial census", st)
}
}
@@ -269,8 +268,7 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false,
walkRules{})
st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false)
if !errors.Is(err, context.Canceled) {
t.Fatalf("syncScan on a cancelled context = %v, want %v",
err, context.Canceled)
@@ -303,7 +301,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) {
db := openTestDB(t)
st, err := syncScan(newWalkClock(hashCancelAtDone), db,
[]string{dir}, walkCancelWorkers, false, walkRules{})
[]string{dir}, walkCancelWorkers, false)
if !errors.Is(err, context.Canceled) {
t.Fatalf("syncScan cancelled mid-hash = %v, want %v",
err, context.Canceled)
@@ -346,10 +344,9 @@ func storedPaths(t *testing.T, path string) []string {
// TestRunScanInterrupted calls the scan entrypoint with a context that
// is already cancelled, as when a signal arrives at once. It must return
// errInterrupted promptly, with its line on stderr after the one naming
// the database and nothing on stdout, leave the database valid and as
// it was, and leave nothing in the way of the next scan, which must
// bring the database up to date.
// errInterrupted promptly with its one line on stderr and nothing on
// stdout, leave the database valid and as it was, and leave nothing in
// the way of the next scan, which must bring the database up to date.
func TestRunScanInterrupted(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
@@ -358,8 +355,7 @@ func TestRunScanInterrupted(t *testing.T) {
stderr := captureStderr(t)
dir := buildSmokeTree(t)
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
false)
err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
if err != nil {
t.Fatal(err)
}
@@ -383,7 +379,7 @@ func TestRunScanInterrupted(t *testing.T) {
defer close(done)
err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
false, false)
false)
}()
awaitReturn(t, done, "runScan")
@@ -393,8 +389,7 @@ func TestRunScanInterrupted(t *testing.T) {
err, errInterrupted)
}
want := "scan: database " + path + " (from " + databaseEnv + ")\n" +
"scan: interrupted after 0 files\n"
want := "scan: interrupted after 0 files\n"
if got := stderr()[shown:]; got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
@@ -410,8 +405,7 @@ func TestRunScanInterrupted(t *testing.T) {
got, before)
}
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
false)
err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
if err != nil {
t.Fatal(err)
}
@@ -437,7 +431,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
dir := buildWalkCancelTree(t)
err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
walkCancelWorkers, false, false)
walkCancelWorkers, false)
if !errors.Is(err, errInterrupted) {
t.Fatalf("runScan interrupted mid-hash = %v, want %v",
err, errInterrupted)
@@ -621,8 +615,7 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) {
// Unbuffered and unread: on a cancelled scan every send gives up.
events := make(chan walkEvent)
subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false,
walkRules{}, events)
subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events)
if len(subs) != 0 {
t.Errorf("cancelled walkOneDir returned %+v to descend into, "+
"want none", subs)
@@ -642,8 +635,7 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) {
missing := filepath.Join(t.TempDir(), "missing")
jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false,
walkRules{})
jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false)
for range 64 {
jobs <- dirJob{path: missing}
@@ -674,7 +666,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{})
jobs, subdirs, events := startWalkWorkers(ctx, 1, false)
// Fill the hand-back channel to its capacity — one slot per worker
// — so the worker's own hand-back is certain to block.
@@ -763,9 +755,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
var hashed atomic.Bool
hash := func(path string, size int64) (
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
) {
hash := func(path string, size int64) (string, string, string, error) {
hashed.Store(true)
return hashSignature(path, size)
@@ -805,9 +795,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
// The scan is cancelled while the worker hashes, so the worker has
// already passed the check that drops queued runs.
hash := func(path string, size int64) (
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
) {
hash := func(path string, size int64) (string, string, string, error) {
cancel()
return hashSignature(path, size)
db.go
+143 -294
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
@@ -12,7 +11,6 @@ import (
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"golang.org/x/sys/unix"
@@ -37,10 +35,6 @@ const schemaVersion = 1
// scan.
const dbDirPerm = 0o755
// dbFilePerm is the mode for a database file created by scan, the mode
// SQLite gives a database file it creates.
const dbFilePerm = 0o644
// lockFilePerm is the mode for the scan lock file. Anyone who can open
// the file can hold the lock and keep every scan from running, so it
// is open to its owner only.
@@ -49,22 +43,21 @@ const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second. head, tail and content each hold the raw 32 bytes of a
// SHA-256.
// second.
const createTableSQL = `
CREATE TABLE files (
path BLOB PRIMARY KEY,
size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL,
head BLOB NOT NULL,
tail BLOB NOT NULL,
content BLOB NOT NULL
head TEXT NOT NULL,
tail TEXT NOT NULL,
content TEXT NOT NULL
) WITHOUT ROWID
`
// createIndexSQL indexes the records by signature, so report can read
// them in signature order without sorting the table.
// createIndexSQL indexes the records by signature, so report can have
// SQLite group them without sorting the whole table.
const createIndexSQL = `
CREATE INDEX files_signature ON files (size, head, tail, content)
`
@@ -81,8 +74,7 @@ ON CONFLICT (path) DO UPDATE SET
content = excluded.content
`
// errNoDatabase reports a missing database file for report, trees and
// info.
// errNoDatabase reports a missing database file for report/trees.
var errNoDatabase = errors.New(
"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
@@ -94,57 +86,14 @@ var errSchemaVersion = errors.New("unsupported database schema version")
// database.
var errScanRunning = errors.New("another scan is running")
// errHashSize reports a record whose head, tail or content is not the
// 32 bytes of a SHA-256.
var errHashSize = errors.New("hash is not 32 bytes")
// databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the
// path, with the working directory, its symlinks resolved, put in front
// of it when it is relative, and where the path came from, "default" or
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
// stderr.
func databasePath() (string, string, error) {
path, source := defaultDatabasePath, "default"
// set and non-empty, the compiled-in default otherwise.
func databasePath() string {
if p := os.Getenv(databaseEnv); p != "" {
path, source = p, "from "+databaseEnv
return p
}
if filepath.IsAbs(path) {
return path, source, nil
}
// The working directory as the kernel reports it, with no symlink
// in it, as README "Database" says. os.Getwd may return PWD
// instead, which can name it through a symlink.
wd, err := unix.Getwd()
if err != nil {
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
}
// Not filepath.Abs or filepath.Join: they clean the path, which
// drops each ".." together with the directory name before it. When
// that directory is a symlink, the cleaned path names another file.
// The working directory ends in "/" only when it is the root.
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
}
// createDatabaseDirectory creates the directory the database at path
// goes in, and any directory above it, if missing. path is absolute, as
// databasePath returns it. The directory is path with its last element
// cut off as text, by filepath.Split. filepath.Dir would clean it,
// dropping each ".." together with the directory name before it; when
// that name is a symlink, the cleaned path names another directory than
// the one the system reaches when it opens path.
func createDatabaseDirectory(path string) error {
dir, _ := filepath.Split(path)
err := os.MkdirAll(dir, dbDirPerm)
if err != nil {
return fmt.Errorf("create database directory: %w", err)
}
return nil
return defaultDatabasePath
}
// scanParams are the connection parameters for scan: read-write, with
@@ -154,9 +103,9 @@ const scanParams = "_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(WAL)" +
"&_pragma=synchronous(NORMAL)"
// reportParams are the connection parameters for report, trees and
// info: read-only, with the same busy timeout. They set no journal
// mode, because setting one is a write.
// reportParams are the connection parameters for report and trees:
// read-only, with the same busy timeout. They set no journal mode,
// because setting one is a write.
const reportParams = "mode=ro" +
"&_pragma=busy_timeout(10000)" +
"&_pragma=query_only(1)"
@@ -191,37 +140,19 @@ func openDB(path, params string) (*sql.DB, error) {
// lockScanDatabase takes the lock that keeps a second scan off the
// database at path: an exclusive flock(2) on the file beside it named
// by the database's path with symlinks resolved and ".lock" appended,
// so every path to one database names the same lock file. If missing,
// the database's parent directory and an empty database file are
// created first, so the lock file is named the same way before the
// database is set up as after. A lock held by another scan fails at
// once instead of waiting. The lock lasts until the returned file is
// closed or the process ends. The lock file is never deleted: a scan
// that deleted it would let the next scan lock a new file while
// another still holds the old one.
// path with ".lock" appended, created along with the database's parent
// directory if missing. A lock held by another scan fails at once
// instead of waiting. The lock lasts until the returned file is closed
// or the process ends. The file is never deleted: a scan that deleted
// it would let the next scan lock a new file while another still holds
// the old one.
func lockScanDatabase(path string) (*os.File, error) {
err := createDatabaseDirectory(path)
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
if err != nil {
return nil, err
return nil, fmt.Errorf("create database directory: %w", err)
}
// Like SQLite, this follows a symlink at path and creates its
// target. SQLite opens an empty file as a new database.
//nolint:gosec // the operator chooses the database path
dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
if err != nil {
return nil, fmt.Errorf("create database: %w", err)
}
_ = dbFile.Close()
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return nil, fmt.Errorf("resolve database path: %w", err)
}
lockPath := resolved + ".lock"
lockPath := path + ".lock"
//nolint:gosec // the operator chooses the database path
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
@@ -247,9 +178,9 @@ func lockScanDatabase(path string) (*os.File, error) {
// openScanDatabase opens the database for the scan subcommand, creating
// the file, its parent directory, and the schema as needed.
func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
err := createDatabaseDirectory(path)
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
if err != nil {
return nil, err
return nil, fmt.Errorf("create database directory: %w", err)
}
db, err := openDB(path, scanParams)
@@ -267,42 +198,6 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
return db, nil
}
// databaseFiles returns, by device and inode, the files of the database
// at path that the scan keeps open: the database file, its -wal and
// -shm files and its .lock file, each named from the database's path
// with symlinks resolved, as SQLite and lockScanDatabase name them. The
// scan calls it once the database and the lock file are open, so that
// its walk passes over them however it reaches them, through a
// symlinked directory included. A file that does not exist then is left
// out, since no walk can reach it. There is no -journal file while the
// database is in WAL mode.
func databaseFiles(path string) ([]fileRec, error) {
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return nil, fmt.Errorf("resolve database path: %w", err)
}
var files []fileRec
for _, p := range []string{
resolved, resolved + "-wal", resolved + "-shm", resolved + ".lock",
} {
fi, err := os.Lstat(p)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return nil, fmt.Errorf("database: %w", err)
}
dev, ino := inodeOfInfo(fi)
files = append(files, fileRec{path: p, dev: dev, ino: ino})
}
return files, nil
}
// closeScanDatabase switches the database at path from WAL back to
// rollback-journal mode and closes it. Out of WAL mode the database
// file alone holds the whole database, so a reader needs no -wal or
@@ -321,9 +216,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) {
_ = db.Close()
}
// openReportDatabase opens an existing database for the report, trees
// and info subcommands. A missing database file is an error directing
// the user to run scan first; the schema version must match exactly.
// openReportDatabase opens an existing database for the report and
// trees subcommands. A missing database file is an error directing the
// user to run scan first; the schema version must match exactly.
func openReportDatabase(ctx context.Context,
path string,
) (*sql.DB, error) {
@@ -462,32 +357,10 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
return v, nil
}
// loadFileRows gives total the number of records, then streams every
// record to fn in path order: byte order, which is the order of the
// primary key, so SQLite does not sort. The count and the records are
// read in one transaction, so they agree while a scan is committing.
func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
fn func(r scanRec),
) error {
// Everything goes through tx: db has one connection, so a query on
// db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = tx.Rollback() }()
var n int64
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
total(n)
rows, err := tx.QueryContext(ctx,
// loadFileRows streams every record to fn in path order: byte order,
// which is the order of the primary key, so SQLite does not sort.
func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
rows, err := db.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
"FROM files ORDER BY path")
if err != nil {
@@ -498,24 +371,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
for rows.Next() {
var (
path, head, tail, content []byte
sec, nsec int64
r scanRec
path []byte
sec, nsec int64
r scanRec
)
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
&r.content)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
r.mtime = time.Unix(sec, nsec)
err = r.setHashes(head, tail, content)
if err != nil {
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
}
fn(r)
}
@@ -527,166 +395,88 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
return nil
}
// signatureRowsSQL selects every record's signature and path in
// signature order, largest size first. That is the order of the
// files_signature index read backwards, so SQLite reads the index alone
// and returns the first row at once, with nothing to sort.
const signatureRowsSQL = `
SELECT size, head, tail, content, path
FROM files
ORDER BY size DESC, head DESC, tail DESC, content DESC
// dupeRowsSQL selects every record in a duplicate group, with the
// group's first path. A group is the records with a content hash that
// share a size, head, tail, and content, when there are two or more of
// them. The rows come in report order: groups by size descending, then
// by first path, and each group's paths ascending.
const dupeRowsSQL = `
SELECT g.first, f.path, f.size
FROM files AS f
JOIN (
SELECT size, head, tail, content, MIN(path) AS first
FROM files
WHERE content <> ''
GROUP BY size, head, tail, content
HAVING COUNT(*) > 1
) AS g USING (size, head, tail, content)
ORDER BY f.size DESC, g.first, f.path
`
// loadSignatureRows gives total the number of records, then streams
// every record's signature and path to fn, without its mtime, in the
// order of signatureRowsSQL. The count and the records are read in one
// transaction, so they agree while a scan is committing. An error from
// fn stops the reading and is returned as it is.
func loadSignatureRows(ctx context.Context, db *sql.DB,
total func(n int64), fn func(r scanRec) error,
) error {
// loadDupeRows streams the rows of dupeRowsSQL to fn and returns the
// number of records in the database. The count and the rows are read
// in one transaction, so they agree while a scan is committing. An
// error from fn stops the reading and is returned as it is.
func loadDupeRows(ctx context.Context, db *sql.DB,
fn func(first, path string, size int64) error,
) (int, error) {
// Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return fmt.Errorf("read records: %w", err)
return 0, fmt.Errorf("read records: %w", err)
}
defer func() { _ = tx.Rollback() }()
var n int64
var records int
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records)
if err != nil {
return fmt.Errorf("count records: %w", err)
return 0, fmt.Errorf("read records: %w", err)
}
total(n)
rows, err := tx.QueryContext(ctx, signatureRowsSQL)
rows, err := tx.QueryContext(ctx, dupeRowsSQL)
if err != nil {
return fmt.Errorf("read records: %w", err)
return 0, fmt.Errorf("read records: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
path, head, tail, content []byte
r scanRec
first, path []byte
size int64
)
err = rows.Scan(&r.size, &head, &tail, &content, &path)
err = rows.Scan(&first, &path, &size)
if err != nil {
return fmt.Errorf("read record: %w", err)
return 0, fmt.Errorf("read record: %w", err)
}
r.path = string(path)
err = r.setHashes(head, tail, content)
err = fn(string(first), string(path), size)
if err != nil {
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
}
err = fn(r)
if err != nil {
return err
return 0, err
}
}
err = rows.Err()
if err != nil {
return fmt.Errorf("read records: %w", err)
return 0, fmt.Errorf("read records: %w", err)
}
return nil
return records, nil
}
// setHashes sets r's head, tail and content to the values read from
// those columns. Each must be the raw 32 bytes of a SHA-256.
func (r *scanRec) setHashes(head, tail, content []byte) error {
for _, b := range [][]byte{head, tail, content} {
if len(b) != sha256.Size {
return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
}
}
r.head = [sha256.Size]byte(head)
r.tail = [sha256.Size]byte(tail)
r.content = [sha256.Size]byte(content)
return nil
}
// countRecords returns the number of records in the database.
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return 0, fmt.Errorf("count records: %w", err)
}
return n, nil
}
// databaseCountsSQL counts the records and adds up their sizes, in one
// pass over the table. A SUM over no rows is NULL, so on an empty table
// the SUM falls back to 0.
const databaseCountsSQL = `
SELECT COUNT(*), COALESCE(SUM(size), 0)
FROM files
`
// databaseCounts is what info reads from the database: its schema
// version and the counts of databaseCountsSQL.
type databaseCounts struct {
schemaVersion int
records int64 // every record
size int64 // the sizes of all records added up
}
// readDatabaseCounts reads the schema version, then the counts of
// databaseCountsSQL, in one read-only transaction, so both describe the
// database at one moment while a scan is committing.
func readDatabaseCounts(ctx context.Context,
db *sql.DB,
) (databaseCounts, error) {
var c databaseCounts
// Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return c, fmt.Errorf("count records: %w", err)
}
defer func() { _ = tx.Rollback() }()
err = tx.QueryRowContext(ctx, "PRAGMA user_version").
Scan(&c.schemaVersion)
if err != nil {
return c, fmt.Errorf("read schema version: %w", err)
}
err = tx.QueryRowContext(ctx, databaseCountsSQL).
Scan(&c.records, &c.size)
if err != nil {
return c, fmt.Errorf("count records: %w", err)
}
return c, nil
}
// loadFileMeta streams every record's path, size and mtime to fn. Scan
// change detection needs no hash values, and skipping the hash columns
// keeps the scan's in-memory index small on multi-million-file
// databases.
// loadFileMeta streams every record's path, size, mtime, and whether
// it carries hashes to fn. Scan change detection needs no hash
// values, and skipping the hash columns keeps the scan's in-memory
// index small on multi-million-file databases.
func loadFileMeta(ctx context.Context, db *sql.DB,
fn func(path string, size int64, mtime time.Time),
fn func(path string, size int64, mtime time.Time, hashed bool),
) error {
rows, err := db.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec FROM files")
"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files")
if err != nil {
return fmt.Errorf("read records: %w", err)
}
@@ -697,14 +487,74 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
var (
path []byte
size, sec, nsec int64
hashed int64
)
err = rows.Scan(&path, &size, &sec, &nsec)
err = rows.Scan(&path, &size, &sec, &nsec, &hashed)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
fn(string(path), size, time.Unix(sec, nsec))
fn(string(path), size, time.Unix(sec, nsec), hashed != 0)
}
err = rows.Err()
if err != nil {
return fmt.Errorf("read records: %w", err)
}
return nil
}
// contentCandidatesSQL selects every record of at least headTailMin
// bytes whose size, head, and tail equal another record's, in each
// group (the records sharing a size, head, and tail) where at least one
// record has no content hash, with whether each record has one. SQLite
// does the grouping, so no other record's hashes are loaded into
// memory; the rows come ordered by size, head, and tail, so each
// group's rows arrive together.
const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
FROM files AS f
JOIN (
SELECT size, head, tail
FROM files
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
) AS g USING (size, head, tail)
ORDER BY size, head, tail
`
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
// each record, without its content hash, and whether it has one.
func loadContentCandidates(ctx context.Context, db *sql.DB,
fn func(r scanRec, hashed bool),
) error {
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
path []byte
sec, nsec int64
r scanRec
hashed int64
)
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
&hashed)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
r.mtime = time.Unix(sec, nsec)
fn(r, hashed != 0)
}
err = rows.Err()
@@ -788,8 +638,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
r.mtime.Unix(), r.mtime.Nanosecond(),
r.head[:], r.tail[:], r.content[:])
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
if err != nil {
return fmt.Errorf("upsert %s: %w", r.path, err)
}
db_test.go
+25 -357
View File
@@ -2,11 +2,9 @@ package main
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
@@ -15,19 +13,11 @@ import (
"time"
)
// testDBPath returns a database path inside a fresh temp dir, with
// symlinks resolved as the scan lock resolves them, so the lock file
// is this path plus ".lock" even where the temp dir is reached through
// a symlink.
// testDBPath returns a database path inside a fresh temp dir.
func testDBPath(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return filepath.Join(dir, "db.sqlite")
return filepath.Join(t.TempDir(), "db.sqlite")
}
// openTestDB creates a fresh scan database in a temp dir.
@@ -45,47 +35,16 @@ func openTestDB(t *testing.T) *sql.DB {
}
func TestDatabasePath(t *testing.T) {
// README §Database: SFDUPES_DATABASE when set and non-empty, the
// default otherwise, with the working directory, its symlinks
// resolved, put in front of a relative path. Nothing is cleaned, so
// a ".." stays.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
t.Setenv(databaseEnv, "")
if got := databasePath(); got != defaultDatabasePath {
t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
}
t.Chdir(dir)
t.Setenv(databaseEnv, "/custom/place.sqlite")
fromEnv := "from " + databaseEnv
cases := []struct {
env, path, source string
}{
{"", defaultDatabasePath, "default"},
{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
fromEnv},
{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
}
for _, tc := range cases {
t.Setenv(databaseEnv, tc.env)
path, source, err := databasePath()
if err != nil || path != tc.path || source != tc.source {
t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
databaseEnv, tc.env, path, source, err, tc.path, tc.source)
}
}
// In the root directory the path gets one slash in front, not two.
t.Chdir("/")
t.Setenv(databaseEnv, "place.sqlite")
path, _, err := databasePath()
if err != nil || path != "/place.sqlite" {
t.Errorf("databasePath() in / = %q, %v; want %q",
path, err, "/place.sqlite")
if got := databasePath(); got != "/custom/place.sqlite" {
t.Errorf("databasePath() = %q, want the env override", got)
}
}
@@ -120,116 +79,6 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
}
}
// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
// through a symlink to a directory, then "..", to a missing directory.
// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
// places scan creates the database's directory, must each create the
// database there, and nothing at srv/new, where the path leads once
// cleaned as text.
func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
t.Parallel()
creates := map[string]func(ctx context.Context, path string) error{
"lockScanDatabase": func(_ context.Context, path string) error {
lock, err := lockScanDatabase(path)
if err == nil {
_ = lock.Close()
}
return err
},
"openScanDatabase": func(ctx context.Context, path string) error {
db, err := openScanDatabase(ctx, path)
if err == nil {
_ = db.Close()
}
return err
},
}
for name, create := range creates {
t.Run(name, func(t *testing.T) {
t.Parallel()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "data", "x"),
filepath.Join(dir, "srv", "link"))
if err != nil {
t.Fatal(err)
}
err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
if err != nil {
t.Fatalf("%s: %v", name, err)
}
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
if err != nil {
t.Errorf("database the path leads to: %v", err)
}
_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("srv/new: stat error %v, want it missing", err)
}
})
}
}
// TestDatabaseFiles opens a database through a symlink to its file:
// databaseFiles finds the database file and its -wal and -shm files
// beside the symlink's target, where SQLite keeps them, and leaves out
// the .lock file, which only a scan creates.
func TestDatabaseFiles(t *testing.T) {
t.Parallel()
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
db, err := openScanDatabase(t.Context(), link)
if err != nil {
t.Fatal(err)
}
defer func() { _ = db.Close() }()
files, err := databaseFiles(link)
if err != nil {
t.Fatal(err)
}
got := make([]string, 0, len(files))
for _, f := range files {
got = append(got, f.path)
}
want := []string{path, path + "-wal", path + "-shm"}
if !slices.Equal(got, want) {
t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
}
}
func TestOpenDatabaseUnversionedForeign(t *testing.T) {
t.Parallel()
@@ -396,189 +245,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
defer func() { _ = reportDB.Close() }()
err = loadFileRows(t.Context(), reportDB, func(int64) {},
func(scanRec) {})
err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
if err != nil {
t.Fatalf("loadFileRows: %v", err)
}
}
// TestLoadRowsCountAgreeDuringScan has a scan commit records after
// loadFileRows, as trees calls it, or loadSignatureRows, as report
// calls it, has counted the records and before it reads them, as when
// trees or report runs during a scan: each must read exactly as many
// records as it counted.
func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
t.Parallel()
loads := map[string]func(context.Context, *sql.DB, func(int64),
func(scanRec)) error{
"loadFileRows": loadFileRows,
"loadSignatureRows": func(ctx context.Context, db *sql.DB,
total func(int64), fn func(scanRec),
) error {
return loadSignatureRows(ctx, db, total, func(r scanRec) error {
fn(r)
return nil
})
},
}
for name, load := range loads {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := testDBPath(t)
scanDB, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = scanDB.Close() }()
recs := smokeTreeRecs()
err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
if err != nil {
t.Fatal(err)
}
reportDB, err := openReportDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = reportDB.Close() }()
var counted, read int64
err = load(t.Context(), reportDB, func(n int64) {
counted = n
err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
if err != nil {
t.Error(err)
}
}, func(scanRec) { read++ })
if err != nil {
t.Fatal(err)
}
if counted != 3 || read != counted {
t.Errorf("counted %d records and read %d, want 3 and 3",
counted, read)
}
})
}
}
// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
// content is not the 32 bytes of a SHA-256: either one byte, or the 64
// bytes of a SHA-256 written as hex text. loadFileRows and
// loadSignatureRows must each fail on it with an error naming its path.
func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
t.Parallel()
hexText := strings.Repeat("ab", sha256.Size)
cases := []struct {
name string
update string
value any
}{
{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
{"hex-text head", "UPDATE files SET head = ?", hexText},
{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
{"hex-text tail", "UPDATE files SET tail = ?", hexText},
{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
{"hex-text content", "UPDATE files SET content = ?", hexText},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
t.Parallel()
db := openTestDB(t)
err := applyChanges(t.Context(), db,
[]scanRec{{path: "/a/f"}}, nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = db.ExecContext(t.Context(), c.update, c.value)
if err != nil {
t.Fatal(err)
}
loads := map[string]func() error{
"loadFileRows": func() error {
return loadFileRows(t.Context(), db, func(int64) {},
func(scanRec) {})
},
"loadSignatureRows": func() error {
return loadSignatureRows(t.Context(), db, func(int64) {},
func(scanRec) error { return nil })
},
}
for name, load := range loads {
err := load()
if !errors.Is(err, errHashSize) ||
!strings.Contains(err.Error(), "/a/f") {
t.Errorf("%s: err = %v, want %v naming /a/f",
name, err, errHashSize)
}
}
})
}
}
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
t.Parallel()
db := openTestDB(t)
rows, err := db.QueryContext(t.Context(),
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
if err != nil {
t.Fatal(err)
}
defer func() { _ = rows.Close() }()
var plan []string
for rows.Next() {
var (
id, parent, unused int
detail string
)
err = rows.Scan(&id, &parent, &unused, &detail)
if err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
want := []string{"SCAN files USING COVERING INDEX files_signature"}
if !slices.Equal(plan, want) {
t.Errorf("query plan = %q, want %q", plan, want)
}
}
func TestApplyChangesRoundTrip(t *testing.T) {
t.Parallel()
@@ -589,20 +261,17 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// written.
recs := []scanRec{
{
size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
tail: testHash("t2"), content: testHash("c2"),
path: "/a/tab\tnew\nline",
size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
content: "c2", path: "/a/tab\tnew\nline",
},
{
size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
content: "c1", path: "/a/x",
},
}
prog := newProgress(progressParams{label: "update", total: 4})
defer prog.finish()
err := applyChanges(t.Context(), db, recs, nil, prog)
err := applyChanges(t.Context(), db, recs, nil,
newProgress("update", 2))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
@@ -616,12 +285,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// An upsert for an existing path updates in place; a delete
// removes exactly its path.
upd := scanRec{
size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
path: "/a/x",
}
err = applyChanges(t.Context(), db, []scanRec{upd},
[]string{"/a/tab\tnew\nline"}, prog)
[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
@@ -644,15 +313,13 @@ func TestApplyChangesBatching(t *testing.T) {
recs := make([]scanRec, 0, n)
for i := range n {
recs = append(recs, scanRec{
size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
path: fmt.Sprintf("/batch/%07d", i),
})
}
prog := newProgress(progressParams{label: "update", total: int64(2 * n)})
defer prog.finish()
err := applyChanges(t.Context(), db, recs, nil, prog)
err := applyChanges(t.Context(), db, recs, nil,
newProgress("update", int64(n)))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
@@ -666,7 +333,8 @@ func TestApplyChangesBatching(t *testing.T) {
deletes = append(deletes, r.path)
}
err = applyChanges(t.Context(), db, nil, deletes, prog)
err = applyChanges(t.Context(), db, nil, deletes,
newProgress("update", int64(n)))
if err != nil {
t.Fatalf("applyChanges deletes: %v", err)
}
exclude.go
-195
View File
@@ -1,195 +0,0 @@
package main
// This file holds two rules for the walk (README "Rules for the walk"):
// the default excludes, the paths a scan's walk does not enter below an
// operand unless scan is given --no-default-excludes, and the
// filesystems that are not persistent, on which the scan never records
// or reads anything.
import (
"bufio"
"errors"
"fmt"
"io"
"io/fs"
"path/filepath"
"slices"
"strings"
"golang.org/x/sys/unix"
)
// The systems that have default excludes, as runtime.GOOS names them.
const (
goosLinux = "linux"
goosDarwin = "darwin"
goosWindows = "windows"
)
// The fields of a line of mountInfoPath that readMountInfo reads,
// counted from 0 (see proc(5)): the device ID, as major:minor, is the
// third field, and after the sixth come optional fields, then a field
// holding "-", then the filesystem type.
const (
deviceField = 2
optionalFieldsStart = 6
)
// typesPerHelpLine is how many filesystem types the scan command's help
// lists on each line.
const typesPerHelpLine = 6
// errMountInfoLine reports a line of mountInfoPath without the fields
// readMountInfo reads.
var errMountInfoLine = errors.New("unexpected line")
// walkRules holds the rules for the walk that a scan sets up as it
// starts. excludes are the default excludes, or none with
// --no-default-excludes. nonPersistent maps the device ID of every
// mounted filesystem that is not persistent to its type; the scan
// records and reads nothing on one of those, whatever the flags.
// databaseFiles are the scan's own database files, which the walk
// passes over (see databaseFiles in db.go).
type walkRules struct {
excludes []string
nonPersistent map[uint64]string
databaseFiles []fileRec
}
// defaultExcludes returns the default excludes of the system goos
// names, as runtime.GOOS does: a fixed list on Linux, macOS and
// Windows, and none on any other system.
func defaultExcludes(goos string) []string {
switch goos {
case goosLinux:
return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
case goosDarwin:
// A walk of / reaches everything in /System/Volumes/Data through
// the firmlinks, so walking it as well would record each of
// those files twice.
return []string{
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
"/System/Volumes/VM", "/System/Volumes/Data",
}
case goosWindows:
// Relative to the root of the drive an operand lies on. The
// first three are the page, swap and hibernation files.
return []string{
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
`Users\*\AppData\Local\Temp`,
}
default:
return nil
}
}
// isDefaultExclude reports whether path is one of excludes, in which a
// * matches one path component.
func isDefaultExclude(path string, excludes []string) bool {
for _, pattern := range excludes {
// The patterns are fixed, so none is malformed.
if ok, _ := filepath.Match(pattern, path); ok {
return true
}
}
return false
}
// nonPersistentFilesystemTypes returns the filesystem types that are
// not persistent: filesystems that the kernel makes up or that are kept
// in memory. devfs is the macOS filesystem of /dev; the others are
// Linux names.
func nonPersistentFilesystemTypes() []string {
return []string{
"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs",
"cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore",
"bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs",
"binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs",
}
}
// nonPersistentLine returns the line the scan prints for path when fi,
// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
// when it is on any other.
func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
dev, ok := deviceOfInfo(fi)
if !ok {
return ""
}
fsType, found := rules.nonPersistent[dev]
if !found {
return ""
}
return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)",
path, fsType)
}
// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
// returns the device ID of every mounted filesystem that is not
// persistent, with its type. The device ID is the one each line gives,
// not that of the file now at its mount point, so a mount that a later
// mount hides never adds the device of another filesystem.
func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
types := nonPersistentFilesystemTypes()
devices := make(map[uint64]string)
lines := bufio.NewScanner(mountinfo)
for lines.Scan() {
fields := strings.Fields(lines.Text())
sep := slices.Index(fields, "-")
if sep < optionalFieldsStart || sep+1 >= len(fields) {
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
}
var major, minor uint32
_, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor)
if err != nil {
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
}
fsType := fields[sep+1]
if slices.Contains(types, fsType) {
devices[unix.Mkdev(major, minor)] = fsType
}
}
return devices, lines.Err()
}
// defaultExcludesHelp describes, for the scan command's help, the
// default excludes of the system goos names and, where the scan lists
// the mounted filesystems, the types it never records anything on.
func defaultExcludesHelp(goos string) string {
var help strings.Builder
dirs := defaultExcludes(goos)
if len(dirs) == 0 {
help.WriteString("This system has no default excludes.\n")
} else {
help.WriteString("Below an operand, the walk does not enter these " +
"default excludes, unless\n--no-default-excludes is given:\n\n")
for _, dir := range dirs {
fmt.Fprintf(&help, " %s\n", dir)
}
}
if goos == goosLinux || goos == goosDarwin {
help.WriteString("\nWith or without --no-default-excludes, the scan " +
"records nothing on a\nfilesystem that is not persistent, " +
"one of these types:\n\n")
types := nonPersistentFilesystemTypes()
for line := range slices.Chunk(types, typesPerHelpLine) {
fmt.Fprintf(&help, " %s\n", strings.Join(line, ", "))
}
}
return help.String()
}
exclude_darwin.go
-52
View File
@@ -1,52 +0,0 @@
//go:build darwin
package main
import (
"slices"
"golang.org/x/sys/unix"
)
// nonPersistentFilesystems returns the device ID of every mounted
// filesystem that is not persistent, with its type, from the system's
// list of mounted filesystems. The device ID is read with lstat on the
// mount point. Where one filesystem is mounted over another on the same
// mount point, the later one in the list counts. A mount point that
// lstat cannot reach is left out.
func nonPersistentFilesystems() (map[uint64]string, error) {
// The first call counts the filesystems, the second lists them.
n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
if err != nil {
return nil, err
}
list := make([]unix.Statfs_t, n)
n, err = unix.Getfsstat(list, unix.MNT_NOWAIT)
if err != nil {
return nil, err
}
mounts := make(map[string]string, n)
for _, st := range list[:n] {
mounts[unix.ByteSliceToString(st.Mntonname[:])] =
unix.ByteSliceToString(st.Fstypename[:])
}
types := nonPersistentFilesystemTypes()
devices := make(map[uint64]string)
for mountPoint, fsType := range mounts {
if !slices.Contains(types, fsType) {
continue
}
if dev, ok := deviceOfPath(mountPoint); ok {
devices[dev] = fsType
}
}
return devices, nil
}
exclude_other.go
-36
View File
@@ -1,36 +0,0 @@
//go:build !darwin
package main
import (
"fmt"
"os"
"runtime"
)
// mountInfoPath is the file in which Linux lists the mounts the reading
// process sees.
const mountInfoPath = "/proc/self/mountinfo"
// nonPersistentFilesystems returns the device ID of every mounted
// filesystem that is not persistent, with its type: on Linux read from
// mountInfoPath (see readMountInfo), and on any other system none.
func nonPersistentFilesystems() (map[uint64]string, error) {
if runtime.GOOS != goosLinux {
return map[uint64]string{}, nil
}
f, err := os.Open(mountInfoPath)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
devices, err := readMountInfo(f)
if err != nil {
return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
}
return devices, nil
}
exclude_test.go
-515
View File
@@ -1,515 +0,0 @@
package main
import (
"bytes"
"errors"
"maps"
"os"
"path/filepath"
"runtime"
"slices"
"strings"
"testing"
"golang.org/x/sys/unix"
)
func TestDefaultExcludes(t *testing.T) {
t.Parallel()
// README "Rules for the walk": the Linux and macOS lists, and none
// on any other system. The Windows list exists in the code only: the
// binary does not build for Windows, so the README leaves it out.
cases := map[string][]string{
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
goosDarwin: {
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
"/System/Volumes/VM", "/System/Volumes/Data",
},
goosWindows: {
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
`Users\*\AppData\Local\Temp`,
},
"freebsd": nil,
}
for goos, want := range cases {
if got := defaultExcludes(goos); !slices.Equal(got, want) {
t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
}
}
}
func TestIsDefaultExclude(t *testing.T) {
t.Parallel()
// A * matches one path component, no more and no fewer.
excludes := []string{"/scratch", "/home/*/cache"}
cases := map[string]bool{
"/scratch": true,
"/scratch/x": false,
"/home/u/cache": true,
"/home/u/v/cache": false,
"/home/cache": false,
}
for path, want := range cases {
if got := isDefaultExclude(path, excludes); got != want {
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
}
}
}
func TestReadMountInfo(t *testing.T) {
t.Parallel()
// The second-to-last line has no optional fields. The tmpfs on
// /data/a/b is hidden by the root filesystem's /srv, mounted over
// /data/a after it: the tmpfs still gives its own device, and the
// root filesystem's device is not listed.
mountinfo := strings.Join([]string{
"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
"25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw",
"26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw",
"27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
}, "\n")
got, err := readMountInfo(strings.NewReader(mountinfo))
if err != nil {
t.Fatal(err)
}
want := map[uint64]string{
unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs",
unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs",
}
if !maps.Equal(got, want) {
t.Errorf("readMountInfo = %v, want %v", got, want)
}
for _, line := range []string{
"22 1 8:1 / / rw,relatime\n",
"22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n",
} {
_, err = readMountInfo(strings.NewReader(line))
if !errors.Is(err, errMountInfoLine) {
t.Errorf("readMountInfo(%q) = %v, want %v", line, err,
errMountInfoLine)
}
}
}
func TestNonPersistentFilesystems(t *testing.T) {
t.Parallel()
if runtime.GOOS != goosLinux {
t.Skip("the test knows a mount only on Linux: /proc")
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfPath("/proc")
if !ok {
t.Fatal("no device for /proc")
}
if got := nonPersistent[dev]; got != "proc" {
t.Errorf("filesystem of /proc = %q, want %q", got, "proc")
}
}
func TestDefaultExcludesHelp(t *testing.T) {
t.Parallel()
for _, goos := range []string{goosLinux, goosDarwin, goosWindows} {
help := defaultExcludesHelp(goos)
for _, dir := range defaultExcludes(goos) {
if !strings.Contains(help, "\n "+dir+"\n") {
t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
}
}
// Only Linux and macOS list their mounted filesystems.
for _, fsType := range nonPersistentFilesystemTypes() {
listed := strings.Contains(help, " "+fsType+",") ||
strings.Contains(help, " "+fsType+"\n")
if listed != (goos != goosWindows) {
t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
}
}
}
want := "This system has no default excludes.\n"
if got := defaultExcludesHelp("freebsd"); got != want {
t.Errorf("help for freebsd = %q, want %q", got, want)
}
// scan's help shows those of the system it runs on.
var stdout, stderr bytes.Buffer
run([]string{cmdScan, "-h"}, &stdout, &stderr)
help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
if !strings.Contains(stderr.String(), help) {
t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
}
}
// TestScanDoesNotEnterDefaultExclude checks that the walk does not
// enter a default exclude below an operand. It says so on a line of its
// own and does not count it as skipped. The records below it count as
// outside the operands: the record of a file gone from it is kept,
// while a file added to it is not recorded.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
excluded := filepath.Join(dir, "tmp")
// A scan with no default excludes records the file in excluded.
gone := writeFile(t, excluded, "gone", pattern(1, 100))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
unseen := writeFile(t, excluded, "new", pattern(2, 10))
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.skipped != 0 {
t.Errorf("skipped = %d, want 0", st.skipped)
}
recs := dbRecords(t, db)
recordByPath(t, recs, gone)
if slices.Contains(recordPaths(recs), unseen) {
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
}
line := "\nscan: skipped " + excluded + " (default exclude)\n"
if !strings.Contains(stderr(), line) {
t.Errorf("stderr = %q, want %q", stderr(), line)
}
}
// TestScanWalksOperandInDefaultExclude checks that an operand at or
// inside a default exclude is walked, also when it lies under another
// operand, whose walk does not enter the default exclude.
func TestScanWalksOperandInDefaultExclude(t *testing.T) {
t.Parallel()
dir := t.TempDir()
excluded := filepath.Join(dir, "tmp")
f := writeFile(t, excluded, "sub/f", pattern(1, 10))
sub := filepath.Dir(f)
cases := map[string][]string{
"at it": {excluded},
"inside it": {sub},
"inside it under an operand": {dir, sub},
}
for name, roots := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
db := openTestDB(t)
st, err := syncScan(t.Context(), db, roots, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.added != 1 {
t.Errorf("added = %d, want %s recorded once", st.added, f)
}
recordByPath(t, dbRecords(t, db), f)
})
}
}
// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default
// exclude given as an operand under another operand, which the walk of
// that other operand does not enter, gets no line saying it was
// skipped: it is walked as an operand of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
excluded := filepath.Join(dir, "tmp")
f := writeFile(t, excluded, "f", pattern(1, 10))
st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
walkRules{excludes: []string{excluded}})
if err != nil {
t.Fatal(err)
}
if st.added != 1 {
t.Errorf("added = %d, want %s recorded once", st.added, f)
}
recordByPath(t, dbRecords(t, db), f)
if strings.Contains(stderr(), "scan: skipped") {
t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
}
}
// TestRunScanTempDirOperand scans a tree in the tests' temporary
// directory, which on Linux lies inside the default exclude /tmp unless
// TMPDIR names another place, with the system's own default excludes:
// the scan walks the whole tree and reports no default exclude.
func TestRunScanTempDirOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := t.TempDir()
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
t.Skip(dir + " is not inside a default exclude")
}
f := writeFile(t, dir, "sub/f", pattern(1, 10))
stderr := scanOK(t, dir)
if strings.Contains(stderr, "scan: skipped") {
t.Errorf("stderr = %q, want no line saying a path was skipped",
stderr)
}
assertRecorded(t, path, f, true)
}
// TestRunScanNoDefaultExcludes scans /var through the command. Its
// /var/tmp, on a persistent filesystem, is a default exclude on Linux:
// without --no-default-excludes the walk does not enter it and says so,
// and with that flag the walk records a file the test puts there.
func TestRunScanNoDefaultExcludes(t *testing.T) {
if runtime.GOOS != goosLinux {
t.Skip("only Linux has /var/tmp among its default excludes")
}
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
rules := walkRules{nonPersistent: nonPersistent}
for _, dir := range []string{"/var", "/var/tmp"} {
fi, lstatErr := os.Lstat(dir)
if lstatErr != nil || !fi.IsDir() ||
rules.nonPersistentLine(dir, fi) != "" {
t.Skip(dir + " is not a directory on a persistent filesystem here")
}
}
//nolint:usetesting // the directory must be inside /var/tmp
tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-")
if err != nil {
t.Skip("cannot write to /var/tmp here: " + err.Error())
}
t.Cleanup(func() { _ = os.RemoveAll(tmp) })
f := writeFile(t, tmp, "f", pattern(1, 10))
path := testDBPath(t)
t.Setenv(databaseEnv, path)
line := "scan: skipped /var/tmp (default exclude)\n"
if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
}
assertRecorded(t, path, f, false)
stderr := scanOK(t, "--no-default-excludes", "/var")
if strings.Contains(stderr, line) {
t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q",
stderr, line)
}
assertRecorded(t, path, f, true)
}
// TestRunScanSkipsNonPersistentOperand scans, through the command, a
// directory on a filesystem that is not persistent, with and without
// --no-default-excludes: either way it is skipped, with a line saying
// so, and nothing in it is recorded.
func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
shm, fsType, _ := shmTempDir(t)
writeFile(t, shm, "f", pattern(1, 10))
t.Setenv(databaseEnv, testDBPath(t))
line := "scan: skipped " + shm + " (" + fsType +
", not a persistent filesystem)\n"
summary := "scan: 0 files seen"
for _, flags := range [][]string{nil, {"--no-default-excludes"}} {
stderr := scanOK(t, append(flags, shm)...)
if !strings.Contains(stderr, line) ||
!strings.Contains(stderr, summary) {
t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr,
line, summary)
}
}
}
// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
// that is not persistent beside one on the tests' temporary directory.
// The records an earlier scan left in it count as outside the operands:
// the record of a file gone from it is kept, while a file added to it
// is not recorded.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
shm, fsType, rules := shmTempDir(t)
stderr := captureStderr(t)
dir := t.TempDir()
db := openTestDB(t)
// A scan that, without the list of filesystems, records gone.
gone := writeFile(t, shm, "gone", pattern(1, 100))
syncTree(t, db, shm)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
unseen := writeFile(t, shm, "new", pattern(2, 10))
st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
if err != nil {
t.Fatal(err)
}
if st.skipped != 0 {
t.Errorf("skipped = %d, want 0", st.skipped)
}
recs := dbRecords(t, db)
recordByPath(t, recs, gone)
if slices.Contains(recordPaths(recs), unseen) {
t.Errorf("%s recorded", unseen)
}
line := "scan: skipped " + shm + " (" + fsType +
", not a persistent filesystem)\n"
if !strings.Contains(stderr(), line) {
t.Errorf("stderr = %q, want %q", stderr(), line)
}
}
// TestWalkLeavesOutNonPersistent starts the walk itself in a directory
// on a filesystem that is not persistent, which a scan never does: it
// drops such an operand first. Below a directory on a persistent
// filesystem, one on another takes a mount, which a test cannot make.
// The walk enters no directory and records no file there, and names
// each.
func TestWalkLeavesOutNonPersistent(t *testing.T) {
t.Parallel()
shm, fsType, rules := shmTempDir(t)
f := writeFile(t, shm, "f", pattern(1, 10))
sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10)))
var leftOut []string
for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) {
if ev.rec.path != "" {
t.Errorf("walk recorded %s", ev.rec.path)
}
if ev.leftOut == "" {
continue
}
leftOut = append(leftOut, ev.leftOut)
want := "scan: skipped " + ev.leftOut + " (" + fsType +
", not a persistent filesystem)"
if ev.warn != want {
t.Errorf("line = %q, want %q", ev.warn, want)
}
}
slices.Sort(leftOut)
if want := []string{f, sub}; !slices.Equal(leftOut, want) {
t.Errorf("walk left out %q, want %q", leftOut, want)
}
}
// shmTempDir returns a new directory in /dev/shm, removed when the test
// ends, the type of its filesystem, and the rules holding the real
// filesystems that are not persistent. It skips the test where /dev/shm
// is on a persistent filesystem, or cannot be written to.
func shmTempDir(t *testing.T) (string, string, walkRules) {
t.Helper()
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfPath("/dev/shm")
fsType, found := nonPersistent[dev]
if !ok || !found {
t.Skip("/dev/shm is not a memory-backed filesystem here")
}
//nolint:usetesting // the directory must be inside /dev/shm
dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-")
if err != nil {
t.Skip("cannot write to /dev/shm here: " + err.Error())
}
t.Cleanup(func() { _ = os.RemoveAll(dir) })
return dir, fsType, walkRules{nonPersistent: nonPersistent}
}
// assertRecorded checks that the database at dbPath holds a record for
// path when want is set, and none when it is not.
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
t.Helper()
db, err := openDB(dbPath, reportParams)
if err != nil {
t.Fatal(err)
}
defer func() { _ = db.Close() }()
if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want {
t.Errorf("%s recorded: %v, want %v", path, got, want)
}
}
go.mod
-2
View File
@@ -24,5 +24,3 @@ require (
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
)
retract v0.0.1 // early build without a database; does not work as the README describes
info.go
-125
View File
@@ -1,125 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
// infoPhase names one phase of info with its place among all of them,
// as "[2/2] count", for the phase's progress display. The list here is
// the one place the phases are counted.
func infoPhase(name string) string {
return phaseLabel([]string{openPhase, "count"}, name)
}
// runInfo implements the info subcommand: it prints the database's
// details on stdout (see writeInfo). It opens the database as report
// does, read-only and without the scan lock, and never touches the
// scanned filesystem. It names the database on stderr before it opens
// it. Any database problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and count (count the records,
// with one query). Each display ends before the next phase starts, and
// before an error is returned to be printed. The details are written
// once the last display has ended, so no display is drawn while they
// are written.
func runInfo(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "info: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: infoPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil {
return err
}
defer func() { _ = db.Close() }()
// One query counts everything, so the phase never has a total.
prog = newProgress(progressParams{
label: infoPhase("count"), total: -1,
waitingFor: countingRecords,
})
counts, err := readDatabaseCounts(ctx, db)
prog.finish()
if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err)
}
return writeInfo(stdout, dbPath, source, counts)
}
// writeInfo writes the details of the database at dbPath to stdout,
// one per line, as a name, a tab and the value (README "`info` mode"):
// the path and source as databasePath returns them, the sizes of the
// database file and of its -wal file when there is one, when the
// database file was last changed, and the schema version and record
// counts that readDatabaseCounts read.
func writeInfo(stdout io.Writer, dbPath, source string,
counts databaseCounts,
) error {
// SQLite keeps the -wal file beside the database file, named from
// its path with symlinks resolved.
file, err := filepath.EvalSymlinks(dbPath)
if err != nil {
return fmt.Errorf("database: %w", err)
}
dbFile, err := os.Stat(file)
if err != nil {
return fmt.Errorf("database: %w", err)
}
walFile, err := os.Stat(file + "-wal")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("database: %w", err)
}
var b strings.Builder
fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source)
fmt.Fprintf(&b, "database file size\t%d (%s)\n",
dbFile.Size(), humanBytes(dbFile.Size()))
if walFile != nil {
fmt.Fprintf(&b, "-wal file size\t%d (%s)\n",
walFile.Size(), humanBytes(walFile.Size()))
}
fmt.Fprintf(&b, "modified\t%s\n",
dbFile.ModTime().UTC().Format(time.RFC3339))
fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
fmt.Fprintf(&b, "records\t%d\n", counts.records)
fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
counts.size, humanBytes(counts.size))
_, err = io.WriteString(stdout, b.String())
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
return nil
}
info_test.go
-166
View File
@@ -1,166 +0,0 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// infoOpenLine is the plain line of info's open phase when it opens the
// database at once; it prints it as the phase starts and as it ends.
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"remain unknown"
// infoCountLine is the plain line of info's count phase when the count
// takes under a second; it prints it as the phase starts and as it
// ends.
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
"remain unknown"
// infoRecs are four records: a duplicate pair, a file of 20 MiB and
// another small file.
func infoRecs() []scanRec {
h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
return []scanRec{
{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
}
}
// TestRunInfo runs info on an empty and on a filled database, with
// stdout and stderr the same file that is not a terminal, and checks
// everything written there: the line naming the database, each phase's
// line as it starts and as it ends, and then the details, each on a
// line of its own. No -wal file is beside a database scan has closed.
func TestRunInfo(t *testing.T) {
cases := map[string]struct {
recs []scanRec
want string // the details after the database file's time
}{
"empty": {
want: "schema version\t1\n" +
"records\t0\n" +
"size of recorded files\t0 (0 B)\n",
},
"filled": {
recs: infoRecs(),
want: "schema version\t1\n" +
"records\t4\n" +
"size of recorded files\t20971537 (20.0 MiB)\n",
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
path := seedDatabase(t, tc.recs)
t.Setenv(databaseEnv, path)
file, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
stderr := captureStderr(t)
code := run([]string{cmdInfo}, os.Stderr, os.Stderr)
if code != exitOK {
t.Fatalf("run(info) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
named := path + " (from " + databaseEnv + ")\n"
want := "info: database " + named +
infoOpenLine + "\n" + infoOpenLine + "\n" +
infoCountLine + "\n" + infoCountLine + "\n" +
"database\t" + named +
fmt.Sprintf("database file size\t%d (%s)\n",
file.Size(), humanBytes(file.Size())) +
"modified\t" + file.ModTime().UTC().Format(time.RFC3339) +
"\n" + tc.want
if got := stderr(); got != want {
t.Errorf("stdout and stderr = %q, want %q", got, want)
}
})
}
}
// TestRunInfoDuringScan runs info while a scan holds the lock and has
// the database open in WAL mode, with records committed to the -wal
// file only. info must count those records and show the -wal file's
// size, also when the database path is a symlink: SQLite keeps the
// -wal file beside the symlink's target.
func TestRunInfoDuringScan(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
db, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
err = applyChanges(t.Context(), db, infoRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
wal, err := os.Stat(path + "-wal")
if err != nil {
t.Fatal(err)
}
for _, named := range []string{path, link} {
t.Run(filepath.Base(named), func(t *testing.T) {
t.Setenv(databaseEnv, named)
stdout := runStdout(t, cmdInfo)
for _, want := range []string{
"database\t" + named + " (from " + databaseEnv + ")\n",
fmt.Sprintf("\n-wal file size\t%d (%s)\n",
wal.Size(), humanBytes(wal.Size())),
"\nrecords\t4\n",
} {
if !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
}
})
}
}
func TestRunInfoNeedsOnlyReadAccess(t *testing.T) {
// README "`info` mode": like report, info needs only read access to
// the database file and none to its directory.
path := testDBPath(t)
t.Setenv(databaseEnv, path)
scanFixture(t)
makeReadOnly(t, path)
stdout := runStdout(t, cmdInfo)
if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
if strings.Contains(stdout, "-wal") {
t.Errorf("stdout = %q, want no -wal file between scans", stdout)
}
}
main.go
+39 -89
View File
@@ -1,19 +1,20 @@
// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
// KiB, and of the content. The content hash covers the whole file under
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
// file is never read in full. scan maintains a persistent SQLite
// database of file signatures (SFDUPES_DATABASE, default
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
// hashed in full. A larger file is compared on the hashes of its first
// and last 64 KiB, and only when those match another file's is its
// content hash computed and compared: of the whole file when it is
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
// or larger. scan maintains a persistent SQLite database of file
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
// that the reporting subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
// sfdupes scan [--workers N] [-x] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes --version
//
// See README.md for the complete specification.
@@ -26,7 +27,6 @@ import (
"io"
"os"
"runtime"
"runtime/debug"
"github.com/spf13/cobra"
)
@@ -44,7 +44,6 @@ const (
cmdScan = "scan"
cmdReport = "report"
cmdTrees = "trees"
cmdInfo = "info"
)
// errNoSubcommand is returned by the root command when it is invoked
@@ -57,32 +56,10 @@ var errNoSubcommand = errors.New("no subcommand")
var errWorkersBelowOne = errors.New("--workers must be at least 1")
// Version is the build version, injected at link time via -ldflags
// (see the Makefile); empty when the linker set none, as for a binary
// built by go install. buildVersion turns it into what --version prints.
// (see the Makefile); "dev" for a plain go build.
//
//nolint:gochecknoglobals // written only by the linker
var Version string
// buildVersion returns the version that sfdupes --version prints
// (README "Install"): the version the linker set, when it set one;
// otherwise the main module's version that Go recorded in the binary,
// a tag such as v0.0.2 or the pseudo-version of an @main build; and
// "dev" when neither is known. info is what debug.ReadBuildInfo
// returns, nil when the binary carries none. Go records "(devel)" for
// a build that has no version, such as one inside a copy of the source
// without git.
func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
if linkerVersion != "" {
return linkerVersion
}
if info == nil || info.Main.Version == "" ||
info.Main.Version == "(devel)" {
return "dev"
}
return info.Main.Version
}
var Version = "dev"
func main() {
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
@@ -96,8 +73,8 @@ func main() {
// exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees and info subcommands
// write their data to stdout.
// runs before the process ends. The report and trees subcommands write
// their data to stdout.
func run(args []string, stdout, stderr io.Writer) int {
// A nil slice makes cobra fall back to os.Args, which would let a
// test binary's own flags reach the command tree.
@@ -137,10 +114,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
var showVersion bool
printVersion := runE(func(context.Context, []string) error {
info, _ := debug.ReadBuildInfo()
_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
buildVersion(Version, info))
_, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version)
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
@@ -174,6 +148,30 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
root.Flags().BoolVarP(&showVersion, "version", "v", false,
"print the version to stdout")
var (
scanWorkers int
scanOneFS bool
)
scanCmd := &cobra.Command{
Use: cmdScan + " [--workers N] [-x] PATH...",
Short: "Walk trees and synchronize the scan database",
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, _ []string) error {
return checkScanWorkers(cmd, scanWorkers)
},
RunE: runE(func(ctx context.Context, args []string) error {
ctx, stop := interruptContext(ctx)
defer stop()
return runScan(ctx, args, scanWorkers, scanOneFS)
}),
}
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
"concurrent workers for the walk, hash, and content phases")
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
"do not cross filesystem boundaries")
reportCmd := &cobra.Command{
Use: cmdReport,
Short: "Read the scan database and print the file-level duplicates report",
@@ -192,59 +190,11 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
}),
}
infoCmd := &cobra.Command{
Use: cmdInfo,
Short: "Read the scan database and print its details",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runInfo(ctx, stdout)
}),
}
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
root.AddCommand(scanCmd, reportCmd, treesCmd)
return root
}
// newScanCommand builds the scan subcommand, with its --workers, -x and
// --no-default-excludes flags. Its help lists the default excludes of
// the system it runs on, and the filesystem types it never records
// anything on.
func newScanCommand() *cobra.Command {
var (
scanWorkers int
scanOneFS bool
scanNoDefaultExcludes bool
)
short := "Walk trees and synchronize the scan database"
scanCmd := &cobra.Command{
Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
Short: short,
Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
Args: cobra.MinimumNArgs(1),
PreRunE: func(cmd *cobra.Command, _ []string) error {
return checkScanWorkers(cmd, scanWorkers)
},
RunE: runE(func(ctx context.Context, args []string) error {
ctx, stop := interruptContext(ctx)
defer stop()
return runScan(ctx, args, scanWorkers, scanOneFS,
scanNoDefaultExcludes)
}),
}
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
"concurrent workers for the walk and hash phases")
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
"do not cross filesystem boundaries")
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
false, "enter the default excludes, the paths listed above")
return scanCmd
}
// checkScanWorkers rejects a scan --workers value below 1. That is a
// usage error reported in one line: cobra prints the returned message
// without the usage text, and run exits 2.
main_test.go
+37 -585
View File
@@ -4,12 +4,10 @@ import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"runtime/debug"
"slices"
"strconv"
"strings"
@@ -20,36 +18,6 @@ import (
// error prints and a runtime failure must not.
const usageMarker = "Usage:"
// TestMain stops the tests at once, saying why, when their temporary
// directory is not on a persistent filesystem: scan records nothing
// there (README "Rules for the walk"), so most scan tests would fail
// without saying why.
func TestMain(m *testing.M) {
nonPersistent, err := nonPersistentFilesystems()
if err != nil {
fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err)
os.Exit(1)
}
dir := os.TempDir()
fi, err := os.Stat(dir)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
dev, ok := deviceOfInfo(fi)
if fsType, found := nonPersistent[dev]; ok && found {
fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+
"and scan records nothing there: set TMPDIR to a directory on a "+
"persistent filesystem\n", dir, fsType)
os.Exit(1)
}
os.Exit(m.Run())
}
// walSuffixes are the SQLite sidecar files a WAL-mode database keeps
// while it is open. A clean close checkpoints the WAL and removes
// both; finding either afterwards means the database was never closed.
@@ -316,38 +284,23 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) {
}
func TestRunMissingDatabaseIsFatal(t *testing.T) {
// README §Database: report, trees and info need an existing
// database; a missing one exits 1 with a message telling the user to
// run scan, after the line naming the database.
openLines := map[string]string{
cmdReport: reportOpenLine,
cmdTrees: treesOpenLine,
cmdInfo: infoOpenLine,
}
for name, openLine := range openLines {
// README §Database: report and trees need an existing database; a
// missing one exits 1 with a message telling the user to run scan.
for _, name := range []string{cmdReport, cmdTrees} {
t.Run(name, func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
var stdout bytes.Buffer
var stdout, stderr bytes.Buffer
// run gets os.Stderr, where the subcommand prints its line,
// so the capture shows the two in the order printed.
stderr := captureStderr(t)
code := run([]string{name}, &stdout, os.Stderr)
code := run([]string{name}, &stdout, &stderr)
if code != exitFatal {
t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
}
// Each shows its open phase, as it starts and as it ends,
// before the error.
want := name + ": database " + path + " (from " + databaseEnv +
")\n" + openLine + "\n" + openLine + "\n" +
"sfdupes: " + path + ": no database (run \"sfdupes " +
want := "sfdupes: " + path + ": no database (run \"sfdupes " +
"scan\" first, or set " + databaseEnv + ")\n"
if got := stderr(); got != want {
if got := stderr.String(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
@@ -358,256 +311,6 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) {
}
}
func TestRunNamesRelativeDatabase(t *testing.T) {
// README §Database: each subcommand's first line on stderr names the
// database, a relative SFDUPES_DATABASE made absolute against the
// working directory. scan calls it new until a scan has created it.
path := testDBPath(t)
t.Chdir(filepath.Dir(path))
t.Setenv(databaseEnv, filepath.Base(path))
dir := t.TempDir()
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("first scan stderr = %q, want it to start %q", got, want)
}
want = "scan: database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("second scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + path + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
// README §Settings: the path names the file exactly. With link a
// symlink to the directory real/sub, link/../db.sqlite names
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
// use that file, relative or absolute, and its first line must name
// the path as set, with the working directory put in front of a
// relative one.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Chdir(dir)
err = os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "real", "sub"),
filepath.Join(dir, "link"))
if err != nil {
t.Fatal(err)
}
named := dir + "/link/../db.sqlite"
from := " (from " + databaseEnv + ")\n"
tree := t.TempDir()
t.Setenv(databaseEnv, "link/../db.sqlite")
want := "scan: new database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("relative scan stderr = %q, want it to start %q", got, want)
}
// The absolute path names the database the relative one created.
t.Setenv(databaseEnv, named)
want = "scan: database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("absolute scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + named + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
_, err = os.Stat(filepath.Join(dir, "real", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
// With no file at the cleaned path, report and trees succeeded on
// the database in real.
_, err = os.Stat(filepath.Join(dir, "db.sqlite"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("database at the cleaned path: stat error %v, want it "+
"missing", err)
}
}
func TestRunScanFromSymlinkedWorkingDirectory(t *testing.T) {
// README §Database: a relative path gets the working directory, its
// symlinks resolved, put in front of it. With home/proj a symlink
// to data/proj, entered as the working directory through the
// symlink, ../new/db.sqlite names data/new/db.sqlite. scan must
// create that directory and file, and nothing beside home/proj.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Join(dir, "data", "proj"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(dir, "home"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
if err != nil {
t.Fatal(err)
}
// t.Chdir also sets PWD to the path through the symlink, as a
// shell does after cd.
t.Chdir(filepath.Join(dir, "home", "proj"))
t.Setenv(databaseEnv, "../new/db.sqlite")
want := "scan: new database " + dir + "/data/proj/../new/db.sqlite" +
" (from " + databaseEnv + ")\n"
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
_, err = os.Lstat(filepath.Join(dir, "home", "new"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("home/new: stat error %v, want it missing", err)
}
}
func TestRunEscapesDatabasePath(t *testing.T) {
// README §Database: each subcommand's first line escapes the
// database path like the paths in a report, so a tab, newline or
// backslash in it keeps the line one line.
dir := t.TempDir()
t.Setenv(databaseEnv, dir+"/a\tb\nc\\d.sqlite")
escaped := dir + `/a\tb\nc\\d.sqlite`
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + escaped + from
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + escaped + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunNamesDefaultDatabase(t *testing.T) {
// README §Database: with SFDUPES_DATABASE empty, each subcommand's
// first line on stderr names the default database. A database may
// exist at the default path on the machine running the test, so
// nothing here may create or read one: scan gets an operand that
// does not exist, which stops it before it takes the lock, and
// report, trees and info a cancelled context, which stops them
// before they read anything. They show their open phase after the
// line.
t.Setenv(databaseEnv, "")
// Whether scan calls the database new depends on that machine too.
scanWant := "scan: database " + defaultDatabasePath + " (default)\n"
_, err := os.Stat(defaultDatabasePath)
if errors.Is(err, fs.ErrNotExist) {
scanWant = "scan: new database " + defaultDatabasePath +
" (default)\n"
}
missing := filepath.Join(t.TempDir(), "nope")
cases := map[string]struct {
run func() error
want string
}{
cmdScan: {
run: func() error {
return runScan(t.Context(), []string{missing}, 1, false, false)
},
want: scanWant,
},
cmdReport: {
run: func() error {
return runReport(cancelledContext(t), io.Discard)
},
want: "report: database " + defaultDatabasePath + " (default)\n" +
reportOpenLine + "\n" + reportOpenLine + "\n",
},
cmdTrees: {
run: func() error {
return runTrees(cancelledContext(t), io.Discard)
},
want: "trees: database " + defaultDatabasePath + " (default)\n" +
treesOpenLine + "\n" + treesOpenLine + "\n",
},
cmdInfo: {
run: func() error {
return runInfo(cancelledContext(t), io.Discard)
},
want: "info: database " + defaultDatabasePath + " (default)\n" +
infoOpenLine + "\n" + infoOpenLine + "\n",
},
}
for name, tc := range cases {
stderr := captureStderr(t)
err = tc.run()
if err == nil {
t.Errorf("%s: error = nil, want it stopped", name)
}
if got := stderr(); got != tc.want {
t.Errorf("%s stderr = %q, want %q", name, got, tc.want)
}
}
}
func TestRunUsageErrors(t *testing.T) {
// Usage errors keep exiting 2 with cobra's own report on stderr.
cases := map[string]struct {
@@ -618,7 +321,6 @@ func TestRunUsageErrors(t *testing.T) {
"scan without paths": {[]string{cmdScan}, usageMarker},
"report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
"unknown subcommand": {[]string{"nope"}, "unknown command"},
}
@@ -648,7 +350,7 @@ func TestRunUsageErrors(t *testing.T) {
}
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
// README §Settings: --workers below 1 is a usage error reported in
// README §scan mode: --workers below 1 is a usage error reported in
// one line on stderr, before the scan opens the database.
for _, workers := range []string{"0", "-1"} {
t.Run(workers, func(t *testing.T) {
@@ -686,8 +388,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
func TestRunHelp(t *testing.T) {
t.Parallel()
// README §Settings: help goes to stderr, exits 0, and leaves stdout
// empty.
// README §Subcommands: help goes to stderr, exits 0, and leaves
// stdout empty.
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
for _, args := range cases {
@@ -713,10 +415,8 @@ func TestRunHelp(t *testing.T) {
func TestRunVersion(t *testing.T) {
t.Parallel()
// README §Settings: the version is one line on stdout, with nothing
// on stderr, and exits 0.
info, _ := debug.ReadBuildInfo()
// README §Subcommands: the version is one line on stdout, with
// nothing on stderr, and exits 0.
for _, arg := range []string{"--version", "-v"} {
var stdout, stderr bytes.Buffer
@@ -725,7 +425,7 @@ func TestRunVersion(t *testing.T) {
t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
}
want := "sfdupes " + buildVersion(Version, info) + "\n"
want := "sfdupes " + Version + "\n"
if got := stdout.String(); got != want {
t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
}
@@ -736,40 +436,6 @@ func TestRunVersion(t *testing.T) {
}
}
func TestBuildVersion(t *testing.T) {
t.Parallel()
// README "Install": the linker's version wins; otherwise the main
// module's version that Go recorded; "dev" when neither is known.
tagged := &debug.BuildInfo{Main: debug.Module{Version: "v0.0.2"}}
pseudo := &debug.BuildInfo{Main: debug.Module{
Version: "v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1",
}}
devel := &debug.BuildInfo{Main: debug.Module{Version: "(devel)"}}
cases := []struct {
name string
linker string
info *debug.BuildInfo
want string
}{
{"linker set", "v0.0.2-3-g4e4a94d", tagged, "v0.0.2-3-g4e4a94d"},
{"linker set, no build info", "4e4a94d", nil, "4e4a94d"},
{"build info with a tag", "", tagged, "v0.0.2"},
{"build info with a pseudo-version", "", pseudo,
"v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1"},
{"(devel)", "", devel, "dev"},
{"no build info", "", nil, "dev"},
}
for _, c := range cases {
if got := buildVersion(c.linker, c.info); got != c.want {
t.Errorf("%s: buildVersion(%q, ...) = %q, want %q",
c.name, c.linker, got, c.want)
}
}
}
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
t.Parallel()
@@ -928,21 +594,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
}
func TestRunScanSkipsZFSOperand(t *testing.T) {
zfs := filepath.Join(t.TempDir(), ".zfs")
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
// No scan records a file inside a .zfs directory, so the record
// beneath the operand is written straight into the database.
path := seedDatabase(t, []scanRec{{size: 10, path: f}})
path := testDBPath(t)
t.Setenv(databaseEnv, path)
zfs := filepath.Join(t.TempDir(), ".zfs")
snapshot := filepath.Join(zfs, "snapshot", "hourly")
f := writeFile(t, snapshot, "f", pattern(1, 10))
// An operand beneath a .zfs directory is walked, because it is not
// itself named .zfs.
scanOK(t, snapshot)
assertOperandSkipped(t, path, zfs, ".zfs directory", f)
}
// assertOperandSkipped scans operand alone and checks that it is skipped
// as kind: a warning naming it, one skip in the summary, exit 0, and the
// record for kept, stored beneath operand before this scan, still in the
// database at dbPath.
// record for kept, which an earlier scan stored beneath operand, still
// in the database at dbPath.
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
kept string,
) {
@@ -971,121 +640,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
recordByPath(t, dbRecords(t, db), kept)
}
// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
// directory, first alone and then under another operand. Each time it
// is warned about and counted as skipped, and nothing in it is walked.
// The other operand is scanned as usual.
func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := t.TempDir()
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
writeFile(t, hourly, "f", pattern(1, 10))
f := writeFile(t, dir, "f", pattern(2, 10))
warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
for _, tc := range []struct {
operands []string
summary string
}{
{
[]string{hourly},
"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n",
},
{
[]string{dir, hourly},
"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n",
},
} {
stderr := scanOK(t, tc.operands...)
if !strings.Contains(stderr, warning) ||
!strings.Contains(stderr, tc.summary) {
t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
stderr, warning, tc.summary)
}
}
db, err := openDB(path, reportParams)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
// by the path given, counted as skipped, and not walked.
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
dir := t.TempDir()
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
snaps := filepath.Join(dir, "snaps")
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(snaps, "daily")
stderr := scanOK(t, daily)
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n"
if !strings.Contains(stderr, warning) ||
!strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
}
}
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
// a symlink to a directory outside any .zfs directory. The operand is
// scanned as usual, and its file is recorded by the path given.
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := t.TempDir()
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
data := filepath.Join(dir, "data")
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(data, "daily")
stderr := scanOK(t, daily)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
db, err := openDB(path, reportParams)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
}
func TestRunReportSucceeds(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
@@ -1107,16 +661,10 @@ func TestRunReportSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want)
}
// Its phases come between the two lines; see
// TestRunReportProgressNamesEachPhase.
first := "report: database " + path + " (from " + databaseEnv + ")\n"
last := "\nreport: 2 records read, 1 duplicate groups, 1 dupe files, " +
want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " +
"300 B reclaimable\n"
if got := stderr(); !strings.HasPrefix(got, first) ||
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
if got := stderr(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
assertNoSidecars(t, path)
@@ -1146,16 +694,10 @@ func TestRunTreesSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want)
}
// Its phases come between the two lines; see
// TestRunTreesProgressNamesEachPhase.
first := "trees: database " + path + " (from " + databaseEnv + ")\n"
last := "\ntrees: 2 records read, 1 duplicate tree groups, " +
"1 dupe trees, 300 B reclaimable\n"
if got := stderr(); !strings.HasPrefix(got, first) ||
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " +
"300 B reclaimable\n"
if got := stderr(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
assertNoSidecars(t, path)
@@ -1234,7 +776,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
}
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
// README §Settings: the path names the database file exactly. In
// README §Database: the path names the database file exactly. In
// SQLite's connection string an unescaped ? or # would end the file
// name and % would start an escape, and a path starting with //
// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1280,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
func TestRunSecondScanFails(t *testing.T) {
// README §Database: while one scan holds the lock, a second scan
// fails at once, naming the lock file, without opening the
// fails at once, naming the lock file, without creating the
// database.
path := testDBPath(t)
t.Setenv(databaseEnv, path)
@@ -1306,99 +848,10 @@ func TestRunSecondScanFails(t *testing.T) {
t.Errorf("stdout = %q, want nothing (data only)", got)
}
// Taking the lock created the database file, empty; opening it as
// a database would have written to it.
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
_, err := os.Stat(path)
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("stat %s = %v, want the database not created", path, err)
}
if info.Size() != 0 {
t.Errorf("database size = %d, want 0 (not opened)", info.Size())
}
}
// assertSecondScanFails runs a scan against the database
// SFDUPES_DATABASE names while the test holds the lock on it. It fails
// the test unless that scan fails at once, naming the lock file beside
// path, the database's real path.
func assertSecondScanFails(t *testing.T, path string) {
t.Helper()
var stderr bytes.Buffer
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
if code != exitFatal {
t.Errorf("run(scan) = %d, want %d", code, exitFatal)
}
want := "sfdupes: another scan is running (lock held on " +
path + ".lock)\n"
if got := stderr.String(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
}
}
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
// README §Database: every path to one database names the same lock
// file, so while a scan holds it through one path, a scan through
// another fails, naming the lock file beside the database's real
// path.
t.Run("symlink to the database", func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
scanFixture(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
t.Setenv(databaseEnv, link)
assertSecondScanFails(t, path)
})
t.Run("symlink to its directory, no database yet", func(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link")
err := os.Symlink(filepath.Dir(path), link)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
assertSecondScanFails(t, path)
})
t.Run("symlink whose target does not exist yet", func(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
// The first scan takes the lock through the link before the
// database exists, then creates the database through it.
holdScanLock(t, link)
db, err := openScanDatabase(t.Context(), link)
if err != nil {
t.Fatal(err)
}
closeScanDatabase(t.Context(), db, link)
t.Setenv(databaseEnv, link)
assertSecondScanFails(t, path)
})
}
func TestRunScanReleasesLock(t *testing.T) {
@@ -1447,7 +900,7 @@ func TestRunReportsDuringScan(t *testing.T) {
func TestRunStdoutClosedIsFatal(t *testing.T) {
// README §Error handling: a stdout write failure exits 1, reported
// in one line on stderr.
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
for _, name := range []string{cmdReport, cmdTrees} {
t.Run(name, func(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
@@ -1497,7 +950,6 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
cases := map[string]func(context.Context, io.Writer) error{
cmdReport: runReport,
cmdTrees: runTrees,
cmdInfo: runInfo,
}
for name, fn := range cases {
progress.go
+111 -474
View File
@@ -2,321 +2,122 @@ package main
import (
"fmt"
"io"
"math"
"os"
"slices"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/schollz/progressbar/v3"
"golang.org/x/term"
)
// plainInterval is the time between progress lines while a phase runs,
// when stderr is not a TTY.
// plainInterval is the minimum time between progress lines when stderr
// is not a TTY.
const plainInterval = 5 * time.Second
// redrawInterval is the time between redraws of the progress display on
// a TTY.
const redrawInterval = 100 * time.Millisecond
// barThrottle caps how often the TTY progress bar redraws.
const barThrottle = 100 * time.Millisecond
// rateWindow is the time the rate a display shows covers: the items
// completed over the last rateWindow, which is the time since the line
// before when stderr is not a TTY.
const rateWindow = 5 * time.Second
// stallTime is how long a phase goes with no item completed before its
// line says it has stalled, and where it waits, in place of the rate.
const stallTime = 10 * time.Second
// spinnerType selects the progressbar library's spinner style.
const spinnerType = 14
// barWidth is the width of a phase's bar between its two ends, as in
// the example line in README "Progress".
const barWidth = 10
// barCell is one column of the filled part of a bar.
const barCell = "█"
// barColumns is the width of what percentBar returns: the percent in
// four columns and its sign, a space, and the bar with its two ends.
const barColumns = barWidth + 8
// walkSpinnerType selects the progressbar library's spinner style used
// for the walk pass, which has no known total.
const walkSpinnerType = 14
// percentScale converts a fraction to a percentage.
const percentScale = 100
// fallbackWidth is the width taken for a terminal whose width cannot be
// read, as the progressbar library takes it.
const fallbackWidth = 80
// spinnerMargin is the columns of a line besides its text: the spinner
// and the three spaces the library draws around the text, and the
// terminal's last column, which every line stays clear of so that no
// terminal wraps it.
const spinnerMargin = 5
// countingRecords is what the display of a phase that reads the records
// says while it counts them, before it has its total.
const countingRecords = "counting the records"
// openPhase is the name of the phase in which report, trees and info
// open the database, and openingDatabase what its display says while
// it does.
const (
openPhase = "open"
openingDatabase = "opening the database"
)
// stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool {
return term.IsTerminal(int(os.Stderr.Fd()))
}
// terminalWidth returns the width of the terminal on stderr, or
// fallbackWidth, as the progressbar library does, when it cannot read
// one.
func terminalWidth() int {
width, _, err := term.GetSize(int(os.Stderr.Fd()))
if err != nil || width <= 0 {
return fallbackWidth
}
return width
}
// phaseLabel names phase with its place among phases, as "[2/4] walk".
func phaseLabel(phases []string, phase string) string {
return fmt.Sprintf("[%d/%d] %s",
slices.Index(phases, phase)+1, len(phases), phase)
}
// progressParams describe one phase's progress display as it starts.
type progressParams struct {
label string // the phase and its place, from phaseLabel
unit string // what the phase counts, as "files"
total int64 // the items the phase will complete; -1 when not known
estimate bool // total is an estimate, which the count may pass
waitingFor string // what the phase waits for until setTotal is called
stalledOn string // where the phase waits, as "disk/db"; see stallTime
}
// drawnCount is the count of items completed as of one draw, and when
// that draw was.
type drawnCount struct {
at time.Time
count int64
}
// progress renders one phase's progress on stderr. On a TTY the
// progressbar library draws its spinner and then line, fitted to the
// terminal's width. A phase with an exact total ends the line with its
// percent and bar, at the terminal's right edge, when the terminal has
// room for them. It is drawn again every redrawInterval, so its elapsed
// time keeps moving while the phase waits for its next item. When
// stderr is not a TTY it emits no ANSI redraws: it prints line as the
// phase starts, every plainInterval while it runs, and as it ends.
// progress renders one scan pass's progress on stderr. On a TTY it
// delegates to the progressbar library (spinner style when the total is
// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
// stderr is not a TTY it emits no ANSI redraws: it prints a plain
// one-line update as the pass starts, then no more often than every
// plainInterval.
//
// The draws come from a goroutine of their own, so mu guards the fields
// below it. Every method must be called from the main goroutine. A nil
// *progress is a valid no-display receiver: every method is a no-op, so
// batched database flushes during the hash phase can reuse the update
// phase's helpers without rendering anything, and its writer writes
// with no display to clear.
// All methods must be called from the main goroutine only. On a TTY
// the library also redraws a spinner from its own goroutine, several
// times a second, so its count and elapsed time stay current while a
// pass waits for its next item. A nil *progress is a valid
// no-display receiver: every method is a no-op, so batched database
// flushes during the streaming pass can reuse the update-pass helpers
// without rendering anything.
type progress struct {
now func() time.Time // reads the time
width func() int // reads the terminal's width; nil when not a TTY
stop chan struct{} // closed by finish to end the draws
done chan struct{} // closed once the draws have ended
mu sync.Mutex
bar *progressbar.ProgressBar // what the library draws; nil until then
widest int // the widest line bar has drawn, in columns
label string
unit string
total int64
estimate bool
waitingFor string
stalledOn string
count int64
start time.Time // when the phase started, for the elapsed time
rateStart time.Time // when the count started, for the time remaining
lastItem time.Time // when the last item was completed, or the count started
// drawn holds the count at each draw over the last rateWindow, oldest
// first; the rate counts from the first. It keeps the draw before the
// latest even when that is older, as the line before is when lines
// are plainInterval apart and a tick comes late.
drawn []drawnCount
label string
total int64 // -1 when unknown (walk pass)
bar *progressbar.ProgressBar
count int64
start time.Time
last time.Time
}
// newProgress starts the display of one phase on stderr.
func newProgress(params progressParams) *progress {
func newProgress(label string, total int64) *progress {
p := &progress{label: label, total: total, start: time.Now()}
if stderrIsTTY() {
return startProgress(params, terminalWidth, redrawInterval, time.Now)
p.bar = newBar(label, total)
return p
}
return startProgress(params, nil, plainInterval, time.Now)
}
// startProgress shows the phase params describes at once, and again
// every interval until finish: with the progressbar library on a
// terminal, whose width each draw reads by calling width, or as a plain
// line when width is nil. The display reads the time by calling now.
func startProgress(params progressParams, width func() int,
interval time.Duration, now func() time.Time,
) *progress {
start := now()
p := &progress{
now: now, width: width,
stop: make(chan struct{}), done: make(chan struct{}),
label: params.label, unit: params.unit, total: params.total,
estimate: params.estimate, waitingFor: params.waitingFor,
stalledOn: params.stalledOn, start: start, rateStart: start,
lastItem: start, drawn: []drawnCount{{at: start}},
}
// Show the zero state at once: the first item may take minutes,
// and a phase must never look hung.
p.draw()
go func() {
defer close(p.done)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-p.stop:
return
case <-ticker.C:
p.draw()
}
}
}()
// Print the zero state at once: the first item may take minutes,
// and a pass must never look hung.
p.last = p.start
fmt.Fprintln(os.Stderr, p.plainLine())
return p
}
// draw shows the phase's progress: on a TTY in place of the line
// before, and otherwise as a plain line. It first adds the count to
// drawn, then drops from drawn the draws older than rateWindow, except
// the one before this draw.
func (p *progress) draw() {
p.mu.Lock()
defer p.mu.Unlock()
now := p.now()
p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
p.drawn = p.drawn[1:]
}
if p.width == nil {
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
return
}
width := p.width()
text := p.line(now, width-spinnerMargin, true)
// Before each draw the library erases its line with as many spaces as
// the widest line it has drawn. A display gets a new one when the
// terminal has been made too narrow for that erase, which would wrap
// and leave a line behind at every draw.
if p.bar == nil || p.widest >= width {
p.newBar(width)
}
p.bar.Describe(text)
// The library drew the spinner, the text and the spaces around it,
// less the terminal's last column, which spinnerMargin keeps clear. A
// bar's cells take one column each but more than one byte.
p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
}
// newBar has the library make what draw draws: its spinner, followed by
// the text draw gives it and nothing of the library's own. p.mu must be
// held.
func (p *progress) newBar(width int) {
if p.bar != nil {
// Erase what was drawn before, short of the last column of a
// terminal that may have been made narrower since: the library's
// own erase is as wide as the widest line it drew.
blank := strings.Repeat(" ", min(p.widest, width-1))
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
}
// A total of -1 gives the library's spinner.
p.bar = progressbar.NewOptions64(-1,
// newBar builds the TTY display for newProgress.
func newBar(label string, total int64) *progressbar.ProgressBar {
opts := []progressbar.Option{
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSpinnerType(spinnerType),
// The spinner turns at each draw, not on a timer of the
// library's own.
progressbar.OptionSetSpinnerChangeInterval(0),
// The text carries the elapsed time and the time remaining.
progressbar.OptionSetElapsedTime(false),
progressbar.OptionSetPredictTime(false),
)
p.widest = 0
}
// percentBar returns the percent and the bar that end the text of a
// phase with an exact total, as " 25% |██ |": the bar's
// barWidth columns fill in proportion to the items done. A count past
// the total, as when a scan adds records while trees reads them, shows
// as 100%. p.mu must be held.
func (p *progress) percentBar() string {
done := min(float64(p.count)/float64(p.total), 1)
filled := strings.Repeat(barCell, int(done*barWidth))
return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
}
// setTotal gives a phase that started without a total its total, once
// it is known. The rate, the time remaining and the time with no item
// completed count from then on: the time spent waiting for the total
// says nothing about how fast items go.
func (p *progress) setTotal(total int64) {
if p == nil {
return
progressbar.OptionSetDescription(label),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetItsString("files"),
progressbar.OptionSetElapsedTime(true),
progressbar.OptionThrottle(barThrottle),
// Render at zero immediately: a phase must be visible the
// moment it starts, even before its first item completes.
progressbar.OptionSetRenderBlankState(true),
}
if total >= 0 {
opts = append(opts,
progressbar.OptionSetPredictTime(true),
progressbar.OptionShowElapsedTimeOnFinish(),
)
} else {
opts = append(opts,
progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(walkSpinnerType),
)
}
now := p.now()
p.mu.Lock()
defer p.mu.Unlock()
p.total = total
p.rateStart, p.lastItem = now, now
p.drawn = []drawnCount{{at: now, count: p.count}}
return progressbar.NewOptions64(total, opts...)
}
// increment records one completed item, which the display shows at its
// next draw.
// increment records one completed item and refreshes the display.
func (p *progress) increment() {
if p == nil {
return
}
now := p.now()
p.mu.Lock()
defer p.mu.Unlock()
p.count++
p.lastItem = now
if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
}
}
// warnf prints a one-line warning to stderr without corrupting the
// display. The whole message is escaped like a report's path columns,
// so a path holding a newline cannot split the warning.
// warnf prints a one-line warning to stderr without corrupting the bar.
// The whole message is escaped like a report's path columns, so a path
// holding a newline cannot split the warning.
func (p *progress) warnf(format string, args ...any) {
if p == nil {
return
@@ -324,210 +125,51 @@ func (p *progress) warnf(format string, args ...any) {
msg := escapePath(fmt.Sprintf(format, args...))
p.mu.Lock()
defer p.mu.Unlock()
if p.bar != nil && p.total < 0 {
// The library also redraws a spinner from its own goroutine, so
// a direct write could land inside a redraw. The bar prints the
// warning itself, just before its next redraw.
_, _ = progressbar.Bprintln(p.bar, msg)
return
}
if p.bar != nil {
// The warning takes the display's line; the next draw shows
// the display again, on the line below.
_ = p.bar.Clear()
}
fmt.Fprintln(os.Stderr, msg)
}
// progressWriter writes a phase's output to out, which may be the file
// the phase's display is drawn on, as stdout is in a plain "sfdupes
// trees" on a terminal. It holds what is written to it and writes that
// to out in pieces of about ioBufSize bytes. When out is the file
// stderr is, each piece is written under the display's lock with the
// display cleared first, as warnf does for a warning: no draw lands in
// the middle of a piece, and the next draw shows the display again, on
// the line below. Every write to it must end with a newline, as a row
// written with one fmt.Fprintf does, so that each piece ends a line.
// Flush writes what it holds once the last row is written.
type progressWriter struct {
// p is the display to clear, nil unless out is the file stderr is.
p *progress
out io.Writer
buf []byte
}
// writer returns a progressWriter writing to out around p's display.
// It clears the display and takes its lock only when out is the file
// stderr is: the same device and inode, as os.SameFile compares them,
// which is one terminal, or one pipe or file that both are sent to.
// Otherwise it writes to out without the lock, so the display keeps
// being drawn while a write to out waits for a slow reader.
func (p *progress) writer(out io.Writer) *progressWriter {
w := &progressWriter{out: out}
if f, ok := out.(*os.File); ok {
outInfo, outErr := f.Stat()
errInfo, errErr := os.Stderr.Stat()
if outErr == nil && errErr == nil && os.SameFile(outInfo, errInfo) {
w.p = p
}
}
return w
}
// Write holds b, and writes what it holds to out once that comes to
// ioBufSize bytes.
func (w *progressWriter) Write(b []byte) (int, error) {
w.buf = append(w.buf, b...)
if len(w.buf) < ioBufSize {
return len(b), nil
}
return len(b), w.Flush()
}
// Flush writes what w holds to out. When out is the file stderr is, the
// display is cleared first and not drawn again until it is written.
func (w *progressWriter) Flush() error {
if w.p != nil {
w.p.mu.Lock()
defer w.p.mu.Unlock()
if w.p.bar != nil {
_ = w.p.bar.Clear()
}
}
_, err := w.out.Write(w.buf)
w.buf = w.buf[:0]
return err
}
// finish ends the phase's display, drawn once more as the phase ends.
// A phase stopped short of its total, as an interrupted one is, shows
// the count it stopped at rather than a full bar.
// finish terminates the pass's display. A bar whose pass stopped short
// of its total, as an interrupted one does, is left as last drawn; the
// library's Finish would fill it up.
func (p *progress) finish() {
if p == nil {
return
}
close(p.stop)
<-p.done
if p.bar != nil {
if p.total >= 0 && p.count < p.total {
_ = p.bar.Exit()
} else {
_ = p.bar.Finish()
}
p.draw()
if p.width != nil {
// The display stays on the TTY as drawn; what follows starts
// on a line of its own.
fmt.Fprintln(os.Stderr)
return
}
fmt.Fprintln(os.Stderr, p.plainLine())
}
// line formats the phase's progress as of now in at most room columns,
// as
//
// [3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
//
// where "files" is the phase's unit. A phase with no total shows the
// items done instead, or what it waits for. The rate is that since the
// first draw in drawn, and the time remaining comes from the rate since
// the count started. Once no item has been completed for stallTime, the
// rate, which would read 0, gives way to the word "stalled" and where
// the phase waits, as "stalled disk/db".
//
// When bar is true, as on a terminal, a phase with an exact total ends
// the line with its percent and bar, in place of the percent after its
// count. A total of 0 leaves the bar nothing to fill.
//
// The phase and its place, the progress, the elapsed time and the time
// remaining are kept whole. A line wider than room first leaves out the
// rate, the words of a stalled phase, or what the phase waits for. Next
// it leaves out the bar. Only if it is still wider is it cut at room.
// Every part but the bar is ASCII, so its length is its width in
// columns. p.mu must be held.
func (p *progress) line(now time.Time, room int, bar bool) string {
elapsed := now.Sub(p.start).Round(time.Second)
// head and tail hold the parts kept whole; extra, between them, is
// the part left out.
var head, extra, tail string
if p.total < 0 && p.waitingFor != "" {
head, extra = p.label+": ", p.waitingFor+", "
tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
} else {
since := p.drawn[0]
rate := perSecond(p.count-since.count, now.Sub(since.at))
average := perSecond(p.count, now.Sub(p.rateStart))
head = p.label + ": " + p.itemsDone(true) + ", "
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
if now.Sub(p.lastItem) >= stallTime {
extra = "stalled " + p.stalledOn + ", "
}
}
if bar && p.total > 0 && !p.estimate {
// Beside the bar, which shows the percent, the count shows none.
// The line is padded to the room the bar leaves it, so the bar
// stays at room's right edge however long the line is.
beside := room - barColumns
barHead := p.label + ": " + p.itemsDone(false) + ", "
switch {
case len(barHead+extra+tail) <= beside:
return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
p.percentBar())
case len(barHead+tail) <= beside:
return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
}
// The bar gives way. The rate, or the words of a stalled phase,
// gave way before it and stay out.
extra = ""
}
full, kept := head+extra+tail, head+tail
switch {
case len(full) <= room:
return full
case len(kept) <= room:
return kept
}
return kept[:max(room, 0)]
}
// perSecond returns count items over d as items a second, or 0 when no
// time has passed.
func perSecond(count int64, d time.Duration) float64 {
if d <= 0 {
return 0
}
return float64(count) / d.Seconds()
}
// itemsDone formats the items done, as "[1234/5000] 25%", with "~"
// before a total that is an estimate and without the percent when
// percent is false, or as "1234 files", in the phase's unit, when the
// phase has no total. p.mu must be held.
func (p *progress) itemsDone(percent bool) string {
// plainLine formats a non-TTY progress line.
func (p *progress) plainLine() string {
elapsed := time.Since(p.start)
if p.total < 0 {
return fmt.Sprintf("%d %s", p.count, p.unit)
}
approx := ""
if p.estimate {
approx = "~"
}
done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
if !percent {
return done
return fmt.Sprintf("%s: %d files, elapsed %s",
p.label, p.count, elapsed.Round(time.Second))
}
pct := float64(percentScale)
@@ -535,24 +177,19 @@ func (p *progress) itemsDone(percent bool) string {
pct = percentScale * float64(p.count) / float64(p.total)
}
return fmt.Sprintf("%s %.0f%%", done, pct)
}
// remain returns the time the phase has left at rate items a second, or
// "unknown": with no total, before the first item, and once the count
// passes an estimate. p.mu must be held.
func (p *progress) remain(rate float64) string {
switch {
case p.total < 0, p.estimate && p.count > p.total:
return "unknown"
case p.count >= p.total:
return "0s"
case rate <= 0:
return "unknown"
rate := 0.0
if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
}
left := time.Duration(
float64(p.total-p.count) / rate * float64(time.Second))
eta := "?"
return left.Round(time.Second).String()
if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)
}
progress_test.go
+69 -1020
View File
File diff suppressed because it is too large Load Diff
report.go
+50 -196
View File
@@ -1,13 +1,11 @@
package main
import (
"bufio"
"context"
"crypto/sha256"
"database/sql"
"fmt"
"io"
"os"
"slices"
"strings"
"time"
)
@@ -25,224 +23,80 @@ const minGroupSize = 2
type scanRec struct {
size int64
mtime time.Time
head [sha256.Size]byte
tail [sha256.Size]byte
content [sha256.Size]byte
head string
tail string
content string
path string
}
// reportPhase names one phase of report with its place among all of
// them, as "[2/2] read", for the phase's progress display. The list
// here is the one place the phases are counted.
func reportPhase(name string) string {
return phaseLabel([]string{openPhase, "read"}, name)
}
// runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. It never touches the scanned
// filesystem; its only I/O is the database, stdout, and stderr. It
// names the database on stderr before it opens it. Any database
// problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (count the records,
// then read them in signature order, writing each size's duplicate
// groups to stdout once that size is read; see readReport). Each
// display ends before the next phase starts, and before an error is
// returned to be printed.
// duplicates report as TSV on stdout. SQLite groups and orders the
// records, and each row is written as it is read, so no group is held
// in memory. It never touches the scanned filesystem; its only I/O is
// the database (with SQLite's temporary sort file), stdout, and stderr.
// Any database problem, including a missing database, is fatal.
func runReport(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
dbPath := databasePath()
fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: reportPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil {
return err
}
defer func() { _ = db.Close() }()
// The read phase writes the report to stdout while it reads the
// records, so it may wait on either.
prog = newProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db/out",
})
report, err := readReport(ctx, db, dbPath, stdout, prog)
out := bufio.NewWriterSize(stdout, ioBufSize)
prog.finish()
_, err = fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
var (
groups, dupeFiles int
reclaimable int64
writeErr error
)
records, err := loadDupeRows(ctx, db,
func(first, path string, size int64) error {
// A group's first path is its first row; every other
// path is a dupe.
if path == first {
groups++
return nil
}
_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
escapePath(first), escapePath(path), size)
dupeFiles++
reclaimable += size
return writeErr
})
if writeErr != nil {
return fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil {
return err
return fmt.Errorf("database %s: %w", dbPath, err)
}
err = out.Flush()
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
fmt.Fprintf(os.Stderr,
"report: %d records read, %d duplicate groups, %d dupe files, "+
"%s reclaimable\n",
report.records, report.dupeGroups, report.dupeFiles,
humanBytes(report.reclaimable))
records, groups, dupeFiles, humanBytes(reclaimable))
return nil
}
// readReport gives prog the number of records in db, the database at
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
stdout io.Writer, prog *progress,
) (*reportWriter, error) {
out := prog.writer(stdout)
report := &reportWriter{out: out}
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
var writeErr error
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
prog.increment()
writeErr = report.add(r)
return writeErr
})
if writeErr != nil {
return nil, fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil {
return nil, fmt.Errorf("database %s: %w", dbPath, err)
}
err = report.finish()
if err == nil {
err = out.Flush()
}
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
return report, nil
}
// reportWriter writes the report's rows to out from the records, added
// in signature order, largest size first, as loadSignatureRows reads
// them. The records sharing a signature come together, and so do the
// records of one size, so it holds only the duplicate groups of the
// size being read, and writes them, ordered by first path, once a
// record of a smaller size comes. It counts what the summary line
// reports.
type reportWriter struct {
out io.Writer
sig scanRec // the signature being read; its path is not used
paths []string // the paths read so far with sig
groups [][]string // the duplicate groups of sig.size, each sorted
records, dupeGroups, dupeFiles int
reclaimable int64
}
// add takes the next record. One with the signature being read joins
// its paths; any other ends them, and one of a smaller size first
// writes the groups of the size before.
func (w *reportWriter) add(r scanRec) error {
w.records++
if r.size == w.sig.size && r.head == w.sig.head &&
r.tail == w.sig.tail && r.content == w.sig.content {
w.paths = append(w.paths, r.path)
return nil
}
w.endGroup()
if r.size != w.sig.size {
err := w.writeGroups()
if err != nil {
return err
}
}
w.sig = r
w.paths = append(w.paths, r.path)
return nil
}
// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
func (w *reportWriter) endGroup() {
if len(w.paths) < minGroupSize {
// A path with no duplicate is dropped, and its slice reused.
w.paths = w.paths[:0]
return
}
slices.Sort(w.paths)
w.groups = append(w.groups, w.paths)
w.paths = nil
}
// writeGroups writes the duplicate groups held, all of size w.sig.size,
// ordered by first path: for each group, a row for every path but the
// first. No two groups share a first path, since no two records share
// a path.
func (w *reportWriter) writeGroups() error {
slices.SortFunc(w.groups, func(a, b []string) int {
return strings.Compare(a[0], b[0])
})
for _, g := range w.groups {
for _, path := range g[1:] {
_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
escapePath(g[0]), escapePath(path), w.sig.size)
if err != nil {
return err
}
w.dupeFiles++
w.reclaimable += w.sig.size
}
w.dupeGroups++
}
w.groups = nil
return nil
}
// finish writes the groups of the last size read.
func (w *reportWriter) finish() error {
w.endGroup()
return w.writeGroups()
}
// escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
report_test.go
+60 -244
View File
@@ -2,7 +2,6 @@ package main
import (
"bytes"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
@@ -10,7 +9,6 @@ import (
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"testing"
"time"
@@ -19,32 +17,18 @@ import (
// awkwardDir is a directory name holding every byte the reports escape.
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
// testHash returns a stand-in hash for a test record: the bytes of s,
// then zeros. Stand-ins sort as their strings do, so a test can choose
// the order in which report reads its records.
func testHash(s string) [sha256.Size]byte {
var h [sha256.Size]byte
copy(h[:], s)
return h
}
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
// sorts after it, so awkwardDir coming first shows that sorting uses
// the raw path.
func awkwardPairRecs() []scanRec {
h, tl, c := testHash("h"), testHash("t"), testHash("c")
return []scanRec{
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
}
}
// seedDatabase writes recs into a fresh database, closes it as scan
// does, out of WAL mode, and returns its path.
// seedDatabase writes recs into a fresh database and returns its path.
func seedDatabase(t *testing.T, recs []scanRec) string {
t.Helper()
@@ -60,7 +44,10 @@ func seedDatabase(t *testing.T, recs []scanRec) string {
t.Fatal(err)
}
closeScanDatabase(t.Context(), db, path)
err = db.Close()
if err != nil {
t.Fatal(err)
}
return path
}
@@ -73,39 +60,25 @@ type dupeGroup struct {
}
// dupeGroups returns the duplicate groups report reads from db, in
// report order, from the rows it writes: a group's rows come together,
// each naming the group's first path.
// report order.
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
t.Helper()
var out bytes.Buffer
_, err := readReport(t.Context(), db, "", &out, nil)
if err != nil {
t.Fatal(err)
}
var groups []dupeGroup
rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
for _, row := range rows[1:] { // after the header
fields := strings.Split(row, "\t")
if len(fields) != 3 {
t.Fatalf("row %q, want first, dupe and size", row)
}
_, err := loadDupeRows(t.Context(), db,
func(first, path string, size int64) error {
if path == first {
groups = append(groups, dupeGroup{size: size})
}
size, err := strconv.ParseInt(fields[2], 10, 64)
if err != nil {
t.Fatal(err)
}
g := &groups[len(groups)-1]
g.paths = append(g.paths, path)
if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] {
groups = append(groups,
dupeGroup{size: size, paths: []string{fields[0]}})
}
g := &groups[len(groups)-1]
g.paths = append(g.paths, fields[1])
return nil
})
if err != nil {
t.Fatal(err)
}
return groups
@@ -145,160 +118,38 @@ func TestRunReportEscapesPaths(t *testing.T) {
}
func TestReportStdoutFailsWhileReading(t *testing.T) {
t.Parallel()
// Every two records are a duplicate pair of a size of their own, and
// a pair's row is written once a smaller size is read. Each row
// holds two paths longer than dir, so the rows come to more than
// twice the stdout buffer, and stdout fails while records are still
// being read, not once every record is.
// Each row holds two paths longer than dir, so the report is more
// than twice the stdout buffer and stdout fails while rows are
// still being read, not at the final flush.
dir := "/" + strings.Repeat("d", 4096)
recs := make([]scanRec, 2*ioBufSize/len(dir))
recs := make([]scanRec, ioBufSize/len(dir))
for i := range recs {
recs[i] = scanRec{
size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
size: 1, head: "h", tail: "t", content: "c",
path: fmt.Sprintf("%s/%d", dir, i),
}
}
db := openTestDB(t)
t.Setenv(databaseEnv, seedDatabase(t, recs))
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatal(err)
}
// Counts the records read, and draws nothing.
prog := &progress{now: time.Now}
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
err := runReport(t.Context(), failingWriter{})
if !errors.Is(err, errWriteFailed) ||
!strings.HasPrefix(err.Error(), "write stdout: ") {
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
}
if prog.count >= int64(len(recs)) {
t.Errorf("stdout failed after all %d records were read, want it "+
"to stop the read", prog.count)
}
}
// reportOpenLine is the plain line of report's open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"remain unknown"
// smokeReport is report's stdout for the smoke-test records.
const smokeReport = "first\tdupe\tsize\n" +
"/d/t1/f1\t/d/t2/f1\t3000\n" +
"/d/t1/f1\t/d/t3/f1\t3000\n" +
"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"
// TestRunReportProgressNamesEachPhase runs report on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the two as it starts and another as
// it ends; read counts the 6 records before it reads them. The summary
// comes last, and stdout holds the report alone.
func TestRunReportProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdReport}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(report) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
if got := stdout.String(); got != smokeReport {
t.Errorf("stdout = %q, want %q", got, smokeReport)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"report: database " + path + " (from " + databaseEnv + ")",
reportOpenLine,
reportOpenLine,
"[2/2] read: counting the records, elapsed 0s, remain unknown",
"[2/2] read: [6/6] 100%, ",
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
"6.1 KiB reclaimable",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestReadReportOnTerminal writes the smoke-test report to the terminal
// its read phase's display is drawn on, as a plain "sfdupes report"
// does: the terminal must show the header and every row whole, each on
// a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestReadReportOnTerminal(t *testing.T) {
stderr := captureStderr(t)
db := openTestDB(t)
err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords,
}, terminalWidth, time.Hour, time.Now)
_, err = readReport(t.Context(), db, "", os.Stderr, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
}
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
// README §Constraints: identical database contents give identical
// output, whatever order the records were inserted in.
a, b := testHash("a"), testHash("b")
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
recs = append(recs,
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
scanRec{size: 50, path: "/x/unhashed"},
)
reversed := slices.Clone(recs)
@@ -401,19 +252,16 @@ func TestWarnfEscapes(t *testing.T) {
func TestDupeGroups(t *testing.T) {
t.Parallel()
h, tl, c := testHash("h"), testHash("t"), testHash("c")
bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
u := testHash("u")
recs := []scanRec{
{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
// Same size as the /z group but a different head hash.
{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
// A singleton signature must not form a group.
{size: 7, head: u, tail: u, content: u, path: "/lonely"},
{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
}
groups := dupeGroupsOf(t, recs)
@@ -437,13 +285,16 @@ func TestDupeGroups(t *testing.T) {
func TestDupeGroupsContentSeparates(t *testing.T) {
t.Parallel()
// Same size, head, and tail: /b differs from the others only in its
// content hash, so it joins no group, while /a and /c group.
h, tl := testHash("h"), testHash("t")
// Same size, head, and tail, but different content hashes: the final
// rung keeps them apart, so no group forms. Matching content groups.
// Records without a content hash never group, not even with each
// other.
recs := []scanRec{
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
{size: 100, head: "h", tail: "t", path: "/d"},
{size: 100, head: "h", tail: "t", path: "/e"},
}
groups := dupeGroupsOf(t, recs)
@@ -462,15 +313,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
// mtime is informational only; records differing only in mtime
// still group together.
h, tl, c := testHash("h"), testHash("t"), testHash("c")
recs := []scanRec{
{
size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
content: c, path: "/m/1",
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
content: "c", path: "/m/1",
},
{
size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
content: c, path: "/m/2",
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
content: "c", path: "/m/2",
},
}
@@ -485,12 +335,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
// The hashes sort opposite to the first paths, so ordering the
// groups by hash instead of by first path fails this test.
a, b := testHash("a"), testHash("b")
recs := []scanRec{
{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
}
groups := dupeGroupsOf(t, recs)
@@ -505,47 +354,14 @@ func TestDupeGroupsTieBreak(t *testing.T) {
}
}
func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
t.Parallel()
// report reads the groups of size 50 in signature order, here /a,
// /z, /k, which is neither the order of their first paths nor its
// reverse, and must write them ordered by first path. The group of
// size 7 comes after them all, though its first path sorts first.
a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
recs := []scanRec{
{size: 50, head: a, tail: a, content: a, path: "/k/2"},
{size: 50, head: a, tail: a, content: a, path: "/k/1"},
{size: 50, head: b, tail: b, content: b, path: "/z/1"},
{size: 50, head: b, tail: b, content: b, path: "/z/2"},
{size: 50, head: c, tail: c, content: c, path: "/a/2"},
{size: 50, head: c, tail: c, content: c, path: "/a/1"},
{size: 7, head: d, tail: d, content: d, path: "/0/2"},
{size: 7, head: d, tail: d, content: d, path: "/0/1"},
}
groups := dupeGroupsOf(t, recs)
got := make([]string, 0, len(groups))
for _, g := range groups {
got = append(got, g.paths[0])
}
want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
if !slices.Equal(got, want) {
t.Errorf("groups' first paths = %q, want %q", got, want)
}
}
func TestDupeGroupsDeterministic(t *testing.T) {
t.Parallel()
a, b := testHash("a"), testHash("b")
recs := []scanRec{
{size: 1, head: a, tail: a, content: a, path: "/p/1"},
{size: 1, head: a, tail: a, content: a, path: "/p/2"},
{size: 2, head: b, tail: b, content: b, path: "/q/1"},
{size: 2, head: b, tail: b, content: b, path: "/q/2"},
{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
}
forward := dupeGroupsOf(t, recs)
scan.go
+492 -547
View File
File diff suppressed because it is too large Load Diff
scan_test.go
+696 -880
View File
File diff suppressed because it is too large Load Diff
script/bootstrap
+11 -34
View File
@@ -2,12 +2,10 @@
# script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already
# installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes nothing is present. The
# installed node is used only when its major version is the pinned one;
# otherwise (node missing, or another major) the pinned version is
# installed via nvm (installing nvm itself first, from a hash-verified
# release archive, never curl | sh), and yarn and the packages are
# installed under it.
# or apk (detected in that order); assumes nothing is present. Node is
# used directly if installed; otherwise it is installed at a pinned
# version via nvm (installing nvm itself first, from a hash-verified
# release archive, never curl | sh).
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -102,52 +100,31 @@ ensure_nvm() {
rm -rf "$tmp"
}
# True when the node on PATH has the pinned major version. Only the
# major is compared: the Dockerfile stages start from a node 22 alpine
# image whose exact version is not the pin, and nvm cannot install a
# prebuilt node on alpine. Another major is not used: the pinned yarn 1
# prints a deprecation warning under node 24. script/fmt and
# script/fmt-check carry this function unchanged, to run the yarn
# installed here.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
ensure_node() {
if node_is_pinned_major; then
echo "bootstrap: using node $(node --version)"
return 0
fi
if ! missing node; then return 0; fi
ensure_nvm
nvm_sh "nvm install $NODE_VERSION"
nvm_sh "nvm use $NODE_VERSION >/dev/null && \
echo \"bootstrap: using node \$(node --version)\""
}
ensure_yarn() {
if ! node_is_pinned_major; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
return 0
fi
if ! missing yarn; then return 0; fi
if ! missing corepack; then
corepack enable
corepack prepare "yarn@$YARN_VERSION" --activate
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
else
npm install -g "yarn@$YARN_VERSION"
fi
}
install_js_deps() {
if node_is_pinned_major; then
yarn install --frozen-lockfile
else
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
yarn install --frozen-lockfile"
else
yarn install --frozen-lockfile
fi
}
script/cibuild
+4 -7
View File
@@ -2,12 +2,9 @@
# script/cibuild: run the CI build. It bootstraps first: a CI runner
# checks out and runs this and nothing else, and script/fmt-check runs
# the formatter on the host, which a pristine checkout cannot do.
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
# line, since a cached gate phase is a check that did not run; unlike
# --no-cache, it keeps the layers above that line and the cache mounts.
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
# before every step that lints, tests or compiles; a stage without that
# line is served from cache on an unchanged tree, with no warning.
# --no-cache for the same reason as script/docker: the gate phases the
# final stage depends on are RUN steps, and a cached one is a check that
# did not run.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -25,7 +22,7 @@ main() {
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
}
script/docker
+3 -7
View File
@@ -1,12 +1,8 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname.
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
# line, since a cached gate phase is a check that did not run; unlike
# --no-cache, it keeps the layers above that line and the cache mounts.
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
# before every step that lints, tests or compiles; a stage without that
# line is served from cache on an unchanged tree, with no warning.
# --no-cache because the gate phases the final stage depends on are RUN
# steps, and a cached one is a check that did not run.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -22,7 +18,7 @@ main() {
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
}
script/fmt
+6 -16
View File
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same
# function as in script/bootstrap, which uses it to decide where it
# installs yarn.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
# script/bootstrap installs node and yarn under nvm and leaves neither
# on the PATH of the shell that called it, so resolve the pinned
# toolchain here the way bootstrap's own install step does. nvm is a
# bash script, hence the subshell.
run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
if command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt: no yarn; run script/bootstrap first" >&2
exit 1
fi
script/fmt-check
+6 -16
View File
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same
# function as in script/bootstrap, which uses it to decide where it
# installs yarn.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
# script/bootstrap installs node and yarn under nvm and leaves neither
# on the PATH of the shell that called it, so resolve the pinned
# toolchain here the way bootstrap's own install step does. nvm is a
# bash script, hence the subshell.
run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
if command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt-check: no yarn; run script/bootstrap first" >&2
exit 1
fi
script/lint
+4 -9
View File
@@ -5,14 +5,9 @@
# make its answer untrustworthy.
#
# The phase is not the last stage in the file, so it is built only when
# --target names it. A new CACHEBUST value reruns every RUN step after
# the ARG CACHEBUST line, since a cached lint layer is a lint that did
# not run; unlike --no-cache, it keeps the layers above that line and
# the cache mounts. This relies on every stage of the Dockerfile
# declaring ARG CACHEBUST before every step that lints, tests or
# compiles; a stage without that line is served from cache on an
# unchanged tree, with no warning.
# --output type=cacheonly writes no image, since nothing uses one.
# --target names it. --no-cache because a cached lint layer is a lint
# that did not run. --output type=cacheonly writes no image, since
# nothing uses one.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -20,7 +15,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
docker build --no-cache \
--target lint \
--output type=cacheonly .
}
script/test
+3 -8
View File
@@ -2,13 +2,8 @@
# script/test: run the test suite. Testing is a phase of the Dockerfile
# and this builds that phase alone, on the same terms as script/lint:
# --target because a phase that is not the last stage is built only when
# named, and a new CACHEBUST value because a cached test layer is a test
# that did not run; unlike --no-cache, it keeps the layers above the ARG
# CACHEBUST line and the cache mounts. This relies on every stage of the
# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
# or compiles; a stage without that line is served from cache on an
# unchanged tree, with no warning. --output type=cacheonly writes no
# image, since nothing uses one.
# named, and --no-cache because a cached test layer is a test that did
# not run. --output type=cacheonly writes no image, since nothing uses one.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -16,7 +11,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
docker build --no-cache \
--target test \
--output type=cacheonly .
}
trees.go
+40 -121
View File
@@ -1,9 +1,9 @@
package main
import (
"bufio"
"context"
"crypto/sha256"
"database/sql"
"fmt"
"io"
"os"
@@ -24,132 +24,44 @@ type treeNode struct {
totalSize int64
}
// treesPhase names one phase of trees with its place among all of them,
// as "[2/4] read", for the phase's progress display. The list here is
// the one place the phases are counted.
func treesPhase(name string) string {
return phaseLabel([]string{openPhase, "read", "group", "write"}, name)
}
// runTrees implements the trees subcommand: it reads every record from
// the database in path order, reconstructs the directory hierarchy from
// the record paths, computes a Merkle-style digest per directory, and
// prints maximal duplicate-tree groups as TSV on stdout. It never
// touches the scanned filesystem; its only I/O is the database, stdout,
// and stderr. It names the database on stderr before it opens it. Any
// database problem, including a missing database, is fatal.
//
// The work runs in four phases, each under a progress display of its
// own on stderr: open (open the database), read (count the records,
// then read them into the hierarchy), group (group the directories by
// digest) and write (write the groups to stdout). Each display ends
// before the next phase starts, and before an error is returned to be
// printed.
// and stderr. Any database problem, including a missing database, is
// fatal.
func runTrees(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
dbPath := databasePath()
fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: treesPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil {
return err
}
defer func() { _ = db.Close() }()
prog = newProgress(progressParams{
label: treesPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords, stalledOn: "db",
records := 0
tree := newTreeBuilder()
err = loadFileRows(ctx, db, func(r scanRec) {
records++
tree.add(r)
})
records, super, allDirs, err := readTree(ctx, db, prog)
prog.finish()
if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err)
}
// Grouping works in memory, so it waits on nothing but the
// processor.
prog = newProgress(progressParams{
label: treesPhase("group"), unit: "directories",
total: int64(len(allDirs)), stalledOn: "cpu",
})
dupes := collectTreeGroups(allDirs, super, prog)
prog.finish()
prog = newProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
stalledOn: "out",
})
dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
prog.finish()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
}
// readTree gives prog the number of records in db as its total, then
// reads every record in path order into the directory hierarchy,
// counting each on prog. It returns the number of records read, the
// super-root and every directory.
func readTree(ctx context.Context, db *sql.DB,
prog *progress,
) (int, *treeNode, []*treeNode, error) {
records := 0
tree := newTreeBuilder()
err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
records++
prog.increment()
tree.add(r)
})
if err != nil {
return 0, nil, nil, err
}
super, allDirs := tree.finish()
return records, super, allDirs, nil
}
dupes := collectTreeGroups(allDirs, super)
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
// a row for every member of each group but its first, counting each
// group on prog once its rows are written. stdout may be the terminal
// prog is drawn on, so the lines go through prog's writer. It returns
// the number of rows (dupe trees) and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress,
) (int, int64, error) {
out := prog.writer(stdout)
out := bufio.NewWriterSize(stdout, ioBufSize)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
_, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err)
return fmt.Errorf("write stdout: %w", err)
}
dupeTrees := 0
@@ -163,22 +75,25 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize)
if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err)
return fmt.Errorf("write stdout: %w", err)
}
dupeTrees++
reclaimable += first.totalSize
}
prog.increment()
}
err = out.Flush()
if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err)
return fmt.Errorf("write stdout: %w", err)
}
return dupeTrees, reclaimable, nil
fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
}
// treeBuilder reconstructs the directory hierarchy from records added
@@ -286,23 +201,28 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
// name and its signature (size, head, tail, content); mtime is
// excluded.
func fileEntry(name string, r scanRec) string {
content := r.content
// A record without a content hash has unknown content (README
// "Database"): give it a signature no other file can share, so
// trees containing it never compare equal. Real hashes are hex, so
// the NUL-prefixed form cannot collide.
if content == "" {
content = "unhashed\x00" + r.path
}
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
"\x00" + string(r.content[:])
"\x00" + r.head + "\x00" + r.tail + "\x00" + content
}
// collectTreeGroups groups directories by digest, counting each one on
// prog as it is grouped, and returns every maximal group with two or
// more members, each group's members sorted by path, groups ordered by
// tree size descending then by first path ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
prog *progress,
) [][]*treeNode {
// collectTreeGroups groups directories by digest and returns every
// maximal group with two or more members, each group's members sorted
// by path, groups ordered by tree size descending then by first path
// ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d)
prog.increment()
}
var dupes [][]*treeNode
@@ -338,8 +258,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
// digest is the SHA-256 of its child entries — files serialized with
// name and signature, subdirectories with name and recursive digest —
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
// and every hash and digest is 32 bytes, so NUL delimiters are
// unambiguous.
// so NUL delimiters are unambiguous.
func (n *treeNode) computeDigest() {
slices.Sort(n.entries)
trees_test.go
+30 -126
View File
@@ -3,25 +3,23 @@ package main
import (
"bytes"
"database/sql"
"os"
"slices"
"strings"
"testing"
"time"
)
// treesOpenLine is the plain line of trees' open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
"remain unknown"
// Signature hashes shared by the smoke-test records.
const (
f1Head = "f1h"
f1Tail = "f1t"
f1Content = "f1c"
f2Head = "f2h"
f2Tail = "f2t"
f2Content = "f2c"
)
// smokeTreeRecs mirrors the tree of the test under README "Definition
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
// by one filename.
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
// /d/t2 are identical, /d/t3 differs from them only by one filename.
func smokeTreeRecs() []scanRec {
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
return []scanRec{
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
@@ -40,7 +38,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
tree := newTreeBuilder()
err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
err := loadFileRows(t.Context(), db, tree.add)
if err != nil {
t.Fatal(err)
}
@@ -147,14 +145,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
for _, top := range []string{"/a", "/c"} {
for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
content := testHash("c")
content := "c"
if p == "/b-x/f" {
content = testHash("other")
content = "other"
}
recs = append(recs, scanRec{
size: 1, head: testHash("h"), tail: testHash("t"),
content: content, path: top + p,
size: 1, head: "h", tail: "t", content: content, path: top + p,
})
}
}
@@ -173,7 +170,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
t.Fatalf("directory paths = %q, want %q", got, want)
}
groups := collectTreeGroups(dirs, super, nil)
groups := collectTreeGroups(dirs, super)
gotGroups := groupPaths(groups)
wantGroups := [][]string{{"/a", "/c"}}
@@ -206,98 +203,6 @@ func TestRunTreesEscapesPaths(t *testing.T) {
}
}
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the four as it starts and another as
// it ends. read counts the 6 records before it reads them, group counts
// the 8 directories, and write the 1 group. The summary comes last, and
// stdout holds the report alone.
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdTrees}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(trees) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
if got := stdout.String(); got != report {
t.Errorf("stdout = %q, want %q", got, report)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"trees: database " + path + " (from " + databaseEnv + ")",
treesOpenLine,
treesOpenLine,
"[2/4] read: counting the records, elapsed 0s, remain unknown",
"[2/4] read: [6/6] 100%, ",
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
"[3/4] group: [8/8] 100%, ",
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
"[4/4] write: [1/1] 100%, ",
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
// terminal its write phase's display is drawn on, as a plain "sfdupes
// trees" does: the terminal must show the header and the row whole,
// each on a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
stderr := captureStderr(t)
super, dirs := treeOf(t, smokeTreeRecs())
dupes := collectTreeGroups(dirs, super, nil)
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
}, terminalWidth, time.Hour, time.Now)
_, _, err := writeTreeGroups(os.Stderr, dupes, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
}
func TestTreeDigests(t *testing.T) {
t.Parallel()
@@ -327,10 +232,11 @@ func TestTreeDigests(t *testing.T) {
func TestTreeDigestContentSensitivity(t *testing.T) {
t.Parallel()
same, c := testHash("same"), testHash("c")
const sharedTail = "same"
recs := []scanRec{
{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
}
_, dirs := treeOf(t, recs)
@@ -348,7 +254,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
super, dirs := treeOf(t, smokeTreeRecs())
groups := collectTreeGroups(dirs, super, nil)
groups := collectTreeGroups(dirs, super)
want := [][]string{{"/d/t1", "/d/t2"}}
if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -372,14 +278,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
super, dirs := treeOf(t, recs)
forward := groupPaths(collectTreeGroups(dirs, super, nil))
forward := groupPaths(collectTreeGroups(dirs, super))
reversed := slices.Clone(recs)
slices.Reverse(reversed)
superR, dirsR := treeOf(t, reversed)
backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
backward := groupPaths(collectTreeGroups(dirsR, superR))
if !slices.EqualFunc(forward, backward, slices.Equal) {
t.Fatalf("output depends on record order: %v vs %v",
forward, backward)
@@ -391,15 +297,14 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
// Identical sibling dirs share a parent, so their group cannot be
// implied by a parent group and must be reported.
h, tl, c := testHash("h"), testHash("t"), testHash("c")
recs := []scanRec{
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
}
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil))
got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
@@ -413,16 +318,15 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
// /p/a and /q/b contain identical x subtrees, but /p/a has an
// extra file, so the parents' digests differ and the x group must
// be reported.
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
recs := []scanRec{
{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
}
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil))
got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/a/x", "/q/b/x"}}
if !slices.EqualFunc(got, want, slices.Equal) {