1 Commits
Author SHA1 Message Date
sneak c8c7882cc7 Walk an operand nested on another filesystem with -x (closes #106)
check / check (push) Canceled after 0s
With -x, an operand under another operand is kept and walked on its
own when it is a directory on a different filesystem than the deepest
kept operand above it, or a regular file in such a directory, since
that operand's walk does not reach it. Below a kept operand, the rule
keeping records under a directory -x did not enter covers only
directories its own walk did not enter, so records of files gone from
it are deleted.

The new tests stub the devices and share their scan steps with the
test for #99, whose comment no
longer claims it runs every phase. The startWalk comment is rewrapped.

Model: opus-5-5
2026-10-14 19:07:23 +00:00
40 changed files with 1272 additions and 4227 deletions
+5
View File
@@ -0,0 +1,5 @@
{
"worktree": {
"bgIsolation": "none"
}
}
+6 -73
View File
@@ -1,79 +1,12 @@
# .dockerignore does NOT use .gitignore semantics. Docker matches with
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
# `/` and an unprefixed pattern is anchored at the context root. Every
# depth-independent pattern therefore needs `**/`, or `config/.env` and
# `certs/server.key` still ship while this file reads as solved. Only
# genuinely root-anchored entries go unprefixed. Never transplant these
# into .gitignore, where `**/` is wrong.
#
# Matching is case-sensitive, so secrets use character ranges rather
# than an ALL-CAPS twin, which would still miss `Server.Key`.
#
# Extend with this repo's own host-built artifacts, written anchored:
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
# deletes the package directory from the context.
# .git is sent without its config. Without a VERSION build argument the # .git is sent without its config. Without a VERSION build argument the
# stage that compiles runs `git describe --tags --always` on .git, which # stage that compiles runs `git describe --tags --always` on .git, which
# does not need .git/config; that file can hold a credential, such as a # does not need .git/config; that file can hold a credential, such as a
# password in a remote URL or the token the CI checkout step stores there. # password in a remote URL or the token the CI checkout step stores there.
# Each submodule keeps a config with the same exposure in its git directory .git/config
# under .git/modules/, nested again for a submodule's own submodules, or in
# its own .git directory when it keeps one.
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
# `deploy/config`, `config/lib`) loses its whole git directory, because
# `**/.git/modules/**/config` also matches that segment's directory
# under .git/modules/. Go's version stamping then fails the build;
# nothing leaks. Name such a submodule without that segment:
# `git submodule add --name`.
**/.git/config
**/.git/modules/**/config
# Agent scratch: one full checkout of the repo per in-flight agent.
# Anchored because it occurs once where agents run at the repo root.
# KNOWN GAP: a repo running agents in subdirectories still ships
# `services/api/.claude/` and must add its own anchored entry.
.claude .claude
.DS_Store
# Environment files. `*.env` covers bare `.env` and the `prod.env` sfdupes
# convention. Re-include a committed template with a negation if the *.log
# build needs one: `!docs/example.env`. *.out
**/*.[eE][nN][vV] *.test
**/.[eE][nN][vV].*
**/.[eE][nN][vV][rR][cC]
# Private keys and the bundles carrying them. Public certificates
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
**/*.[pP][eE][mM]
**/*.[kK][eE][yY]
**/*.[pP]12
**/*.[pP][fF][xX]
**/[iI][dD]_[rR][sS][aA]
**/[iI][dD]_[dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
**/[iI][dD]_[eE][dD]25519
**/[iI][dD]_[eE][dD]25519_[sS][kK]
# Dependencies: restored inside the image, never copied in.
**/node_modules
# OS metadata.
**/.DS_Store
**/Thumbs.db
# Editor state: never a build input, and it churns COPY.
**/*.swp
**/*.swo
**/*~
**/*.bak
**/.idea
**/.vscode
**/*.sublime-*
# This repository's host-built artifacts: the binary `make build` writes,
# and test binaries, coverage output and logs.
/sfdupes
/*.test
/*.out
/*.log
-3
View File
@@ -13,6 +13,3 @@ indent_style = tab
[*.go] [*.go]
indent_style = tab indent_style = tab
# This repository's own sections, such as one for another language it
# uses, go below this comment, and a re-vendor keeps them.
-9
View File
@@ -1,20 +1,11 @@
name: check name: check
on: [push] on: [push]
# Free the shared runner: a new push cancels only the same branch's older run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Free the shared runner from a hung build.
timeout-minutes: 20
steps: steps:
# actions/checkout v4.2.2, 2026-02-22 # actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
# script/cibuild needs no token, so none is left in .git/config.
with: with:
persist-credentials: false
# All history and tags, so git describe finds the version tag.
fetch-depth: 0 fetch-depth: 0
- run: script/cibuild - run: script/cibuild
+12 -37
View File
@@ -11,50 +11,25 @@ Thumbs.db
.vscode/ .vscode/
*.sublime-* *.sublime-*
# Agent scratch (worktrees of this repo, created and destroyed by
# in-flight tooling). Unanchored: .gitignore patterns already match at
# every depth, so no prefix is wanted here. This is not a .dockerignore
# entry and must not be given a `**/` prefix on the way into one.
.claude/
# Node # Node
node_modules/ node_modules/
# Secrets. Unanchored like every entry above, so each matches at every # Environment / secrets
# depth. Matching is case-sensitive on Linux, so names use character .env
# ranges rather than a lowercase form that misses `Server.Key`. .env.*
*.pem
*.key
# Environment files. `*.env` covers bare `.env` and the `prod.env` # Go build artifacts
# convention. Only the templates `example.env` and `sample.env` are
# re-included below. A repository that commits any other template adds
# its own negation at the end of this file, for example `!.env.example`.
*.[eE][nN][vV]
.[eE][nN][vV].*
.[eE][nN][vV][rR][cC]
!example.env
!sample.env
# Private keys and the bundles carrying them.
*.[pP][eE][mM]
*.[kK][eE][yY]
*.[pP]12
*.[pP][fF][xX]
[iI][dD]_[rR][sS][aA]
[iI][dD]_[dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
[iI][dD]_[eE][dD]25519
[iI][dD]_[eE][dD]25519_[sS][kK]
# This repository's own entries, such as its build outputs, go below
# this comment, and a re-vendor keeps them. Anchor a binary built at the
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
/sfdupes /sfdupes
*.log
*.out
*.test *.test
*.out
*.log
# A scan database lists every path it scanned. # Local scan data
*.sqlite *.sqlite
*.sqlite-shm *.sqlite-shm
*.sqlite-wal *.sqlite-wal
# Agent worktrees
.claude/worktrees/
-3
View File
@@ -17,7 +17,6 @@ linters:
disable: disable:
# Genuinely incompatible with project patterns # Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields - exhaustruct # Requires all struct fields
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
- godot # Requires comments to end with periods - godot # Requires comments to end with periods
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
@@ -25,8 +24,6 @@ linters:
# silenced by disabling that name, not by enabling the successor. # silenced by disabling that name, not by enabling the successor.
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- gomodguard # Deprecated, replaced by gomodguard_v2 - gomodguard # Deprecated, replaced by gomodguard_v2
# Misses findings at random in v2.14.0; back once a pinned release fixes it
- canonicalheader
settings: settings:
lll: lll:
line-length: 88 line-length: 88
+136 -59
View File
@@ -1,80 +1,159 @@
# Every stage declares ARG CACHEBUST first: script/ passes a new value # Lint stage — fast feedback on formatting and lint issues
# on every build, which reruns every RUN step after it. The steps that # golangci/golangci-lint:v2.12.2, 2026-08-07
# compile mount Go's build cache, which survives from build to build. FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
# Lint phase, built alone by script/lint. The tools are invoked directly
# rather than through `make lint`, which runs docker itself and so cannot
# run inside a build step.
# golangci/golangci-lint:v2.14.0, 2026-10-07
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
ARG CACHEBUST
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a # Cache-buster for the gate layers, and only for them: on an unchanged
# variable first so that its own exit status, as when it cannot parse a # tree Docker would serve the gates below from cache and the build would
# file, still fails the step. # exit 0 having run nothing. script/cibuild and script/docker pass a
RUN files="$(gofmt -s -l .)" && \ # fresh CHECK_EPOCH; a build without one, such as a bare
# `docker build .`, fails at the check right after the ARG.
#
# ARG is per-stage, so the markdown and build stages declare it again.
# Each gate RUN must reference the value: BuildKit hashes the expanded
# command, so a declared but unreferenced ARG invalidates nothing. Keep
# it below the dependency layers so they stay cached.
ARG CHECK_EPOCH
RUN if [ -z "${CHECK_EPOCH}" ]; then \
echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \
exit 1; \
fi
# These gates call the tools directly, not through `make lint` or
# `make fmt-check`: both run docker, which cannot run inside a docker
# build. This step is the gofmt half of `make fmt-check`; the markdown
# stage is its prettier half. gofmt's output is assigned to a variable
# first so that its own exit status, as when it cannot parse a file,
# still fails the step.
RUN echo "gate gofmt, epoch ${CHECK_EPOCH}" && \
files="$(gofmt -s -l .)" && \
if [ -n "$files" ]; then \ if [ -n "$files" ]; then \
echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \ echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \
fi fi
# Validates .golangci.yml against the schema the pinned binary embeds. # Fails the build when the FROM above and the one in Dockerfile.lint pin
RUN golangci-lint config verify --config .golangci.yml # different linter images.
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \
script/verify-lint-image-pin
# Same config-schema check Dockerfile.lint runs, kept here so this build
# gates on exactly what script/lint gates on. It validates against a
# schema the pinned binary embeds, so it needs no network.
RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \
golangci-lint config verify --config .golangci.yml
RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \
golangci-lint run --config .golangci.yml ./... golangci-lint run --config .golangci.yml ./...
# Test phase, built alone by script/test. -race needs cgo and so a C # Prettier stage: the prettier that formats this repository's Markdown,
# compiler, which the Debian Go image ships and the alpine one does not. # never installed on a host. script/fmt and script/fmt-check build this
# -count=1 because the mounted cache also holds test results, which go # stage alone and run it with the repository mounted on /src. prettier
# test would otherwise replay. # is installed in /tools so that the repository, mounted or copied onto
# # /src, cannot hide it.
# The tests run as nobody: several of them make a file unreadable and # node:22-alpine, 2026-02-22
# expect reading it to fail, and root reads it anyway. nobody has no home FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS prettier
# directory, so HOME is /tmp, and Go keeps its build cache in WORKDIR /tools
# /tmp/.cache/go-build. That is where the cache is mounted, owned by # yarn.lock pins prettier by hash, and --frozen-lockfile fails rather
# nobody (65534) so the tests can write to it. # than install anything yarn.lock does not name.
# golang:1.25-trixie, 2026-10-04 COPY package.json yarn.lock ./
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test RUN yarn install --frozen-lockfile
ARG CACHEBUST ENV PATH=/tools/node_modules/.bin:$PATH
USER nobody
ENV HOME=/tmp
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies are # Markdown stage: the Markdown half of `make fmt-check`, as a gate.
# what make BuildKit build them first, so this stage cannot run unless FROM prettier AS markdown
# lint and test passed. COPY . .
# Second per-stage declaration of the gate cache-buster and its check;
# see the lint stage above.
ARG CHECK_EPOCH
RUN if [ -z "${CHECK_EPOCH}" ]; then \
echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \
exit 1; \
fi
RUN echo "gate prettier, epoch ${CHECK_EPOCH}" && \
prettier --check '**/*.md' --tab-width 4 --prose-wrap always
# Build stage
# golang:1.25-alpine, 2026-07-23 # golang:1.25-alpine, 2026-07-23
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null # We never build or run as root. Create an unprivileged user and point
COPY --from=test /src/go.sum /dev/null # HOME and the build cache at its home so go build and go test can write
RUN apk add --no-cache git make # it when we drop to it below.
# A tar-stream context keeps the sender's file owners, which git refuses. #
RUN git config --system --add safe.directory /src # The module cache stays at the base image's default /go/pkg/mod and
# belongs to root: script/bootstrap fills it as root. Do not move it
# into the home and hand it over with `chown -R`: that walks every file
# in it, which took from about 80 s to over ten minutes on a shared
# host, depending on load.
RUN adduser -D -u 1000 builder
ENV HOME=/home/builder
ENV GOPATH=/home/builder/go
ENV GOMODCACHE=/go/pkg/mod
ENV GOCACHE=/home/builder/.cache/go-build
WORKDIR /src WORKDIR /src
# No-op file copies whose only purpose is the build-graph edge: they
# make this stage depend on the lint and markdown stages, so BuildKit
# finishes those gates before compilation and tests start. Remove one
# and the build silently stops gating on that stage and still exits 0.
COPY --from=lint /src/go.sum /dev/null
COPY --from=markdown /src/go.sum /dev/null
# Install development prerequisites the same way a developer does. Only
# script/ and the dependency manifests are copied first, so this layer
# stays cached until they change. Bootstrap ends in `go mod download`.
COPY script/ script/
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN script/bootstrap
COPY . .
# Hand builder only what it writes to, without walking the module cache.
# This layer stays cached with bootstrap.
# - /src itself: make build writes the binary into it, and git refuses
# a repository whose top directory belongs to another user.
# - the module cache's cache/download directory itself, not what is in
# it: Go only reads the downloaded modules, but make build saves its
# lookup of this module's own version from git there, in a new
# directory named after the module path.
# - builder's home: the go commands bootstrap ran as root left Go's
# telemetry files there, a few small files.
RUN chown builder:builder /src /go/pkg/mod/cache/download && \
chown -R builder:builder /home/builder
# The sources are handed to builder as they are copied, so no layer has
# to walk them. Then drop root before running any checks or builds.
COPY --chown=builder:builder . .
USER builder
# Fail the build unless the branch is green. Runs as non-root: root
# would bypass the chmod(0) the permission-denied tests rely on.
#
# The gate is `make test`, not `make check`, which runs docker; lint and
# the format checks ran in the lint and markdown stages above. `make`,
# not the script directly, because the Makefile's
# `export CGO_ENABLED = 0` applies only to what it invokes.
#
# Third per-stage declaration of the gate cache-buster and its check;
# see the lint stage above. It is placed after USER so the drop to the
# unprivileged user still happens before the checks run.
ARG CHECK_EPOCH
RUN if [ -z "${CHECK_EPOCH}" ]; then \
echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \
exit 1; \
fi
RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test
# The version stamped into the binary: the VERSION build argument when # The version stamped into the binary: the VERSION build argument when
# one is given, otherwise `git describe --tags --always` of the .git in # one is given, otherwise `git describe --tags --always` of the .git in
# the build context. A context that carries .git and still yields no # the build context (git is installed by script/bootstrap above). A
# version fails the build; with neither, as from a source tarball, it is # context that carries .git and still yields no version fails the build;
# "dev". `make build` rather than `go build`, so the image and a host # with neither, as from a source tarball, it is "dev".
# build share one compile recipe, cgo disabled included.
ARG VERSION ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
[ "$version" = unknown ]; }; then \ [ "$version" = unknown ]; }; then \
echo "no version could be derived although the build context carries .git" >&2; \ echo "no version could be derived although the build context carries .git" >&2; \
@@ -82,11 +161,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
fi; \ fi; \
make build VERSION="$version" make build VERSION="$version"
# Runtime stage, and the last one: a plain `docker build .` builds this # Runtime stage
# stage's chain and nothing else.
# alpine:3.22, 2026-07-23 # alpine:3.22, 2026-07-23
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
ARG CACHEBUST
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
+45
View File
@@ -0,0 +1,45 @@
# Lint-only image, built by script/lint: the repo is copied into the
# pinned golangci-lint image and the linter runs as a build step, so a
# successful build is a clean lint. No bind mount, so it works when the
# docker daemon is remote.
#
# It is separate from the main Dockerfile's lint stage because
# script/lint must not depend on the rest of that build; the two FROM
# lines are kept identical by script/verify-lint-image-pin, run as a
# gate below.
# golangci/golangci-lint:v2.12.2, 2026-08-07
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
WORKDIR /src
# Dependency layers first, so they stay cached across lint runs.
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Cache-buster for the gate layers, and only for them; caching of the
# lint run is waived by ruling. On an unchanged tree the gates below
# would be served from cache and this build would exit 0 in under a
# second having run no linter. script/lint passes a fresh value on every
# invocation.
#
# Each gate RUN must reference the value: BuildKit hashes the expanded
# command, so a declared but unreferenced ARG invalidates nothing. Keep
# it below the dependency layers so they stay cached.
ARG CHECK_EPOCH
# Fails the build when the FROM above and the main Dockerfile's lint
# stage pin different linter images.
RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \
script/verify-lint-image-pin
# Validates .golangci.yml against golangci-lint's JSON schema, which the
# pinned binary embeds: measured under `--network none`, it passes a
# valid config and rejects an invalid one. No gate step makes a network
# call, but `go mod download` above needs the network on a cold cache.
RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \
golangci-lint config verify --config .golangci.yml
RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \
golangci-lint run --config .golangci.yml ./...
+4 -1
View File
@@ -6,7 +6,7 @@ BINARY := sfdupes
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
LDFLAGS := -X main.Version=$(VERSION) LDFLAGS := -X main.Version=$(VERSION)
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean .PHONY: sfdupes build bootstrap setup test test-race lint fmt fmt-check check docker hooks clean
# Standard targets are thin shims; the implementations live in script/ # Standard targets are thin shims; the implementations live in script/
# per the scripts-to-rule-them-all pattern. # per the scripts-to-rule-them-all pattern.
@@ -27,6 +27,9 @@ setup:
test: test:
@script/test @script/test
test-race:
@script/test-race
lint: lint:
@script/lint @script/lint
+177 -309
View File
@@ -38,9 +38,8 @@ database persists between runs; a rescan only hashes files that are new or
changed, or that may have gained a duplicate since the last scan, and removes changed, or that may have gained a duplicate since the last scan, and removes
records for files that no longer exist. `report` reads the database and prints records for files that no longer exist. `report` reads the database and prints
the file-level duplicates report. `trees` reads the same database and prints the the file-level duplicates report. `trees` reads the same database and prints the
duplicate-tree report. `info` prints the database's details: its path, the sizes duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation
of its files and the number of its records. A missing/invalid subcommand — or a with no `PATH` operand — prints a usage message and exits 2.
`scan` invocation with no `PATH` operand — prints a usage message and exits 2.
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
@@ -97,12 +96,10 @@ As a file `/etc/cron.d/sfdupes`:
the first scan. To keep it elsewhere, set the first scan. To keep it elsewhere, set
`SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line. `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line.
- `scan` writes nothing to stdout. Its stderr, appended here to - `scan` writes nothing to stdout. Its stderr, appended here to
`/var/log/sfdupes.log`, holds a first line naming the database (see `/var/log/sfdupes.log`, holds a plain progress line as each phase starts and
"Database"), plain progress lines naming each phase and its place among the then at most every 5 seconds, a warning for each path it skips, and the
five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it summary line (see "Progress" and "`scan` mode"). The log grows with every
runs, and one as it ends), a warning for each path it skips, and the summary scan; rotate it like any other.
line (see "Progress" and "`scan` mode"). The log grows with every scan; rotate
it like any other.
- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A - Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A
scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the
reason among the last lines of the log; `tail` prints them, and cron mails reason among the last lines of the log; `tail` prints them, and cron mails
@@ -110,9 +107,9 @@ As a file `/etc/cron.d/sfdupes`:
- A scan still running when the next one starts carries on. The new one fails at - A scan still running when the next one starts carries on. The new one fails at
once, and the lines cron mails include once, and the lines cron mails include
`sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`.
- `report`, `trees` and `info` need only read access to the database (see - `report` and `trees` need only read access to the database (see "Database").
"Database"). Under the usual umask of `022` the first scan creates it readable Under the usual umask of `022` the first scan creates it readable by every
by every user, so an unprivileged user can run them against root's database. user, so an unprivileged user can run them against root's database.
### Reading the reports ### Reading the reports
@@ -200,13 +197,12 @@ Goals, in order:
under resilver). Holding one small record (path, size, mtime) per file in under resilver). Holding one small record (path, size, mtime) per file in
memory during a scan is acceptable; holding every file's hashes is not (they memory during a scan is acceptable; holding every file's hashes is not (they
stay in the database). The reporting commands do not hold every file's stay in the database). The reporting commands do not hold every file's
hashes either: `report` reads the records in signature order and holds the hashes either: `report` lets SQLite group and order the records and writes
duplicate paths of one size at a time, so its memory grows with the each row as it reads it, so its memory does not grow with the database, and
duplicates of the size that has the most of them, and `trees` reads the `trees` reads the records in path order and keeps each directory's path,
records in path order and keeps each directory's path, digest and totals, digest and totals, plus the hashes of only the files in the directories
plus the hashes of only the files in the directories holding the record holding the record being read, so its memory grows with the number of
being read, so its memory grows with the number of directories and with the directories and with the size of the largest directory.
size of the largest directory.
3. **Scan incrementally, analyze offline.** The expensive filesystem scan 3. **Scan incrementally, analyze offline.** The expensive filesystem scan
maintains a persistent database; an unchanged file is never read again on a maintains a persistent database; an unchanged file is never read again on a
rescan, except to compute its content hash once a file of 10 MiB or more rescan, except to compute its content hash once a file of 10 MiB or more
@@ -236,7 +232,7 @@ Goals, in order:
### Subcommands ### Subcommands
Four subcommands, all implemented: Three subcommands, all implemented:
1. `scan` — walk the filesystem and synchronize the database: one signature 1. `scan` — walk the filesystem and synchronize the database: one signature
record per regular file. record per regular file.
@@ -244,14 +240,11 @@ Four subcommands, all implemented:
3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy 3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy
from the database records, compute a Merkle-style digest per directory, and from the database records, compute a Merkle-style digest per directory, and
report maximal groups of identical trees. report maximal groups of identical trees.
4. `info` — the database's details: its path, the sizes of its files, its
schema version and the counts of its records.
``` ```
sfdupes scan [--workers N] [-x] PATH... sfdupes scan [--workers N] [-x] PATH...
sfdupes report > dupes.tsv sfdupes report > dupes.tsv
sfdupes trees > dupetrees.tsv sfdupes trees > dupetrees.tsv
sfdupes info
sfdupes --version sfdupes --version
sfdupes [command] --help sfdupes [command] --help
``` ```
@@ -266,36 +259,16 @@ stdout.
### Database ### Database
All four subcommands operate on a single SQLite database file: All three subcommands operate on a single SQLite database file:
- Location: the value of the `SFDUPES_DATABASE` environment variable when set - Location: the value of the `SFDUPES_DATABASE` environment variable when set
and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no
command-line flag. The path names the file exactly, whatever characters it command-line flag. The path names the file exactly, whatever characters it
holds (`?`, `#` and `%` included); a relative path is relative to the working holds (`?`, `#` and `%` included); a relative path is relative to the working
directory. directory.
- Each subcommand first prints one line on stderr naming the database it uses, - `scan` creates the database (and its parent directory) on first use. `report`
before anything else it prints (progress displays included), whether or not and `trees` require an existing database; a missing database file is a fatal
stderr is a terminal. The line gives the path with the working directory put error (exit 1) telling the user to run `scan` first.
in front of it when it is relative, and otherwise as set (a `..` in it stays,
so the line names the same file), escaped like the paths in a report (see
"Report output format"), and says where the path came from:
```
scan: new database /var/lib/sfdupes/db.sqlite (default)
trees: database /srv/tmp/sfdupes/db.db (from SFDUPES_DATABASE)
```
The working directory put in front is the one `pwd -P` prints, with its
symlinks resolved. After a `cd` through a symlink to a directory, a path
starting `../` therefore names a file beside the directory the symlink leads
to, as it does for any other program. `scan` says `new database` when no
file is at that path as it starts. `report`, `trees` and `info` print the
line before they open the database, so the error for a missing database
comes after it.
- `scan` creates the database (and its parent directory) on first use. `report`,
`trees` and `info` require an existing database; a missing database file is a
fatal error (exit 1) telling the user to run `scan` first.
- Only one `scan` runs against a database at a time. For its whole run, `scan` - Only one `scan` runs against a database at a time. For its whole run, `scan`
holds an exclusive `flock(2)` lock on a lock file beside the database, named holds an exclusive `flock(2)` lock on a lock file beside the database, named
by appending `.lock` to the database's path with symlinks resolved by appending `.lock` to the database's path with symlinks resolved
@@ -311,8 +284,8 @@ All four subcommands operate on a single SQLite database file:
on. The lock file is created on first use, open to its owner only, and left in on. The lock file is created on first use, open to its owner only, and left in
place: a leftover file blocks nothing, because the lock ends with the process place: a leftover file blocks nothing, because the lock ends with the process
holding it however it ends, a fatal error or an interrupt included, and holding it however it ends, a fatal error or an interrupt included, and
deleting the file while a scan runs would let a second scan start. `report`, deleting the file while a scan runs would let a second scan start. `report`
`trees` and `info` never take the lock, so they run during a scan. and `trees` never take the lock, so they run during a scan.
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so - While `scan` runs, the database is in WAL journal mode with a busy timeout, so
running a report while a cron `scan` is in progress is safe. The filesystem is running a report while a cron `scan` is in progress is safe. The filesystem is
authoritative; the database is an eventually-consistent reflection of it. authoritative; the database is an eventually-consistent reflection of it.
@@ -328,27 +301,13 @@ All four subcommands operate on a single SQLite database file:
needs the database to itself: a `scan` that starts while a report is still needs the database to itself: a `scan` that starts while a report is still
reading waits for it up to the 10-second busy timeout, then fails; a `scan` reading waits for it up to the 10-second busy timeout, then fails; a `scan`
that ends while a report has the database open warns and leaves the database that ends while a report has the database open warns and leaves the database
in WAL mode until the next scan. `report` writes its rows while it reads the in WAL mode until the next scan. `report` writes each row as it reads it, so
records, so it is still reading while its output is paused (a pager, a stalled it is still reading while its output is paused (a pager, a stalled pipe), and
pipe), and a `scan` started then fails after the busy timeout. a `scan` started then fails after the busy timeout.
- A `scan` writes to the database and the files beside it at these points (its - `report` and `trees` open the database read-only and need only read access to
phases are described under "`scan` mode"): the database file, and no write access to its directory. While the database is
- As it starts, before the walk: the directory, an empty database file and in WAL mode they also read the `-wal` and `-shm` files beside it, which SQLite
the `.lock` file, each only if missing. creates with the database file's permissions.
- Then, still before the walk: the schema, in a new database, and the `-wal`
and `-shm` files, which stay beside the database while the scan runs.
- While it loads the existing records (`load`) and walks: no records. A
first scan's database holds none, however long its walk has run.
- During the hash phase: the first records, committed 10,000 at a time.
- In the update phase: the rest of the hashed records, the records for
size-unique files, and the deletions.
- In the content phase: content hashes, again committed 10,000 at a time.
- At the end, as it closes the database: the `-wal` and `-shm` files go
away, unless a report still has the database open (see above).
- `report`, `trees` and `info` open the database read-only and need only read
access to the database file, and no write access to its directory. While the
database is in WAL mode they also read the `-wal` and `-shm` files beside it,
which SQLite creates with the database file's permissions.
- Schema (`PRAGMA user_version` is the schema version, currently 1; a database - Schema (`PRAGMA user_version` is the schema version, currently 1; a database
with any other version is a fatal error. `scan` creates the schema and sets with any other version is a fatal error. `scan` creates the schema and sets
the version in one transaction, so a first scan stopped while doing so leaves the version in one transaction, so a first scan stopped while doing so leaves
@@ -384,8 +343,8 @@ All four subcommands operate on a single SQLite database file:
a file of 10 MiB or more, `content` stays empty until the content phase of a a file of 10 MiB or more, `content` stays empty until the content phase of a
scan (see "`scan` mode" below) has read the file. A record with an empty scan (see "`scan` mode" below) has read the file. A record with an empty
`content` is never part of a duplicate group, though it still defines the `content` is never part of a duplicate group, though it still defines the
file for tree reconstruction. The `files_signature` index lets `report` read file for tree reconstruction. The `files_signature` index lets SQLite group
the records in signature order without sorting the table. the records by signature for `report` without sorting the whole table.
### Duplicate detection ### Duplicate detection
@@ -440,10 +399,9 @@ scan and are enumerated concurrently: every operand seeds the shared walk worker
pool. Overlapping operands are harmless — an operand that duplicates another or pool. Overlapping operands are harmless — an operand that duplicates another or
lies under another is dropped before walking, so every file is reached exactly lies under another is dropped before walking, so every file is reached exactly
once and produces one database record. With `-x`, though, an operand under once and produces one database record. With `-x`, though, an operand under
another one is kept and walked on its own when any directory from it (from its another one is kept and walked on its own when it is a directory on a different
directory, for a regular file) up to but not including the operand covering it filesystem than the operand covering it, or a regular file in such a directory,
is on a different filesystem than that operand, because that operand's walk does because the walk of the covering operand does not reach it.
not enter such a directory and so does not reach it.
An operand that is a symlink (never followed, not even as an operand), socket, An operand that is a symlink (never followed, not even as an operand), socket,
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
@@ -452,8 +410,9 @@ drops it from the scanned operands before reading the database. Another operand
beneath it is still scanned. The records stored beneath it are not deleted: they beneath it is still scanned. The records stored beneath it are not deleted: they
are treated like any other record outside the scanned operands, including the are treated like any other record outside the scanned operands, including the
content-phase exception below. If it lies under another operand, they are under content-phase exception below. If it lies under another operand, they are under
that operand instead, and are treated like any other record there. This is not that operand instead, and are deleted like any other record there that this scan
an error: a scan whose every operand is dropped walks nothing and exits 0. did not verify. This is not an error: a scan whose every operand is dropped
walks nothing and exits 0.
`scan` synchronizes the database with the filesystem state under the scanned `scan` synchronizes the database with the filesystem state under the scanned
operands: operands:
@@ -498,15 +457,10 @@ operands:
schedule into the same database. Records below an operand at or inside such a schedule into the same database. Records below an operand at or inside such a
directory are not kept this way, since that operand's own walk verifies them. directory are not kept this way, since that operand's own walk verifies them.
`scan` runs **five sequential phases over the whole scan**. Parallelism lives `scan` runs **four sequential phases over the whole scan**. Parallelism lives
inside the walk, hash and content phases; batched database writes begin during inside each phase; batched database writes begin during the hash phase:
the hash phase:
1. **load** — read the existing records. Each record under the operands is kept 1. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
in memory, by path, with its size, its mtime and whether it has hashes, for
the walk to compare each file with; the size of every other record joins the
size census.
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
with the walk worker pool: every operand seeds the shared queue, and each with the walk worker pool: every operand seeds the shared queue, and each
worker reads one directory at a time, handing discovered subdirectories back worker reads one directory at a time, handing discovered subdirectories back
to the queue and running `lstat` on each regular file as it is discovered to the queue and running `lstat` on each regular file as it is discovered
@@ -516,7 +470,7 @@ the hash phase:
or busy pools. The walk builds the size census and resolves unchanged or busy pools. The walk builds the size census and resolves unchanged
already-hashed files on the fly; every other file is carried to the hash already-hashed files on the fly; every other file is carried to the hash
phase as a (path, size, mtime) record. phase as a (path, size, mtime) record.
3. **hash** — with the census complete, each carried file's size decides its 2. **hash** — with the census complete, each carried file's size decides its
fate. Size-unique files are never read: new or changed ones are recorded fate. Size-unique files are never read: new or changed ones are recorded
without hashes in the update phase, unchanged unhashed ones simply keep without hashes in the update phase, unchanged unhashed ones simply keep
their records. Every file with a shared size is hashed by the worker pool as their records. Every file with a shared size is hashed by the worker pool as
@@ -532,10 +486,10 @@ the hash phase:
**while hashing runs**, so a scan interrupted after hours keeps everything **while hashing runs**, so a scan interrupted after hours keeps everything
hashed so far and the next scan resumes cheaply, skipping records already hashed so far and the next scan resumes cheaply, skipping records already
written. written.
4. **update** — commit the final partial batch, the hash-less records for 3. **update** — commit the final partial batch, the hash-less records for
size-unique new and changed files, and the deletions for records the scan size-unique new and changed files, and the deletions for records the scan
did not verify (vanished files, plus paths that failed to stat or hash). did not verify (vanished files, plus paths that failed to stat or hash).
5. **content** — find every record of 10 MiB or more without a `content` hash 4. **content** — find every record of 10 MiB or more without a `content` hash
whose size, `head`, and `tail` equal another record's, anywhere in the whose size, `head`, and `tail` equal another record's, anywhere in the
database: records from this scan and records stored by earlier scans, inside database: records from this scan and records stored by earlier scans, inside
or outside the scanned operands. SQLite finds them, so only the records to or outside the scanned operands. SQLite finds them, so only the records to
@@ -577,10 +531,9 @@ Rules for the walk:
Per-file errors never abort the run; the final summary reports how many were Per-file errors never abort the run; the final summary reports how many were
skipped. As specified above, a skipped path that has a database record from an skipped. As specified above, a skipped path that has a database record from an
earlier scan loses that record, unless it failed only in the content phase, or earlier scan loses that record, unless it failed only in the content phase, or
is an operand dropped before the database was read whose record counts as is an operand dropped before the database was read that lies under no other
outside the scanned operands; an unreadable directory subtree likewise loses operand; an unreadable directory subtree likewise loses its records (accepted:
its records (accepted: the database mirrors what the latest scan could the database mirrors what the latest scan could actually verify).
actually verify).
Concurrency: the walk phase (which also stats files), the hash phase, and the Concurrency: the walk phase (which also stats files), the hash phase, and the
content phase each use a worker pool of `--workers` workers (default content phase each use a worker pool of `--workers` workers (default
@@ -589,9 +542,8 @@ files. `--workers` must be at least 1: a smaller value is a usage error,
reported in one line on stderr with exit 2 before anything is scanned. All three reported in one line on stderr with exit 2 before anything is scanned. All three
phases are seek-bound on spinning disks, so raising `--workers` well past the phases are seek-bound on spinning disks, so raising `--workers` well past the
core count can help on pools with many spindles. The main goroutine owns core count can help on pools with many spindles. The main goroutine owns
partitioning, database writes, and the progress counts, which each progress partitioning, database writes, and progress rendering; progress display must
display draws from a goroutine of its own; progress display must never block the never block the workers.
workers.
`scan` writes nothing to stdout. The summary line on stderr reports the files `scan` writes nothing to stdout. The summary line on stderr reports the files
seen this run broken down by disposition, plus skips: seen this run broken down by disposition, plus skips:
@@ -609,8 +561,11 @@ total.)
**`report` must never touch the filesystem being analyzed.** It does not stat, **`report` must never touch the filesystem being analyzed.** It does not stat,
open, or otherwise access any path that appears in the records; its only I/O is open, or otherwise access any path that appears in the records; its only I/O is
reading the database and writing stdout/stderr. `report` must produce identical reading the database, writing stdout/stderr, and the temporary file SQLite sorts
output whether or not the scanned filesystem is still mounted. in when the duplicate rows do not fit in memory. SQLite puts that file in
`$SQLITE_TMPDIR` or `$TMPDIR` when set, otherwise in `/var/tmp` (or `/tmp`), and
deletes it as soon as it has opened it. `report` must produce identical output
whether or not the scanned filesystem is still mounted.
Processing: Processing:
@@ -624,16 +579,6 @@ Processing:
by `first` path ascending. Output must be fully deterministic for a given by `first` path ascending. Output must be fully deterministic for a given
database state. database state.
`report` runs in two phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `read` counts the records, then reads
them in signature order, largest size first. That is the order of the
`files_signature` index read backwards, so SQLite sorts nothing and returns the
first record at once. The records sharing a signature come together, and so do
the records of one size, so `report` holds only the duplicate groups of the size
it is reading, and writes them, ordered by `first` path, once it reaches a
smaller size. Its memory holds at most the duplicate paths of one size at a
time, never every record.
#### Report output format #### Report output format
TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a
@@ -707,12 +652,6 @@ Processing:
- Order groups by total tree size descending, tie-broken by `first` path - Order groups by total tree size descending, tie-broken by `first` path
ascending. Output must be fully deterministic for a given input. ascending. Output must be fully deterministic for a given input.
`trees` runs in four phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `read` counts the records, then reads
them in path order, building the hierarchy and computing the digests as it goes;
`group` groups the directories by digest and picks the maximal groups; `write`
writes the groups to stdout.
#### Trees output format #### Trees output format
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
@@ -731,167 +670,57 @@ Summary to stderr: records read, number of duplicate-tree groups, number of dupe
trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human
units. units.
### `info` mode
`info` takes no positional arguments and prints the database's details on
stdout, one per line: a name, a tab, and the value. It opens the database the
way `report` does, read-only and without the scan lock, so it runs during a scan
and needs only read access (see "Database"). Like `report`, it never touches the
scanned filesystem.
```
database /var/lib/sfdupes/db.sqlite (default)
database file size 2147483648 (2.0 GiB)
-wal file size 4124152 (3.9 MiB)
modified 2026-10-16T12:02:50Z
schema version 1
records 12345678
records with head and tail hashes 1234567
records with content hash 123456
size of recorded files 150000000000000 (136.4 TiB)
```
- `database` is the path and where it came from, as on the line every subcommand
prints first on stderr (see "Database").
- `database file size` and `-wal file size` are in bytes, then in human units.
The `-wal` line is there only when a `-wal` file is beside the database, as
while a scan runs.
- `modified` is the time the database file was last changed, in UTC. While the
database is in WAL mode, the newest changes may be in the `-wal` file alone.
- `schema version` is the database's `PRAGMA user_version`. A database with
another version than this build's is refused, as by `report`.
- `records` is the number of records. `records with head and tail hashes` and
`records with content hash` count the records that have those hashes (see
"Database"), and `size of recorded files` adds up the sizes of all records, in
bytes, then in human units.
`info` runs in two phases, each with its own progress display on stderr (see
"Progress"): `open` opens the database; `count` reads the schema version and
counts the records with one query, a single pass over the table, both in one
read-only transaction, so the numbers agree with each other while a scan is
committing. `info` writes its lines once the `count` display has ended, so on a
terminal they come after the displays.
### Progress ### Progress
Use the progress-bar library for all progress of `scan`, `report`, `trees` and Use the progress-bar library for all scan progress; rendering in the style of
`info`; rendering in the style of `pv` is the model. All progress goes to `pv` is the model. All progress goes to stderr.
stderr.
`scan`, `report`, `trees` and `info` must always show that they are working and Each phase gets its own display, rendered the moment the phase starts — a scan
not hung. Each phase gets its own display from the moment it starts, naming the must never look hung. Loading the existing-record index (`load`) and the walk
phase and its place among the command's phases and showing its progress, its have no known totals while running: show a live count, rate, and elapsed time
elapsed time and its estimated time remaining (ETA). (spinner-style, no percentage or ETA). The content phase's display (`content`)
starts the same way, counting the records checked while SQLite finds the files
to read and `lstat` checks them, then shows a bar once reading starts. The hash
and update phases, and the content phase's reads, have exact totals — only files
that actually need hashing appear in the hash and content totals, so their ETAs
are meaningful. Required elements for the bars with known totals:
A scan has five phases (`[1/5] load`, `[2/5] walk`, `[3/5] hash`,
`[4/5] update`, `[5/5] content`):
- `load` counts the records first, then reads them against that count. While the
count runs, the display says it is counting the records.
- `walk` has no exact total. Its estimate is the number of records the last scan
left under the operands, known once `load` is done and shown with a `~`. Once
the walk passes the estimate, or when there are no such records, as on a first
scan, the ETA is unknown.
- `hash` and `update` have exact totals. Only files that actually need hashing
appear in the hash total, so its ETA is meaningful.
- `content` first counts the records whose files it checks with `lstat`, which
SQLite does by adding up the number of records with each matching size,
`head`, and `tail`, without fetching the records. It then says it is searching
the database until SQLite returns the first record; the checks, and their rate
and ETA, start there. The files to read then get a display of their own, with
their count as its total.
`trees` has four phases (`[1/4] open`, `[2/4] read`, `[3/4] group`,
`[4/4] write`, see "`trees` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `read` counts the records first, then reads them against that count. While the
count runs, the display says it is counting the records.
- `group` has the directories `read` found as its total, and `write` the
duplicate-tree groups.
`report` has two phases (`[1/2] open`, `[2/2] read`, see "`report` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `read` counts the records first, then reads them against that count, writing
the rows to stdout as it goes. While the count runs, the display says it is
counting the records.
`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"):
- `open` has no total. While it opens the database, waiting on a busy one
included, the display says it is opening the database.
- `count` has no total either, because one query counts everything: the display
says it is counting the records until the query returns, and its ETA is
unknown.
Each display is one line holding:
- the phase and its place among the phases
- a `[m/n] x%` display (items processed / total items, percent); before a phase
has a total, the items processed, or what the phase is waiting for
- current rate (items/s), naming the items the phase counts: files, records,
directories or groups
- elapsed time - elapsed time
- estimated time remaining, or `unknown` - estimated time remaining
- a `[m/n] x%` display (items processed / total items, percent)
- current rate (items/s)
Example shape (exact layout is flexible, content is not): Example shape (exact layout is flexible, content is not):
``` ```
[3/5] hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54 hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54
[2/5] walk: [1234567/~1200000] 103%, 5120 files/s, elapsed 4m1s, eta unknown
``` ```
Additional requirements: Additional requirements:
- On a terminal, a phase with an exact total (every phase but `scan`'s `walk`,
the `open` phase of `report` and `trees`, and both phases of `info`, once it
has counted what it will do) is drawn as its line followed by the progress-bar
library's bar, 10 columns wide, which shows the percent in place of the line's
own. Any other phase, and a phase whose bar has no room, is drawn as the
library's spinner followed by its line. Each display is drawn again several
times a second, so its elapsed time keeps moving while a phase waits for its
next item.
- On a terminal no line reaches the terminal's last column, however narrow the
terminal is. The phase and its place, the progress, the elapsed time and the
ETA are kept whole: a line too wide for the terminal leaves out its rate first
(or what the phase is waiting for), and a bar with no room beside those four
gives way to the spinner. Only when those four alone are too wide, as on a
terminal narrower than about 70 columns hours into a scan of millions of
files, is the line cut short at the terminal's edge, so the end of the ETA is
lost first. A terminal made narrower while a phase runs gets the same: the
next draw erases the line within the new width and fits the line to it, and
the display goes on redrawing in place.
- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI - When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI
redraws: print the line the moment each phase starts, every 5 seconds while it redraws: print a plain one-line progress update the moment each phase starts,
runs (whether or not an item has completed, and never more often), and once then no more often than every 5 seconds.
more as the phase ends. - Progress updates are driven from the main goroutine and must be non-blocking
- Progress must be non-blocking with respect to the worker pool: the main with respect to the worker pool. On a terminal the spinner-style displays also
goroutine counts the items, and each display draws them from a goroutine of redraw on their own several times a second, so their count and elapsed time
its own. stay current while a phase waits for its next item.
- A warning printed during a phase always lands on a line of its own, never - A warning printed during a phase always lands on a line of its own, never
inside the progress display. So does every line written to stdout when stdout inside the progress display.
and stderr are the same file (the same device and inode): one terminal, as in - A bar whose phase stops short of its total, as an interrupted one does, is
a plain `sfdupes report` or `sfdupes trees`, or one pipe or file that both are left as last drawn rather than filled up.
sent to. When they are not, stdout is written without waiting on the display, - `report` and `trees` modes need no progress display, only their stderr
and the display keeps being drawn while a slow reader of stdout holds up a summaries.
write.
- A phase that stops short of its total, as an interrupted one does, is left
showing the count it stopped at rather than filled up.
### Error handling and exit codes ### Error handling and exit codes
- `0`: success, even if individual files were skipped with warnings. - `0`: success, even if individual files were skipped with warnings.
- `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is - `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is
already running against the same database, the database cannot be already running against the same database, the database cannot be
created/opened/read/written, a missing database for `report`/`trees`/`info`, created/opened/read/written, a missing database for `report`/`trees`, stdout
stdout write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below).
below).
- `2`: usage error (including `scan` with no `PATH` operand, `scan` with - `2`: usage error (including `scan` with no `PATH` operand, `scan` with
`--workers` below 1, and `report`/`trees`/`info` with any positional `--workers` below 1, and `report`/`trees` with any positional argument).
argument).
A stdout write failure, such as a full disk, is reported in one line on stderr A stdout write failure, such as a full disk, is reported in one line on stderr
and exits 1. Two cases never reach sfdupes as a failed write: and exits 1. Two cases never reach sfdupes as a failed write:
@@ -934,67 +763,104 @@ entrypoints are:
- `script/bootstrap` — install everything needed to build and develop this - `script/bootstrap` — install everything needed to build and develop this
repository, idempotently, assuming nothing is present. `git`, `make`, and `go` repository, idempotently, assuming nothing is present. `git`, `make`, and `go`
come from the first of nix, apt, brew, or apk found on the host, and are come from the first of nix, apt, brew, or apk found on the host, and are
presence-checked only. An installed node is used when its major version is 22; presence-checked only. `golangci-lint` and prettier are deliberately **not**
otherwise node 22.17.0 is installed through nvm, which comes from a release installed: they run in Docker (see `script/lint` and `script/fmt`) and never
archive whose sha256 the script checks, and yarn 1.22.22 is activated under it from a host install, so there is no host copy to drift from the pin. A missing
through corepack. With an installed node 22, a yarn already on `PATH` is used `docker` is warned about rather than installed or treated as fatal —
as it is; otherwise yarn 1.22.22 is activated through corepack, or installed everything except linting, formatting and `make test-race` works without it.
with `npm` when there is no corepack. `yarn install --frozen-lockfile` then Ends with `go mod download`.
installs the prettier that `package.json` and `yarn.lock` pin. `golangci-lint`
is never installed: it runs in Docker (see `script/lint`). `docker` is not
installed either; testing, linting and the image build need it. Ends with
`go mod download`.
- `script/setup` — make a fresh clone ready for development: runs - `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit`. `script/bootstrap`, then `script/install-precommit`.
- `script/projectname` — print this project's name (`sfdupes`). Scripts that - `script/projectname` — print this project's name (`sfdupes`). Scripts that
need the name call it, so they stay identical across repositories. need the name call it, so they stay identical across repositories.
- `script/test` — run the test suite under the race detector, with a 90-second - `script/test` — run the test suite with a 30-second timeout and coverage
timeout and coverage enabled, rerunning verbosely on failure so the logs show enabled, rerunning verbosely on failure so the logs show which test failed.
which test failed. It builds the `Dockerfile`'s `test` phase alone and writes - `script/test-race` — run the test suite under the race detector with a
no image. The race detector needs cgo and a C compiler, which the build never 60-second timeout. The detector needs cgo and a C compiler, which the build
uses, so the phase starts from a digest-pinned Debian `golang` image, which never uses, so the tests run in a digest-pinned Debian `golang` image that has
has `gcc`. The tests run as `nobody`, because several of them make a file `gcc`, with the checkout mounted read-only; the container is removed when it
unreadable and root reads it anyway. exits. They run as the calling user, or as `nobody` when that is root, because
- `script/lint` — run the linter. It builds the `Dockerfile`'s `lint` phase several tests make a file unreadable and root reads it anyway; only then must
alone and writes no image: in the digest-pinned `golangci/golangci-lint` the checkout be readable by other users. Not part of `script/check`. Every run
image, the gofmt check, `golangci-lint config verify` and `golangci-lint run` starts with empty caches, so it needs the network and takes minutes, and the
run as build steps, so a successful build is a clean lint. The linter is never mount needs a local docker daemon.
run on the host, which makes a working `docker` the one prerequisite for - `script/lint` — run the linter. It builds `Dockerfile.lint`, which copies the
linting. repository into the digest-pinned `golangci/golangci-lint` image and runs
`golangci-lint config verify` and `golangci-lint run` as build steps, so a
successful build is a clean lint. That exit status is all it produces, so it
runs with `--output=type=cacheonly` and writes no image; a run leaves only
build cache. The linter is never run on the host, which makes a working
`docker` the one prerequisite for linting — and therefore for `make check` and
the pre-commit hook. Offline machines: the gate steps themselves make no
network calls. `golangci-lint run` does not, and neither does
`golangci-lint config verify` — it validates against a schema the pinned
binary embeds, measured under `--network none` to both pass a valid config and
reject an invalid one. The build around them does. `Dockerfile.lint` runs
`go mod download` before the gates and this module has external dependencies,
so a first lint on a machine with a cold BuildKit cache reaches the network
there (as well as pulling the pinned image); under `--network none` it fails
at that step, before any gate. That layer sits above the gates and stays
cached, so once it is warm `script/lint` — and with it `make check` — runs
entirely offline, until `go.mod` or `go.sum` changes and the download layer
goes cold again. Because the daemon only ever sees a build context, this works
when the docker daemon is remote and bind mounts are impossible.
- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every - `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every
Markdown file with prettier, at the settings in `.prettierrc` (4-space Markdown file with prettier, at the settings in `.prettierrc` (4-space
indents, prose wrapped at 80 columns). Both run on the host, prettier through indents, prose wrapped at 80 columns). prettier is pinned by hash through
yarn at the version `yarn.lock` pins. When the node on `PATH` is not node 22, `package.json` and `yarn.lock` and never installed on the host: this builds
this loads the node 22.17.0 that `script/bootstrap` installed through nvm. the `Dockerfile`'s `prettier` stage, a digest-pinned node image into which
- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any `yarn install --frozen-lockfile` installs it, tagged `sfdupes-prettier`, and
unformatted file and exits non-zero instead of writing. gofmt and prettier runs that with the repository mounted, as the calling user. Needs `docker`,
both run every time, and each names itself when it fails. The `Dockerfile`'s and because of the mount, unlike `script/lint`, a local docker daemon.
`lint` phase runs the same gofmt check. - `script/fmt-check` — the read-only counterpart of `script/fmt`, with the
repository mounted read-only: prints any unformatted file and exits non-zero
instead of writing. gofmt and prettier both run every time, and each names
itself when it fails. The `Dockerfile` runs the same two checks as gates: the
gofmt check in its lint stage, prettier in its `markdown` stage.
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in - `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in
that order. Modifies nothing. The first two need `docker`, the third what that order. Modifies nothing. Needs `docker`, because `script/lint` and
`script/bootstrap` installs. `script/fmt-check` do.
- `script/docker` — build the Docker image, tagged with the name from - `script/docker` — build the Docker image, tagged with the name from
`script/projectname`, passing the output of `script/projectname`. The `Dockerfile` runs the gates as build steps, so this
`git describe --tags --always --dirty` (or `unknown` when that is empty) as is also the check a developer or reviewer runs by hand.
the `VERSION` build argument. The `Dockerfile`'s build stage depends on its - `script/cibuild` — build the Docker image untagged. This is what the Gitea
`lint` and `test` phases, so this also runs the tests and the linter. workflow runs on push; because the gates run as build steps, a successful
- `script/cibuild` — run `script/bootstrap`, then `script/check`, then build the build implies the repository is green.
image as `script/docker` does. This is what the Gitea workflow runs on push; a
successful run means every check passed. The tests and the linter run twice:
once in `script/check`, and again as stages of the image build.
- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a - `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a
no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then
`script/check`. `script/check`.
- `script/install-precommit` — install the git pre-commit hook, as - `script/install-precommit` — install the git pre-commit hook that runs
`.git/hooks/pre-commit`, that runs `script/precommit`. `script/precommit`. The hook is written to the common git directory, so the
main checkout and every worktree share it.
- `script/verify-lint-image-pin` — fail unless the `golangci/golangci-lint`
reference in `Dockerfile.lint` and the one in the `Dockerfile` lint stage are
the same image at the same digest, naming both if not. The linter is pinned in
those two files and nothing else keeps them in sync, so a bump applied to one
alone would leave `make lint` and the `Dockerfile`'s fail-fast lint stage
checking the same tree against different rulesets, both green. The guard
restates neither pin — a third copy would be the same drift one file further
out — and runs as a gate in both files, so `make lint`, `make check` and
`make docker` all catch it.
Every `docker build` in `script/` passes a new `CACHEBUST` build argument, which `script/verify-linter-pin` used to live here. It compared a linter binary
every stage of the `Dockerfile` declares first, so every `RUN` step runs again against a version pin in `script/bootstrap`, and both of its subjects are gone:
each time. On an unchanged tree Docker would otherwise serve the gate steps from no linter binary is copied between build stages any more, and bootstrap pins no
its cache, and the build would pass having run no test and no linter. Go's build version because it installs no linter. The drift it existed to catch has moved
cache is kept between builds on the steps that compile, so only what changed is from binary-versus-pin to pin-versus-pin, which is what
compiled again; the tests run with `-count=1`, so none reports a stored result. `script/verify-lint-image-pin` above checks.
Every run downloads the Go modules again and needs the network.
`script/lint`, `script/docker` and `script/cibuild` all pass a freshly computed
`CHECK_EPOCH` build argument, and the gate steps in `Dockerfile.lint` and
`Dockerfile` reference it. Without that, an unchanged tree lets Docker serve the
gate layers from cache and the build exits 0 having executed no tests and no
lint — a green it never earned, and one this repository has produced twice.
`CHECK_EPOCH` invalidates the gate layers on every run while leaving the pinned
base images and the dependency layers cached. Each script's value carries the
process id as well as the epoch, because two runs land inside the same second
easily and a bare epoch would cache the second one. Each `Dockerfile` stage with
gates fails when the value is empty, so a bare `docker build .` stops with
`CHECK_EPOCH is unset; build via script/cibuild or script/docker` instead of
serving the gates from cache.
## Build ## Build
@@ -1006,15 +872,17 @@ compile recipe:
the default target. the default target.
- `make bootstrap` — install the build and development dependencies. - `make bootstrap` — install the build and development dependencies.
- `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook. - `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook.
- `make test` — run the test suite under the race detector, in Docker (90-second - `make test` — run the test suite (30-second timeout; reruns with `-v` on
timeout; reruns with `-v` on failure; see `script/test`); requires `docker`. failure).
- `make lint` — run `golangci-lint` with the repo config and the gofmt check, in - `make test-race` — run the test suite under the race detector, in Docker (see
Docker (see `script/lint`); requires `docker`. `script/test-race`); requires `docker`. Not part of `make check`.
- `make lint` — run `golangci-lint` with the repo config, in Docker (see
`script/lint`); requires `docker`.
- `make fmt` / `make fmt-check` — format the Go sources and the Markdown / - `make fmt` / `make fmt-check` — format the Go sources and the Markdown /
verify formatting without writing, on the host; requires `go` and what verify formatting without writing; requires `docker`, for prettier (see
`make bootstrap` installs (see `script/fmt`). `script/fmt`).
- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires - `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires
`docker` and what `make bootstrap` installs. `docker`, via `lint` and `fmt-check`.
- `make docker` — build the Docker image, which runs the gates as build stages. - `make docker` — build the Docker image, which runs the gates as build stages.
- `make hooks` — install the pre-commit hook. - `make hooks` — install the pre-commit hook.
- `make clean` — remove the binary. - `make clean` — remove the binary.
+88 -436
View File
@@ -1,6 +1,6 @@
--- ---
title: Repository Policies title: Repository Policies
last_modified: 2026-10-15 last_modified: 2026-07-06
--- ---
This document covers repository structure, tooling, and workflow standards. Code This document covers repository structure, tooling, and workflow standards. Code
@@ -54,37 +54,21 @@ style conventions are in separate documents:
`cibuild`. `script/bootstrap` installs all dependencies idempotently and `cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the (detected in that order; apt runs noninteractive). For node it uses the
installed node only when its major version is the pinned one; otherwise (node installed node if present; otherwise it installs a PINNED node version via
missing, or another major, such as the node 24 the CI runner image ships) it nvm, first installing nvm itself if missing — from a hash-verified GitHub
installs the PINNED node version via nvm and installs yarn and the packages release archive (never `curl | sh`), with bash installed as an explicit
under it. nvm itself is installed first if missing, from a hash-verified prerequisite since nvm requires bash. yarn is then pinned via
GitHub release archive (never `curl | sh`), with bash installed as an explicit `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
prerequisite since nvm requires bash. Only the major version is compared always exact versions. `script/cibuild` runs the CI build: it changes to the
because the `Dockerfile` stages start from a node image whose exact version is repo root and runs `docker build .`; the Gitea workflow calls it. Four further
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned scripts are our own extensions to the standard: `script/check` runs
via `corepack prepare yarn@<version> --activate`. Never install "latest" or `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes what the git pre-commit hook runs, and it calls `script/check`;
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the `script/install-precommit` installs the git pre-commit hook (the `make hooks`
image with the version; the Gitea workflow calls it. **`script/cibuild` runs target shims to it); and `script/projectname` (literally that filename) simply
`script/bootstrap` first**, because the workflow checks out the repo and runs outputs the project's name. Scripts that need the name call
nothing else, while `script/fmt-check` runs the formatter on the host: on a `script/projectname` — e.g. `script/docker` assembles its image tag from it —
pristine checkout with nothing installed the run dies there, after the so those scripts stay byte-identical across all repos. Repo-type-specific
containerised gates have passed. **The bootstrap alone is not enough**: when
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
under another node. The host entrypoints that need yarn — `script/fmt` and
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
pinned node. A runner carrying nothing but docker and git then gets through
`script/check`. Four further scripts are our own extensions to the standard:
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
`script/precommit` is what the git pre-commit hook runs, and it calls
`script/check`; `script/install-precommit` installs the git pre-commit hook
(the `make hooks` target shims to it); and `script/projectname` (literally
that filename) simply outputs the project's name. Scripts that need the name
call `script/projectname` — e.g. `script/docker` assembles its image tag from
it — so those scripts stay byte-identical across all repos. Repo-type-specific
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/precommit`, not in the hook itself. Model scripts are at `script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
@@ -105,257 +89,87 @@ style conventions are in separate documents:
contributor should be able to understand the entire development workflow by contributor should be able to understand the entire development workflow by
reading the Makefile. reading the Makefile.
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a - Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
`lint` phase and a `test` phase, with the final stage depending on both so the as a build step so the build fails if the branch is not green. For non-server
image cannot be built unless they pass. For non-server repos the final stage repos, the Dockerfile should bring up a development environment and run
brings up a development environment; for server repos it is the runtime image. `make check`. For server repos, `make check` should run as an early build
The gate phases and the build stage start from their pinned base images and stage before the final image is assembled. Dockerfiles install development
install what those images lack either inline, as the canonical Go `Dockerfile` prerequisites by running `script/bootstrap` rather than duplicating installs
below does for `git`, or by running `script/bootstrap`, as the `prompts` inline; COPY `script/` and the dependency manifests (`package.json` +
repo's own `Dockerfile` does for its yarn packages. The development `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
environment stage installs development prerequisites by running layer stays cached until dependencies change.
`script/bootstrap` rather than duplicating its installs inline. A stage that
runs `script/bootstrap` COPYs `script/` and the dependency manifests
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is - **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
no separate lint file. `script/lint` and `script/test` each build one phase repos use a multistage build where linting runs in an independent stage based
and nothing else: on the `golangci/golangci-lint` image (pinned by hash). This stage runs
`make fmt-check` and `make lint` before the full build begins. The build stage
then declares an explicit dependency on the lint stage via
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
linting before proceeding to compilation and tests. This ensures lint failures
surface in seconds rather than minutes, without blocking on dependency
download or compilation in the build stage.
```sh The standard pattern for a Go repo Dockerfile is:
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target lint --output type=cacheonly .
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target test --output type=cacheonly .
```
**A stage that is not the last one in the file is built only when the final
stage's chain depends on it, or when `--target` names it.** That is why the
two gates are always invoked by name here, and why the final stage carries a
`COPY --from=` of a harmless file from each of them: without that edge a
plain `docker build .` builds the last stage alone and exits 0 having linted
and tested nothing.
**The gate builds write no image.** With `--output type=cacheonly` the phase
runs and a failing step fails the build, but the result is not exported.
Nothing uses those images, and writing one out is slow: a Go test phase's
image holds the toolchain. A build given neither `--output` nor `-t` writes
an untagged image and leaves it dangling, on every developer host and every
CI runner. `script/cibuild` and `script/docker` build the image that ships
and tag it, so each build replaces the previous image; each assigns the tag
on its own line before the build, so `set -e` stops it where
`script/projectname` fails.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
themselves a `docker build` and would recurse into a daemon that does not
exist in a build step. Formatting is the exception and stays on the host:
`script/fmt` writes the working tree, and `script/fmt-check` is its
read-only twin.
**No lint verdict may come from a host invocation of the linter.** On a
shared host golangci-lint reads a result cache keyed on file content rather
than location, so a second checkout of the same content is served the first
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own golangci-lint cache, its
own `TMPDIR` and a digest-pinned binary, so neither is reachable.
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
invalidates a `COPY` layer only when the copied content changes, so on an
unchanged tree the check `RUN` is served from cache, nothing executes, and the
build still exits 0. Every `docker build` in `script/` therefore passes
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
id, so no two builds share a value, not even the three in one
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
`script/docker` are the four, and there is no fifth — `script/check` runs the
two gate phases and `script/fmt-check`, and builds no image of its own. Every
stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
`FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
are compared by content and need nothing. A stage without the `ARG` line is
served from cache on an unchanged tree, so a repository re-vendors the scripts
and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
it reruns every step too, but it also empties every cache mount, and Go's
build cache, which the canonical Go `Dockerfile` below mounts so that compiled
packages survive between check builds, would start empty every time. A bare
`docker build .` is not evidence that anything ran: a sub-second build
reporting success is a cache hit, not a result. Never invalidate by pruning —
`docker builder prune` and friends destroy a build cache shared with every
other build on the host. When a check is added or changed, prove it works by
planting a defect it must catch and watching the run fail on it, then revert
the defect. A green run alone shows neither that the check ran nor that it
covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by
hash), so lint failures surface in seconds rather than after a full compile,
and the test phase is based on the Debian Go image. The canonical Go repo
`Dockerfile`:
```dockerfile ```dockerfile
# Every stage declares ARG CACHEBUST first: script/ passes a new value # Lint stage — fast feedback on formatting and lint issues
# on every build, which reruns every RUN step after it. The steps that
# compile mount Go's build cache, which survives from build to build.
# Lint phase
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint FROM golangci/golangci-lint@sha256:... AS lint
ARG CACHEBUST
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN make fmt-check
golangci-lint run --config .golangci.yml ./... RUN make lint
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Build stage
# image ships and the alpine one does not. -count=1 because the mounted
# cache also holds test results, which go test would otherwise replay.
# golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test
ARG CACHEBUST
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \
go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD # golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder FROM golang@sha256:... AS builder
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src WORKDIR /src
# Force BuildKit to run the lint stage before proceeding
COPY --from=lint /src/go.sum /dev/null
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN make test
# The VERSION build arg when one is given, otherwise ARG VERSION=dev
# `git describe --tags --always` on the .git in the build context. With RUN CGO_ENABLED=0 go build -trimpath \
# .git present, a version that is still empty, dev or unknown fails the -ldflags="-s -w -X main.Version=${VERSION}" \
# build: git is missing or could not read the checkout. -o /app ./cmd/app/
ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \
VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \
exit 1 ;; \
esac; \
fi; \
CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/
# Runtime stage, and the last one # Runtime stage
FROM alpine@sha256:... FROM alpine@sha256:...
ARG CACHEBUST
COPY --from=builder /app /usr/local/bin/app COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"] ENTRYPOINT ["app"]
``` ```
Key points: Key points:
- The lint phase uses the `golangci/golangci-lint` image directly (it has - The lint stage uses the `golangci/golangci-lint` image directly (it
both Go and the linter), so nothing needs installing. includes both Go and the linter), so there is no need to install the
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only linter separately.
purpose is the ordering edge. BuildKit runs stages in parallel by default, - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
and a stage nothing depends on is not built at all, so without these two a stage dependency. BuildKit runs stages in parallel by default; without
lines a red gate would not fail the build. this line, the build stage would not wait for lint to finish and a lint
- Keep the runtime stage last, and if you add a stage after it, give it the failure might not fail the overall build.
same two copies. A plain `docker build .` builds the last stage's chain
and nothing else.
- Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
it for root), and only on the steps that compile: `golangci-lint run`,
both `go test` runs and `go build`. It keeps compiled packages between
check builds, and Go reuses one only when all its inputs are unchanged, so
it saves the compile without skipping a check. Nothing else is kept:
golangci-lint's own cache would replay stored findings in place of a lint
run, and Go builds from whatever files a kept module cache holds without
checking them again (only `go mod verify` does), so a file changed there
would go unnoticed. Go's build cache also holds test results, which is why
both `go test` runs carry `-count=1`.
- If the project uses `//go:embed` directives that reference build artifacts - If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must (e.g. a web frontend compiled in a separate stage), the lint stage must
create placeholder files so the embed directives resolve. Example: create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
- If the project requires CGO or system libraries for linting, install them The lint stage should not depend on the actual build output — it exists to
in the lint phase. The `golangci/golangci-lint` image is Debian-based and fail fast.
has no `apk`, so install with `apt-get` under the Debian package name - If the project requires CGO or system libraries for linting (e.g.
(`libvips-dev`, where alpine says `vips-dev`), and delete the package `vips-dev`), install them in the lint stage with `apk add`.
lists in the same `RUN`, so the layer does not keep them: - The build stage runs `make test` after compilation setup. Tests run in the
build stage, not the lint stage, because they may require compiled
```dockerfile artifacts or heavier dependencies.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
```
- `.dockerignore` lets `.git` into the build context. It keeps out every git
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
repository's own, each submodule's under `.git/modules/`, and that of a
submodule keeping its own `.git` directory. `git describe` does not need
them, and each can hold a credential: a password in a remote URL, or the
token the CI checkout step stores there. A submodule whose name has a
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
git directory to `**/.git/modules/**/config`, and Go's version stamping
then fails the build: give it a name without that segment
(`git submodule add --name`). The stage that compiles has `git` (the
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
takes the version from the `VERSION` build argument when one is given,
otherwise from `git describe --tags --always`. That gives the tag on a
tagged commit; on a later commit, the tag, the number of commits since it
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
tag is reachable. The stage that compiles also marks its working directory
safe for git (`git config --system --add safe.directory /src`): a context
sent as a tar stream keeps the sender's file owners, and git refuses a
checkout owned by another user, so the version would come out empty.
`ARG VERSION` has no default, and the build fails if the context carries
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
`docker build .` with no build arguments must succeed; a Dockerfile that
refuses an empty build argument drops that refusal and keeps the argument.
A checkout whose `.git` is a file (a linked worktree, or a repository
checked out as a submodule) is the exception: that file points to a git
directory outside the build context, so the build cannot read the version
and a plain `docker build .` fails; pass the version with
`--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
do.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` on push, and checks out the repo as its only other step, runs `script/cibuild` (which runs `docker build .`) on push. Since the
with `persist-credentials: false`: `script/cibuild` needs no token, and Dockerfile already runs `make check`, a successful build implies all checks
without it the checkout leaves the job's token in `.git/config` for every pass.
later step. The checkout step also sets `fetch-depth: 0`, which fetches the
tags `git describe` needs: by default it clones shallow with no tags, and a
tagged repository's CI build would stamp a bare short commit id. The
workflow's `concurrency` block groups runs by workflow and branch
(`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
so a new push cancels the older run on the same branch, queued or running, and
no other: runs for replaced commits do not hold up the shared runner.
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
so a successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
second time. That is the price of the rule above, and it is worth paying: the
image that ships is built from a run of its own gates rather than from a cache
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
shared runner after 20 minutes. That allows for the three Docker builds
described above (the test phase, the lint phase, then the image), each held to
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
limited to `main` by a `branches` list under `on: push` cannot be checked by
review: to try a change to it, add the feature branch to that list and push,
then remove the branch from the list again before merging. Keep any job in it
that publishes behind `if: github.ref_name == 'main'`, so the run from the
feature branch publishes nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for - Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -375,21 +189,14 @@ style conventions are in separate documents:
module under test to verify it compiles/parses. There is no excuse for module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op. `make test` to be a no-op.
- `make test` must complete in under 60 seconds. That is the hard cap, and a - `make test` must complete in under 20 seconds. Add a 30-second timeout in the
suite that exceeds it fails. Under 20 seconds is the target. A suite between Makefile.
20 and 60 seconds is still green, but the overage must be filed as an
improvement bug against that repo. Add a 90-second timeout to the test
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
hard cap so that it catches a genuinely hung test rather than a merely slow
one.
- **The test command should use the conditional verbose rerun pattern.** Run - **`make test` should use the conditional verbose rerun pattern.** Run tests
tests without `-v` (verbose) first. If tests fail, automatically rerun with without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
`-v` to show full output. This keeps CI logs and `docker build` output clean show full output. This keeps CI logs and `docker build` output clean on
on success (just package/suite summaries) while providing full diagnostic success (just package/suite summaries) while providing full diagnostic detail
detail on failure (every test case, every assertion). The command lives in the on failure (every test case, every assertion). The general shell pattern:
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
Makefile form below is the same pattern for any repo-local invocation:
```makefile ```makefile
test: test:
@@ -402,25 +209,11 @@ style conventions are in separate documents:
```makefile ```makefile
test: test:
@go test -count=1 -timeout 90s -race -cover ./... || \ @go test -timeout 30s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 30s -race -v ./...; exit 1; }
``` ```
`-count=1` is required on both invocations: it defeats Go's test _result_
cache, so neither run can report a stored pass in place of running the
tests. It leaves the build cache alone, so it costs the runtime of the suite
and no recompilation.
That cache is Go's own, separate from Docker's layer cache. Go stores a
passing result in its cache directory (`GOCACHE`), and when the same tests
run again on unchanged code it prints that result, marked `(cached)`,
without running them. The directory lasts from one run to the next on a
developer's machine, where this target runs, and in the `test` phase of the
`Dockerfile`, which mounts it to keep compiled packages between check
builds, so the phase's `go test` runs carry `-count=1` too. A new
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
Python example: Python example:
```makefile ```makefile
@@ -446,89 +239,10 @@ style conventions are in separate documents:
must be in `.gitignore`. No exceptions. must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
`node_modules/`, and the repo's own build outputs. Fetch the standard Fetch the standard `.gitignore` from
`.gitignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
a new repo. A repo's `.gitignore` is the standard file followed by the repo's a new repo.
own entries, such as its binaries; a re-vendor replaces the standard part and
keeps those entries. These patterns are written to `.gitignore`'s own
semantics, in which an unanchored pattern already matches at every depth; they
are not a `.dockerignore` and must not be transplanted into one unmodified.
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
across unmodified leaves secrets in the build context.** Docker matches with
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
therefore excludes only the copies at the repository root, while `config/.env`
and `certs/server.key` still reach the context and can land in an image layer
— which is more dangerous than a short file with no secret patterns at all,
because it reads as solved and stops anyone looking. Give every
depth-independent pattern the `**/` prefix and leave only genuinely
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
binary, written `/myapp` and never `**/myapp`, which would also match
`cmd/myapp/` and delete the package directory from the context. Matching is
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
also catches something the build needs, re-include it with a negation
(`!docs/example.env`); deleting the pattern reopens the exposure for every
other file it covers. Fetch the standard `.dockerignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
it with the repo's own artifacts.
- **In-repo agent scratch belongs in both files, written to each file's own
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
additional checkout of the repo — so under `COPY . .` the build context
inflates by a multiple of the repo and another session's unreviewed work can
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
prefix, because the prefixed form would also delete any nested directory of
that name from the build. Anchoring carries a known gap that the canonical
`.dockerignore` states in its own comment, since consuming repos receive the
file and not the tracker: the directory is created in the agent's working
directory, so a repo running agents in subdirectories still ships
`services/api/.claude/` and must add its own anchored entry there.
- **A plain `docker build .` of a clone stamps the version that
`git describe --tags --always` gives**, derived from the `.git` in the build
context as the canonical `Dockerfile` above shows. Without its failure check,
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
and the build would still exit 0. `script/docker` and `script/cibuild` pass
the version they compute on the host; it takes precedence. They do this
byte-identically across repos:
```sh
# The version and the tag each get their own line: a failing command
# substitution inside an argument does not trip `set -e`, so the inline
# form degrades to an empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$(script/projectname)"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \
-t "$tag" .
```
`--always` makes an untagged repo yield an abbreviated commit hash rather
than failing, and the `[ -n "$version" ]` line is the single place the
fallback is applied — a live check that fires on a build from an export with
no `.git` and on a repository with no commits yet. Do not fold it into the
substitution as `|| echo unknown`, which makes the guard unreachable. The
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
the scripts stay byte-identical. One consequence for CI: the standard
checkout action clones shallow and fetches no tags, so the canonical
`.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step.
- **Verify `.dockerignore` by enumerating the image, not by reading the
patterns.** Plant files at the root _and_ at least two directories deep, build
a probe image that does `COPY . .`, and list what actually landed
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
size is not a substitute: a nested secret is a few bytes, and BuildKit
transfers only the delta from the previous build.
- **No build artifacts in version control.** Code-derived data (compiled - **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the bundles, minified output, generated assets) must never be committed to the
@@ -544,61 +258,9 @@ style conventions are in separate documents:
- Make all changes on a feature branch. You can do whatever you want on a - Make all changes on a feature branch. You can do whatever you want on a
feature branch. feature branch.
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must - `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
_NEVER_ be modified by an agent: fetch it from manually by the user. Fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
byte-identical, so that no repo can quietly loosen its own linting. Linter
configuration changes are made to the canonical copy in the `prompts` repo and
reach consuming repos by re-vendoring; an agent may open a PR against
canonical, which only the user merges. One list is exempt from byte-identity,
because it cannot be written once for every repo: the `deny` list of the
`test-support` depguard rule, where a repo names its own test-support packages
by full import path. A repo adds entries there and changes nothing else, and a
re-vendor carries its entries forward. The canonical golangci-lint version is
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
image
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
directive must not name a newer Go minor version than the one golangci-lint
was built with, or golangci-lint refuses to lint it: this release lints
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
installs golangci-lint on the host. A repo sets the lint phase digest to the
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
the two prompted the change: the canonical copy can name linters that an older
golangci-lint rejects, and a newer golangci-lint can add linters that
`default: all` switches on until the canonical copy disables them.
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
`PATH` only, so on an already-provisioned machine the pin is inert and a
version bump is a silent no-op — while the Dockerfile, installing into a clean
image, gets the pinned version, so a local `make check` and `make docker` can
disagree about what the tool even is. The canonical form:
- compares the installed version against the pin over the **whole** version
token; a parser that stops at the first `-` reports `2.12.2` for a host
running `2.12.2-rc1` and skips the install;
- treats absent, non-zero, empty or unrecognised `--version` output as a
mismatch, so the failure direction is a redundant install and never a
skipped one;
- after installing, re-resolves the binary the way callers do — `hash -r`,
then through `PATH`, not through the directory the installer wrote to —
and fails naming the resolved path, since an install that a shadowing
binary hides succeeds while changing nothing any caller sees;
- is actually called, and prints the version on both success paths: a
function defined and never invoked has the same exit status and the same
empty output as one that worked.
Node is the exception to the whole-token comparison: the canonical
`script/bootstrap` compares only its major version, because the node 22
alpine image the `Dockerfile` stages start from is not the exact pin, and
nvm cannot install a prebuilt node on alpine.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
a `go.mod` tool dependency or through a `tools.go` file, either of which
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
- When pinning images or packages by hash, add a comment above the reference - When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD). with the version and date (YYYY-MM-DD).
@@ -709,21 +371,15 @@ style conventions are in separate documents:
Never edit existing migrations after release. Never edit existing migrations after release.
- All repos should have an `.editorconfig` enforcing the project's indentation - All repos should have an `.editorconfig` enforcing the project's indentation
settings: the standard file from settings.
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
tabs for `Makefile` and Go files, followed by the repo's own sections, such as
one for another language it uses. A re-vendor replaces the standard part and
keeps those sections.
- Avoid putting files in the repo root unless necessary. Root should contain - Avoid putting files in the repo root unless necessary. Root should contain
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
and language-specific config). Everything else goes in a subdirectory. language-specific config). Everything else goes in a subdirectory. Canonical
Canonical subdirectory names: subdirectory names:
- `bin/` — executable scripts and tools - `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose - `cmd/` — Go command entrypoints
body is a single call into `internal/` or `pkg/`, no project logic in
`cmd/`
- `configs/` — configuration templates and examples - `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform) - `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root) - `docs/` — documentation and markdown (README.md stays in root)
@@ -750,7 +406,3 @@ style conventions are in separate documents:
- Go: `go.mod`, `go.sum`, `.golangci.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml` - Python: `pyproject.toml`
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
is never committed under a file or directory named after one agent tool, such
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
+97 -123
View File
@@ -29,79 +29,9 @@
# Completed Steps # Completed Steps
- new subcommand `info` prints the database's details on stdout, one per line as
a name, a tab and the value: its path and where it came from, the sizes of the
database file and of its `-wal` file when there is one, when the database file
was last changed, the schema version, the number of records, how many have
head and tail hashes and how many a content hash, and the sizes of all records
added up. It opens the database as `report` does, and shows two phases on
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
from one read-only transaction, the counts from one query (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/117)
- `report` shows its work on stderr in two phases, `[1/2] open` and
`[2/2] read`, with the progress display `scan` and `trees` use, from the
moment it starts until the summary line: `open` says it is opening the
database, also while it waits on a busy one; `read` counts the records, then
reads them against that count in signature order, which SQLite takes from the
`files_signature` index without sorting, so the first record comes at once.
`report` forms the duplicate groups itself, holding only those of the size it
is reading, and writes them ordered by first path before it reads the next
size. stdout and the summary line are unchanged, and SQLite no longer needs a
temporary file to sort in (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/114)
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
from the moment it starts until the summary line: `open` says it is opening
the database, also while it waits on a busy one; `read` counts the records,
then reads them against that count; `group` counts the directories it groups,
and `write` the groups it writes. A display's rate names what its phase
counts: files, records, directories or groups. stdout and the summary line are
unchanged. When stdout and stderr are the same file, as one terminal is, none
of stdout's lines shares a line with the display; when they are not, the
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/113)
- each `scan` phase's progress display names the phase and its place among the
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
time and ETA: `load` counts the records first, the walk's total is estimated
from the records under the operands, and the content phase counts the records
it checks from SQLite's grouping alone, then says it is searching the database
until the first one is returned. On a terminal a phase with an exact total
draws its line followed by the library's bar, and any other a spinner followed
by the line; every display is drawn again several times a second. No line
reaches the terminal's last column, however narrow, also after the terminal is
made narrower during a phase: a line too wide leaves out its rate, then its
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
only when those alone do not fit, as below about 70 columns late in a large
scan. In plain mode the line is printed every 5 seconds whether or not an item
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
- `scan`, `report` and `trees` first print a line on stderr naming the database
they use, as an absolute path, and whether that path is the default or comes
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
"Database" says when `scan` writes each file (2026-10-16,
https://git.eeqj.de/sneak/sfdupes/issues/111)
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
that write no image, and `make test` runs the suite under the race detector,
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
every build, while the steps that compile keep Go's build cache and the tests
run with `-count=1`; prettier runs on the host, from the node and yarn
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
and the build stage installs `git` and `make` itself; a new push cancels the
workflow's older run on the same branch, and a run stops after 20 minutes;
`.claude/settings.json` is deleted (2026-10-15,
https://git.eeqj.de/sneak/sfdupes/issues/95)
- with `-x`, an operand under another operand is walked on its own instead of - with `-x`, an operand under another operand is walked on its own instead of
being dropped when a directory from it (from its directory, for a regular being dropped when it is a directory on a different filesystem than that
file) up to but not including that operand is on a different filesystem than operand, or a regular file in such a directory, and the records of files gone
that operand, whose walk then cannot reach it, and the records of files gone
from below it are deleted (2026-10-14, from below it are deleted (2026-10-14,
https://git.eeqj.de/sneak/sfdupes/issues/106) https://git.eeqj.de/sneak/sfdupes/issues/106)
@@ -129,26 +59,19 @@
https://git.eeqj.de/sneak/sfdupes/issues/12) https://git.eeqj.de/sneak/sfdupes/issues/12)
- cut the narration from `TODO.md` Completed Steps and from the comments in - cut the narration from `TODO.md` Completed Steps and from the comments in
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since `script/` and both Dockerfiles; §Workflow now branches from and merges to
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
- `make test-race` ran the test suite under the race detector in a cgo-enabled - `make test-race` runs the test suite under the race detector in a cgo-enabled
container, outside `make check` (2026-10-04, container, outside `make check` (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/18). Since https://git.eeqj.de/sneak/sfdupes/issues/18)
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
`make test-race` is gone
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`, - a bare `docker build .` fails with a message naming `script/cibuild` and
rather than serve the gates from cache (2026-10-04, `script/docker` instead of serving the gates from cache (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/39). Since https://git.eeqj.de/sneak/sfdupes/issues/39)
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
`script/` pass a new `CACHEBUST` value, so theirs always run
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks - `make fmt` and `make fmt-check` run prettier over all Markdown, in Docker, and
it; all Markdown reformatted (2026-10-04, CI checks it; all Markdown reformatted (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/19) https://git.eeqj.de/sneak/sfdupes/issues/19)
- `script/lint` writes no image, so a run no longer leaves an untagged one - `script/lint` writes no image, so a run no longer leaves an untagged one
@@ -191,12 +114,9 @@
- README documents install, Docker, a daily cron scan and how to read and check - README documents install, Docker, a daily cron scan and how to read and check
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home - the `Dockerfile` build stage keeps the Go module cache out of `builder`'s home
and copied the sources with `--chown`, so no `chown -R` walked them and copies the sources with `--chown`, so no `chown -R` walks them
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43)
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
nothing changes owner: the build stage only compiles, as root, and the tests
run as `nobody` in the `test` phase
- `--version` prints `sfdupes VERSION` to stdout; README documents it and - `--version` prints `sfdupes VERSION` to stdout; README documents it and
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
@@ -260,46 +180,97 @@
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and - fix the lint-image pin comments and `FROM` form in `Dockerfile` and
`Dockerfile.lint` (2026-08-10, branch `next`, closes `Dockerfile.lint` (2026-08-10, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`, `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
without the false `(Debian-based)` note or the tag. Since without the false `(Debian-based)` note or the tag; digest unchanged.
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase `script/verify-lint-image-pin` still matches the tagless form, and a tag on
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and one side only is caught as a plain mismatch.
`script/verify-lint-image-pin`, which compared the two pins, are gone.
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, - run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
owner ruling the linter is never installed on a host, and owner ruling the linter is never installed on a host. `Dockerfile.lint` copies
`golangci-lint config verify` runs before `golangci-lint run`. the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and
`script/bootstrap` stopped installing or pinning the linter, and runs `golangci-lint config verify` and `golangci-lint run` as build steps;
`script/verify-linter-pin` was retired. Since `script/lint` builds it. `script/bootstrap` no longer installs or pins the
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the linter, and warns rather than fails when `docker` is absent;
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build `ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`.
stage depends on through `COPY --from=lint /src/go.sum /dev/null`; `script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside in both files, compares their two `FROM` lines and restates neither pin.
an image build may run docker, so the phase calls `golangci-lint` directly. Traps: an unchanged tree lets a lint build pass in under a second having run
no linter, so every gate `RUN` references `ARG CHECK_EPOCH` (BuildKit hashes
the expanded command) and `script/lint` passes `"$(date +%s)-$$"`, the PID
because two runs land in the same second easily. Nothing inside an image build
may shell out to docker, so the `Dockerfile` lint stage calls `golangci-lint`
directly and the build stage runs `make test` and `make fmt-check` instead of
`make check`, through `make` because the Makefile's `export CGO_ENABLED = 0`
only reaches what it invokes. `COPY --from=lint /src/go.sum /dev/null`
replaces the copied linter binary as the only edge making the build stage wait
for lint; dropping it would end fail-fast linting under a still-green build.
`golangci-lint config verify`, included per the ruling, validates from an
embedded schema with no network call, but `go mod download` above the gates
still needs the network on a cold cache. Verified: `make lint` green with no
`golangci-lint` on `PATH`; two back-to-back `script/lint` runs on an untouched
tree both ran the linter (27.7s and 28.7s in the lint step, `COPY . .`
`CACHED` above); a planted unused variable failed `script/lint`, and failed
`make docker` at `[lint 9/9]` with the build stage stopped at
`[builder 3/12]`; the drift guard fails on a tag-only, a digest-only and an
unreadable reference, naming both sides; under `--network none` config verify
passes a valid config and rejects an invalid one; `make docker` green in 5m35s
with all six gates run under one epoch (lint 37.6s, test 25.2s reporting
`ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`); in the
builder image with the Go test cache off, `--user 0:0` still fails
`TestScanHardlinkRunFailsTogether` where the unprivileged user passes. Noted
for follow-up, not fixed here: `golangci-lint` warns that `gomodguard` is
deprecated since v2.12.0 in favour of `gomodguard_v2`.
- install the Docker build stage's prerequisites by running `script/bootstrap` - install the Docker build stage's prerequisites by running `script/bootstrap`
instead of `apk add --no-cache make` inline (2026-08-09, branch instead of `apk add --no-cache make` inline (2026-08-09, branch
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42), `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42):
with `script/verify-linter-pin` failing the build unless the linter copied the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
from the lint stage was the version `script/bootstrap` pinned. Builds then which ends in `go mod download`, so the separate call to it is gone.
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as `COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap
https://git.eeqj.de/sneak/sfdupes/issues/43. Since layer: it is the only edge making this stage depend on the lint stage, so
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and deleting it would end fail-fast linting silently. A new
`make` with `apk add --no-cache` and only compiles; the `lint` and `test` `script/verify-linter-pin`, run in the build stage before bootstrap, fails the
phases are the gates build naming both versions unless that copied binary is the version
`script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip.
`$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. Everything
added sits above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still
precede `make check`. Verified: the guard fails the build with both versions
named when the lint stage's linter is faked to another version, and passes an
unmodified build; bootstrap runs clean under Alpine's `sh` and `apk`, finding
the copied linter already at the pin; a second build served the bootstrap and
dependency layers `CACHED` while both gates ran with a fresh epoch; a planted
`unused` finding failed the build at the lint gate in 48.9s with the build
stage's `make check` never starting; and the suite run in the image as
`--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the drop to the
unprivileged user is still needed. That last check needs the Go test cache
off: as root it first reported `ok ... (cached)`, reusing the build-time
result. Build times on a noisy shared host: 2m13s on an unchanged tree, 2m17s
and 4m29s after a source change, 5m14s cold, which breaches the policy
ceiling; `chown -R builder:builder /src /home/builder` walks the module cache
and alone varied from 77s to 210s across those builds, and `main` measured
5m03s cold with a 209s `chown`. Filed as
https://git.eeqj.de/sneak/sfdupes/issues/43
- bust the Docker layer cache for the gate steps, so `script/cibuild` and - bust the Docker layer cache for the gate steps, so `script/cibuild` and
`script/docker` cannot report a green they did not earn (2026-08-09, branch `script/docker` cannot report a green they did not earn (2026-08-09, branch
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the `cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker `Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
served them from cache and the build exited 0 having run nothing. The fix was served them from cache and the build exited 0 having run nothing. Both scripts
a `CHECK_EPOCH` build argument; since now pass `--build-arg CHECK_EPOCH="$(date +%s)"`. `ARG` is per stage and the
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/` gates span two stages, so it is declared in both; BuildKit hashes the expanded
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile` command, so each gate `RUN` echoes the epoch, which also logs it as evidence
declares first. Run as root, the tests fail the layer ran. It sits below the dependency layers so they stay cached.
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)` Verified under `BUILDKIT_PROGRESS=plain`, each script run twice back to back
the test relies on, so the `test` phase runs them as `nobody` on an unchanged tree: all three gates ran on all four runs with a fresh epoch
(`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s then 53.4s), and
thirteen steps were still served `CACHED`. With a planted `unused` finding the
build failed at `make lint` in 36.1s and the build-stage `make check` never
started. Run as root, the same image fails `TestScanHardlinkRunFailsTogether`,
because root reads through the `chmod(0)` the test relies on, so the build
stage must drop to the unprivileged `builder` user. Local fix only;
propagating it to the canonical templates is
https://git.eeqj.de/sneak/prompts/issues/26
- check the installed golangci-lint version in `script/bootstrap` instead of - check the installed golangci-lint version in `script/bootstrap` instead of
only its presence (2026-08-09, branch `bootstrap-version-check`, closes only its presence (2026-08-09, branch `bootstrap-version-check`, closes
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
@@ -455,3 +426,6 @@ Accepted divergences (no action):
- flat single-package layout with `.go` files in the repo root — fine for a - flat single-package layout with `.go` files in the repo root — fine for a
small single-binary tool per the Go styleguide; the tracker audit agrees small single-binary tool per the Go styleguide; the tracker audit agrees
- `make test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go
builds) and the race detector requires cgo, so the detector runs in a separate
cgo-enabled container, `make test-race`, which is not part of `make check`
+4 -6
View File
@@ -344,10 +344,9 @@ func storedPaths(t *testing.T, path string) []string {
// TestRunScanInterrupted calls the scan entrypoint with a context that // TestRunScanInterrupted calls the scan entrypoint with a context that
// is already cancelled, as when a signal arrives at once. It must return // is already cancelled, as when a signal arrives at once. It must return
// errInterrupted promptly, with its line on stderr after the one naming // errInterrupted promptly with its one line on stderr and nothing on
// the database and nothing on stdout, leave the database valid and as // stdout, leave the database valid and as it was, and leave nothing in
// it was, and leave nothing in the way of the next scan, which must // the way of the next scan, which must bring the database up to date.
// bring the database up to date.
func TestRunScanInterrupted(t *testing.T) { func TestRunScanInterrupted(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
@@ -390,8 +389,7 @@ func TestRunScanInterrupted(t *testing.T) {
err, errInterrupted) err, errInterrupted)
} }
want := "scan: database " + path + " (from " + databaseEnv + ")\n" + want := "scan: interrupted after 0 files\n"
"scan: interrupted after 0 files\n"
if got := stderr()[shown:]; got != want { if got := stderr()[shown:]; got != want {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
+68 -198
View File
@@ -11,7 +11,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"strconv" "strconv"
"strings"
"time" "time"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
@@ -61,8 +60,8 @@ CREATE TABLE files (
) WITHOUT ROWID ) WITHOUT ROWID
` `
// createIndexSQL indexes the records by signature, so report can read // createIndexSQL indexes the records by signature, so report can have
// them in signature order without sorting the table. // SQLite group them without sorting the whole table.
const createIndexSQL = ` const createIndexSQL = `
CREATE INDEX files_signature ON files (size, head, tail, content) CREATE INDEX files_signature ON files (size, head, tail, content)
` `
@@ -79,8 +78,7 @@ ON CONFLICT (path) DO UPDATE SET
content = excluded.content content = excluded.content
` `
// errNoDatabase reports a missing database file for report, trees and // errNoDatabase reports a missing database file for report/trees.
// info.
var errNoDatabase = errors.New( var errNoDatabase = errors.New(
"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")") "no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
@@ -93,36 +91,13 @@ var errSchemaVersion = errors.New("unsupported database schema version")
var errScanRunning = errors.New("another scan is running") var errScanRunning = errors.New("another scan is running")
// databasePath resolves the database location: SFDUPES_DATABASE when // databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the // set and non-empty, the compiled-in default otherwise.
// path, with the working directory, its symlinks resolved, put in front func databasePath() string {
// of it when it is relative, and where the path came from, "default" or
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
// stderr.
func databasePath() (string, string, error) {
path, source := defaultDatabasePath, "default"
if p := os.Getenv(databaseEnv); p != "" { if p := os.Getenv(databaseEnv); p != "" {
path, source = p, "from "+databaseEnv return p
} }
if filepath.IsAbs(path) { return defaultDatabasePath
return path, source, nil
}
// The working directory as the kernel reports it, with no symlink
// in it. os.Getwd may return PWD instead, which can name it through
// a symlink; a ".." after that symlink then leads somewhere else
// once the path is cleaned, as scan cleans it to create the
// database's directory.
wd, err := unix.Getwd()
if err != nil {
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
}
// Not filepath.Abs or filepath.Join: they clean the path, which
// drops each ".." together with the directory name before it. When
// that directory is a symlink, the cleaned path names another file.
// The working directory ends in "/" only when it is the root.
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
} }
// scanParams are the connection parameters for scan: read-write, with // scanParams are the connection parameters for scan: read-write, with
@@ -132,9 +107,9 @@ const scanParams = "_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(WAL)" + "&_pragma=journal_mode(WAL)" +
"&_pragma=synchronous(NORMAL)" "&_pragma=synchronous(NORMAL)"
// reportParams are the connection parameters for report, trees and // reportParams are the connection parameters for report and trees:
// info: read-only, with the same busy timeout. They set no journal // read-only, with the same busy timeout. They set no journal mode,
// mode, because setting one is a write. // because setting one is a write.
const reportParams = "mode=ro" + const reportParams = "mode=ro" +
"&_pragma=busy_timeout(10000)" + "&_pragma=busy_timeout(10000)" +
"&_pragma=query_only(1)" "&_pragma=query_only(1)"
@@ -263,9 +238,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) {
_ = db.Close() _ = db.Close()
} }
// openReportDatabase opens an existing database for the report, trees // openReportDatabase opens an existing database for the report and
// and info subcommands. A missing database file is an error directing // trees subcommands. A missing database file is an error directing the
// the user to run scan first; the schema version must match exactly. // user to run scan first; the schema version must match exactly.
func openReportDatabase(ctx context.Context, func openReportDatabase(ctx context.Context,
path string, path string,
) (*sql.DB, error) { ) (*sql.DB, error) {
@@ -404,32 +379,10 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
return v, nil return v, nil
} }
// loadFileRows gives total the number of records, then streams every // loadFileRows streams every record to fn in path order: byte order,
// record to fn in path order: byte order, which is the order of the // which is the order of the primary key, so SQLite does not sort.
// primary key, so SQLite does not sort. The count and the records are func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
// read in one transaction, so they agree while a scan is committing. rows, err := db.QueryContext(ctx,
func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
fn func(r scanRec),
) error {
// Everything goes through tx: db has one connection, so a query on
// db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = tx.Rollback() }()
var n int64
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return fmt.Errorf("count records: %w", err)
}
total(n)
rows, err := tx.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+ "SELECT path, size, mtime, mtime_nsec, head, tail, content "+
"FROM files ORDER BY path") "FROM files ORDER BY path")
if err != nil { if err != nil {
@@ -464,140 +417,77 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
return nil return nil
} }
// signatureRowsSQL selects every record's signature and path in // dupeRowsSQL selects every record in a duplicate group, with the
// signature order, largest size first. That is the order of the // group's first path. A group is the records with a content hash that
// files_signature index read backwards, so SQLite reads the index alone // share a size, head, tail, and content, when there are two or more of
// and returns the first row at once, with nothing to sort. // them. The rows come in report order: groups by size descending, then
const signatureRowsSQL = ` // by first path, and each group's paths ascending.
SELECT size, head, tail, content, path const dupeRowsSQL = `
FROM files SELECT g.first, f.path, f.size
ORDER BY size DESC, head DESC, tail DESC, content DESC FROM files AS f
JOIN (
SELECT size, head, tail, content, MIN(path) AS first
FROM files
WHERE content <> ''
GROUP BY size, head, tail, content
HAVING COUNT(*) > 1
) AS g USING (size, head, tail, content)
ORDER BY f.size DESC, g.first, f.path
` `
// loadSignatureRows gives total the number of records, then streams // loadDupeRows streams the rows of dupeRowsSQL to fn and returns the
// every record's signature and path to fn, without its mtime, in the // number of records in the database. The count and the rows are read
// order of signatureRowsSQL. The count and the records are read in one // in one transaction, so they agree while a scan is committing. An
// transaction, so they agree while a scan is committing. An error from // error from fn stops the reading and is returned as it is.
// fn stops the reading and is returned as it is. func loadDupeRows(ctx context.Context, db *sql.DB,
func loadSignatureRows(ctx context.Context, db *sql.DB, fn func(first, path string, size int64) error,
total func(n int64), fn func(r scanRec) error, ) (int, error) {
) error {
// Everything goes through tx: the report connection is the only // Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever. // one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
defer func() { _ = tx.Rollback() }() defer func() { _ = tx.Rollback() }()
var n int64 var records int
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n) err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records)
if err != nil { if err != nil {
return fmt.Errorf("count records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
total(n) rows, err := tx.QueryContext(ctx, dupeRowsSQL)
rows, err := tx.QueryContext(ctx, signatureRowsSQL)
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() { _ = rows.Close() }()
for rows.Next() { for rows.Next() {
var ( var (
path []byte first, path []byte
r scanRec size int64
) )
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path) err = rows.Scan(&first, &path, &size)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return 0, fmt.Errorf("read record: %w", err)
} }
r.path = string(path) err = fn(string(first), string(path), size)
err = fn(r)
if err != nil { if err != nil {
return err return 0, err
} }
} }
err = rows.Err() err = rows.Err()
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return 0, fmt.Errorf("read records: %w", err)
} }
return nil return records, nil
}
// countRecords returns the number of records in the database.
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
if err != nil {
return 0, fmt.Errorf("count records: %w", err)
}
return n, nil
}
// databaseCountsSQL counts the records, those with head and tail
// hashes and those with a content hash, and adds up their sizes, in one
// pass over the table. A SUM over no rows is NULL, so on an empty table
// each SUM falls back to 0.
const databaseCountsSQL = `
SELECT COUNT(*),
COALESCE(SUM(head <> '' AND tail <> ''), 0),
COALESCE(SUM(content <> ''), 0),
COALESCE(SUM(size), 0)
FROM files
`
// databaseCounts is what info reads from the database: its schema
// version and the counts of databaseCountsSQL.
type databaseCounts struct {
schemaVersion int
records int64 // every record
headTail int64 // records with head and tail hashes
content int64 // records with a content hash
size int64 // the sizes of all records added up
}
// readDatabaseCounts reads the schema version, then the counts of
// databaseCountsSQL, in one read-only transaction, so both describe the
// database at one moment while a scan is committing.
func readDatabaseCounts(ctx context.Context,
db *sql.DB,
) (databaseCounts, error) {
var c databaseCounts
// Everything goes through tx: the report connection is the only
// one, so a query on db would wait for tx forever.
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return c, fmt.Errorf("count records: %w", err)
}
defer func() { _ = tx.Rollback() }()
err = tx.QueryRowContext(ctx, "PRAGMA user_version").
Scan(&c.schemaVersion)
if err != nil {
return c, fmt.Errorf("read schema version: %w", err)
}
err = tx.QueryRowContext(ctx, databaseCountsSQL).
Scan(&c.records, &c.headTail, &c.content, &c.size)
if err != nil {
return c, fmt.Errorf("count records: %w", err)
}
return c, nil
} }
// loadFileMeta streams every record's path, size, mtime, and whether // loadFileMeta streams every record's path, size, mtime, and whether
@@ -638,46 +528,26 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
return nil return nil
} }
// contentGroupsSQL selects each group (the records sharing a size, // contentCandidatesSQL selects every record of at least headTailMin
// head, and tail) of records of at least headTailMin bytes that has more // bytes whose size, head, and tail equal another record's, in each
// than one record and at least one record without a content hash, with // group (the records sharing a size, head, and tail) where at least one
// the number of its records. // record has no content hash, with whether each record has one. SQLite
const contentGroupsSQL = ` // does the grouping, so no other record's hashes are loaded into
SELECT size, head, tail, COUNT(*) AS records // memory; the rows come ordered by size, head, and tail, so each
FROM files // group's rows arrive together.
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
`
// contentCandidatesSQL selects every record of the groups
// contentGroupsSQL selects, with whether each record has a content
// hash. SQLite does the grouping, so no other record's hashes are
// loaded into memory; the rows come ordered by size, head, and tail, so
// each group's rows arrive together.
const contentCandidatesSQL = ` const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> '' SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
FROM files AS f FROM files AS f
JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail) JOIN (
SELECT size, head, tail
FROM files
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
) AS g USING (size, head, tail)
ORDER BY size, head, tail ORDER BY size, head, tail
` `
// countContentCandidates returns the number of rows of
// contentCandidatesSQL, adding up the records of the groups
// contentGroupsSQL selects, so it neither joins nor sorts.
func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
err := db.QueryRowContext(ctx,
"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
headTailMin).Scan(&n)
if err != nil {
return 0, fmt.Errorf("count content candidates: %w", err)
}
return n, nil
}
// loadContentCandidates streams the rows of contentCandidatesSQL to fn: // loadContentCandidates streams the rows of contentCandidatesSQL to fn:
// each record, without its content hash, and whether it has one. // each record, without its content hash, and whether it has one.
func loadContentCandidates(ctx context.Context, db *sql.DB, func loadContentCandidates(ctx context.Context, db *sql.DB,
+15 -224
View File
@@ -43,47 +43,16 @@ func openTestDB(t *testing.T) *sql.DB {
} }
func TestDatabasePath(t *testing.T) { func TestDatabasePath(t *testing.T) {
// README §Database: SFDUPES_DATABASE when set and non-empty, the t.Setenv(databaseEnv, "")
// default otherwise, with the working directory, its symlinks
// resolved, put in front of a relative path. Nothing is cleaned, so if got := databasePath(); got != defaultDatabasePath {
// a ".." stays. t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
} }
t.Chdir(dir) t.Setenv(databaseEnv, "/custom/place.sqlite")
fromEnv := "from " + databaseEnv if got := databasePath(); got != "/custom/place.sqlite" {
cases := []struct { t.Errorf("databasePath() = %q, want the env override", got)
env, path, source string
}{
{"", defaultDatabasePath, "default"},
{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
fromEnv},
{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
}
for _, tc := range cases {
t.Setenv(databaseEnv, tc.env)
path, source, err := databasePath()
if err != nil || path != tc.path || source != tc.source {
t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
databaseEnv, tc.env, path, source, err, tc.path, tc.source)
}
}
// In the root directory the path gets one slash in front, not two.
t.Chdir("/")
t.Setenv(databaseEnv, "place.sqlite")
path, _, err := databasePath()
if err != nil || path != "/place.sqlite" {
t.Errorf("databasePath() in / = %q, %v; want %q",
path, err, "/place.sqlite")
} }
} }
@@ -284,128 +253,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
defer func() { _ = reportDB.Close() }() defer func() { _ = reportDB.Close() }()
err = loadFileRows(t.Context(), reportDB, func(int64) {}, err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
func(scanRec) {})
if err != nil { if err != nil {
t.Fatalf("loadFileRows: %v", err) t.Fatalf("loadFileRows: %v", err)
} }
} }
// TestLoadRowsCountAgreeDuringScan has a scan commit records after
// loadFileRows, as trees calls it, or loadSignatureRows, as report
// calls it, has counted the records and before it reads them, as when
// trees or report runs during a scan: each must read exactly as many
// records as it counted.
func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
t.Parallel()
loads := map[string]func(context.Context, *sql.DB, func(int64),
func(scanRec)) error{
"loadFileRows": loadFileRows,
"loadSignatureRows": func(ctx context.Context, db *sql.DB,
total func(int64), fn func(scanRec),
) error {
return loadSignatureRows(ctx, db, total, func(r scanRec) error {
fn(r)
return nil
})
},
}
for name, load := range loads {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := testDBPath(t)
scanDB, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = scanDB.Close() }()
recs := smokeTreeRecs()
err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
if err != nil {
t.Fatal(err)
}
reportDB, err := openReportDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = reportDB.Close() }()
var counted, read int64
err = load(t.Context(), reportDB, func(n int64) {
counted = n
err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
if err != nil {
t.Error(err)
}
}, func(scanRec) { read++ })
if err != nil {
t.Fatal(err)
}
if counted != 3 || read != counted {
t.Errorf("counted %d records and read %d, want 3 and 3",
counted, read)
}
})
}
}
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
t.Parallel()
db := openTestDB(t)
rows, err := db.QueryContext(t.Context(),
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
if err != nil {
t.Fatal(err)
}
defer func() { _ = rows.Close() }()
var plan []string
for rows.Next() {
var (
id, parent, unused int
detail string
)
err = rows.Scan(&id, &parent, &unused, &detail)
if err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
want := []string{"SCAN files USING COVERING INDEX files_signature"}
if !slices.Equal(plan, want) {
t.Errorf("query plan = %q, want %q", plan, want)
}
}
func TestApplyChangesRoundTrip(t *testing.T) { func TestApplyChangesRoundTrip(t *testing.T) {
t.Parallel() t.Parallel()
@@ -425,10 +278,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
}, },
} }
prog := newProgress(progressParams{label: "update", total: 4}) err := applyChanges(t.Context(), db, recs, nil,
defer prog.finish() newProgress("update", 2))
err := applyChanges(t.Context(), db, recs, nil, prog)
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -447,7 +298,7 @@ func TestApplyChangesRoundTrip(t *testing.T) {
} }
err = applyChanges(t.Context(), db, []scanRec{upd}, err = applyChanges(t.Context(), db, []scanRec{upd},
[]string{"/a/tab\tnew\nline"}, prog) []string{"/a/tab\tnew\nline"}, newProgress("update", 2))
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -475,10 +326,8 @@ func TestApplyChangesBatching(t *testing.T) {
}) })
} }
prog := newProgress(progressParams{label: "update", total: int64(2 * n)}) err := applyChanges(t.Context(), db, recs, nil,
defer prog.finish() newProgress("update", int64(n)))
err := applyChanges(t.Context(), db, recs, nil, prog)
if err != nil { if err != nil {
t.Fatalf("applyChanges: %v", err) t.Fatalf("applyChanges: %v", err)
} }
@@ -492,7 +341,8 @@ func TestApplyChangesBatching(t *testing.T) {
deletes = append(deletes, r.path) deletes = append(deletes, r.path)
} }
err = applyChanges(t.Context(), db, nil, deletes, prog) err = applyChanges(t.Context(), db, nil, deletes,
newProgress("update", int64(n)))
if err != nil { if err != nil {
t.Fatalf("applyChanges deletes: %v", err) t.Fatalf("applyChanges deletes: %v", err)
} }
@@ -501,62 +351,3 @@ func TestApplyChangesBatching(t *testing.T) {
t.Fatalf("records = %d, want 0", len(got)) t.Fatalf("records = %d, want 0", len(got))
} }
} }
// TestCountContentCandidates checks that countContentCandidates counts
// the records the content phase's search returns: those of each group
// of two or more records of headTailMin or more with a head, where at
// least one has no content hash.
func TestCountContentCandidates(t *testing.T) {
t.Parallel()
db := openTestDB(t)
// Each group's records: size, head, and content, one per record.
groups := []struct {
size int64
head string
contents []string
}{
{headTailMin, "h", []string{"", "c", "c"}}, // returned
{headTailMin + 1, "h", []string{"", ""}}, // returned
{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
{headTailMin + 3, "h", []string{""}}, // one record
{headTailMin - 1, "h", []string{"", ""}}, // too small
{headTailMin + 4, "", []string{"", ""}}, // no head
}
var recs []scanRec
for _, g := range groups {
for _, content := range g.contents {
recs = append(recs, scanRec{
size: g.size, mtime: time.Unix(1, 0), head: g.head,
tail: "t", content: content,
path: fmt.Sprintf("/r/%d", len(recs)),
})
}
}
err := applyChanges(t.Context(), db, recs, nil, nil)
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
got, err := countContentCandidates(t.Context(), db)
if err != nil {
t.Fatalf("countContentCandidates: %v", err)
}
var returned int64
err = loadContentCandidates(t.Context(), db,
func(scanRec, bool) { returned++ })
if err != nil {
t.Fatalf("loadContentCandidates: %v", err)
}
if got != 5 || returned != 5 {
t.Errorf("count = %d, records returned = %d, want 5 and 5",
got, returned)
}
}
-128
View File
@@ -1,128 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
// infoPhase names one phase of info with its place among all of them,
// as "[2/2] count", for the phase's progress display. The list here is
// the one place the phases are counted.
func infoPhase(name string) string {
return phaseLabel([]string{openPhase, "count"}, name)
}
// runInfo implements the info subcommand: it prints the database's
// details on stdout (see writeInfo). It opens the database as report
// does, read-only and without the scan lock, and never touches the
// scanned filesystem. It names the database on stderr before it opens
// it. Any database problem, including a missing database, is fatal.
//
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and count (count the records,
// with one query). Each display ends before the next phase starts, and
// before an error is returned to be printed. The details are written
// once the last display has ended, so no display is drawn while they
// are written.
func runInfo(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "info: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: infoPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil {
return err
}
defer func() { _ = db.Close() }()
// One query counts everything, so the phase never has a total.
prog = newProgress(progressParams{
label: infoPhase("count"), total: -1,
waitingFor: countingRecords,
})
counts, err := readDatabaseCounts(ctx, db)
prog.finish()
if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err)
}
return writeInfo(stdout, dbPath, source, counts)
}
// writeInfo writes the details of the database at dbPath to stdout,
// one per line, as a name, a tab and the value (README "`info` mode"):
// the path and source as databasePath returns them, the sizes of the
// database file and of its -wal file when there is one, when the
// database file was last changed, and the schema version and record
// counts that readDatabaseCounts read.
func writeInfo(stdout io.Writer, dbPath, source string,
counts databaseCounts,
) error {
// SQLite keeps the -wal file beside the database file, named from
// its path with symlinks resolved.
file, err := filepath.EvalSymlinks(dbPath)
if err != nil {
return fmt.Errorf("database: %w", err)
}
dbFile, err := os.Stat(file)
if err != nil {
return fmt.Errorf("database: %w", err)
}
walFile, err := os.Stat(file + "-wal")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("database: %w", err)
}
var b strings.Builder
fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source)
fmt.Fprintf(&b, "database file size\t%d (%s)\n",
dbFile.Size(), humanBytes(dbFile.Size()))
if walFile != nil {
fmt.Fprintf(&b, "-wal file size\t%d (%s)\n",
walFile.Size(), humanBytes(walFile.Size()))
}
fmt.Fprintf(&b, "modified\t%s\n",
dbFile.ModTime().UTC().Format(time.RFC3339))
fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
fmt.Fprintf(&b, "records\t%d\n", counts.records)
fmt.Fprintf(&b, "records with head and tail hashes\t%d\n",
counts.headTail)
fmt.Fprintf(&b, "records with content hash\t%d\n", counts.content)
fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
counts.size, humanBytes(counts.size))
_, err = io.WriteString(stdout, b.String())
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
return nil
}
-168
View File
@@ -1,168 +0,0 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// infoOpenLine is the plain line of info's open phase when it opens the
// database at once; it prints it as the phase starts and as it ends.
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"eta unknown"
// infoCountLine is the plain line of info's count phase when the count
// takes under a second; it prints it as the phase starts and as it
// ends.
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
"eta unknown"
// infoRecs are four records: a duplicate pair with every hash, a file
// of 20 MiB with head and tail hashes but no content hash yet, and a
// file never hashed.
func infoRecs() []scanRec {
return []scanRec{
{size: 5, head: "h", tail: "t", content: "c", path: "/d/a"},
{size: 5, head: "h", tail: "t", content: "c", path: "/d/b"},
{size: 20 << 20, head: "bh", tail: "bt", path: "/d/big"},
{size: 7, path: "/d/unique"},
}
}
// TestRunInfo runs info on an empty and on a filled database, with
// stdout and stderr the same file that is not a terminal, and checks
// everything written there: the line naming the database, each phase's
// line as it starts and as it ends, and then the details, each on a
// line of its own. No -wal file is beside a database scan has closed.
func TestRunInfo(t *testing.T) {
cases := map[string]struct {
recs []scanRec
want string // the details after the database file's time
}{
"empty": {
want: "schema version\t1\n" +
"records\t0\n" +
"records with head and tail hashes\t0\n" +
"records with content hash\t0\n" +
"size of recorded files\t0 (0 B)\n",
},
"filled": {
recs: infoRecs(),
want: "schema version\t1\n" +
"records\t4\n" +
"records with head and tail hashes\t3\n" +
"records with content hash\t2\n" +
"size of recorded files\t20971537 (20.0 MiB)\n",
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
path := seedDatabase(t, tc.recs)
t.Setenv(databaseEnv, path)
file, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
stderr := captureStderr(t)
code := run([]string{cmdInfo}, os.Stderr, os.Stderr)
if code != exitOK {
t.Fatalf("run(info) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
named := path + " (from " + databaseEnv + ")\n"
want := "info: database " + named +
infoOpenLine + "\n" + infoOpenLine + "\n" +
infoCountLine + "\n" + infoCountLine + "\n" +
"database\t" + named +
fmt.Sprintf("database file size\t%d (%s)\n",
file.Size(), humanBytes(file.Size())) +
"modified\t" + file.ModTime().UTC().Format(time.RFC3339) +
"\n" + tc.want
if got := stderr(); got != want {
t.Errorf("stdout and stderr = %q, want %q", got, want)
}
})
}
}
// TestRunInfoDuringScan runs info while a scan holds the lock and has
// the database open in WAL mode, with records committed to the -wal
// file only. info must count those records and show the -wal file's
// size, also when the database path is a symlink: SQLite keeps the
// -wal file beside the symlink's target.
func TestRunInfoDuringScan(t *testing.T) {
path := testDBPath(t)
link := filepath.Join(t.TempDir(), "link.sqlite")
err := os.Symlink(path, link)
if err != nil {
t.Fatal(err)
}
db, err := openScanDatabase(t.Context(), path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
err = applyChanges(t.Context(), db, infoRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
holdScanLock(t, path)
wal, err := os.Stat(path + "-wal")
if err != nil {
t.Fatal(err)
}
for _, named := range []string{path, link} {
t.Run(filepath.Base(named), func(t *testing.T) {
t.Setenv(databaseEnv, named)
stdout := runStdout(t, cmdInfo)
for _, want := range []string{
"database\t" + named + " (from " + databaseEnv + ")\n",
fmt.Sprintf("\n-wal file size\t%d (%s)\n",
wal.Size(), humanBytes(wal.Size())),
"\nrecords\t4\n",
} {
if !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
}
})
}
}
func TestRunInfoNeedsOnlyReadAccess(t *testing.T) {
// README "`info` mode": like report, info needs only read access to
// the database file and none to its directory.
path := testDBPath(t)
t.Setenv(databaseEnv, path)
scanFixture(t)
makeReadOnly(t, path)
stdout := runStdout(t, cmdInfo)
if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) {
t.Errorf("stdout = %q, want it to hold %q", stdout, want)
}
if strings.Contains(stdout, "-wal") {
t.Errorf("stdout = %q, want no -wal file between scans", stdout)
}
}
+24 -41
View File
@@ -8,14 +8,13 @@
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB // under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
// or larger. scan maintains a persistent SQLite database of file // or larger. scan maintains a persistent SQLite database of file
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) // signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
// that the other subcommands read. // that the reporting subcommands read.
// //
// Usage: // Usage:
// //
// sfdupes scan [--workers N] [-x] PATH... // sfdupes scan [--workers N] [-x] PATH...
// sfdupes report > dupes.tsv // sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv // sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes --version // sfdupes --version
// //
// See README.md for the complete specification. // See README.md for the complete specification.
@@ -45,7 +44,6 @@ const (
cmdScan = "scan" cmdScan = "scan"
cmdReport = "report" cmdReport = "report"
cmdTrees = "trees" cmdTrees = "trees"
cmdInfo = "info"
) )
// errNoSubcommand is returned by the root command when it is invoked // errNoSubcommand is returned by the root command when it is invoked
@@ -75,8 +73,8 @@ func main() {
// exit code. It is the program's single exit point: the subcommands // exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup — // return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL — // above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees and info subcommands // runs before the process ends. The report and trees subcommands write
// write their data to stdout. // their data to stdout.
func run(args []string, stdout, stderr io.Writer) int { func run(args []string, stdout, stderr io.Writer) int {
// A nil slice makes cobra fall back to os.Args, which would let a // A nil slice makes cobra fall back to os.Args, which would let a
// test binary's own flags reach the command tree. // test binary's own flags reach the command tree.
@@ -150,41 +148,6 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
root.Flags().BoolVarP(&showVersion, "version", "v", false, root.Flags().BoolVarP(&showVersion, "version", "v", false,
"print the version to stdout") "print the version to stdout")
reportCmd := &cobra.Command{
Use: cmdReport,
Short: "Read the scan database and print the file-level duplicates report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runReport(ctx, stdout)
}),
}
treesCmd := &cobra.Command{
Use: cmdTrees,
Short: "Read the scan database and print the duplicate-tree report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runTrees(ctx, stdout)
}),
}
infoCmd := &cobra.Command{
Use: cmdInfo,
Short: "Read the scan database and print its details",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runInfo(ctx, stdout)
}),
}
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
return root
}
// newScanCommand builds the scan subcommand, with its --workers and -x
// flags.
func newScanCommand() *cobra.Command {
var ( var (
scanWorkers int scanWorkers int
scanOneFS bool scanOneFS bool
@@ -209,7 +172,27 @@ func newScanCommand() *cobra.Command {
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
"do not cross filesystem boundaries") "do not cross filesystem boundaries")
return scanCmd reportCmd := &cobra.Command{
Use: cmdReport,
Short: "Read the scan database and print the file-level duplicates report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runReport(ctx, stdout)
}),
}
treesCmd := &cobra.Command{
Use: cmdTrees,
Short: "Read the scan database and print the duplicate-tree report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runTrees(ctx, stdout)
}),
}
root.AddCommand(scanCmd, reportCmd, treesCmd)
return root
} }
// checkScanWorkers rejects a scan --workers value below 1. That is a // checkScanWorkers rejects a scan --workers value below 1. That is a
+15 -294
View File
@@ -284,38 +284,23 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) {
} }
func TestRunMissingDatabaseIsFatal(t *testing.T) { func TestRunMissingDatabaseIsFatal(t *testing.T) {
// README §Database: report, trees and info need an existing // README §Database: report and trees need an existing database; a
// database; a missing one exits 1 with a message telling the user to // missing one exits 1 with a message telling the user to run scan.
// run scan, after the line naming the database. for _, name := range []string{cmdReport, cmdTrees} {
openLines := map[string]string{
cmdReport: reportOpenLine,
cmdTrees: treesOpenLine,
cmdInfo: infoOpenLine,
}
for name, openLine := range openLines {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
var stdout bytes.Buffer var stdout, stderr bytes.Buffer
// run gets os.Stderr, where the subcommand prints its line, code := run([]string{name}, &stdout, &stderr)
// so the capture shows the two in the order printed.
stderr := captureStderr(t)
code := run([]string{name}, &stdout, os.Stderr)
if code != exitFatal { if code != exitFatal {
t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
} }
// Each shows its open phase, as it starts and as it ends, want := "sfdupes: " + path + ": no database (run \"sfdupes " +
// before the error.
want := name + ": database " + path + " (from " + databaseEnv +
")\n" + openLine + "\n" + openLine + "\n" +
"sfdupes: " + path + ": no database (run \"sfdupes " +
"scan\" first, or set " + databaseEnv + ")\n" "scan\" first, or set " + databaseEnv + ")\n"
if got := stderr(); got != want { if got := stderr.String(); got != want {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
@@ -326,256 +311,6 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) {
} }
} }
func TestRunNamesRelativeDatabase(t *testing.T) {
// README §Database: each subcommand's first line on stderr names the
// database, a relative SFDUPES_DATABASE made absolute against the
// working directory. scan calls it new until a scan has created it.
path := testDBPath(t)
t.Chdir(filepath.Dir(path))
t.Setenv(databaseEnv, filepath.Base(path))
dir := t.TempDir()
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("first scan stderr = %q, want it to start %q", got, want)
}
want = "scan: database " + path + from
if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
t.Errorf("second scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + path + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
// README §Database: the path names the file exactly. With link a
// symlink to the directory real/sub, link/../db.sqlite names
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
// use that file, relative or absolute, and its first line must name
// the path as set, with the working directory put in front of a
// relative one.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Chdir(dir)
err = os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "real", "sub"),
filepath.Join(dir, "link"))
if err != nil {
t.Fatal(err)
}
named := dir + "/link/../db.sqlite"
from := " (from " + databaseEnv + ")\n"
tree := t.TempDir()
t.Setenv(databaseEnv, "link/../db.sqlite")
want := "scan: new database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("relative scan stderr = %q, want it to start %q", got, want)
}
// The absolute path names the database the relative one created.
t.Setenv(databaseEnv, named)
want = "scan: database " + named + from
if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
t.Errorf("absolute scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + named + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
_, err = os.Stat(filepath.Join(dir, "real", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
// With no file at the cleaned path, report and trees succeeded on
// the database in real.
_, err = os.Stat(filepath.Join(dir, "db.sqlite"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("database at the cleaned path: stat error %v, want it "+
"missing", err)
}
}
func TestRunScanFromSymlinkedWorkingDirectory(t *testing.T) {
// README §Database: a relative path gets the working directory, its
// symlinks resolved, put in front of it. With home/proj a symlink
// to data/proj, entered as the working directory through the
// symlink, ../new/db.sqlite names data/new/db.sqlite. scan must
// create that directory and file, and nothing beside home/proj.
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Join(dir, "data", "proj"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(filepath.Join(dir, "home"), 0o750)
if err != nil {
t.Fatal(err)
}
err = os.Symlink(filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
if err != nil {
t.Fatal(err)
}
// t.Chdir also sets PWD to the path through the symlink, as a
// shell does after cd.
t.Chdir(filepath.Join(dir, "home", "proj"))
t.Setenv(databaseEnv, "../new/db.sqlite")
want := "scan: new database " + dir + "/data/proj/../new/db.sqlite" +
" (from " + databaseEnv + ")\n"
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
if err != nil {
t.Errorf("database the path names: %v", err)
}
_, err = os.Lstat(filepath.Join(dir, "home", "new"))
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("home/new: stat error %v, want it missing", err)
}
}
func TestRunEscapesDatabasePath(t *testing.T) {
// README §Database: each subcommand's first line escapes the
// database path like the paths in a report, so a tab, newline or
// backslash in it keeps the line one line.
dir := t.TempDir()
t.Setenv(databaseEnv, dir+"/a\tb\nc\\d.sqlite")
escaped := dir + `/a\tb\nc\\d.sqlite`
from := " (from " + databaseEnv + ")\n"
want := "scan: new database " + escaped + from
if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
t.Errorf("scan stderr = %q, want it to start %q", got, want)
}
for _, name := range []string{cmdReport, cmdTrees} {
stderr := captureStderr(t)
code := run([]string{name}, io.Discard, os.Stderr)
want = name + ": database " + escaped + from
if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
"start %q", name, code, got, exitOK, want)
}
}
}
func TestRunNamesDefaultDatabase(t *testing.T) {
// README §Database: with SFDUPES_DATABASE empty, each subcommand's
// first line on stderr names the default database. A database may
// exist at the default path on the machine running the test, so
// nothing here may create or read one: scan gets an operand that
// does not exist, which stops it before it takes the lock, and
// report, trees and info a cancelled context, which stops them
// before they read anything. They show their open phase after the
// line.
t.Setenv(databaseEnv, "")
// Whether scan calls the database new depends on that machine too.
scanWant := "scan: database " + defaultDatabasePath + " (default)\n"
_, err := os.Stat(defaultDatabasePath)
if errors.Is(err, fs.ErrNotExist) {
scanWant = "scan: new database " + defaultDatabasePath +
" (default)\n"
}
missing := filepath.Join(t.TempDir(), "nope")
cases := map[string]struct {
run func() error
want string
}{
cmdScan: {
run: func() error {
return runScan(t.Context(), []string{missing}, 1, false)
},
want: scanWant,
},
cmdReport: {
run: func() error {
return runReport(cancelledContext(t), io.Discard)
},
want: "report: database " + defaultDatabasePath + " (default)\n" +
reportOpenLine + "\n" + reportOpenLine + "\n",
},
cmdTrees: {
run: func() error {
return runTrees(cancelledContext(t), io.Discard)
},
want: "trees: database " + defaultDatabasePath + " (default)\n" +
treesOpenLine + "\n" + treesOpenLine + "\n",
},
cmdInfo: {
run: func() error {
return runInfo(cancelledContext(t), io.Discard)
},
want: "info: database " + defaultDatabasePath + " (default)\n" +
infoOpenLine + "\n" + infoOpenLine + "\n",
},
}
for name, tc := range cases {
stderr := captureStderr(t)
err = tc.run()
if err == nil {
t.Errorf("%s: error = nil, want it stopped", name)
}
if got := stderr(); got != tc.want {
t.Errorf("%s stderr = %q, want %q", name, got, tc.want)
}
}
}
func TestRunUsageErrors(t *testing.T) { func TestRunUsageErrors(t *testing.T) {
// Usage errors keep exiting 2 with cobra's own report on stderr. // Usage errors keep exiting 2 with cobra's own report on stderr.
cases := map[string]struct { cases := map[string]struct {
@@ -586,7 +321,6 @@ func TestRunUsageErrors(t *testing.T) {
"scan without paths": {[]string{cmdScan}, usageMarker}, "scan without paths": {[]string{cmdScan}, usageMarker},
"report with args": {[]string{cmdReport, "x"}, usageMarker}, "report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker}, "trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker}, "unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
"unknown subcommand": {[]string{"nope"}, "unknown command"}, "unknown subcommand": {[]string{"nope"}, "unknown command"},
} }
@@ -927,16 +661,10 @@ func TestRunReportSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want) t.Errorf("stdout = %q, want %q", got, want)
} }
// Its phases come between the two lines; see want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " +
// TestRunReportProgressNamesEachPhase.
first := "report: database " + path + " (from " + databaseEnv + ")\n"
last := "\nreport: 2 records read, 1 duplicate groups, 1 dupe files, " +
"300 B reclaimable\n" "300 B reclaimable\n"
if got := stderr(); got != want {
if got := stderr(); !strings.HasPrefix(got, first) || t.Errorf("stderr = %q, want %q", got, want)
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
} }
assertNoSidecars(t, path) assertNoSidecars(t, path)
@@ -966,16 +694,10 @@ func TestRunTreesSucceeds(t *testing.T) {
t.Errorf("stdout = %q, want %q", got, want) t.Errorf("stdout = %q, want %q", got, want)
} }
// Its phases come between the two lines; see want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " +
// TestRunTreesProgressNamesEachPhase. "300 B reclaimable\n"
first := "trees: database " + path + " (from " + databaseEnv + ")\n" if got := stderr(); got != want {
last := "\ntrees: 2 records read, 1 duplicate tree groups, " + t.Errorf("stderr = %q, want %q", got, want)
"1 dupe trees, 300 B reclaimable\n"
if got := stderr(); !strings.HasPrefix(got, first) ||
!strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to start %q and end %q",
got, first, last)
} }
assertNoSidecars(t, path) assertNoSidecars(t, path)
@@ -1267,7 +989,7 @@ func TestRunReportsDuringScan(t *testing.T) {
func TestRunStdoutClosedIsFatal(t *testing.T) { func TestRunStdoutClosedIsFatal(t *testing.T) {
// README §Error handling: a stdout write failure exits 1, reported // README §Error handling: a stdout write failure exits 1, reported
// in one line on stderr. // in one line on stderr.
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} { for _, name := range []string{cmdReport, cmdTrees} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t)) t.Setenv(databaseEnv, testDBPath(t))
@@ -1317,7 +1039,6 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
cases := map[string]func(context.Context, io.Writer) error{ cases := map[string]func(context.Context, io.Writer) error{
cmdReport: runReport, cmdReport: runReport,
cmdTrees: runTrees, cmdTrees: runTrees,
cmdInfo: runInfo,
} }
for name, fn := range cases { for name, fn := range cases {
+115 -397
View File
@@ -2,289 +2,122 @@ package main
import ( import (
"fmt" "fmt"
"io"
"math"
"os" "os"
"slices"
"strings"
"sync"
"time" "time"
"github.com/schollz/progressbar/v3" "github.com/schollz/progressbar/v3"
"golang.org/x/term" "golang.org/x/term"
) )
// plainInterval is the time between progress lines while a phase runs, // plainInterval is the minimum time between progress lines when stderr
// when stderr is not a TTY. // is not a TTY.
const plainInterval = 5 * time.Second const plainInterval = 5 * time.Second
// redrawInterval is the time between redraws of the progress display on // barThrottle caps how often the TTY progress bar redraws.
// a TTY. const barThrottle = 100 * time.Millisecond
const redrawInterval = 100 * time.Millisecond
// spinnerType selects the progressbar library's spinner style. // walkSpinnerType selects the progressbar library's spinner style used
const spinnerType = 14 // for the walk pass, which has no known total.
const walkSpinnerType = 14
// barWidth is the width of the progressbar library's bar between its
// two ends, as in the example line in README "Progress".
const barWidth = 10
// percentScale converts a fraction to a percentage. // percentScale converts a fraction to a percentage.
const percentScale = 100 const percentScale = 100
// fallbackWidth is the width taken for a terminal whose width cannot be
// read, as the progressbar library takes it.
const fallbackWidth = 80
// spinnerMargin is the columns of a spinner's line besides its text:
// the spinner and the three spaces the library draws around the text,
// and the terminal's last column, which every line stays clear of so
// that no terminal wraps it.
const spinnerMargin = 5
// barMargin is the columns of a bar's line besides its text: the
// percent and the bar the library draws after the text, the spaces
// around them, and the terminal's last column.
const barMargin = barWidth + 11
// countingRecords is what the display of a phase that reads the records
// says while it counts them, before it has its total.
const countingRecords = "counting the records"
// openPhase is the name of the phase in which report, trees and info
// open the database, and openingDatabase what its display says while
// it does.
const (
openPhase = "open"
openingDatabase = "opening the database"
)
// stderrIsTTY reports whether stderr is attached to a terminal. // stderrIsTTY reports whether stderr is attached to a terminal.
func stderrIsTTY() bool { func stderrIsTTY() bool {
return term.IsTerminal(int(os.Stderr.Fd())) return term.IsTerminal(int(os.Stderr.Fd()))
} }
// terminalWidth returns the width of the terminal on stderr, or // progress renders one scan pass's progress on stderr. On a TTY it
// fallbackWidth, as the progressbar library does, when it cannot read // delegates to the progressbar library (spinner style when the total is
// one. // unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
func terminalWidth() int { // stderr is not a TTY it emits no ANSI redraws: it prints a plain
width, _, err := term.GetSize(int(os.Stderr.Fd())) // one-line update as the pass starts, then no more often than every
if err != nil || width <= 0 { // plainInterval.
return fallbackWidth
}
return width
}
// phaseLabel names phase with its place among phases, as "[2/5] walk".
func phaseLabel(phases []string, phase string) string {
return fmt.Sprintf("[%d/%d] %s",
slices.Index(phases, phase)+1, len(phases), phase)
}
// progressParams describe one phase's progress display as it starts.
type progressParams struct {
label string // the phase and its place, from phaseLabel
unit string // what the phase counts, as "files"
total int64 // the items the phase will complete; -1 when not known
estimate bool // total is an estimate, which the count may pass
waitingFor string // what the phase waits for until setTotal is called
}
// progress renders one phase's progress on stderr. On a TTY the
// progressbar library draws line, fitted to the terminal's width: after
// the library's spinner, or, for a phase with an exact total, before the
// library's bar when the terminal has room for the bar beside it. It is
// drawn again every redrawInterval, so its elapsed time keeps moving
// while the phase waits for its next item. When stderr is not a TTY it
// emits no ANSI redraws: it prints line as the phase starts, every
// plainInterval while it runs, and as it ends.
// //
// The draws come from a goroutine of their own, so mu guards the fields // All methods must be called from the main goroutine only. On a TTY
// below it. Every method must be called from the main goroutine. A nil // the library also redraws a spinner from its own goroutine, several
// *progress is a valid no-display receiver: every method is a no-op, so // times a second, so its count and elapsed time stay current while a
// batched database flushes during the hash phase can reuse the update // pass waits for its next item. A nil *progress is a valid
// phase's helpers without rendering anything, and its writer writes // no-display receiver: every method is a no-op, so batched database
// with no display to clear. // flushes during the streaming pass can reuse the update-pass helpers
// without rendering anything.
type progress struct { type progress struct {
width func() int // reads the terminal's width; nil when not a TTY label string
stop chan struct{} // closed by finish to end the draws total int64 // -1 when unknown (walk pass)
done chan struct{} // closed once the draws have ended bar *progressbar.ProgressBar
count int64
mu sync.Mutex start time.Time
bar *progressbar.ProgressBar // what the library draws; nil until then last time.Time
withBar bool // bar is the library's bar, not its spinner
widest int // the widest line bar has drawn, in columns
label string
unit string
total int64
estimate bool
waitingFor string
count int64
start time.Time // when the phase started, for the elapsed time
rateStart time.Time // when the count started, for the rate
} }
// newProgress starts the display of one phase on stderr. func newProgress(label string, total int64) *progress {
func newProgress(params progressParams) *progress { p := &progress{label: label, total: total, start: time.Now()}
if stderrIsTTY() { if stderrIsTTY() {
return startProgress(params, terminalWidth, redrawInterval) p.bar = newBar(label, total)
return p
} }
return startProgress(params, nil, plainInterval) // Print the zero state at once: the first item may take minutes,
} // and a pass must never look hung.
p.last = p.start
// startProgress shows the phase params describes at once, and again fmt.Fprintln(os.Stderr, p.plainLine())
// every interval until finish: with the progressbar library on a
// terminal, whose width each draw reads by calling width, or as a plain
// line when width is nil.
func startProgress(params progressParams, width func() int,
interval time.Duration,
) *progress {
now := time.Now()
p := &progress{
width: width, stop: make(chan struct{}), done: make(chan struct{}),
label: params.label, unit: params.unit, total: params.total,
estimate: params.estimate, waitingFor: params.waitingFor,
start: now, rateStart: now,
}
// Show the zero state at once: the first item may take minutes,
// and a phase must never look hung.
p.draw()
go func() {
defer close(p.done)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-p.stop:
return
case <-ticker.C:
p.draw()
}
}
}()
return p return p
} }
// draw shows the phase's progress: on a TTY in place of the line // newBar builds the TTY display for newProgress.
// before, and otherwise as a plain line. func newBar(label string, total int64) *progressbar.ProgressBar {
func (p *progress) draw() { opts := []progressbar.Option{
p.mu.Lock()
defer p.mu.Unlock()
if p.width == nil {
text, _ := p.line(math.MaxInt, true)
fmt.Fprintln(os.Stderr, text)
return
}
width := p.width()
// A phase with an exact total gets the library's bar when its line,
// without the percent the bar shows, fits whole beside the bar. A
// total of 0 leaves the bar nothing to fill.
text, withBar := "", false
if p.total > 0 && !p.estimate {
text, withBar = p.line(width-barMargin, false)
}
margin := barMargin
if !withBar {
margin = spinnerMargin
text, _ = p.line(width-margin, true)
}
// The library makes a bar or a spinner, and before each draw erases
// its line with as many spaces as the widest line it has drawn. A
// display gets a new one when it turns from one to the other, as its
// line grows or the terminal is resized, and when the terminal has
// been made too narrow for that erase, which would wrap and leave a
// line behind at every draw.
if p.bar == nil || p.withBar != withBar || p.widest >= width {
p.newBar(withBar, width)
}
if withBar {
_ = p.bar.Set64(p.count)
}
p.bar.Describe(text)
// The library drew the text and its margin, less the terminal's last
// column, which the margin keeps clear.
p.widest = max(p.widest, len(text)+margin-1)
}
// newBar has the library make what draw draws: a bar for the phase's
// total when withBar is set, and a spinner when not. Either shows the
// text draw gives it, and nothing of its own besides the spinner, or
// the bar and its percent. p.mu must be held.
func (p *progress) newBar(withBar bool, width int) {
if p.bar != nil {
// Erase what was drawn before, short of the last column of a
// terminal that may have been made narrower since: the library's
// own erase is as wide as the widest line it drew.
blank := strings.Repeat(" ", min(p.widest, width-1))
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
}
total := int64(-1) // the library's spinner
if withBar {
total = p.total
}
p.bar = progressbar.NewOptions64(total,
progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetWidth(barWidth), progressbar.OptionSetDescription(label),
progressbar.OptionSpinnerType(spinnerType), progressbar.OptionShowCount(),
// The spinner turns at each draw, not on a timer of the progressbar.OptionShowIts(),
// library's own. progressbar.OptionSetItsString("files"),
progressbar.OptionSetSpinnerChangeInterval(0), progressbar.OptionSetElapsedTime(true),
// The text carries the elapsed time and the ETA. progressbar.OptionThrottle(barThrottle),
progressbar.OptionSetElapsedTime(false), // Render at zero immediately: a phase must be visible the
progressbar.OptionSetPredictTime(false), // moment it starts, even before its first item completes.
) progressbar.OptionSetRenderBlankState(true),
p.withBar, p.widest = withBar, 0 }
} if total >= 0 {
opts = append(opts,
// setTotal gives a phase that started without a total its total, once progressbar.OptionSetPredictTime(true),
// it is known. The rate, and so the ETA, count from then on: the time progressbar.OptionShowElapsedTimeOnFinish(),
// spent waiting for the total says nothing about how fast items go. )
func (p *progress) setTotal(total int64) { } else {
if p == nil { opts = append(opts,
return progressbar.OptionSetPredictTime(false),
progressbar.OptionSpinnerType(walkSpinnerType),
)
} }
p.mu.Lock() return progressbar.NewOptions64(total, opts...)
defer p.mu.Unlock()
p.total = total
p.rateStart = time.Now()
} }
// increment records one completed item, which the display shows at its // increment records one completed item and refreshes the display.
// next draw.
func (p *progress) increment() { func (p *progress) increment() {
if p == nil { if p == nil {
return return
} }
p.mu.Lock()
defer p.mu.Unlock()
p.count++ p.count++
if p.bar != nil {
_ = p.bar.Add(1)
return
}
if time.Since(p.last) >= plainInterval {
p.last = time.Now()
fmt.Fprintln(os.Stderr, p.plainLine())
}
} }
// warnf prints a one-line warning to stderr without corrupting the // warnf prints a one-line warning to stderr without corrupting the bar.
// display. The whole message is escaped like a report's path columns, // The whole message is escaped like a report's path columns, so a path
// so a path holding a newline cannot split the warning. // holding a newline cannot split the warning.
func (p *progress) warnf(format string, args ...any) { func (p *progress) warnf(format string, args ...any) {
if p == nil { if p == nil {
return return
@@ -292,168 +125,58 @@ func (p *progress) warnf(format string, args ...any) {
msg := escapePath(fmt.Sprintf(format, args...)) msg := escapePath(fmt.Sprintf(format, args...))
p.mu.Lock() if p.bar != nil && p.total < 0 {
defer p.mu.Unlock() // The library also redraws a spinner from its own goroutine, so
// a direct write could land inside a redraw. The bar prints the
// warning itself, just before its next redraw.
_, _ = progressbar.Bprintln(p.bar, msg)
return
}
if p.bar != nil { if p.bar != nil {
// The warning takes the display's line; the next draw shows
// the display again, on the line below.
_ = p.bar.Clear() _ = p.bar.Clear()
} }
fmt.Fprintln(os.Stderr, msg) fmt.Fprintln(os.Stderr, msg)
}
// progressWriter writes a phase's output to out, which may be the file // The library never draws a finished bar again, so the warning for
// the phase's display is drawn on, as stdout is in a plain "sfdupes // a pass's last item would erase it for good: draw its last frame
// trees" on a terminal. It holds what is written to it and writes that // again below the warning.
// to out in pieces of about ioBufSize bytes. When out is the file if p.bar != nil && p.bar.IsFinished() {
// stderr is, each piece is written under the display's lock with the fmt.Fprint(os.Stderr, p.bar.String())
// display cleared first, as warnf does for a warning: no draw lands in
// the middle of a piece, and the next draw shows the display again, on
// the line below. Every write to it must end with a newline, as a row
// written with one fmt.Fprintf does, so that each piece ends a line.
// Flush writes what it holds once the last row is written.
type progressWriter struct {
// p is the display to clear, nil unless out is the file stderr is.
p *progress
out io.Writer
buf []byte
}
// writer returns a progressWriter writing to out around p's display.
// It clears the display and takes its lock only when out is the file
// stderr is: the same device and inode, as os.SameFile compares them,
// which is one terminal, or one pipe or file that both are sent to.
// Otherwise it writes to out without the lock, so the display keeps
// being drawn while a write to out waits for a slow reader.
func (p *progress) writer(out io.Writer) *progressWriter {
w := &progressWriter{out: out}
if f, ok := out.(*os.File); ok {
outInfo, outErr := f.Stat()
errInfo, errErr := os.Stderr.Stat()
if outErr == nil && errErr == nil && os.SameFile(outInfo, errInfo) {
w.p = p
}
} }
return w
} }
// Write holds b, and writes what it holds to out once that comes to // finish terminates the pass's display. A bar whose pass stopped short
// ioBufSize bytes. // of its total, as an interrupted one does, is left as last drawn; the
func (w *progressWriter) Write(b []byte) (int, error) { // library's Finish would fill it up.
w.buf = append(w.buf, b...)
if len(w.buf) < ioBufSize {
return len(b), nil
}
return len(b), w.Flush()
}
// Flush writes what w holds to out. When out is the file stderr is, the
// display is cleared first and not drawn again until it is written.
func (w *progressWriter) Flush() error {
if w.p != nil {
w.p.mu.Lock()
defer w.p.mu.Unlock()
if w.p.bar != nil {
_ = w.p.bar.Clear()
}
}
_, err := w.out.Write(w.buf)
w.buf = w.buf[:0]
return err
}
// finish ends the phase's display, drawn once more as the phase ends.
// A phase stopped short of its total, as an interrupted one is, shows
// the count it stopped at rather than a full bar.
func (p *progress) finish() { func (p *progress) finish() {
if p == nil { if p == nil {
return return
} }
close(p.stop) if p.bar != nil {
<-p.done if p.total >= 0 && p.count < p.total {
_ = p.bar.Exit()
p.draw() } else {
_ = p.bar.Finish()
if p.width != nil {
// The display stays on the TTY as drawn; what follows starts
// on a line of its own.
fmt.Fprintln(os.Stderr)
}
}
// line formats the phase's progress in at most room columns, as
//
// [3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
//
// where "files" is the phase's unit, without the percent when percent
// is false, as before the library's bar, which shows its own. A phase
// with no total shows the items done instead, or what it waits for.
//
// The phase and its place, the progress, the elapsed time and the ETA
// are kept whole. A line wider than room leaves out the rate, or what
// the phase waits for; only if it is still wider is it cut at room, and
// line reports that it does not fit. Every part is ASCII, so its length
// is its width in columns. p.mu must be held.
func (p *progress) line(room int, percent bool) (string, bool) {
elapsed := time.Since(p.start).Round(time.Second)
// head and tail hold the parts kept whole; extra, between them, is
// the part left out.
var head, extra, tail string
if p.total < 0 && p.waitingFor != "" {
head, extra = p.label+": ", p.waitingFor
tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
} else {
rate := 0.0
if s := time.Since(p.rateStart).Seconds(); s > 0 {
rate = float64(p.count) / s
} }
head = p.label + ": " + p.itemsDone(percent) + ", " fmt.Fprintln(os.Stderr)
extra = fmt.Sprintf("%.0f %s/s", rate, p.unit)
tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(rate)) return
} }
full, kept := head+extra+", "+tail, head+tail fmt.Fprintln(os.Stderr, p.plainLine())
switch {
case len(full) <= room:
return full, true
case len(kept) <= room:
return kept, true
}
return kept[:max(room, 0)], false
} }
// itemsDone formats the items done, as "[1234/5000] 25%", with "~" // plainLine formats a non-TTY progress line.
// before a total that is an estimate and without the percent when func (p *progress) plainLine() string {
// percent is false, or as "1234 files", in the phase's unit, when the elapsed := time.Since(p.start)
// phase has no total. p.mu must be held.
func (p *progress) itemsDone(percent bool) string {
if p.total < 0 { if p.total < 0 {
return fmt.Sprintf("%d %s", p.count, p.unit) return fmt.Sprintf("%s: %d files, elapsed %s",
} p.label, p.count, elapsed.Round(time.Second))
approx := ""
if p.estimate {
approx = "~"
}
done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
if !percent {
return done
} }
pct := float64(percentScale) pct := float64(percentScale)
@@ -461,24 +184,19 @@ func (p *progress) itemsDone(percent bool) string {
pct = percentScale * float64(p.count) / float64(p.total) pct = percentScale * float64(p.count) / float64(p.total)
} }
return fmt.Sprintf("%s %.0f%%", done, pct) rate := 0.0
} if s := elapsed.Seconds(); s > 0 {
rate = float64(p.count) / s
// eta returns the time the phase has left at rate items a second, or
// "unknown": with no total, before the first item, and once the count
// passes an estimate. p.mu must be held.
func (p *progress) eta(rate float64) string {
switch {
case p.total < 0, p.estimate && p.count > p.total:
return "unknown"
case p.count >= p.total:
return "0s"
case rate <= 0:
return "unknown"
} }
remaining := time.Duration( eta := "?"
float64(p.total-p.count) / rate * float64(time.Second))
return remaining.Round(time.Second).String() if rate > 0 && p.count <= p.total {
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
eta = remaining.Round(time.Second).String()
}
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
p.label, p.count, p.total, pct, rate,
elapsed.Round(time.Second), eta)
} }
+78 -588
View File
@@ -1,24 +1,16 @@
package main package main
import ( import (
"fmt"
"io"
"math"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
"unicode/utf8"
) )
// testRedraw is the redraw interval the tests give a display, so that // spinnerIdle comfortably outlasts the 100ms interval at which the
// many redraws fit in a few milliseconds. // progressbar library redraws a spinner from its own goroutine.
const testRedraw = time.Millisecond const spinnerIdle = 500 * time.Millisecond
// testDeadline bounds how long a test waits for a display to draw what
// it expects.
const testDeadline = 5 * time.Second
//nolint:paralleltest // replaces the process-wide os.Stderr //nolint:paralleltest // replaces the process-wide os.Stderr
func TestStderrIsTTYFalseForNonTerminals(t *testing.T) { func TestStderrIsTTYFalseForNonTerminals(t *testing.T) {
@@ -62,483 +54,60 @@ func TestStderrIsTTYFalseForNonTerminals(t *testing.T) {
} }
} }
func TestPhaseLabel(t *testing.T) { // TestNewProgressPrintsBeforeFirstItem checks that each pass shows its
t.Parallel() // zero state the moment it starts when stderr is not a terminal, and
// that the next line still waits for plainInterval.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestNewProgressPrintsBeforeFirstItem(t *testing.T) {
stderr := captureStderr(t)
phases := []string{"load", "walk", "hash"} newProgress("walk", -1).increment()
if got := phaseLabel(phases, "walk"); got != "[2/3] walk" { newProgress("hash", 10).increment()
t.Errorf("phaseLabel = %q, want %q", got, "[2/3] walk")
want := "walk: 0 files, elapsed 0s\n" +
"hash: [0/10] 0% 0 files/s elapsed 0s eta ?\n"
if got := stderr(); got != want {
t.Errorf("stderr = %q, want %q", got, want)
} }
} }
// TestProgressLine checks the line a display shows in each state, ten // newWalkSpinner returns the walk pass's terminal display, writing to
// seconds into the phase. // os.Stderr whether or not it is a terminal, and stops the library's
func TestProgressLine(t *testing.T) { // redraws when the test ends.
t.Parallel() func newWalkSpinner(t *testing.T) *progress {
ago := time.Now().Add(-10 * time.Second)
cases := []struct {
p *progress
want string
}{{
p: &progress{
label: scanPhase("load"), total: -1,
waitingFor: countingRecords,
},
want: "[1/5] load: counting the records, elapsed 10s, eta unknown",
}, {
p: &progress{
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
},
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
}, {
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
want: "[3/5] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
}, {
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
want: "[4/5] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
}, {
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: 100,
estimate: true, count: 20,
},
want: "[2/5] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
}, {
// Past the estimate the walk cannot tell how long it has left.
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: 100,
estimate: true, count: 120,
},
want: "[2/5] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
"eta unknown",
}, {
// A first scan's walk has no estimate.
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
},
want: "[2/5] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
}, {
// The rate is in what the phase counts.
p: &progress{
label: treesPhase("group"), unit: "directories", total: 100,
count: 20,
},
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
"eta 40s",
}}
for _, tc := range cases {
tc.p.start, tc.p.rateStart = ago, ago
if got, _ := tc.p.line(math.MaxInt, true); got != tc.want {
t.Errorf("line = %q, want %q", got, tc.want)
}
}
}
// TestLineFitsRoom checks how a line is fitted to the room it has, ten
// seconds into the phase: it leaves out the rate, or what the phase
// waits for, and is cut only when what is left is still too wide.
func TestLineFitsRoom(t *testing.T) {
t.Parallel()
ago := time.Now().Add(-10 * time.Second)
hash := &progress{
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
start: ago, rateStart: ago,
}
content := &progress{
label: scanPhase("content"), total: -1,
waitingFor: "searching the database", start: ago, rateStart: ago,
}
cases := []struct {
p *progress
room int
percent bool
want string
fits bool
}{{
p: hash, room: 57, percent: true,
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
fits: true,
}, {
p: hash, room: 56, percent: true,
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40s",
fits: true,
}, {
p: hash, room: 45, percent: true,
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40",
fits: false,
}, {
// Before the library's bar, which shows the percent.
p: hash, room: 57, percent: false,
want: "[3/5] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
fits: true,
}, {
p: content, room: 40, percent: true,
want: "[5/5] content: elapsed 10s, eta unknown",
fits: true,
}}
for _, tc := range cases {
got, fits := tc.p.line(tc.room, tc.percent)
if got != tc.want || fits != tc.fits {
t.Errorf("line(%d, %t) = %q, %t, want %q, %t",
tc.room, tc.percent, got, fits, tc.want, tc.fits)
}
}
}
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
// and so its ETA, from when the total becomes known, not from when the
// phase started waiting for it.
func TestSetTotalStartsRate(t *testing.T) {
t.Parallel()
waited := time.Now().Add(-20 * time.Second)
p := &progress{
label: scanPhase("load"), unit: scanUnit, total: -1, start: waited,
rateStart: waited,
}
p.setTotal(100)
// Ten seconds after the total came, without waiting ten seconds,
// 20 records are read.
p.start = p.start.Add(-10 * time.Second)
p.rateStart = p.rateStart.Add(-10 * time.Second)
p.count = 20
want := "[1/5] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
if got, _ := p.line(math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
}
// awaitOutput waits until stderr holds want at least times times, and
// fails the test if it never does.
func awaitOutput(t *testing.T, stderr func() string, want string,
times int,
) {
t.Helper() t.Helper()
deadline := time.Now().Add(testDeadline) p := &progress{
for strings.Count(stderr(), want) < times { label: "walk", total: -1, start: time.Now(),
if time.Now().After(deadline) { bar: newBar("walk", -1),
t.Fatalf("stderr = %q, want %q at least %d times",
stderr(), want, times)
}
time.Sleep(testRedraw)
} }
t.Cleanup(p.finish)
return p
} }
// TestPlainProgressPrintsAtStartAndEnd checks that when stderr is not a // TestProgressWarningsOnOwnLines drives the terminal display of the walk
// terminal a phase prints its line the moment it starts, before its // pass through a run of warnings with no items between them, as when the
// first item, nothing for the items it completes before plainInterval // walk meets many unreadable paths, for several of the spinner's
// has passed, and its line again as it ends. // redraws: every warning must land on a line of its own, never inside a
// // redraw.
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
stderr := captureStderr(t)
p := newProgress(progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 1000,
})
for range 1000 {
p.increment()
}
p.finish()
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
want := "[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
if len(lines) != 2 || lines[0] != want ||
!strings.HasPrefix(lines[1], "[3/5] hash: [1000/1000] 100%, ") {
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
lines, want)
}
}
// TestPlainProgressPrintsWhileWaiting checks that a phase waiting with
// no completed items still prints a line every interval.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
stderr := captureStderr(t)
p := startProgress(progressParams{
label: scanPhase("load"), total: -1,
waitingFor: countingRecords,
}, nil, testRedraw)
awaitOutput(t, stderr,
"[1/5] load: counting the records, elapsed 0s, eta unknown\n", 3)
p.finish()
}
// lastFrame returns what a terminal shows of the frames a display drew:
// the last one. The library starts each frame with a carriage return
// and erases the previous one with spaces first.
func lastFrame(out string) string {
var shown string
for frame := range strings.SplitSeq(out, "\r") {
if strings.TrimSpace(frame) != "" {
shown = frame
}
}
return shown
}
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
// phase, spinner or bar, keeps being drawn while the phase waits for its
// next item, with the elapsed time as of now, and shows every item
// completed before the wait. stderr is not a terminal here, so the
// displays take it to be fallbackWidth wide.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
stderr := captureStderr(t)
cases := []struct {
params progressParams
want string // how the line shows a count of 50
}{
{
progressParams{label: scanPhase("walk"), unit: scanUnit, total: -1},
"50 files,",
},
{
progressParams{label: scanPhase("hash"), unit: scanUnit, total: 100},
"[50/100],",
},
}
for _, tc := range cases {
before := len(stderr())
since := func() string { return stderr()[before:] }
p := startProgress(tc.params, terminalWidth, testRedraw)
for range 50 {
p.increment()
}
// An hour into the phase, without waiting an hour.
p.mu.Lock()
p.start = p.start.Add(-time.Hour)
p.mu.Unlock()
awaitOutput(t, since, "elapsed 1h0m0s", 1)
p.finish()
if shown := lastFrame(since()); !strings.Contains(shown, tc.want) {
t.Errorf("terminal shows %q, want %q", shown, tc.want)
}
}
}
// drawLate draws the display params describes on a terminal width
// columns wide as it starts and once more as it ends, with count items
// done two and a half hours into the phase, and returns what it wrote
// to stderr.
func drawLate(stderr func() string, params progressParams, width int,
count int64,
) string {
before := len(stderr())
// No timed draws come between the two.
p := startProgress(params, func() int { return width }, time.Hour)
// Without counting the items one by one or waiting the hours.
p.mu.Lock()
p.count = count
p.start = p.start.Add(-150 * time.Minute)
p.rateStart = p.start
p.mu.Unlock()
p.finish()
return stderr()[before:]
}
// TestTerminalLinesKeepETA checks the line each display shows on an
// 80-column terminal at a large scan's counts and times, and the line
// of trees' read phase at the same counts: the rate is left out, so
// that the elapsed time and the ETA stay whole.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalLinesKeepETA(t *testing.T) {
stderr := captureStderr(t)
cases := []struct {
params progressParams
want string
}{{
// Past its estimate, the walk cannot tell how long it has left.
params: progressParams{
label: scanPhase("walk"), unit: scanUnit, total: 1200000,
estimate: true,
},
want: "[2/5] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
"eta unknown",
}, {
params: progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
},
want: "[3/5] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
}, {
params: progressParams{
label: scanPhase("content"), unit: scanUnit, total: 9876543,
},
want: "[5/5] content: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
}, {
params: progressParams{
label: treesPhase("read"), unit: "records", total: 9876543,
},
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
}}
for _, tc := range cases {
out := drawLate(stderr, tc.params, 80, 1234567)
if shown := lastFrame(out); !strings.Contains(shown, tc.want) {
t.Errorf("terminal shows %q, want %q", shown, tc.want)
}
}
}
// TestTerminalLinesFitWidth draws a large scan's bar display and a small
// scan's, two and a half hours into the phase, on terminals from 120
// columns down to 40, and checks that every line stays clear of the
// terminal's last column. A line that reaches it wraps, and every
// redraw then leaves a line behind. The bar is drawn where it has room,
// as at 120 columns, and left out where it has none.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalLinesFitWidth(t *testing.T) {
stderr := captureStderr(t)
for _, width := range []int{120, 100, 80, 72, 60, 50, 40} {
for _, total := range []int64{9876543, 20000} {
params := progressParams{
label: scanPhase("content"), unit: scanUnit, total: total,
}
out := drawLate(stderr, params, width, total/8)
if width == 120 && !strings.Contains(out, "% |") {
t.Errorf("no bar on a terminal %d wide: %q", width, out)
}
lines := strings.FieldsFuncSeq(out, func(r rune) bool {
return r == '\r' || r == '\n'
})
for line := range lines {
if n := utf8.RuneCountInString(line); n >= width {
t.Errorf("a line of %d columns on a terminal %d wide: %q",
n, width, line)
}
}
}
}
}
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
// and a half hours into the phase, on a terminal 120 columns wide, then
// makes the terminal narrower than that line and draws it again a few
// times. Nothing written after that, the erase before each redraw
// included, may reach the narrower terminal's last column: it would
// wrap, and every redraw would leave a line behind. At these widths the
// spinner stays a spinner and the bar stays a bar.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
stderr := captureStderr(t)
cases := []struct {
params progressParams
narrow int
}{
{
progressParams{label: scanPhase("walk"), unit: scanUnit, total: -1},
60,
},
{
progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
},
90,
},
}
for _, tc := range cases {
width := 120
// No timed draws: the test draws.
p := startProgress(tc.params, func() int { return width }, time.Hour)
p.mu.Lock()
p.count = 1234567
p.start = p.start.Add(-150 * time.Minute)
p.rateStart = p.start
p.mu.Unlock()
p.draw()
width = tc.narrow
before := len(stderr())
p.draw()
p.draw()
p.finish()
lines := strings.FieldsFuncSeq(stderr()[before:], func(r rune) bool {
return r == '\r' || r == '\n'
})
for line := range lines {
if n := utf8.RuneCountInString(line); n >= width {
t.Errorf("%d columns written on a terminal made %d wide: %q",
n, width, line)
}
}
}
}
// TestProgressWarningsOnOwnLines drives the terminal displays, spinner
// and bar, through a run of warnings with no items between them, as when
// the walk meets many unreadable paths, for many redraws: every warning
// must land on a line of its own, never inside a redraw.
// //
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressWarningsOnOwnLines(t *testing.T) { func TestProgressWarningsOnOwnLines(t *testing.T) {
stderr := captureStderr(t) stderr := captureStderr(t)
p := newWalkSpinner(t)
cases := []progressParams{ // No pause between warnings: one written straight to stderr is
{label: scanPhase("walk"), total: -1}, // garbled only if a redraw lands while it is being written.
{label: scanPhase("hash"), total: 10},
}
issued := 0 issued := 0
for start := time.Now(); time.Since(start) < spinnerIdle; issued++ {
for _, params := range cases { p.warnf("warning")
p := startProgress(params, terminalWidth, testRedraw)
// No pause between warnings: one written straight to stderr is
// garbled only if a redraw lands while it is being written.
for start := time.Now(); time.Since(start) < 50*testRedraw; issued++ {
p.warnf("warning")
}
p.finish()
} }
// The spinner prints the warnings at its next redraw.
time.Sleep(spinnerIdle)
// A terminal shows each line as the text after its last carriage // A terminal shows each line as the text after its last carriage
// return. // return.
shown := 0 shown := 0
@@ -560,162 +129,83 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
} }
} }
// TestProgressWriterRowsOnOwnLines drives the terminal displays, // TestSpinnerShowsCountAfterBurst checks that once a burst of items
// spinner and bar, through a run of rows written through the display's // faster than the redraw limit is over, the walk display shows every
// writer to the same terminal, as stdout is in a plain "sfdupes trees", // item completed while it waits for the next one.
// each written out at once, for many redraws: the terminal must show
// every row whole, in order, on a line of its own.
// //
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressWriterRowsOnOwnLines(t *testing.T) { func TestSpinnerShowsCountAfterBurst(t *testing.T) {
stderr := captureStderr(t) stderr := captureStderr(t)
p := newWalkSpinner(t)
cases := []progressParams{ for range 50 {
{ p.increment()
label: treesPhase("open"), total: -1,
waitingFor: "opening the database",
},
{label: treesPhase("write"), unit: "groups", total: 10},
} }
var want []string time.Sleep(spinnerIdle)
for _, params := range cases { if shown := lastFrame(stderr()); !strings.Contains(shown, "(50/-,") {
p := startProgress(params, terminalWidth, testRedraw) t.Errorf("terminal shows %q, want a count of 50", shown)
out := p.writer(os.Stderr)
// No pause between rows: one is garbled only if a redraw lands
// on its line.
for start := time.Now(); time.Since(start) < 50*testRedraw; {
row := fmt.Sprintf("row %d", len(want))
want = append(want, row)
_, err := fmt.Fprintln(out, row)
if err == nil {
err = out.Flush()
}
if err != nil {
t.Fatal(err)
}
}
p.finish()
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
if strings.Contains(line, "row ") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
}
if len(shown) != len(want) {
t.Fatalf("terminal shows %d rows, want %d", len(shown), len(want))
}
for i := range want {
if shown[i] != want[i] {
t.Fatalf("terminal shows %q, want %q", shown[i], want[i])
}
} }
} }
// TestProgressWriterStalledStdoutKeepsLines writes a row larger than a // lastFrame returns what a terminal shows of the frames a bar drew: the
// pipe holds through the display's writer to a pipe nobody reads yet, // last one. The library starts each frame with a carriage return and
// as stdout is when its reader has stopped while stderr goes to a file: // erases the previous one with spaces first.
// while the write waits, the display must go on printing its line. func lastFrame(out string) string {
// var shown string
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressWriterStalledStdoutKeepsLines(t *testing.T) {
stderr := captureStderr(t)
r, w, err := os.Pipe() for frame := range strings.SplitSeq(out, "\r") {
if err != nil { if strings.TrimSpace(frame) != "" {
t.Fatal(err) shown = frame
}
t.Cleanup(func() {
_ = w.Close()
_ = r.Close()
})
p := startProgress(progressParams{label: treesPhase("write"), total: 1},
nil, testRedraw)
out := p.writer(w)
written := make(chan error, 1)
go func() {
_, writeErr := fmt.Fprintln(out, strings.Repeat("x", ioBufSize))
written <- writeErr
}()
// Pass or fail, the pipe is then read so that the write ends, and
// the display ends before stderr is put back.
t.Cleanup(func() {
go func() {
_, _ = io.Copy(io.Discard, r)
}()
writeErr := <-written
if writeErr != nil {
t.Error(writeErr)
} }
p.finish()
})
// The first byte read shows that the write has started; the rest
// of the row waits until the pipe is read.
_, err = r.Read(make([]byte, 1))
if err != nil {
t.Fatal(err)
} }
line := "[4/4] write: " return shown
awaitOutput(t, stderr, line, strings.Count(stderr(), line)+3)
} }
// TestTerminalStoppedShortKeepsCount checks that the terminal display // TestBarStoppedShortKeepsCount checks that the terminal display of a
// of a phase that stops before its total, as an interrupted one does, // pass that stops before its total, as an interrupted one does, is left
// is left showing the count it stopped at rather than a full bar. // as last drawn instead of being filled up.
// //
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalStoppedShortKeepsCount(t *testing.T) { func TestBarStoppedShortKeepsCount(t *testing.T) {
stderr := captureStderr(t) stderr := captureStderr(t)
p := startProgress(progressParams{label: scanPhase("hash"), total: 10}, p := &progress{
terminalWidth, redrawInterval) label: "hash", total: 10, start: time.Now(),
bar: newBar("hash", 10),
}
p.increment() p.increment()
// Past the redraw limit, so the bar draws the next count.
time.Sleep(2 * barThrottle)
p.increment() p.increment()
p.finish() p.finish()
if shown := lastFrame(stderr()); !strings.Contains(shown, "[2/10],") { if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/10,") {
t.Errorf("terminal shows %q, want a count of 2 of 10", shown) t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
} }
} }
// TestWarningAfterLastItemKeepsBar checks that a warning printed after // TestWarningAfterLastItemKeepsBar checks that a warning printed after
// a phase's last item, as for a failed read of the last file, leaves the // a pass's last item, as for a failed read of the last file, leaves the
// phase's full bar on the terminal below the warning. // pass's finished bar on the terminal below the warning.
// //
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestWarningAfterLastItemKeepsBar(t *testing.T) { func TestWarningAfterLastItemKeepsBar(t *testing.T) {
stderr := captureStderr(t) stderr := captureStderr(t)
p := startProgress(progressParams{label: scanPhase("hash"), total: 2}, p := &progress{
terminalWidth, testRedraw) label: "hash", total: 2, start: time.Now(),
bar: newBar("hash", 2),
}
p.increment() p.increment()
p.increment() p.increment()
awaitOutput(t, stderr, "[2/2],", 1)
p.warnf("warning") p.warnf("warning")
p.finish() p.finish()
if shown := lastFrame(stderr()); !strings.Contains(shown, "[2/2],") { if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") {
t.Errorf("terminal shows %q, want a count of 2 of 2", shown) t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
} }
} }
+47 -196
View File
@@ -1,12 +1,11 @@
package main package main
import ( import (
"bufio"
"context" "context"
"database/sql"
"fmt" "fmt"
"io" "io"
"os" "os"
"slices"
"strings" "strings"
"time" "time"
) )
@@ -30,222 +29,74 @@ type scanRec struct {
path string path string
} }
// reportPhase names one phase of report with its place among all of
// them, as "[2/2] read", for the phase's progress display. The list
// here is the one place the phases are counted.
func reportPhase(name string) string {
return phaseLabel([]string{openPhase, "read"}, name)
}
// runReport implements the report subcommand: it prints the file-level // runReport implements the report subcommand: it prints the file-level
// duplicates report as TSV on stdout. It never touches the scanned // duplicates report as TSV on stdout. SQLite groups and orders the
// filesystem; its only I/O is the database, stdout, and stderr. It // records, and each row is written as it is read, so no group is held
// names the database on stderr before it opens it. Any database // in memory. It never touches the scanned filesystem; its only I/O is
// problem, including a missing database, is fatal. // the database (with SQLite's temporary sort file), stdout, and stderr.
// // Any database problem, including a missing database, is fatal.
// The work runs in two phases, each under a progress display of its
// own on stderr: open (open the database) and read (count the records,
// then read them in signature order, writing each size's duplicate
// groups to stdout once that size is read; see readReport). Each
// display ends before the next phase starts, and before an error is
// returned to be printed.
func runReport(ctx context.Context, stdout io.Writer) error { func runReport(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath() dbPath := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: reportPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath) db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
prog = newProgress(progressParams{ out := bufio.NewWriterSize(stdout, ioBufSize)
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords,
})
report, err := readReport(ctx, db, dbPath, stdout, prog)
prog.finish() _, err = fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return fmt.Errorf("write stdout: %w", err)
}
var (
groups, dupeFiles int
reclaimable int64
writeErr error
)
records, err := loadDupeRows(ctx, db,
func(first, path string, size int64) error {
// A group's first path is its first row; every other
// path is a dupe.
if path == first {
groups++
return nil
}
_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
escapePath(first), escapePath(path), size)
dupeFiles++
reclaimable += size
return writeErr
})
if writeErr != nil {
return fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil { if err != nil {
return err return fmt.Errorf("database %s: %w", dbPath, err)
}
err = out.Flush()
if err != nil {
return fmt.Errorf("write stdout: %w", err)
} }
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
"report: %d records read, %d duplicate groups, %d dupe files, "+ "report: %d records read, %d duplicate groups, %d dupe files, "+
"%s reclaimable\n", "%s reclaimable\n",
report.records, report.dupeGroups, report.dupeFiles, records, groups, dupeFiles, humanBytes(reclaimable))
humanBytes(report.reclaimable))
return nil return nil
} }
// readReport gives prog the number of records in db, the database at
// dbPath, as its total, then reads every record in signature order,
// counting each on prog, and writes the report to stdout: the header
// line, then the rows of each size's duplicate groups once that size is
// read. stdout may be the terminal prog is drawn on, so the lines go
// through prog's writer. It returns what it wrote, with the totals for
// the summary line.
func readReport(ctx context.Context, db *sql.DB, dbPath string,
stdout io.Writer, prog *progress,
) (*reportWriter, error) {
out := prog.writer(stdout)
report := &reportWriter{out: out}
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
var writeErr error
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
prog.increment()
writeErr = report.add(r)
return writeErr
})
if writeErr != nil {
return nil, fmt.Errorf("write stdout: %w", writeErr)
}
if err != nil {
return nil, fmt.Errorf("database %s: %w", dbPath, err)
}
err = report.finish()
if err == nil {
err = out.Flush()
}
if err != nil {
return nil, fmt.Errorf("write stdout: %w", err)
}
return report, nil
}
// reportWriter writes the report's rows to out from the records, added
// in signature order, largest size first, as loadSignatureRows reads
// them. The records sharing a signature come together, and so do the
// records of one size, so it holds only the duplicate groups of the
// size being read, and writes them, ordered by first path, once a
// record of a smaller size comes. It counts what the summary line
// reports.
type reportWriter struct {
out io.Writer
sig scanRec // the signature being read; its path is not used
paths []string // the paths read so far with sig
groups [][]string // the duplicate groups of sig.size, each sorted
records, dupeGroups, dupeFiles int
reclaimable int64
}
// add takes the next record. One with the signature being read joins
// its paths; any other ends them, and one of a smaller size first
// writes the groups of the size before.
func (w *reportWriter) add(r scanRec) error {
w.records++
// A record without a content hash has unknown content (README
// "Database"), so it is never a duplicate.
if r.content == "" {
return nil
}
if r.size == w.sig.size && r.head == w.sig.head &&
r.tail == w.sig.tail && r.content == w.sig.content {
w.paths = append(w.paths, r.path)
return nil
}
w.endGroup()
if r.size != w.sig.size {
err := w.writeGroups()
if err != nil {
return err
}
}
w.sig = r
w.paths = append(w.paths, r.path)
return nil
}
// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
func (w *reportWriter) endGroup() {
if len(w.paths) < minGroupSize {
// A path with no duplicate is dropped, and its slice reused.
w.paths = w.paths[:0]
return
}
slices.Sort(w.paths)
w.groups = append(w.groups, w.paths)
w.paths = nil
}
// writeGroups writes the duplicate groups held, all of size w.sig.size,
// ordered by first path: for each group, a row for every path but the
// first. No two groups share a first path, since no two records share
// a path.
func (w *reportWriter) writeGroups() error {
slices.SortFunc(w.groups, func(a, b []string) int {
return strings.Compare(a[0], b[0])
})
for _, g := range w.groups {
for _, path := range g[1:] {
_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
escapePath(g[0]), escapePath(path), w.sig.size)
if err != nil {
return err
}
w.dupeFiles++
w.reclaimable += w.sig.size
}
w.dupeGroups++
}
w.groups = nil
return nil
}
// finish writes the groups of the last size read.
func (w *reportWriter) finish() error {
w.endGroup()
return w.writeGroups()
}
// escapePath returns a path as it is written in a report column (README // escapePath returns a path as it is written in a report column (README
// "Report output format"): a backslash, tab, newline or carriage return // "Report output format"): a backslash, tab, newline or carriage return
// becomes \\, \t, \n or \r, and every other byte is kept as it is. // becomes \\, \t, \n or \r, and every other byte is kept as it is.
+24 -189
View File
@@ -9,7 +9,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -29,8 +28,7 @@ func awkwardPairRecs() []scanRec {
} }
} }
// seedDatabase writes recs into a fresh database, closes it as scan // seedDatabase writes recs into a fresh database and returns its path.
// does, out of WAL mode, and returns its path.
func seedDatabase(t *testing.T, recs []scanRec) string { func seedDatabase(t *testing.T, recs []scanRec) string {
t.Helper() t.Helper()
@@ -46,7 +44,10 @@ func seedDatabase(t *testing.T, recs []scanRec) string {
t.Fatal(err) t.Fatal(err)
} }
closeScanDatabase(t.Context(), db, path) err = db.Close()
if err != nil {
t.Fatal(err)
}
return path return path
} }
@@ -59,39 +60,25 @@ type dupeGroup struct {
} }
// dupeGroups returns the duplicate groups report reads from db, in // dupeGroups returns the duplicate groups report reads from db, in
// report order, from the rows it writes: a group's rows come together, // report order.
// each naming the group's first path.
func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup { func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
t.Helper() t.Helper()
var out bytes.Buffer
_, err := readReport(t.Context(), db, "", &out, nil)
if err != nil {
t.Fatal(err)
}
var groups []dupeGroup var groups []dupeGroup
rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n") _, err := loadDupeRows(t.Context(), db,
for _, row := range rows[1:] { // after the header func(first, path string, size int64) error {
fields := strings.Split(row, "\t") if path == first {
if len(fields) != 3 { groups = append(groups, dupeGroup{size: size})
t.Fatalf("row %q, want first, dupe and size", row) }
}
size, err := strconv.ParseInt(fields[2], 10, 64) g := &groups[len(groups)-1]
if err != nil { g.paths = append(g.paths, path)
t.Fatal(err)
}
if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] { return nil
groups = append(groups, })
dupeGroup{size: size, paths: []string{fields[0]}}) if err != nil {
} t.Fatal(err)
g := &groups[len(groups)-1]
g.paths = append(g.paths, fields[1])
} }
return groups return groups
@@ -131,147 +118,26 @@ func TestRunReportEscapesPaths(t *testing.T) {
} }
func TestReportStdoutFailsWhileReading(t *testing.T) { func TestReportStdoutFailsWhileReading(t *testing.T) {
t.Parallel() // Each row holds two paths longer than dir, so the report is more
// than twice the stdout buffer and stdout fails while rows are
// Every two records are a duplicate pair of a size of their own, and // still being read, not at the final flush.
// a pair's row is written once a smaller size is read. Each row
// holds two paths longer than dir, so the rows come to more than
// twice the stdout buffer, and stdout fails while records are still
// being read, not once every record is.
dir := "/" + strings.Repeat("d", 4096) dir := "/" + strings.Repeat("d", 4096)
recs := make([]scanRec, 2*ioBufSize/len(dir)) recs := make([]scanRec, ioBufSize/len(dir))
for i := range recs { for i := range recs {
recs[i] = scanRec{ recs[i] = scanRec{
size: int64(i/2) + 1, head: "h", tail: "t", content: "c", size: 1, head: "h", tail: "t", content: "c",
path: fmt.Sprintf("%s/%d", dir, i), path: fmt.Sprintf("%s/%d", dir, i),
} }
} }
db := openTestDB(t) t.Setenv(databaseEnv, seedDatabase(t, recs))
err := applyChanges(t.Context(), db, recs, nil, nil) err := runReport(t.Context(), failingWriter{})
if err != nil {
t.Fatal(err)
}
// Counts the records read, and draws nothing.
prog := &progress{}
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
if !errors.Is(err, errWriteFailed) || if !errors.Is(err, errWriteFailed) ||
!strings.HasPrefix(err.Error(), "write stdout: ") { !strings.HasPrefix(err.Error(), "write stdout: ") {
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed) t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
} }
if prog.count >= int64(len(recs)) {
t.Errorf("stdout failed after all %d records were read, want it "+
"to stop the read", prog.count)
}
}
// reportOpenLine is the plain line of report's open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
"eta unknown"
// smokeReport is report's stdout for the smoke-test records.
const smokeReport = "first\tdupe\tsize\n" +
"/d/t1/f1\t/d/t2/f1\t3000\n" +
"/d/t1/f1\t/d/t3/f1\t3000\n" +
"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"
// TestRunReportProgressNamesEachPhase runs report on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the two as it starts and another as
// it ends; read counts the 6 records before it reads them. The summary
// comes last, and stdout holds the report alone.
func TestRunReportProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdReport}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(report) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
if got := stdout.String(); got != smokeReport {
t.Errorf("stdout = %q, want %q", got, smokeReport)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"report: database " + path + " (from " + databaseEnv + ")",
reportOpenLine,
reportOpenLine,
"[2/2] read: counting the records, elapsed 0s, eta unknown",
"[2/2] read: [6/6] 100%, ",
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
"6.1 KiB reclaimable",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestReadReportOnTerminal writes the smoke-test report to the terminal
// its read phase's display is drawn on, as a plain "sfdupes report"
// does: the terminal must show the header and every row whole, each on
// a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestReadReportOnTerminal(t *testing.T) {
stderr := captureStderr(t)
db := openTestDB(t)
err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
if err != nil {
t.Fatal(err)
}
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: reportPhase("read"), unit: "records", total: -1,
waitingFor: countingRecords,
}, terminalWidth, time.Hour)
_, err = readReport(t.Context(), db, "", os.Stderr, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
} }
func TestRunReportsIgnoreInsertionOrder(t *testing.T) { func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
@@ -488,37 +354,6 @@ func TestDupeGroupsTieBreak(t *testing.T) {
} }
} }
func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
t.Parallel()
// report reads the groups of size 50 in signature order, here /a,
// /z, /k, which is neither the order of their first paths nor its
// reverse, and must write them ordered by first path. The group of
// size 7 comes after them all, though its first path sorts first.
recs := []scanRec{
{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
}
groups := dupeGroupsOf(t, recs)
got := make([]string, 0, len(groups))
for _, g := range groups {
got = append(got, g.paths[0])
}
want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
if !slices.Equal(got, want) {
t.Errorf("groups' first paths = %q, want %q", got, want)
}
}
func TestDupeGroupsDeterministic(t *testing.T) { func TestDupeGroupsDeterministic(t *testing.T) {
t.Parallel() t.Parallel()
+55 -141
View File
@@ -84,28 +84,14 @@ type fileMeta struct {
hashed bool hashed bool
} }
// scanUnit is what every scan phase counts, as its progress display // runScan implements the scan subcommand: four sequential phases —
// names it. // walk (which stats each file as it is discovered), hash, update,
const scanUnit = "files" // content — that synchronize the persistent database with the
// filesystem state under the PATH operands. Only files whose size at
// scanPhase names one phase of a scan with its place among all of them, // least one other file shares are ever hashed: a size-unique file
// as "[2/5] walk", for the phase's progress display. The list here is // cannot be a duplicate. A file of headTailMin or more gets its content
// the one place the phases are counted. // hash only when its size, head, and tail match another file's. Flag
func scanPhase(name string) string { // parsing and the at-least-one-operand check are done by cobra. The
return phaseLabel(
[]string{"load", "walk", "hash", "update", "content"}, name)
}
// runScan implements the scan subcommand: five sequential phases —
// load (which reads the existing records), walk (which stats each file
// as it is discovered), hash, update, content — that synchronize the
// persistent database with the filesystem state under the PATH
// operands. Only files whose size at least one other file shares are
// ever hashed: a size-unique file cannot be a duplicate. A file of
// headTailMin or more gets its content hash only when its size, head,
// and tail match another file's. Flag parsing and the
// at-least-one-operand check are done by cobra. Before anything else,
// the scan names the database on stderr, saying whether it is new. The
// scan holds the lock on the database for its whole run, so a second // scan holds the lock on the database for its whole run, so a second
// scan fails before it walks the filesystem or opens the database. // scan fails before it walks the filesystem or opens the database.
// Errors are returned rather than exiting, so that the deferred close — // Errors are returned rather than exiting, so that the deferred close —
@@ -118,26 +104,12 @@ func scanPhase(name string) string {
func runScan(ctx context.Context, roots []string, workers int, func runScan(ctx context.Context, roots []string, workers int,
oneFS bool, oneFS bool,
) error { ) error {
dbPath, source, err := databasePath() roots, err := resolveRoots(roots)
if err != nil { if err != nil {
return err return err
} }
// A database file missing now is created by lockScanDatabase below. dbPath := databasePath()
what := "database"
_, err = os.Stat(dbPath)
if errors.Is(err, fs.ErrNotExist) {
what = "new database"
}
fmt.Fprintf(os.Stderr, "scan: %s %s (%s)\n", what, escapePath(dbPath),
source)
roots, err = resolveRoots(roots)
if err != nil {
return err
}
lock, err := lockScanDatabase(dbPath) lock, err := lockScanDatabase(dbPath)
if err != nil { if err != nil {
@@ -232,12 +204,12 @@ func resolveRoots(roots []string) ([]string, error) {
// pruneRoots drops operands already covered by another operand: // pruneRoots drops operands already covered by another operand:
// duplicates and any operand lying under another one. With -x, an // duplicates and any operand lying under another one. With -x, an
// operand under another one is kept when the walk of the deepest kept // operand under another one is kept when device puts it, or for a
// operand above it cannot reach it: when device puts any directory from // regular file its directory, on a different filesystem than the
// it (from its directory, for a regular file) up to but not including // deepest kept operand above it, because that operand's walk then does
// that operand on a different filesystem than that operand. Every // not reach it. Every remaining file is then reachable through exactly
// remaining file is then reachable through exactly one root, so walked // one root, so walked paths are unique without keeping a scan-wide set
// paths are unique without keeping a scan-wide set of every path seen. // of every path seen.
func pruneRoots(roots []string, oneFS bool, func pruneRoots(roots []string, oneFS bool,
device func(path string) (uint64, bool), device func(path string) (uint64, bool),
) []string { ) []string {
@@ -267,15 +239,10 @@ func pruneRoots(roots []string, oneFS bool,
continue continue
} }
aboveDev, aboveOK := device(above)
if !aboveOK {
continue
}
// The walk of above enters a directory only when it is on // The walk of above enters a directory only when it is on
// above's filesystem, and then reaches every regular file in it // above's filesystem, and then reaches every regular file in it
// whatever device the file reports, so for a regular-file // whatever device the file reports, so for a regular-file
// operand the directories checked start at its directory. // operand its directory decides.
dir := r dir := r
fi, err := os.Lstat(r) fi, err := os.Lstat(r)
@@ -283,13 +250,11 @@ func pruneRoots(roots []string, oneFS bool,
dir = filepath.Dir(r) dir = filepath.Dir(r)
} }
for ; dir != above; dir = filepath.Dir(dir) { dev, ok := device(dir)
dev, ok := device(dir) aboveDev, aboveOK := device(above)
if ok && dev != aboveDev {
kept = append(kept, r)
break if ok && aboveOK && dev != aboveDev {
} kept = append(kept, r)
} }
} }
@@ -356,18 +321,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
} }
// runPhases synchronizes the database with the filesystem under roots // runPhases synchronizes the database with the filesystem under roots
// in five sequential phases: load (read the existing records), walk // in four sequential phases: walk (enumerate and stat every file,
// (enumerate and stat every file, building a complete size census), // building a complete size census), hash (read only the new or
// hash (read only the new or changed — or previously unhashed — files // changed — or previously unhashed — files whose size at least one
// whose size at least one other file shares, committing results in // other file shares, committing results in batches as they arrive),
// batches as they arrive), update (record the size-unique files without // update (record the size-unique files without reading them, and
// reading them, and delete the records the scan no longer verifies), // delete the records the scan no longer verifies), and content (fill
// and content (fill in the content hash of every record of headTailMin // in the content hash of every record of headTailMin or more whose
// or more whose size, head, and tail match another record's). Records // size, head, and tail match another record's). Records outside the
// outside the roots are never touched, except that the content phase // roots are never touched, except that the content phase fills in
// fills in their content hash. Operands the walk cannot start from are // their content hash. Operands the walk cannot start from are dropped
// dropped first, so the records beneath them are treated as if those // first, so the records beneath them count as outside the roots unless
// operands had not been given. // they lie under another root.
func (s *scanState) runPhases(ctx context.Context, roots []string, func (s *scanState) runPhases(ctx context.Context, roots []string,
workers int, oneFS bool, workers int, oneFS bool,
) error { ) error {
@@ -409,10 +374,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
// walkableRoots returns the operands the walk can start from: regular // walkableRoots returns the operands the walk can start from: regular
// files, and directories not named .zfs. Every other operand is warned // files, and directories not named .zfs. Every other operand is warned
// about, counted as skipped, and dropped. A dropped operand is no // about, counted as skipped, and dropped. A dropped operand is no
// longer a root, so the records stored beneath it are treated as if it // longer a root, so the records stored beneath it count as outside the
// had not been given: those under no other root count as outside the // roots and are not deleted as unverified, unless it lies under another
// roots and are not deleted as unverified. An operand that fails lstat // root. An operand that fails lstat here is kept, and the walk warns
// here is kept, and the walk warns about it. // about it.
func (s *scanState) walkableRoots(roots []string) []string { func (s *scanState) walkableRoots(roots []string) []string {
kept := make([]string, 0, len(roots)) kept := make([]string, 0, len(roots))
@@ -442,21 +407,10 @@ func (s *scanState) walkableRoots(roots []string) []string {
// separately into the same database. // separately into the same database.
func (s *scanState) loadIndex(ctx context.Context, roots []string) error { func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// Indexing tens of millions of records takes real time; without a // Indexing tens of millions of records takes real time; without a
// display the scan looks hung before the walk begins. Counting them // display the scan looks hung before the walk begins.
// first gives the display its total. prog := newProgress("load", -1)
prog := newProgress(progressParams{
label: scanPhase("load"), unit: scanUnit, total: -1,
waitingFor: countingRecords,
})
defer prog.finish() defer prog.finish()
total, err := countRecords(ctx, s.db)
if err != nil {
return err
}
prog.setTotal(total)
s.existing = make(map[string]fileMeta) s.existing = make(map[string]fileMeta)
return loadFileMeta(ctx, s.db, return loadFileMeta(ctx, s.db,
@@ -488,17 +442,7 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
func (s *scanState) walkPhase( func (s *scanState) walkPhase(
events <-chan walkEvent, roots []string, events <-chan walkEvent, roots []string,
) ([]fileRec, []fileRec) { ) ([]fileRec, []fileRec) {
// The records the last scan left under the roots estimate how many prog := newProgress("walk", -1)
// files the walk will find. A first scan has none, and no estimate.
estimate := int64(len(s.existing))
if estimate == 0 {
estimate = -1
}
prog := newProgress(progressParams{
label: scanPhase("walk"), unit: scanUnit, total: estimate,
estimate: true,
})
var ( var (
changed, unhashed []fileRec changed, unhashed []fileRec
@@ -669,11 +613,10 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
} }
// readRuns reads runs with the worker pool, one read per inode run, in // readRuns reads runs with the worker pool, one read per inode run, in
// the order given, under the progress display of the phase named label. // the order given, under a progress display named label. The workers
// The workers compute each run's hashes with hash, and each result goes // compute each run's hashes with hash, and each result goes to record;
// to record; a run that fails to read is warned about and counted as // a run that fails to read is warned about and counted as skipped
// skipped instead. The total counts actual reads, so the display shows // instead. The total counts actual reads, so the bar shows a real ETA.
// a real ETA.
// //
// Returning early — a failed database write, or a cancelled scan — must // Returning early — a failed database write, or a cancelled scan — must
// not strand the pool: the feeder would park forever on a full jobs // not strand the pool: the feeder would park forever on a full jobs
@@ -687,9 +630,7 @@ func (s *scanState) readRuns(ctx context.Context, workers int,
pool := startHashPool(ctx, runs, workers, hash) pool := startHashPool(ctx, runs, workers, hash)
defer pool.stop() defer pool.stop()
prog := newProgress(progressParams{ prog := newProgress(label, int64(len(runs)))
label: scanPhase(label), unit: scanUnit, total: int64(len(runs)),
})
defer prog.finish() defer prog.finish()
for range runs { for range runs {
@@ -776,9 +717,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
s.st.removed = len(deletes) s.st.removed = len(deletes)
total := len(s.batch) + len(s.sentinels) + len(deletes) total := len(s.batch) + len(s.sentinels) + len(deletes)
prog := newProgress(progressParams{ prog := newProgress("update", int64(total))
label: scanPhase("update"), unit: scanUnit, total: int64(total),
})
defer prog.finish() defer prog.finish()
@@ -859,28 +798,16 @@ func (s *scanState) contentPhase(ctx context.Context, workers int) error {
func (s *scanState) contentCandidates( func (s *scanState) contentCandidates(
ctx context.Context, ctx context.Context,
) ([]fileRec, map[string]scanRec, error) { ) ([]fileRec, map[string]scanRec, error) {
// The search and the checks take real time on a large database; // The query and the checks take real time on a large database;
// without a display the scan looks hung before the reads begin. // without a display the scan looks hung before the reads begin.
// The display says it is searching until SQLite returns the first prog := newProgress("content", -1)
// record. The checks start there, against a count of the records
// the search returns, and so do their rate and ETA.
prog := newProgress(progressParams{
label: scanPhase("content"), unit: scanUnit, total: -1,
waitingFor: "searching the database",
})
defer prog.finish() defer prog.finish()
total, err := countContentCandidates(ctx, s.db)
if err != nil {
return nil, nil, err
}
var ( var (
searching = true // no record has been returned yet toRead []fileRec
toRead []fileRec first scanRec // the current group's first record
first scanRec // the current group's first record passed int // the current group's files that passed the check
passed int // the current group's files that passed the check unread []fileRec // those of them without a content hash
unread []fileRec // those of them without a content hash
) )
recs := make(map[string]scanRec) recs := make(map[string]scanRec)
@@ -899,13 +826,7 @@ func (s *scanState) contentCandidates(
passed, unread = 0, nil passed, unread = 0, nil
} }
err = loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) { err := loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) {
if searching {
searching = false
prog.setTotal(total)
}
prog.increment() prog.increment()
if r.size != first.size || r.head != first.head || r.tail != first.tail { if r.size != first.size || r.head != first.head || r.tail != first.tail {
@@ -936,11 +857,6 @@ func (s *scanState) contentCandidates(
return nil, nil, err return nil, nil, err
} }
// A search that returned no record still ends showing its count.
if searching {
prog.setTotal(total)
}
endGroup() endGroup()
return toRead, recs, nil return toRead, recs, nil
@@ -1116,8 +1032,7 @@ func operandWarning(root string, fi fs.FileInfo) string {
// operand becomes an initial job. walkableRoots has already dropped // operand becomes an initial job. walkableRoots has already dropped
// every other operand. One that has changed into something else since // every other operand. One that has changed into something else since
// is warned about and skipped here; it is still a root, so the records // is warned about and skipped here; it is still a root, so the records
// stored beneath it, other than those under another root, are deleted // stored beneath it are deleted as unverified.
// as unverified.
func seedRoot(ctx context.Context, root string, func seedRoot(ctx context.Context, root string,
events chan<- walkEvent, events chan<- walkEvent,
) []dirJob { ) []dirJob {
@@ -1317,8 +1232,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
// .zfs (ZFS snapshot pseudo-dirs would list every file once per // .zfs (ZFS snapshot pseudo-dirs would list every file once per
// snapshot), and with -x never enter a directory on a different // snapshot), and with -x never enter a directory on a different
// filesystem than its operand, sending an event that names it so the // filesystem than its operand, sending an event that names it so the
// scan keeps the records below it, except those below an operand at or // scan keeps the records below it.
// inside it.
func subdirJob(ctx context.Context, p string, e fs.DirEntry, func subdirJob(ctx context.Context, p string, e fs.DirEntry,
parent dirJob, oneFS bool, events chan<- walkEvent, parent dirJob, oneFS bool, events chan<- walkEvent,
) (dirJob, bool) { ) (dirJob, bool) {
+7 -129
View File
@@ -1136,11 +1136,10 @@ func TestSubdirJobStatError(t *testing.T) {
} }
// syncOneFilesystem scans roots into db with -x and returns the scan // syncOneFilesystem scans roots into db with -x and returns the scan
// stats. Every device pruneRoots looks up, and the device each // stats, taking the device of each operand, and of a regular-file
// operand's walk compares its subdirectories with, comes from devices // operand's directory, from devices instead of from lstat, so a test
// instead of from lstat, so a test can put a path on a filesystem that // can put an operand on a filesystem that no directory here is on. It
// no directory here is on. Unlike runPhases, it skips walkableRoots, // runs the steps of runPhases except the content phase.
// the check for a cancelled walk, and the content phase.
func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
devices map[string]uint64, devices map[string]uint64,
) scanStats { ) scanStats {
@@ -1303,64 +1302,6 @@ func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
} }
} }
// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
// with -x an operand on the same filesystem as the operand covering it
// is walked on its own when a directory between them is on another
// filesystem, since the walk of the covering operand does not enter
// that directory: a file added below the nested operand is recorded,
// and the record of a file gone from below it is deleted. A record
// below that directory but outside the nested operand is still kept.
// Both operands are given one device that no directory here is on,
// while mnt keeps its own.
func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
// A scan without -x records both files below mnt.
gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
mnt := filepath.Join(dir, "mnt")
back := filepath.Join(mnt, "back")
syncOneFilesystem(t, db, []string{dir, back},
map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
paths := recordPaths(dbRecords(t, db))
if !slices.Contains(paths, added) {
t.Errorf("no record for %s: the walk did not reach %s", added, back)
}
if slices.Contains(paths, gone) {
t.Errorf("record for %s kept: the file is gone", gone)
}
if !slices.Contains(paths, kept) {
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
kept, dir, mnt)
}
}
// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a // TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
// regular-file operand under another operand is dropped when the file // regular-file operand under another operand is dropped when the file
// reports a different device than that operand but its directory does // reports a different device than that operand but its directory does
@@ -1478,7 +1419,7 @@ func dbRecords(t *testing.T, db *sql.DB) []scanRec {
var recs []scanRec var recs []scanRec
err := loadFileRows(t.Context(), db, func(int64) {}, func(r scanRec) { err := loadFileRows(t.Context(), db, func(r scanRec) {
recs = append(recs, r) recs = append(recs, r)
}) })
if err != nil { if err != nil {
@@ -1550,7 +1491,7 @@ func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) {
super, dirs := dbTree(t, db) super, dirs := dbTree(t, db)
tg := collectTreeGroups(dirs, super, nil) tg := collectTreeGroups(dirs, super)
if len(tg) != 1 { if len(tg) != 1 {
t.Fatalf("len(tree groups) = %d, want 1", len(tg)) t.Fatalf("len(tree groups) = %d, want 1", len(tg))
} }
@@ -1586,69 +1527,6 @@ func TestScanPipeline(t *testing.T) {
assertSmokeTreeGroups(t, dir, db) assertSmokeTreeGroups(t, dir, db)
} }
// TestScanProgressNamesEachPhase checks the plain progress lines of a
// first scan, and of a rescan of one operand holding two of the
// database's records. Each phase's first line names it with its place
// among the five: load counts the records before reading them, the
// content phase searches the database, and the walk estimates its total
// from the records under its operand, or says its ETA is unknown when
// there are none.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestScanProgressNamesEachPhase(t *testing.T) {
stderr := captureStderr(t)
dir := buildSmokeTree(t)
db := openTestDB(t)
syncTree(t, db, dir)
first := stderr()
syncTree(t, db, filepath.Join(dir, "a"))
rescan := stderr()[len(first):]
load := "[1/5] load: counting the records, elapsed 0s, eta unknown"
content := "[5/5] content: searching the database, elapsed 0s, eta unknown"
cases := map[string][]string{
first: {
load, "[2/5] walk: 0 files, 0 files/s, elapsed 0s, eta unknown",
"[3/5] hash: [0/", "[4/5] update: [0/", content,
},
rescan: {
load, "[2/5] walk: [0/~2] 0%, 0 files/s, elapsed 0s, eta unknown",
"[3/5] hash: [0/", "[4/5] update: [0/", content,
},
}
for out, want := range cases {
var got []string
for line := range strings.Lines(out) {
label, _, _ := strings.Cut(line, ":")
if len(got) == 0 || !strings.HasPrefix(got[len(got)-1], label) {
got = append(got, line)
}
}
if len(got) != len(want) {
t.Fatalf("first lines of the phases = %q, want %q", got, want)
}
for i := range want {
if !strings.HasPrefix(got[i], want[i]) {
t.Errorf("phase %d starts %q, want %q", i+1, got[i], want[i])
}
}
}
// The load phase's total is the count of the records.
loaded := fmt.Sprintf("\n[1/5] load: [%d/%d] 100%%, ",
smokeTreeFiles, smokeTreeFiles)
if !strings.Contains(rescan, loaded) {
t.Errorf("rescan stderr = %q, want %q", rescan, loaded)
}
}
func TestSyncScanUnchangedReuse(t *testing.T) { func TestSyncScanUnchangedReuse(t *testing.T) {
t.Parallel() t.Parallel()
@@ -2092,7 +1970,7 @@ func TestTreesUnhashedNeverEqual(t *testing.T) {
for name, unknown := range cases { for name, unknown := range cases {
super, dirs := treeOf(t, append(slices.Clone(shared), unknown...)) super, dirs := treeOf(t, append(slices.Clone(shared), unknown...))
if tg := collectTreeGroups(dirs, super, nil); len(tg) != 0 { if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
t.Errorf("%s: tree groups = %d, want 0 (the files may differ)", t.Errorf("%s: tree groups = %d, want 0 (the files may differ)",
name, len(tg)) name, len(tg))
} }
+16 -107
View File
@@ -1,24 +1,13 @@
#!/bin/sh #!/bin/sh
# script/bootstrap: install all dependencies needed to build and develop # script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already # this repo. Idempotent; assumes nothing is present (not git, make, or
# installed tools are skipped. Base tooling comes from nix, apt, brew, # go). Base tooling comes from nix, apt, brew, or apk (detected in that
# or apk (detected in that order); assumes nothing is present. The # order). golangci-lint and prettier are never installed: they run via
# installed node is used only when its major version is the pinned one; # docker only (script/lint, script/fmt, script/fmt-check).
# otherwise (node missing, or another major) the pinned version is
# installed via nvm (installing nvm itself first, from a hash-verified
# release archive, never curl | sh), and yarn and the packages are
# installed under it.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-07-06
NODE_VERSION="22.17.0"
NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
PKGMGR="" PKGMGR=""
SUDO="" SUDO=""
APT_UPDATED="" APT_UPDATED=""
@@ -51,7 +40,6 @@ pkg_install() {
case "$PKGMGR" in case "$PKGMGR" in
nix) nix-env -iA "nixpkgs.$1" ;; nix) nix-env -iA "nixpkgs.$1" ;;
apt) apt)
# Package lists may be empty (fresh images); refresh once per run.
if [ -z "$APT_UPDATED" ]; then if [ -z "$APT_UPDATED" ]; then
$SUDO env DEBIAN_FRONTEND=noninteractive apt-get update $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update
APT_UPDATED=1 APT_UPDATED=1
@@ -67,103 +55,24 @@ missing() {
! command -v "$1" >/dev/null 2>&1 ! command -v "$1" >/dev/null 2>&1
} }
# verify_sha256 <file> <expected-hash>
verify_sha256() {
if command -v sha256sum >/dev/null 2>&1; then
actual="$(sha256sum "$1" | cut -d' ' -f1)"
else
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
fi
if [ "$actual" != "$2" ]; then
echo "bootstrap: sha256 mismatch for $1" >&2
echo " expected: $2" >&2
echo " actual: $actual" >&2
exit 1
fi
}
# nvm is a bash script; run a command in a bash with nvm loaded
nvm_sh() {
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
}
ensure_nvm() {
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
# nvm prerequisites; nvm itself requires bash
if missing bash; then pkg_install bash bash bash bash; fi
if missing curl; then pkg_install curl curl curl curl; fi
if missing git; then pkg_install git git git git; fi
tmp="$(mktemp -d)"
curl -fsSL -o "$tmp/nvm.tar.gz" \
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
mkdir -p "$HOME/.nvm"
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
rm -rf "$tmp"
}
# True when the node on PATH has the pinned major version. Only the
# major is compared: the Dockerfile stages start from a node 22 alpine
# image whose exact version is not the pin, and nvm cannot install a
# prebuilt node on alpine. Another major is not used: the pinned yarn 1
# prints a deprecation warning under node 24. script/fmt and
# script/fmt-check carry this function unchanged, to run the yarn
# installed here.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
ensure_node() {
if node_is_pinned_major; then
echo "bootstrap: using node $(node --version)"
return 0
fi
ensure_nvm
nvm_sh "nvm install $NODE_VERSION"
nvm_sh "nvm use $NODE_VERSION >/dev/null && \
echo \"bootstrap: using node \$(node --version)\""
}
ensure_yarn() {
if ! node_is_pinned_major; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
return 0
fi
if ! missing yarn; then return 0; fi
if ! missing corepack; then
corepack enable
corepack prepare "yarn@$YARN_VERSION" --activate
else
npm install -g "yarn@$YARN_VERSION"
fi
}
install_js_deps() {
if node_is_pinned_major; then
yarn install --frozen-lockfile
else
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
yarn install --frozen-lockfile"
fi
}
main() { main() {
cd "$ROOT" cd "$ROOT"
if missing make; then pkg_install gnumake make make make; fi # Deliberately unpinned, so presence is the whole check: go.mod
# governs the Go version, and reproducible builds run in the
# digest-pinned Docker images.
if missing git; then pkg_install git git git git; fi if missing git; then pkg_install git git git git; fi
# Go builds the binary and runs gofmt. Presence is the whole check: if missing make; then pkg_install gnumake make make make; fi
# go.mod names the Go version, and the tests and the linter run in
# digest-pinned images.
if missing go; then pkg_install go golang go go; fi if missing go; then pkg_install go golang go go; fi
ensure_node # Warn, do not fail: only the targets named below, and the
ensure_yarn # pre-commit hook, need docker.
install_js_deps if missing docker; then
echo "bootstrap: WARNING: docker not found; make lint, make fmt," >&2
echo "bootstrap: make fmt-check, make check, make docker and" >&2
echo "bootstrap: make test-race require it." >&2
fi
go mod download go mod download
echo "bootstrap complete" echo "bootstrap complete"
+1 -3
View File
@@ -1,8 +1,6 @@
#!/bin/sh #!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own # script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. test and lint are Docker # extension to scripts-to-rule-them-all. Must not modify any files.
# phases; fmt-check is native, because a formatter writes the working
# tree. Must not modify any files.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+10 -23
View File
@@ -1,32 +1,19 @@
#!/bin/sh #!/bin/sh
# script/cibuild: run the CI build. It bootstraps first: a CI runner # script/cibuild: run the CI build. The Gitea workflow runs this on
# checks out and runs this and nothing else, and script/fmt-check runs # push. The Dockerfile runs every gate make check runs, as build steps,
# the formatter on the host, which a pristine checkout cannot do. # so a successful build means the repo is green.
# A new CACHEBUST value reruns every RUN step, since a cached gate phase #
# is a check that did not run; unlike --no-cache, it keeps Go's build cache. # Without a fresh CHECK_EPOCH, a rebuild of an unchanged checkout serves
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST # the gate layers from cache and passes having run none of them. The
# first; a stage without that line is served from cache on an unchanged # process id goes in with the epoch so two runs started in the same
# tree, with no warning. # second still differ.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
"$SCRIPT_DIR/bootstrap" docker build --build-arg CHECK_EPOCH="$(date +%s)-$$" .
"$SCRIPT_DIR/check"
# The version and the tag each get their own line: a failing
# command substitution inside an argument does not trip `set -e`,
# so the inline form degrades silently to an empty constant. The
# VERSION build argument takes precedence over the version a build
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \
-t "$tag" .
} }
main "$@" main "$@"
+8 -17
View File
@@ -1,11 +1,9 @@
#!/bin/sh #!/bin/sh
# script/docker: build the Docker image tagged with the project name. # script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname. # The tag comes from script/projectname. CHECK_EPOCH is passed for the
# A new CACHEBUST value reruns every RUN step, since a cached gate phase # same reason script/cibuild passes it: without a fresh value an
# is a check that did not run; unlike --no-cache, it keeps Go's build cache. # unchanged tree is served from cache and this exits 0 having run no
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST # gate.
# first; a stage without that line is served from cache on an unchanged
# tree, with no warning.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -13,17 +11,10 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
# The version and the tag each get their own line: a failing docker build \
# command substitution inside an argument does not trip `set -e`, --build-arg CHECK_EPOCH="$(date +%s)-$$" \
# so the inline form degrades silently to an empty constant. The -t "$("$SCRIPT_DIR/projectname")" \
# VERSION build argument takes precedence over the version a build .
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \
-t "$tag" .
} }
main "$@" main "$@"
+12 -32
View File
@@ -1,42 +1,22 @@
#!/bin/sh #!/bin/sh
# script/fmt: format all files (writes). # script/fmt: format all files (writes): the Go sources with gofmt, the
# Markdown with prettier. prettier is never installed on the host: it
# runs from the Dockerfile's prettier stage with the repository mounted,
# as the calling user so the files it rewrites keep their owner. The tag
# makes each build replace the previous image instead of leaving another
# one behind.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same
# function as in script/bootstrap, which uses it to decide where it
# installs yarn.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt: no yarn; run script/bootstrap first" >&2
exit 1
fi
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
}
main() { main() {
cd "$ROOT" cd "$ROOT"
gofmt -s -w . gofmt -s -w .
run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always image="$("$SCRIPT_DIR/projectname")-prettier"
docker build -q --target prettier -t "$image" . >/dev/null
docker run --rm --user "$(id -u):$(id -g)" -v "$ROOT:/src" "$image" \
prettier --write '**/*.md' --tab-width 4 --prose-wrap always
} }
main "$@" main "$@"
+15 -36
View File
@@ -1,45 +1,19 @@
#!/bin/sh #!/bin/sh
# script/fmt-check: check formatting (read-only). # script/fmt-check: check formatting (read-only). Same scope as
# script/fmt, but fails instead of writing. gofmt and prettier both run
# every time and each reports its own failure, so the output says which
# one failed.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# True when the node on PATH has the pinned major version: the same
# function as in script/bootstrap, which uses it to decide where it
# installs yarn.
node_is_pinned_major() {
if ! command -v node >/dev/null 2>&1; then return 1; fi
installed="$(node --version)"
installed="${installed#v}"
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
}
# Run the yarn script/bootstrap installed: the yarn on PATH when the
# node on PATH has the pinned major version, and otherwise yarn under
# the pinned node in nvm, which bootstrap leaves off the PATH of the
# shell that called it. nvm is a bash script, hence the subshell.
run_yarn() {
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt-check: no yarn; run script/bootstrap first" >&2
exit 1
fi
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
}
main() { main() {
cd "$ROOT" cd "$ROOT"
status=0 status=0
# gofmt and prettier both run every time, so the output names each # Under set -e a bare assignment would end the script when gofmt
# one that fails. Under set -e a bare assignment would end the # fails (a Go file it cannot parse), and prettier would never run.
# script when gofmt fails (a Go file it cannot parse).
if ! files="$(gofmt -s -l .)"; then if ! files="$(gofmt -s -l .)"; then
echo "gofmt: failed; see its errors above" >&2 echo "gofmt: failed; see its errors above" >&2
status=1 status=1
@@ -50,9 +24,14 @@ main() {
status=1 status=1
fi fi
# run_yarn ends in exec; the subshell returns here afterwards. # Same image as script/fmt; see there.
(run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always) || image="$("$SCRIPT_DIR/projectname")-prettier"
docker build -q --target prettier -t "$image" . >/dev/null
if ! docker run --rm -v "$ROOT:/src:ro" "$image" \
prettier --check '**/*.md' --tab-width 4 --prose-wrap always; then
echo "prettier: Markdown not formatted; run make fmt" >&2
status=1 status=1
fi
exit "$status" exit "$status"
} }
+7 -3
View File
@@ -1,15 +1,19 @@
#!/bin/sh #!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs # script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all. # script/precommit. Our own extension to scripts-to-rule-them-all.
# Hooks are shared between the main checkout and all worktrees, so
# resolve the common git dir instead of assuming .git is a directory.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
hook=".git/hooks/pre-commit" hooks_dir="$(git rev-parse --git-common-dir)/hooks"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit mkdir -p "$hooks_dir"
chmod +x .git/hooks/pre-commit hook="$hooks_dir/pre-commit"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
chmod +x "$hook"
echo "pre-commit hook installed: runs script/precommit" echo "pre-commit hook installed: runs script/precommit"
} }
+18 -16
View File
@@ -1,26 +1,28 @@
#!/bin/sh #!/bin/sh
# script/lint: run the linter. Linting is a phase of the Dockerfile and # script/lint: run the linter. golangci-lint is never installed on a
# this builds that phase alone; the linter is never installed or run on # host: this builds Dockerfile.lint, which copies the repo into the
# a developer host, where a shared result cache and a host-global lock # digest-pinned golangci-lint image and lints as a build step, so a
# make its answer untrustworthy. # successful build is a clean lint. A cold cache needs the network to
# pull the image and for `go mod download`; once warm this runs offline
# until go.mod or go.sum changes.
# #
# The phase is not the last stage in the file, so it is built only when # Without a fresh CHECK_EPOCH docker serves the gate layers from cache
# --target names it. A new CACHEBUST value reruns every RUN step, since # on an unchanged tree and this exits 0 having run no linter. The PID is
# a cached lint layer is a lint that did not run; unlike --no-cache, it # in the value because two lint runs land inside the same second easily.
# keeps Go's build cache. This relies on every stage of the Dockerfile #
# declaring ARG CACHEBUST first; a stage without that line is served # The image is never used, so --output=type=cacheonly writes none;
# from cache on an unchanged tree, with no warning. # without it every run leaves an untagged image behind.
# --output type=cacheonly writes no image, since nothing uses one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build \
--target lint \ --build-arg CHECK_EPOCH="$(date +%s)-$$" \
--output type=cacheonly . --output=type=cacheonly \
-f Dockerfile.lint \
.
} }
main "$@" main "$@"
+2 -2
View File
@@ -1,13 +1,13 @@
#!/bin/sh #!/bin/sh
# script/precommit: run by the git pre-commit hook; fails the commit if # script/precommit: run by the git pre-commit hook; fails the commit if
# checks fail. Our own extension to scripts-to-rule-them-all. # checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
# go mod tidy must be a no-op before the checks run.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
# Go extra: go mod tidy must be a no-op before the checks run.
cd "$ROOT" cd "$ROOT"
go mod tidy go mod tidy
if ! git diff --exit-code -- go.mod go.sum; then if ! git diff --exit-code -- go.mod go.sum; then
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh #!/bin/sh
# script/setup: set up the repo for development after a fresh clone: # script/setup: set up the repo for development after a fresh clone:
# installs dependencies and the git pre-commit hook. # installs dependencies (script/bootstrap) and the git pre-commit hook.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+8 -14
View File
@@ -1,23 +1,17 @@
#!/bin/sh #!/bin/sh
# script/test: run the test suite. Testing is a phase of the Dockerfile # script/test: run the test suite. Reruns verbosely on failure so CI
# and this builds that phase alone, on the same terms as script/lint: # logs show which test failed.
# --target because a phase that is not the last stage is built only when
# named, and a new CACHEBUST value because a cached test layer is a test
# that did not run; unlike --no-cache, it keeps Go's build cache. This
# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
# a stage without that line is served from cache on an unchanged tree,
# with no warning. --output type=cacheonly writes no image, since
# nothing uses one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ go test -timeout 30s -cover ./... || {
--target test \ echo "--- Rerunning with -v for details ---"
--output type=cacheonly . go test -timeout 30s -v ./...
exit 1
}
} }
main "$@" main "$@"
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
# script/test-race: run the test suite under the race detector. Not part
# of script/check.
#
# The race detector needs cgo and a C compiler, which the host build
# never uses, so the tests run in a golang image that has gcc. The
# checkout is mounted read-only, so the docker daemon must be local. The
# container starts with empty caches every time: each run downloads the
# dependencies and compiles them with the detector, which needs the
# network and takes minutes.
#
# The tests run as the calling user, never as root: several of them make
# a file unreadable and expect reading it to fail, and root reads it
# anyway. When the caller is root they run as nobody, and then the
# checkout must be readable by other users. Neither user has a home
# directory in the image, so HOME is /tmp, where Go puts its build cache.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# golang:1.25-trixie, 2026-10-04. Debian rather than the Alpine image the
# Dockerfile builds with, because this one includes gcc.
IMAGE="golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73"
main() {
user="$(id -u):$(id -g)"
if [ "$(id -u)" -eq 0 ]; then
user=65534:65534
fi
docker run --rm \
--user "$user" \
--env HOME=/tmp \
--env CGO_ENABLED=1 \
--volume "$ROOT:/src:ro" \
--workdir /src \
"$IMAGE" \
go test -race -timeout 60s ./...
}
main "$@"
+77
View File
@@ -0,0 +1,77 @@
#!/bin/sh
# script/verify-lint-image-pin: fail unless the golangci-lint image
# referenced by Dockerfile.lint and the one referenced by the main
# Dockerfile's lint stage are the same image at the same digest. Our own
# extension to scripts-to-rule-them-all, not one of its entrypoints; run
# as a gate in both files. Nothing else keeps the two pins in sync, and
# a bump applied to one alone would lint the same tree against different
# rulesets, both green.
#
# Do not hardcode the expected digest here: that is a third copy to keep
# in sync.
#
# A reference that cannot be read is a hard failure, not a skip: two
# empty strings compare equal.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
LINT_DOCKERFILE="Dockerfile.lint"
MAIN_DOCKERFILE="Dockerfile"
# Echo the single golangci-lint image reference in the named Dockerfile.
# Scans every argument of every FROM instruction rather than assuming a
# field position, so `FROM --platform=... img AS stage` reads correctly.
# Exits non-zero, with a diagnosis, unless there is exactly one.
lint_image_ref() {
file="$1"
if [ ! -f "$file" ]; then
echo "verify-lint-image-pin: $file: not found" >&2
return 1
fi
refs="$(
awk '
toupper($1) == "FROM" {
for (i = 2; i <= NF; i++) {
if ($i ~ /^golangci\/golangci-lint[:@]/) {
print $i
}
}
}
' "$file"
)"
count="$(printf '%s' "$refs" | grep -c . || true)"
if [ "$count" -ne 1 ]; then
echo "verify-lint-image-pin: $file: expected exactly one" \
"golangci/golangci-lint FROM reference, found $count" >&2
return 1
fi
printf '%s\n' "$refs"
}
main() {
cd "$ROOT"
lint_ref="$(lint_image_ref "$LINT_DOCKERFILE")"
main_ref="$(lint_image_ref "$MAIN_DOCKERFILE")"
if [ "$lint_ref" != "$main_ref" ]; then
echo "verify-lint-image-pin: the linter image is pinned twice and" \
"the two pins disagree:" >&2
echo "verify-lint-image-pin: $LINT_DOCKERFILE: $lint_ref" >&2
echo "verify-lint-image-pin: $MAIN_DOCKERFILE: $main_ref" >&2
echo "verify-lint-image-pin: bump both FROM lines together so" \
"script/lint and the Dockerfile lint stage keep running the" \
"same linter" >&2
exit 1
fi
echo "verify-lint-image-pin: $LINT_DOCKERFILE and $MAIN_DOCKERFILE" \
"agree on $lint_ref"
}
main "$@"
+28 -114
View File
@@ -1,9 +1,9 @@
package main package main
import ( import (
"bufio"
"context" "context"
"crypto/sha256" "crypto/sha256"
"database/sql"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -24,129 +24,44 @@ type treeNode struct {
totalSize int64 totalSize int64
} }
// treesPhase names one phase of trees with its place among all of them,
// as "[2/4] read", for the phase's progress display. The list here is
// the one place the phases are counted.
func treesPhase(name string) string {
return phaseLabel([]string{openPhase, "read", "group", "write"}, name)
}
// runTrees implements the trees subcommand: it reads every record from // runTrees implements the trees subcommand: it reads every record from
// the database in path order, reconstructs the directory hierarchy from // the database in path order, reconstructs the directory hierarchy from
// the record paths, computes a Merkle-style digest per directory, and // the record paths, computes a Merkle-style digest per directory, and
// prints maximal duplicate-tree groups as TSV on stdout. It never // prints maximal duplicate-tree groups as TSV on stdout. It never
// touches the scanned filesystem; its only I/O is the database, stdout, // touches the scanned filesystem; its only I/O is the database, stdout,
// and stderr. It names the database on stderr before it opens it. Any // and stderr. Any database problem, including a missing database, is
// database problem, including a missing database, is fatal. // fatal.
//
// The work runs in four phases, each under a progress display of its
// own on stderr: open (open the database), read (count the records,
// then read them into the hierarchy), group (group the directories by
// digest) and write (write the groups to stdout). Each display ends
// before the next phase starts, and before an error is returned to be
// printed.
func runTrees(ctx context.Context, stdout io.Writer) error { func runTrees(ctx context.Context, stdout io.Writer) error {
dbPath, source, err := databasePath() dbPath := databasePath()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
escapePath(dbPath), source)
// Opening can wait on a busy database for as long as the busy
// timeout.
prog := newProgress(progressParams{
label: treesPhase(openPhase), total: -1,
waitingFor: openingDatabase,
})
db, err := openReportDatabase(ctx, dbPath) db, err := openReportDatabase(ctx, dbPath)
prog.finish()
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
prog = newProgress(progressParams{ records := 0
label: treesPhase("read"), unit: "records", total: -1, tree := newTreeBuilder()
waitingFor: countingRecords,
err = loadFileRows(ctx, db, func(r scanRec) {
records++
tree.add(r)
}) })
records, super, allDirs, err := readTree(ctx, db, prog)
prog.finish()
if err != nil { if err != nil {
return fmt.Errorf("database %s: %w", dbPath, err) return fmt.Errorf("database %s: %w", dbPath, err)
} }
prog = newProgress(progressParams{
label: treesPhase("group"), unit: "directories",
total: int64(len(allDirs)),
})
dupes := collectTreeGroups(allDirs, super, prog)
prog.finish()
prog = newProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
})
dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
prog.finish()
if err != nil {
return err
}
fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
}
// readTree gives prog the number of records in db as its total, then
// reads every record in path order into the directory hierarchy,
// counting each on prog. It returns the number of records read, the
// super-root and every directory.
func readTree(ctx context.Context, db *sql.DB,
prog *progress,
) (int, *treeNode, []*treeNode, error) {
records := 0
tree := newTreeBuilder()
err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
records++
prog.increment()
tree.add(r)
})
if err != nil {
return 0, nil, nil, err
}
super, allDirs := tree.finish() super, allDirs := tree.finish()
return records, super, allDirs, nil dupes := collectTreeGroups(allDirs, super)
}
// writeTreeGroups writes dupes to stdout as TSV, a header line and then out := bufio.NewWriterSize(stdout, ioBufSize)
// a row for every member of each group but its first, counting each
// group on prog once its rows are written. stdout may be the terminal
// prog is drawn on, so the lines go through prog's writer. It returns
// the number of rows (dupe trees) and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress,
) (int, int64, error) {
out := prog.writer(stdout)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize") _, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
dupeTrees := 0 dupeTrees := 0
@@ -160,22 +75,25 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
escapePath(first.path), escapePath(n.path), escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize) first.fileCount, first.totalSize)
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
dupeTrees++ dupeTrees++
reclaimable += first.totalSize reclaimable += first.totalSize
} }
prog.increment()
} }
err = out.Flush() err = out.Flush()
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return fmt.Errorf("write stdout: %w", err)
} }
return dupeTrees, reclaimable, nil fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil
} }
// treeBuilder reconstructs the directory hierarchy from records added // treeBuilder reconstructs the directory hierarchy from records added
@@ -297,18 +215,14 @@ func fileEntry(name string, r scanRec) string {
"\x00" + r.head + "\x00" + r.tail + "\x00" + content "\x00" + r.head + "\x00" + r.tail + "\x00" + content
} }
// collectTreeGroups groups directories by digest, counting each one on // collectTreeGroups groups directories by digest and returns every
// prog as it is grouped, and returns every maximal group with two or // maximal group with two or more members, each group's members sorted
// more members, each group's members sorted by path, groups ordered by // by path, groups ordered by tree size descending then by first path
// tree size descending then by first path ascending. // ascending.
func collectTreeGroups(allDirs []*treeNode, super *treeNode, func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
prog *progress,
) [][]*treeNode {
groups := make(map[[sha256.Size]byte][]*treeNode) groups := make(map[[sha256.Size]byte][]*treeNode)
for _, d := range allDirs { for _, d := range allDirs {
groups[d.digest] = append(groups[d.digest], d) groups[d.digest] = append(groups[d.digest], d)
prog.increment()
} }
var dupes [][]*treeNode var dupes [][]*treeNode
+7 -107
View File
@@ -3,11 +3,8 @@ package main
import ( import (
"bytes" "bytes"
"database/sql" "database/sql"
"os"
"slices" "slices"
"strings"
"testing" "testing"
"time"
) )
// Signature hashes shared by the smoke-test records. // Signature hashes shared by the smoke-test records.
@@ -20,11 +17,6 @@ const (
f2Content = "f2c" f2Content = "f2c"
) )
// treesOpenLine is the plain line of trees' open phase when it opens
// the database at once; it prints it as the phase starts and as it ends.
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
"eta unknown"
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
// /d/t2 are identical, /d/t3 differs from them only by one filename. // /d/t2 are identical, /d/t3 differs from them only by one filename.
func smokeTreeRecs() []scanRec { func smokeTreeRecs() []scanRec {
@@ -46,7 +38,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
tree := newTreeBuilder() tree := newTreeBuilder()
err := loadFileRows(t.Context(), db, func(int64) {}, tree.add) err := loadFileRows(t.Context(), db, tree.add)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -178,7 +170,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
t.Fatalf("directory paths = %q, want %q", got, want) t.Fatalf("directory paths = %q, want %q", got, want)
} }
groups := collectTreeGroups(dirs, super, nil) groups := collectTreeGroups(dirs, super)
gotGroups := groupPaths(groups) gotGroups := groupPaths(groups)
wantGroups := [][]string{{"/a", "/c"}} wantGroups := [][]string{{"/a", "/c"}}
@@ -211,98 +203,6 @@ func TestRunTreesEscapesPaths(t *testing.T) {
} }
} }
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
// records with stderr not a terminal, and checks every line it prints
// there. After the line naming the database, each phase prints a line
// naming it with its place among the four as it starts and another as
// it ends. read counts the 6 records before it reads them, group counts
// the 8 directories, and write the 1 group. The summary comes last, and
// stdout holds the report alone.
func TestRunTreesProgressNamesEachPhase(t *testing.T) {
path := seedDatabase(t, smokeTreeRecs())
t.Setenv(databaseEnv, path)
stderr := captureStderr(t)
var stdout bytes.Buffer
code := run([]string{cmdTrees}, &stdout, os.Stderr)
if code != exitOK {
t.Fatalf("run(trees) = %d, want %d; stderr: %s",
code, exitOK, stderr())
}
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
if got := stdout.String(); got != report {
t.Errorf("stdout = %q, want %q", got, report)
}
// How each line starts; the rate at the end of a phase varies.
want := []string{
"trees: database " + path + " (from " + databaseEnv + ")",
treesOpenLine,
treesOpenLine,
"[2/4] read: counting the records, elapsed 0s, eta unknown",
"[2/4] read: [6/6] 100%, ",
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, eta unknown",
"[3/4] group: [8/8] 100%, ",
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, eta unknown",
"[4/4] write: [1/1] 100%, ",
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
if len(lines) != len(want) {
t.Fatalf("stderr lines = %q, want %d lines starting %q",
lines, len(want), want)
}
for i := range want {
if !strings.HasPrefix(lines[i], want[i]) {
t.Errorf("stderr line %d = %q, want it to start %q",
i+1, lines[i], want[i])
}
}
}
// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
// terminal its write phase's display is drawn on, as a plain "sfdupes
// trees" does: the terminal must show the header and the row whole,
// each on a line of its own.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestWriteTreeGroupsOnTerminal(t *testing.T) {
stderr := captureStderr(t)
super, dirs := treeOf(t, smokeTreeRecs())
dupes := collectTreeGroups(dirs, super, nil)
// No timed draws: the display is drawn as it starts and as it ends.
p := startProgress(progressParams{
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
}, terminalWidth, time.Hour)
_, _, err := writeTreeGroups(os.Stderr, dupes, p)
p.finish()
if err != nil {
t.Fatal(err)
}
// A terminal shows each line as the text after its last carriage
// return.
var shown []string
for line := range strings.SplitSeq(stderr(), "\n") {
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
}
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
t.Errorf("terminal shows lines %q, want %q first", shown, want)
}
}
func TestTreeDigests(t *testing.T) { func TestTreeDigests(t *testing.T) {
t.Parallel() t.Parallel()
@@ -354,7 +254,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
super, dirs := treeOf(t, smokeTreeRecs()) super, dirs := treeOf(t, smokeTreeRecs())
groups := collectTreeGroups(dirs, super, nil) groups := collectTreeGroups(dirs, super)
want := [][]string{{"/d/t1", "/d/t2"}} want := [][]string{{"/d/t1", "/d/t2"}}
if got := groupPaths(groups); !slices.EqualFunc(got, want, if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -378,14 +278,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
forward := groupPaths(collectTreeGroups(dirs, super, nil)) forward := groupPaths(collectTreeGroups(dirs, super))
reversed := slices.Clone(recs) reversed := slices.Clone(recs)
slices.Reverse(reversed) slices.Reverse(reversed)
superR, dirsR := treeOf(t, reversed) superR, dirsR := treeOf(t, reversed)
backward := groupPaths(collectTreeGroups(dirsR, superR, nil)) backward := groupPaths(collectTreeGroups(dirsR, superR))
if !slices.EqualFunc(forward, backward, slices.Equal) { if !slices.EqualFunc(forward, backward, slices.Equal) {
t.Fatalf("output depends on record order: %v vs %v", t.Fatalf("output depends on record order: %v vs %v",
forward, backward) forward, backward)
@@ -404,7 +304,7 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil)) got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/x1", "/p/x2"}} want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) { if !slices.EqualFunc(got, want, slices.Equal) {
@@ -426,7 +326,7 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
super, dirs := treeOf(t, recs) super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil)) got := groupPaths(collectTreeGroups(dirs, super))
want := [][]string{{"/p/a/x", "/q/b/x"}} want := [][]string{{"/p/a/x", "/q/b/x"}}
if !slices.EqualFunc(got, want, slices.Equal) { if !slices.EqualFunc(got, want, slices.Equal) {