diff --git a/.dockerignore b/.dockerignore index fd9fbce..e1d1d88 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,9 @@ -.git +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +.git/config + .claude .DS_Store sfdupes diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index ee73864..1be9221 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,4 +6,6 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 - run: script/cibuild diff --git a/.golangci.yml b/.golangci.yml index 26b1610..a7a74c2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,14 +10,20 @@ run: linters: default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields - - depguard # Dependency allow/block lists - godot # Requires comments to end with periods - - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 settings: lll: line-length: 88 @@ -28,6 +34,64 @@ linters: max-complexity: 15 dupl: threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..23d67fc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +node_modules/ +yarn.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/Dockerfile b/Dockerfile index a6009f2..b2d7424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,34 +6,36 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -# Cache-buster for the gate layers, and only for them. Docker -# invalidates COPY only when the copied content changes, so on an -# unchanged tree the gates below would be served from cache and the -# build would exit 0 having run nothing. script/cibuild and -# script/docker pass a fresh CHECK_EPOCH on every invocation. +# Cache-buster for the gate layers, and only for them: on an unchanged +# tree Docker would serve the gates below from cache and the build would +# exit 0 having run nothing. script/cibuild and script/docker pass a +# fresh CHECK_EPOCH; a build without one, such as a bare +# `docker build .`, fails at the check right after the ARG. # -# Two properties this depends on. ARG is per-stage, so the build stage -# below declares it again; one declaration here would leave that -# stage's gate cacheable. And each gate RUN must reference the value, -# because BuildKit hashes the expanded command: a declared but -# unreferenced ARG invalidates nothing. -# -# It sits below the dependency layers deliberately. Everything above it -# (the pinned base image, go mod download) keeps its cache; only the -# gates go cold. +# ARG is per-stage, so the markdown and build stages declare it again. +# Each gate RUN must reference the value: BuildKit hashes the expanded +# command, so a declared but unreferenced ARG invalidates nothing. Keep +# it below the dependency layers so they stay cached. ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi -# The linter is invoked directly here, not through `make lint`. That -# target now runs `docker build -f Dockerfile.lint`, and a docker build -# cannot run a docker build: routing the gate through make would mean -# nesting docker inside this image. Same reason `make check` is gone -# from the build stage below. `make fmt-check` stays as it is — it is a -# gate, not the aggregate, and it shells out to nothing. -RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check +# These gates call the tools directly, not through `make lint` or +# `make fmt-check`: both run docker, which cannot run inside a docker +# build. This step is the gofmt half of `make fmt-check`; the markdown +# stage is its prettier half. gofmt's output is assigned to a variable +# first so that its own exit status, as when it cannot parse a file, +# still fails the step. +RUN echo "gate gofmt, epoch ${CHECK_EPOCH}" && \ + files="$(gofmt -s -l .)" && \ + if [ -n "$files" ]; then \ + echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \ + fi -# The FROM above and the one in Dockerfile.lint pin the same linter -# twice, and nothing else keeps them in sync; this fails the build when -# they disagree. See the script for why it restates neither pin. +# Fails the build when the FROM above and the one in Dockerfile.lint pin +# different linter images. RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \ script/verify-lint-image-pin @@ -46,69 +48,118 @@ RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \ RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \ golangci-lint run --config .golangci.yml ./... +# Prettier stage: the prettier that formats this repository's Markdown, +# never installed on a host. script/fmt and script/fmt-check build this +# stage alone and run it with the repository mounted on /src. prettier +# is installed in /tools so that the repository, mounted or copied onto +# /src, cannot hide it. +# node:22-alpine, 2026-02-22 +FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS prettier +WORKDIR /tools +# yarn.lock pins prettier by hash, and --frozen-lockfile fails rather +# than install anything yarn.lock does not name. +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile +ENV PATH=/tools/node_modules/.bin:$PATH +WORKDIR /src + +# Markdown stage: the Markdown half of `make fmt-check`, as a gate. +FROM prettier AS markdown +COPY . . +# Second per-stage declaration of the gate cache-buster and its check; +# see the lint stage above. +ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi +RUN echo "gate prettier, epoch ${CHECK_EPOCH}" && \ + prettier --check '**/*.md' --tab-width 4 --prose-wrap always + # Build stage # golang:1.25-alpine, 2026-07-23 FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder # We never build or run as root. Create an unprivileged user and point -# HOME and the Go caches at its home so go build and go test can write -# their caches when we drop to it below. $GOPATH/bin is deliberately not -# on PATH: script/bootstrap no longer `go install`s anything (the linter -# runs from a pinned image, never from a host install), so nothing lands -# there and adding it would only widen what this image resolves. +# HOME and the build cache at its home so go build and go test can write +# it when we drop to it below. +# +# The module cache stays at the base image's default /go/pkg/mod and +# belongs to root: script/bootstrap fills it as root. Do not move it +# into the home and hand it over with `chown -R`: that walks every file +# in it, which took from about 80 s to over ten minutes on a shared +# host, depending on load. RUN adduser -D -u 1000 builder ENV HOME=/home/builder ENV GOPATH=/home/builder/go +ENV GOMODCACHE=/go/pkg/mod ENV GOCACHE=/home/builder/.cache/go-build WORKDIR /src -# No-op file copy whose only purpose is the build-graph edge: it is what -# makes this stage depend on the lint stage, and so what forces BuildKit -# to finish fmt-check, the pin guard and lint before compilation and -# tests start. Remove it and the fail-fast design dies silently — the -# build stops gating on lint and still exits 0. It replaces a copy of -# the linter binary itself, which is no longer wanted here: nothing in -# this stage runs the linter, because `make lint` is now a docker build -# and a docker build cannot run inside one. +# No-op file copies whose only purpose is the build-graph edge: they +# make this stage depend on the lint and markdown stages, so BuildKit +# finishes those gates before compilation and tests start. Remove one +# and the build silently stops gating on that stage and still exits 0. COPY --from=lint /src/go.sum /dev/null +COPY --from=markdown /src/go.sum /dev/null -# Install development prerequisites the same way a developer does, -# rather than duplicating the installs inline. Only script/ and the -# dependency manifests are copied first, nothing else, so this layer -# stays cached until the scripts or the dependencies change — bootstrap -# ends in `go mod download`, which is why there is no separate -# invocation of it here. +# Install development prerequisites the same way a developer does. Only +# script/ and the dependency manifests are copied first, so this layer +# stays cached until they change. Bootstrap ends in `go mod download`. COPY script/ script/ COPY go.mod go.sum ./ RUN script/bootstrap -COPY . . +# Hand builder only what it writes to, without walking the module cache. +# This layer stays cached with bootstrap. +# - /src itself: make build writes the binary into it, and git refuses +# a repository whose top directory belongs to another user. +# - the module cache's cache/download directory itself, not what is in +# it: Go only reads the downloaded modules, but make build saves its +# lookup of this module's own version from git there, in a new +# directory named after the module path. +# - builder's home: the go commands bootstrap ran as root left Go's +# telemetry files there, a few small files. +RUN chown builder:builder /src /go/pkg/mod/cache/download && \ + chown -R builder:builder /home/builder -# Hand the sources and caches to the unprivileged user, then drop root -# before running any checks or builds. -RUN chown -R builder:builder /src /home/builder +# The sources are handed to builder as they are copied, so no layer has +# to walk them. Then drop root before running any checks or builds. +COPY --chown=builder:builder . . USER builder -# Fail the build unless the branch is green. Runs as non-root so the -# permission-denied test paths are exercised legitimately (root would -# bypass the chmod(0) the tests rely on). +# Fail the build unless the branch is green. Runs as non-root: root +# would bypass the chmod(0) the permission-denied tests rely on. # -# The gates are the individual targets, not `make check`: that aggregate -# runs `script/lint`, which is now a docker build, and nothing inside an -# image build may shell out to docker. Lint is not skipped by this — it -# ran in the lint stage above, which this stage's COPY --from makes a -# prerequisite. `make`, not the scripts directly, because the Makefile's +# The gate is `make test`, not `make check`, which runs docker; lint and +# the format checks ran in the lint and markdown stages above. `make`, +# not the script directly, because the Makefile's # `export CGO_ENABLED = 0` applies only to what it invokes. # -# Second per-stage declaration of the gate cache-buster; see the lint -# stage above for why one is not enough. It is placed after USER so the -# drop to the unprivileged user still happens before the checks run. +# Third per-stage declaration of the gate cache-buster and its check; +# see the lint stage above. It is placed after USER so the drop to the +# unprivileged user still happens before the checks run. ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test -RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check -RUN make build +# The version stamped into the binary: the VERSION build argument when +# one is given, otherwise `git describe --tags --always` of the .git in +# the build context (git is installed by script/bootstrap above). A +# context that carries .git and still yields no version fails the build; +# with neither, as from a source tarball, it is "dev". +ARG VERSION +RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \ + if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ + [ "$version" = unknown ]; }; then \ + echo "no version could be derived although the build context carries .git" >&2; \ + exit 1; \ + fi; \ + make build VERSION="$version" # Runtime stage # alpine:3.22, 2026-07-23 diff --git a/Dockerfile.lint b/Dockerfile.lint index aac1cb9..4d6707f 100644 --- a/Dockerfile.lint +++ b/Dockerfile.lint @@ -1,14 +1,12 @@ -# Lint-only image: this is how the linter runs, everywhere. The repo is -# COPYed into the pinned golangci-lint image and the linter runs as a -# build step, so a successful build IS a clean lint. golangci-lint is -# never installed on a host — one toolchain, pinned by digest, identical -# on a laptop and in CI — and this works even when the docker daemon is -# remote and bind mounts are impossible. +# Lint-only image, built by script/lint: the repo is copied into the +# pinned golangci-lint image and the linter runs as a build step, so a +# successful build is a clean lint. No bind mount, so it works when the +# docker daemon is remote. # -# script/lint builds this file. It is a separate image from the lint -# stage of the main Dockerfile because script/lint must not depend on -# the rest of that build; the two FROM lines are kept identical by -# script/verify-lint-image-pin, run as a gate below. +# It is separate from the main Dockerfile's lint stage because +# script/lint must not depend on the rest of that build; the two FROM +# lines are kept identical by script/verify-lint-image-pin, run as a +# gate below. # golangci/golangci-lint:v2.12.2, 2026-08-07 FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 @@ -20,38 +18,26 @@ RUN go mod download COPY . . -# Cache-buster for the gate layers, and only for them. Caching of the -# lint run is waived by ruling: COPY is invalidated only by changed -# content, so on an unchanged tree the gates below would be served from -# cache and this build would exit 0 in under a second having run no -# linter at all. That exact false green has bitten this repo twice -# already (#32, #39). script/lint passes a fresh value on every +# Cache-buster for the gate layers, and only for them; caching of the +# lint run is waived by ruling. On an unchanged tree the gates below +# would be served from cache and this build would exit 0 in under a +# second having run no linter. script/lint passes a fresh value on every # invocation. # -# Each gate RUN must reference the value, because BuildKit hashes the -# expanded command and not the ARG declaration: a declared but -# unreferenced ARG invalidates nothing. The ARG sits below the -# dependency layers deliberately — everything above it keeps its cache, -# only the gates go cold. +# Each gate RUN must reference the value: BuildKit hashes the expanded +# command, so a declared but unreferenced ARG invalidates nothing. Keep +# it below the dependency layers so they stay cached. ARG CHECK_EPOCH -# The linter version is pinned in two places, here and in the main -# Dockerfile's lint stage. Nothing else keeps them in sync, so a -# half-applied bump is a build failure; see the script. +# Fails the build when the FROM above and the main Dockerfile's lint +# stage pin different linter images. RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \ script/verify-lint-image-pin -# Validates .golangci.yml against golangci-lint's JSON schema. The -# concern about this step was that it fetches that schema over a live, -# unpinned HTTPS call; measured on the pinned image, it does not. The -# binary carries the schema for its own version, so under -# `--network none` this both passes on a valid config and still rejects -# an invalid one with the jsonschema error. That holds for the gate -# steps generally — none of them makes a network call — but not for -# this build as a whole: `go mod download` above needs the network on a -# cold cache, and under `--network none` a first build fails there -# before reaching any gate. That layer stays cached, so only a warm -# cache lints offline, until go.mod or go.sum changes. +# Validates .golangci.yml against golangci-lint's JSON schema, which the +# pinned binary embeds: measured under `--network none`, it passes a +# valid config and rejects an invalid one. No gate step makes a network +# call, but `go mod download` above needs the network on a cold cache. RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \ golangci-lint config verify --config .golangci.yml diff --git a/Makefile b/Makefile index 67c7260..5adfcec 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ BINARY := sfdupes VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -X main.Version=$(VERSION) -.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean +.PHONY: sfdupes build bootstrap setup test test-race lint fmt fmt-check check docker hooks clean # Standard targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern. @@ -27,6 +27,9 @@ setup: test: @script/test +test-race: + @script/test-race + lint: @script/lint diff --git a/README.md b/README.md index 6508cbe..2a1bb62 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,23 @@ ## Description -`sfdupes` is an MIT-licensed Go CLI tool by -[@sneak](https://sneak.berlin) that quickly identifies *candidate* -duplicate files — and, ultimately, entire duplicate directory trees — -across very large filesystems without reading every byte of every file. -Files are considered duplicates when their sizes are equal and they -agree on a short ladder of hashes. A file under 10 MiB is hashed in full -and compared directly. A larger file is gated first on the SHA-256 of -its first 64 KiB and of its last 64 KiB, and only when its size and -both of those match another file's is it read for a content hash to -compare — the SHA-256 of the whole file when it is under 50 MiB, or of -gigabyte-spaced 1 MiB samples when it is 50 MiB or larger. Below 50 MiB -the content hash is proof of identical content; at or above 50 MiB it is -a strong candidate signal rather than proof, because the gaps between -samples are never read. The intended use is -finding duplicate downloads and duplicated directory trees on -multi-terabyte ZFS servers where reading every byte of every file is -prohibitively expensive. `scan` maintains a persistent SQLite database of file -signatures that survives between runs, so it can be run from cron and -the reports can be generated at any time from the most recent scan. +`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that +quickly identifies _candidate_ duplicate files — and, ultimately, entire +duplicate directory trees — across very large filesystems without reading every +byte of every file. Files are considered duplicates when their sizes are equal +and they agree on a short ladder of hashes. A file under 10 MiB is hashed in +full and compared directly. A larger file is gated first on the SHA-256 of its +first 64 KiB and of its last 64 KiB, and only when its size and both of those +match another file's is it read for a content hash to compare — the SHA-256 of +the whole file when it is under 50 MiB, or of gigabyte-spaced 1 MiB samples when +it is 50 MiB or larger. Below 50 MiB the content hash is proof of identical +content; at or above 50 MiB it is a strong candidate signal rather than proof, +because the gaps between samples are never read. The intended use is finding +duplicate downloads and duplicated directory trees on multi-terabyte ZFS servers +where reading every byte of every file is prohibitively expensive. `scan` +maintains a persistent SQLite database of file signatures that survives between +runs, so it can be run from cron and the reports can be generated at any time +from the most recent scan. This README is the complete and authoritative specification. @@ -34,393 +32,558 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite" ./sfdupes trees > dupetrees.tsv ``` -`scan` walks one or more filesystem trees and maintains one database -record per regular file (path, size, mtime, head hash, tail hash, -content hash). The -database persists between runs; a rescan only hashes files that are new -or changed, or that may have gained a duplicate since the last scan, -and removes records for files that no longer exist. -`report` reads the database and prints the file-level duplicates -report. `trees` reads the same database and prints the duplicate-tree -report. A missing/invalid subcommand — or a `scan` invocation with no -`PATH` operand — prints a usage message and exits 2. +`scan` walks one or more filesystem trees and maintains one database record per +regular file (path, size, mtime, head hash, tail hash, content hash). The +database persists between runs; a rescan only hashes files that are new or +changed, or that may have gained a duplicate since the last scan, and removes +records for files that no longer exist. `report` reads the database and prints +the file-level duplicates report. `trees` reads the same database and prints the +duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation +with no `PATH` operand — prints a usage message and exits 2. -The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed -anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a -daily `sfdupes scan` cron job, with the reporting commands run -interactively whenever needed; their results are as fresh as the last -completed scan. +The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere +by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` +cron job, with the reporting commands run interactively whenever needed; their +results are as fresh as the last completed scan. + +### Install + +With Go installed, this builds and installs the current `main` branch: + +```sh +go install sneak.berlin/go/sfdupes@main +``` + +The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. A +binary installed this way reports its version as `dev`; one built from a clone +or into the Docker image carries the git tag or commit it was built from. + +From a clone, `make build` writes the binary to `./sfdupes`: + +```sh +git clone https://git.eeqj.de/sneak/sfdupes.git +cd sfdupes +make build +``` + +Copy the binary to `/usr/local/bin` for the cron job below. + +`make docker` builds the Docker image, tagged `sfdupes`, after running the tests +and the linter (see "Build"). The image runs `sfdupes` as root with the database +at its default path, so a bind mount of `/var/lib/sfdupes` keeps the database +between runs. Mount the scanned tree at the same path inside the container as on +the host; read-only is enough. The database records paths as the container sees +them, so the reports then name the host's paths. + +```sh +make docker +docker run --rm -v /srv:/srv:ro -v /var/lib/sfdupes:/var/lib/sfdupes \ + sfdupes scan /srv +docker run --rm -v /var/lib/sfdupes:/var/lib/sfdupes sfdupes report > dupes.tsv +``` + +### Daily scan from cron + +Run `scan` as root, so that it can read every file: a path it cannot read is +skipped with a warning and loses its database record (see "Rules for the walk"). +As a file `/etc/cron.d/sfdupes`: + +``` +30 3 * * * root /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log +``` + +- The database is `/var/lib/sfdupes/db.sqlite`, created with its directory by + the first scan. To keep it elsewhere, set + `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line. +- `scan` writes nothing to stdout. Its stderr, appended here to + `/var/log/sfdupes.log`, holds a plain progress line as each phase starts and + then at most every 5 seconds, a warning for each path it skips, and the + summary line (see "Progress" and "`scan` mode"). The log grows with every + scan; rotate it like any other. +- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A + scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the + reason among the last lines of the log; `tail` prints them, and cron mails + them to root if the host can send mail. +- A scan still running when the next one starts carries on. The new one fails at + once, and the lines cron mails include + `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. +- `report` and `trees` need only read access to the database (see "Database"). + Under the usual umask of `022` the first scan creates it readable by every + user, so an unprivileged user can run them against root's database. + +### Reading the reports + +Each row of `report` names two copies of one file, and each row of `trees` two +copies of one directory tree (see "Report output format" and "Trees output +format"). In a group of copies, the path that sorts first byte by byte is +`first` and every other path is a `dupe` of it. `first` says nothing about which +copy is the original or the oldest; which copy to keep is your choice. + +A row is a candidate, not proof: + +- The reports read only the database, so they show the files as of the last + scan; a file may have changed or gone since. +- A file of 50 MiB or more is compared only on samples of its content (see + "Duplicate detection"). +- Paths that are hard links to one file are listed as duplicates, but they share + their data, so removing one frees nothing. + +Compare a pair byte for byte before removing either copy. For the row +`/srv/a/big.iso`, `/srv/b/big-copy.iso`, `4294967296`: + +```sh +cmp /srv/a/big.iso /srv/b/big-copy.iso && echo identical +[ /srv/a/big.iso -ef /srv/b/big-copy.iso ] && echo "hard links" +``` + +`cmp` prints nothing and exits 0 only when every byte matches, and otherwise +reports where the files differ. The second line prints `hard links` when the two +paths are the same file, so removing either frees nothing. + +A path holding a backslash, tab, newline or carriage return is escaped in the +reports (see "Report output format"). Undo the escapes before using it. +`printf '%b'` does exactly that, because every backslash in an escaped path +starts one of the four escapes. Command substitution drops trailing newlines, so +print an `x` after the path and remove it afterwards, or a path that ends in a +newline names a different file: + +```sh +p="$(printf '%bx' '/srv/a/tab\tname.txt')"; p="${p%x}" +cmp "$p" /srv/b/tab-copy.txt +``` + +Check a `trees` row with `diff -r`, which compares the two trees file by file +and also names anything present in only one of them, such as an empty directory +or a symlink, which `trees` does not see. ## Rationale -Duplicate finders that hash entire files do not scale to the target -environment: ~10 million files and ~150 TB on possibly slow or busy -disks (a ZFS pool under resilver). sfdupes spends disk I/O only on files -whose size at least one other file shares, since a size-unique file -cannot be a duplicate. Of those, a file under 10 MiB is read in full; a -larger one has its cheap end windows read first, and is read for a -content hash only when its size and both end windows match another -file's — the whole file below 50 MiB, but only gigabyte-spaced samples -at or above 50 MiB, so the largest files are never read in full. This -keeps a full-filesystem sweep tractable, and the signatures are kept in -a persistent database, so the expensive filesystem pass is incremental: -a rescan re-hashes only files whose recorded mtime or size changed, plus -— for its content hash — a file of 10 MiB or more whose size and end -windows have come to match another file's. All analysis happens offline -from the database alone. The end goal is -not individual files but whole duplicated trees — duplicate -extractions, duplicate downloads, copied project trees — which an -operator can consider removing as a unit. +Duplicate finders that hash entire files do not scale to the target environment: +~10 million files and ~150 TB on possibly slow or busy disks (a ZFS pool under +resilver). sfdupes spends disk I/O only on files whose size at least one other +file shares, since a size-unique file cannot be a duplicate. Of those, a file +under 10 MiB is read in full; a larger one has its cheap end windows read first, +and is read for a content hash only when its size and both end windows match +another file's — the whole file below 50 MiB, but only gigabyte-spaced samples +at or above 50 MiB, so the largest files are never read in full. This keeps a +full-filesystem sweep tractable, and the signatures are kept in a persistent +database, so the expensive filesystem pass is incremental: a rescan re-hashes +only files whose recorded mtime or size changed, plus — for its content hash — a +file of 10 MiB or more whose size and end windows have come to match another +file's. All analysis happens offline from the database alone. The end goal is +not individual files but whole duplicated trees — duplicate extractions, +duplicate downloads, copied project trees — which an operator can consider +removing as a unit. ## Design Goals, in order: -1. **Find whole duplicate trees, not just files.** The end goal is to - identify places where the exact same set of files and directories - exists at two or more paths (duplicate extractions, duplicate - downloads, copied project trees), so the operator can consider - removing an entire subtree at once. File-level duplicate detection is - the foundation; tree-level detection is built on top of it. -2. **Spend I/O in proportion to duplicate likelihood.** Only files - whose size at least one other file shares are read at all — a - size-unique file cannot be a duplicate. Those are compared by the - ladder in "Duplicate detection" below: a file under 10 MiB is hashed - in full, while a larger file is gated on cheap 64 KiB end windows - first, and gets a content hash only when its size and both end - windows match another file's. That hash reads the whole file below - 50 MiB but only gigabyte-spaced 1 MiB samples at or above it, so the - very largest files are still never read in full. Scale target: tens of - millions of files, ~150 TB filesystem, possibly slow or busy disks - (ZFS pool under resilver). Holding one small record (path, size, - mtime) per file in memory during a scan is acceptable; holding - every file's hashes is not (they stay in the database). -3. **Scan incrementally, analyze offline.** The expensive filesystem - scan maintains a persistent database; an unchanged file is never - read again on a rescan, except to compute its content hash once a - file of 10 MiB or more comes to match another on size and both end - windows. All analysis (`report`, `trees`) works from - the database alone and must never touch the scanned filesystem - again. `scan` is designed to be cronned; the reports run at any +1. **Find whole duplicate trees, not just files.** The end goal is to identify + places where the exact same set of files and directories exists at two or + more paths (duplicate extractions, duplicate downloads, copied project + trees), so the operator can consider removing an entire subtree at once. + File-level duplicate detection is the foundation; tree-level detection is + built on top of it. +2. **Spend I/O in proportion to duplicate likelihood.** Only files whose size + at least one other file shares are read at all — a size-unique file cannot + be a duplicate. Those are compared by the ladder in "Duplicate detection" + below: a file under 10 MiB is hashed in full, while a larger file is gated + on cheap 64 KiB end windows first, and gets a content hash only when its + size and both end windows match another file's. That hash reads the whole + file below 50 MiB but only gigabyte-spaced 1 MiB samples at or above it, so + the very largest files are still never read in full. Scale target: tens of + millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool + under resilver). Holding one small record (path, size, mtime) per file in + memory during a scan is acceptable; holding every file's hashes is not (they + stay in the database). The reporting commands do not hold every file's + hashes either: `report` lets SQLite group and order the records and writes + each row as it reads it, so its memory does not grow with the database, and + `trees` reads the records in path order and keeps each directory's path, + digest and totals, plus the hashes of only the files in the directories + holding the record being read, so its memory grows with the number of + directories and with the size of the largest directory. +3. **Scan incrementally, analyze offline.** The expensive filesystem scan + maintains a persistent database; an unchanged file is never read again on a + rescan, except to compute its content hash once a file of 10 MiB or more + comes to match another on size and both end windows. All analysis (`report`, + `trees`) works from the database alone and must never touch the scanned + filesystem again. `scan` is designed to be cronned; the reports run at any time against the last completed scan. -4. **Clean stream separation.** Everything on stdout is machine-readable - data. All progress, warnings, and summaries go to stderr. Never mix - them. +4. **Clean stream separation.** Everything on stdout is machine-readable data. + All progress, warnings, summaries, and help and usage text go to stderr. + Never mix them. ### Constraints -- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: - `sfdupes`. -- Dependencies: standard library, `github.com/spf13/cobra` for the - CLI, **one progress-bar library** - (`github.com/schollz/progressbar/v3`), and **one SQLite driver** - (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled). - `github.com/spf13/viper` is permitted if configuration-file support - is ever needed, but is not currently used. No other third-party - deps. -- Cross-compilation is not a concern. Builds run with cgo disabled (the - `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. -- Analysis modes (`report`, `trees`) must be deterministic: identical - database contents, identical output, regardless of the order in - which records were inserted. +- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: `sfdupes`. +- Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one + progress-bar library** (`github.com/schollz/progressbar/v3`), + `golang.org/x/term` to tell whether stderr is a terminal, **one SQLite + driver** (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and + `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). + `github.com/spf13/viper` is permitted if configuration-file support is ever + needed, but is not currently used. No other third-party deps. +- Cross-compilation is not a concern. Builds run with cgo disabled (the + `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. +- Analysis modes (`report`, `trees`) must be deterministic: identical database + contents, identical output, regardless of the order in which records were + inserted. ### Subcommands Three subcommands, all implemented: -1. `scan` — walk the filesystem and synchronize the database: one - signature record per regular file. +1. `scan` — walk the filesystem and synchronize the database: one signature + record per regular file. 2. `report` — file-level duplicate report from the database. -3. `trees` — tree-level duplicate report: reconstruct the directory - hierarchy from the database records, compute a Merkle-style digest - per directory, and report maximal groups of identical trees. +3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy + from the database records, compute a Merkle-style digest per directory, and + report maximal groups of identical trees. ``` sfdupes scan [--workers N] [-x] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv +sfdupes --version +sfdupes [command] --help ``` +`--workers N` sets the size of each `scan` worker pool (default: the number of +CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that +operand's filesystem; both are described under "`scan` mode". +`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and +exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a +subcommand, prints the help text to stderr and exits 0, writing nothing to +stdout. + ### Database All three subcommands operate on a single SQLite database file: -- Location: the value of the `SFDUPES_DATABASE` environment variable - when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. - There is no command-line flag. -- `scan` creates the database (and its parent directory) on first - use. `report` and `trees` require an existing database; a missing - database file is a fatal error (exit 1) telling the user to run - `scan` first. -- The database uses WAL journal mode and a busy timeout, so running a - report while a cron `scan` is in progress is safe. The filesystem - is authoritative; the database is an eventually-consistent - reflection of it. Hashed records are committed in batched - transactions while the scan is still running (keeping the WAL - small and letting concurrent reports observe progress), so a - report may see a scan's changes partially applied, and a scan - that dies partway leaves a valid database holding everything - hashed so far; the next scan skips those records and converges - toward the filesystem. -- Schema (`PRAGMA user_version` is the schema version, currently 1; a - database with any other version is a fatal error): +- Location: the value of the `SFDUPES_DATABASE` environment variable when set + and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no + command-line flag. The path names the file exactly, whatever characters it + holds (`?`, `#` and `%` included); a relative path is relative to the working + directory. +- `scan` creates the database (and its parent directory) on first use. `report` + and `trees` require an existing database; a missing database file is a fatal + error (exit 1) telling the user to run `scan` first. +- Only one `scan` runs against a database at a time. For its whole run, `scan` + holds an exclusive `flock(2)` lock on a lock file beside the database, named + by appending `.lock` to the database's path with symlinks resolved + (`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the + filesystem or opens the database. Every path to one database, through a + symlink to the file or to a directory above it, names the same lock file. Just + before it takes the lock, `scan` creates the database file, empty, if it is + missing (through a symlink at the database path, it creates the symlink's + target), so the lock file is named the same way before the first scan sets up + the database as after. A second `scan` against the same database does not + wait: it fails at once with a one-line error naming the lock file and exits 1, + without walking anything or opening the database, and the running scan carries + on. The lock file is created on first use, open to its owner only, and left in + place: a leftover file blocks nothing, because the lock ends with the process + holding it however it ends, a fatal error or an interrupt included, and + deleting the file while a scan runs would let a second scan start. `report` + and `trees` never take the lock, so they run during a scan. +- While `scan` runs, the database is in WAL journal mode with a busy timeout, so + running a report while a cron `scan` is in progress is safe. The filesystem is + authoritative; the database is an eventually-consistent reflection of it. + Hashed records are committed in batched transactions while the scan is still + running (keeping the WAL small and letting concurrent reports observe + progress), so a report may see a scan's changes partially applied, and a scan + that dies partway leaves a valid database holding every batch committed so far + (an interrupted scan also commits the batch in progress, see "Error handling + and exit codes"); the next scan skips those records and converges toward the + filesystem. +- `scan` switches the database back to rollback-journal mode when it closes it, + so between scans the database file alone holds the whole database. Each switch + needs the database to itself: a `scan` that starts while a report is still + reading waits for it up to the 10-second busy timeout, then fails; a `scan` + that ends while a report has the database open warns and leaves the database + in WAL mode until the next scan. `report` writes each row as it reads it, so + it is still reading while its output is paused (a pager, a stalled pipe), and + a `scan` started then fails after the busy timeout. +- `report` and `trees` open the database read-only and need only read access to + the database file, and no write access to its directory. While the database is + in WAL mode they also read the `-wal` and `-shm` files beside it, which SQLite + creates with the database file's permissions. +- Schema (`PRAGMA user_version` is the schema version, currently 1; a database + with any other version is a fatal error. `scan` creates the schema and sets + the version in one transaction, so a first scan stopped while doing so leaves + an empty database the next scan sets up. A database at version 0 that already + has a `files` table was therefore not made by sfdupes; every subcommand + refuses it with an error telling the user to remove the file and rescan): ```sql CREATE TABLE files ( - path BLOB PRIMARY KEY, -- absolute path, raw bytes - size INTEGER NOT NULL, -- bytes, from lstat - mtime INTEGER NOT NULL, -- Unix seconds, from lstat - head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB - tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB - content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples + path BLOB PRIMARY KEY, -- absolute path, raw bytes + size INTEGER NOT NULL, -- bytes, from lstat + mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat + mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 + head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB + tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB + content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples ) WITHOUT ROWID; + CREATE INDEX files_signature ON files (size, head, tail, content); ``` - Paths are stored as BLOBs because Unix paths are raw bytes, not - guaranteed UTF-8. `mtime` is used only for change detection; it is - not part of the duplicate key. For a file under 10 MiB `head`, `tail`, - and `content` all hold the whole-file hash (that range is hashed in - full, with no end windows); for a larger file `head` and `tail` hold - the first- and last-64 KiB hashes and `content` the whole-file or - sampled hash. All three are empty strings when the file has never - been hashed because its size was unique as of the last scan that - covered it. For a file of 10 MiB or more, `content` stays empty - until the content phase of a scan (see "`scan` mode" below) has - read the file. A record with an empty `content` is never part of a - duplicate group, though it still defines the file for tree - reconstruction. + Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed + UTF-8. `mtime` and `mtime_nsec` hold the file's mtime to the nanosecond: + `mtime` the whole Unix seconds, rounded down, and `mtime_nsec` the + nanoseconds past that second. Split this way they hold any mtime a + filesystem can record, one before 1678 or after 2262 included, which a + single 64-bit count of nanoseconds cannot. They are used only for change + detection and are not part of the duplicate key. For a file under 10 MiB + `head`, `tail`, and `content` all hold the whole-file hash (that range is + hashed in full, with no end windows); for a larger file `head` and `tail` + hold the first- and last-64 KiB hashes and `content` the whole-file or + sampled hash. All three are empty strings when the file has never been + hashed because its size was unique as of the last scan that covered it. For + a file of 10 MiB or more, `content` stays empty until the content phase of a + scan (see "`scan` mode" below) has read the file. A record with an empty + `content` is never part of a duplicate group, though it still defines the + file for tree reconstruction. The `files_signature` index lets SQLite group + the records by signature for `report` without sorting the whole table. ### Duplicate detection -Two files are duplicates only when they agree on every rung of this -ladder; a mismatch at any rung means they are not duplicates. `scan` -stores each file's hashes, and `report` and `trees` group files by the -whole signature — size, `head`, `tail`, and `content` — so the grouping -is exactly this ladder applied across everything scanned into the -database, even across separate scans. +Two files are duplicates only when they agree on every rung of this ladder; a +mismatch at any rung means they are not duplicates. `scan` stores each file's +hashes, and `report` and `trees` group files by the whole signature — size, +`head`, `tail`, and `content` — so the grouping is exactly this ladder applied +across everything scanned into the database, even across separate scans. -1. **Size.** Files of different sizes are never compared. Only files - whose size at least one other file shares are hashed at all. -2. **Under 10 MiB: whole file.** A file smaller than 10 MiB is hashed - in full and compared directly, with no separate end-window step — - small files are cheap to read to the last byte, and doing so makes - the comparison exact. `head`, `tail`, and `content` all hold this - whole-file SHA-256, so such a file's signature is decided entirely - by its size and its content. -3. **10 MiB and above: head and tail.** For a larger file, the SHA-256 - of the first 64 KiB (`head`) and of the last 64 KiB (`tail`) are a - cheap gate that eliminates most same-size pairs before any bulk - reading: the content hash of the next two rungs is computed only for - a file whose size, `head`, and `tail` match another file's, whether - that file is scanned in the same run or stored by an earlier scan. - A stored file that first gains such a match in a later scan gets its - content hash then; until it has one, its `content` is empty and it - is not a duplicate. At 10 MiB and above the two windows never +1. **Size.** Files of different sizes are never compared. Only files whose size + at least one other file shares are hashed at all. +2. **Under 10 MiB: whole file.** A file smaller than 10 MiB is hashed in full + and compared directly, with no separate end-window step — small files are + cheap to read to the last byte, and doing so makes the comparison exact. + `head`, `tail`, and `content` all hold this whole-file SHA-256, so such a + file's signature is decided entirely by its size and its content. +3. **10 MiB and above: head and tail.** For a larger file, the SHA-256 of the + first 64 KiB (`head`) and of the last 64 KiB (`tail`) are a cheap gate that + eliminates most same-size pairs before any bulk reading: the content hash of + the next two rungs is computed only for a file whose size, `head`, and + `tail` match another file's, whether that file is scanned in the same run or + stored by an earlier scan. A stored file that first gains such a match in a + later scan gets its content hash then; until it has one, its `content` is + empty and it is not a duplicate. At 10 MiB and above the two windows never overlap. -4. **10 MiB and above, content below 50 MiB.** The SHA-256 of the - entire file. Agreement here is proof of identical content (barring a - SHA-256 collision). -5. **10 MiB and above, content 50 MiB and above.** A sampled SHA-256: - the 1 MiB window at each gigabyte-aligned offset (0, 1 GiB, 2 GiB, … - while inside the file, the final window truncated at end of file) is - fed, in order, into one hash. This is **deliberately probabilistic** - — the gaps between samples are never read, so two large files that - agree on every sample are reported as duplicates without being read - in full. It is the price of never reading a 150 GB file end to end. - Because size is already part of the signature, only equal-size files - reach this rung, so their sample boundaries always align. +4. **10 MiB and above, content below 50 MiB.** The SHA-256 of the entire file. + Agreement here is proof of identical content (barring a SHA-256 collision). +5. **10 MiB and above, content 50 MiB and above.** A sampled SHA-256: the 1 MiB + window at each gigabyte-aligned offset (0, 1 GiB, 2 GiB, … while inside the + file, the final window truncated at end of file) is fed, in order, into one + hash. This is **deliberately probabilistic** — the gaps between samples are + never read, so two large files that agree on every sample are reported as + duplicates without being read in full. It is the price of never reading a + 150 GB file end to end. Because size is already part of the signature, only + equal-size files reach this rung, so their sample boundaries always align. -`head`, `tail`, and `content` are one column each. A file below 10 MiB -and one at or above it never share a size, and neither do a file below -50 MiB and one at or above it, so a stored value is never ambiguous -between the whole-file, end-window, and sampled forms. +`head`, `tail`, and `content` are one column each. A file below 10 MiB and one +at or above it never share a size, and neither do a file below 50 MiB and one at +or above it, so a stored value is never ambiguous between the whole-file, +end-window, and sampled forms. ### `scan` mode -`scan` requires one or more `PATH` operands naming the trees to scan. -There is no default path; invoking `scan` with no operand is a usage -error (usage message on stderr, exit 2). An operand may be a directory -or a regular file; an operand that does not exist is a fatal error -(exit 1). Because database records persist between runs and are keyed -by absolute path, each operand is resolved to an absolute, lexically -cleaned path (symlinks are not resolved) before walking, so results do -not depend on the working directory. All operands belong to a single -scan and are enumerated concurrently: every operand seeds the shared -walk worker pool. Overlapping operands are harmless — an operand that -duplicates another or lies under another is dropped before walking, -so every file is reached exactly once and produces one database -record. +`scan` requires one or more `PATH` operands naming the trees to scan. There is +no default path; invoking `scan` with no operand is a usage error (usage message +on stderr, exit 2). An operand may be a directory or a regular file; an operand +that does not exist is a fatal error (exit 1). Because database records persist +between runs and are keyed by absolute path, each operand is resolved to an +absolute, lexically cleaned path (symlinks are not resolved) before walking, so +results do not depend on the working directory. All operands belong to a single +scan and are enumerated concurrently: every operand seeds the shared walk worker +pool. Overlapping operands are harmless — an operand that duplicates another or +lies under another is dropped before walking, so every file is reached exactly +once and produces one database record. With `-x`, though, an operand under +another one is kept and walked on its own when any directory from it (from its +directory, for a regular file) up to but not including the operand covering it +is on a different filesystem than that operand, because that operand's walk does +not enter such a directory and so does not reach it. -`scan` synchronizes the database with the filesystem state under the -scanned operands: +An operand that is a symlink (never followed, not even as an operand), socket, +FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints +a one-line warning naming the path and what it is, counts it as skipped, and +drops it from the scanned operands before reading the database. Another operand +beneath it is still scanned. The records stored beneath it are not deleted: they +are treated like any other record outside the scanned operands, including the +content-phase exception below. If it lies under another operand, they are under +that operand instead, and are treated like any other record there. This is not +an error: a scan whose every operand is dropped walks nothing and exits 0. -- Only a file whose size at least one other file shares is ever - read: a size-unique file cannot be a duplicate, so it is recorded - without hashes (`head`, `tail`, and `content` empty). The size - census covers every file walked this scan plus every database - record outside the scanned operands, so a possible duplicate of a - separately scanned tree is still recognized. -- A file not yet in the database is inserted: hashed when its size - is shared, without hashes otherwise. -- A file already in the database is **skipped without reading its - contents** when its lstat size equals the recorded size and its - lstat mtime is not newer than the recorded mtime. This is what - makes a daily rescan cheap. Exception: an unchanged file whose - record lacks hashes is hashed — and its record updated — once its - size becomes shared, so hashing deferred by size-uniqueness - happens as soon as it could matter. Likewise, an unchanged file of - 10 MiB or more whose record has no `content` hash is read for one - by the content phase below once its size, `head`, and `tail` match - another record's. -- A file whose mtime is newer than recorded, or whose size differs, - is processed as if new: re-hashed, or recorded without hashes, - per the shared-size rule. -- A database record whose path lies under one of the scanned operands - but was not successfully processed this run is deleted. This - removes records for deleted files. It also removes records for - paths that failed to stat or hash this run: the database only ever - contains signatures verified by the most recent scan that covered - them (a subsequent successful scan re-adds such files). A failure - in the content phase below removes nothing: the record is left as - it is. -- Database records outside the scanned operands are untouched, so - disjoint trees can be scanned on different schedules into the same - database. The one exception is the content phase below: a stored - file of 10 MiB or more without a `content` hash is read for one, - wherever it lies, once its size, `head`, and `tail` match another - record's. If that file is gone or has changed since its record was - written, the record is left as it is. +`scan` synchronizes the database with the filesystem state under the scanned +operands: -`scan` runs **four sequential phases over the whole scan**. -Parallelism lives inside each phase; batched database writes begin -during the hash phase: +- Only a file whose size at least one other file shares is ever read: a + size-unique file cannot be a duplicate, so it is recorded without hashes + (`head`, `tail`, and `content` empty). The size census covers every file + walked this scan plus every database record outside the scanned operands, so a + possible duplicate of a separately scanned tree is still recognized. +- A file not yet in the database is inserted: hashed when its size is shared, + without hashes otherwise. +- A file already in the database is **skipped without reading its contents** + when its lstat size equals the recorded size and its lstat mtime is not newer + than the recorded mtime. This is what makes a daily rescan cheap. Exception: + an unchanged file whose record lacks hashes is hashed — and its record updated + — once its size becomes shared, so hashing deferred by size-uniqueness happens + as soon as it could matter. Likewise, an unchanged file of 10 MiB or more + whose record has no `content` hash is read for one by the content phase below + once its size, `head`, and `tail` match another record's. +- A file whose mtime is newer than recorded, or whose size differs, is processed + as if new: re-hashed, or recorded without hashes, per the shared-size rule. + Change detection compares the mtime to the nanosecond, as finely as the + filesystem records it, so a same-size rewrite counts as a change whenever the + filesystem gives it a later mtime than recorded, even within the same second. +- A database record whose path lies under one of the scanned operands but was + not successfully processed this run is deleted. This removes records for + deleted files. It also removes records for paths that failed to stat or hash + this run: the database only ever contains signatures verified by the most + recent scan that covered them (a subsequent successful scan re-adds such + files). A failure in the content phase below removes nothing: the record is + left as it is. +- Database records outside the scanned operands are untouched, so disjoint trees + can be scanned on different schedules into the same database. The one + exception is the content phase below: a stored file of 10 MiB or more without + a `content` hash is read for one, wherever it lies, once its size, `head`, and + `tail` match another record's. If that file is gone or has changed since its + record was written, the record is left as it is. +- With `-x`, a record below a directory the walk did not enter because that + directory is on another filesystem than its operand counts as outside the + scanned operands: it is not deleted, and its size joins the size census. A + filesystem mounted under an operand can therefore be scanned on its own + schedule into the same database. Records below an operand at or inside such a + directory are not kept this way, since that operand's own walk verifies them. -1. **walk + stat** — enumerate the trees under all `PATH` operands - concurrently with the walk worker pool: every operand seeds the - shared queue, and each worker reads one directory at a time, - handing discovered subdirectories back to the queue and running - `lstat` on each regular file as it is discovered (while the - directory's metadata is still hot). Sequential directory - enumeration is metadata-latency-bound and takes hours at tens of - millions of files; per-directory parallelism is what makes the - walk tractable on large or busy pools. The walk builds the size - census and resolves unchanged already-hashed files on the fly; - every other file is carried to the hash phase as a (path, size, - mtime) record. -2. **hash** — with the census complete, each carried file's size - decides its fate. Size-unique files are never read: new or - changed ones are recorded without hashes in the update phase, - unchanged unhashed ones simply keep their records. Every file - with a shared size is hashed by the worker pool as described in - "Duplicate detection" above: a file under 10 MiB in full, which - gives its `head`, `tail`, and `content` alike, and a larger file - only in its end windows, which give its `head` and `tail`; its - content hash is left to the content phase. Zero-length files have - constant hashes and are never opened. Files are hashed in **inode - order** (minimizing seeks on spinning disks), and paths that are hard - links to the same inode are **read once**, all sharing the one - result — a hard-link backup farm costs one read per inode, not - per path. The phase total counts actual reads, so progress and - ETA are meaningful. Completed records are committed in batched - transactions **while hashing runs**, so a scan interrupted after - hours keeps everything hashed so far and the next scan resumes - cheaply, skipping records already written. -3. **update** — commit the final partial batch, the hash-less - records for size-unique new and changed files, and the deletions - for records the scan did not verify (vanished files, plus paths - that failed to stat or hash). -4. **content** — find every record of 10 MiB or more without a - `content` hash whose size, `head`, and `tail` equal another - record's, anywhere in the database: records from this scan and - records stored by earlier scans, inside or outside the scanned - operands. SQLite finds them, so only the records to be read are - kept in memory, never every file's hashes. Every record sharing - their size, `head`, and `tail`, including one that already has a - `content` hash, has its file checked with `lstat` first. A file - that is gone, is no longer a regular file, or has changed (a - different size, or an mtime newer than recorded) keeps its record - as it is and does not count as a match for the others. Any other - `lstat` error is warned about and counted as skipped, with the same - result. If such a record has no `content` hash, it stays out of - duplicate groups; if it has one, it is still reported until a scan - covering its own tree updates or removes it. The files that pass - and have no `content` hash are read only if at least two of those - records pass, so a file whose only matches are stale costs no read; - a file that already has a `content` hash is never read again. - They are read by a worker pool as in the hash phase, in inode order - and once per inode, and their content hashes are committed in - batches. A failed read is warned about and counted as skipped; its - record keeps an empty `content`, so it is not a duplicate, and a - later scan tries again. +`scan` runs **four sequential phases over the whole scan**. Parallelism lives +inside each phase; batched database writes begin during the hash phase: + +1. **walk + stat** — enumerate the trees under all `PATH` operands concurrently + with the walk worker pool: every operand seeds the shared queue, and each + worker reads one directory at a time, handing discovered subdirectories back + to the queue and running `lstat` on each regular file as it is discovered + (while the directory's metadata is still hot). Sequential directory + enumeration is metadata-latency-bound and takes hours at tens of millions of + files; per-directory parallelism is what makes the walk tractable on large + or busy pools. The walk builds the size census and resolves unchanged + already-hashed files on the fly; every other file is carried to the hash + phase as a (path, size, mtime) record. +2. **hash** — with the census complete, each carried file's size decides its + fate. Size-unique files are never read: new or changed ones are recorded + without hashes in the update phase, unchanged unhashed ones simply keep + their records. Every file with a shared size is hashed by the worker pool as + described in "Duplicate detection" above: a file under 10 MiB in full, which + gives its `head`, `tail`, and `content` alike, and a larger file only in its + end windows, which give its `head` and `tail`; its content hash is left to + the content phase. Zero-length files have constant hashes and are never + opened. Files are hashed in **inode order** (minimizing seeks on spinning + disks), and paths that are hard links to the same inode are **read once**, + all sharing the one result — a hard-link backup farm costs one read per + inode, not per path. The phase total counts actual reads, so progress and + ETA are meaningful. Completed records are committed in batched transactions + **while hashing runs**, so a scan interrupted after hours keeps everything + hashed so far and the next scan resumes cheaply, skipping records already + written. +3. **update** — commit the final partial batch, the hash-less records for + size-unique new and changed files, and the deletions for records the scan + did not verify (vanished files, plus paths that failed to stat or hash). +4. **content** — find every record of 10 MiB or more without a `content` hash + whose size, `head`, and `tail` equal another record's, anywhere in the + database: records from this scan and records stored by earlier scans, inside + or outside the scanned operands. SQLite finds them, so only the records to + be read are kept in memory, never every file's hashes. Every record sharing + their size, `head`, and `tail`, including one that already has a `content` + hash, has its file checked with `lstat` first. A file that is gone, is no + longer a regular file, or has changed (a different size, or an mtime newer + than recorded) keeps its record as it is and does not count as a match for + the others. Any other `lstat` error is warned about and counted as skipped, + with the same result. If such a record has no `content` hash, it stays out + of duplicate groups; if it has one, it is still reported until a scan + covering its own tree updates or removes it. The files that pass and have no + `content` hash are read only if at least two of those records pass, so a + file whose only matches are stale costs no read; a file that already has a + `content` hash is never read again. They are read by a worker pool as in the + hash phase, in inode order and once per inode, and their content hashes are + committed in batches. A failed read is warned about and counted as skipped; + its record keeps an empty `content`, so it is not a duplicate, and a later + scan tries again. Rules for the walk: -- Only regular files. Skip directories, symlinks (do not follow, - including symlink operands), sockets, FIFOs, and device nodes. -- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; - walking them would list every file once per snapshot). -- Filesystem boundaries are crossed by default. With `-x` - (long form `--one-file-system`, following the GNU `du`/`rsync` - convention), never descend into a directory on a different - filesystem than its `PATH` operand; each operand is bounded by its - own filesystem. -- On any per-path error (permission denied, file vanished between - passes, unreadable): print a one-line warning to stderr, skip the - path, and continue. Per-file errors never abort the run; the final - summary reports how many were skipped. As specified above, a - skipped path that has a database record from an earlier scan loses - that record, unless it failed only in the content phase; an - unreadable directory subtree likewise loses its records (accepted: - the database mirrors what the latest scan could actually verify). +- Only regular files. Skip directories, symlinks (do not follow, including + symlink operands), sockets, FIFOs, and device nodes. An operand that is a + symlink, socket, FIFO, or device node is dropped as described in "`scan` mode" + above. +- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking + them would list every file once per snapshot), not even when it is an operand; + such an operand is dropped the same way. +- Filesystem boundaries are crossed by default. With `-x` (long form + `--one-file-system`, following the GNU `du`/`rsync` convention), never descend + into a directory on a different filesystem than its `PATH` operand; each + operand is bounded by its own filesystem. The records below such a directory, + unless they lie below an operand at or inside that directory, count as outside + the scanned operands, as described in "`scan` mode" above: they are not + deleted, and their sizes join the size census. +- On any per-path error (permission denied, file vanished between passes, + unreadable): print a one-line warning to stderr, skip the path, and continue. + Per-file errors never abort the run; the final summary reports how many were + skipped. As specified above, a skipped path that has a database record from an + earlier scan loses that record, unless it failed only in the content phase, or + is an operand dropped before the database was read whose record counts as + outside the scanned operands; an unreadable directory subtree likewise loses + its records (accepted: the database mirrors what the latest scan could + actually verify). -Concurrency: the walk phase (which also stats files), the hash phase, -and the content phase each use a worker pool of `--workers` workers -(default `runtime.NumCPU()`); the walk parallelizes across -directories, hashing across files. All three phases are seek-bound on -spinning disks, so raising `--workers` well past the core count can -help on pools with many spindles. The main goroutine owns -partitioning, database writes, and progress rendering; progress -display must never block the workers. +Concurrency: the walk phase (which also stats files), the hash phase, and the +content phase each use a worker pool of `--workers` workers (default +`runtime.NumCPU()`); the walk parallelizes across directories, hashing across +files. `--workers` must be at least 1: a smaller value is a usage error, +reported in one line on stderr with exit 2 before anything is scanned. All three +phases are seek-bound on spinning disks, so raising `--workers` well past the +core count can help on pools with many spindles. The main goroutine owns +partitioning, database writes, and progress rendering; progress display must +never block the workers. -`scan` writes nothing to stdout. The summary line on stderr reports the -files seen this run broken down by disposition, plus skips: +`scan` writes nothing to stdout. The summary line on stderr reports the files +seen this run broken down by disposition, plus skips: ``` scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped ``` -(`removed` counts deleted database records, which are not part of the -files-seen total.) +(`removed` counts deleted database records, which are not part of the files-seen +total.) ### `report` mode -`report` reads every record from the database and takes no positional -arguments. +`report` reads every record from the database and takes no positional arguments. -**`report` must never touch the filesystem being analyzed.** It does not -stat, open, or otherwise access any path that appears in the records; its -only I/O is reading the database and writing stdout/stderr. It must -produce identical output whether or not the scanned filesystem is still -mounted. +**`report` must never touch the filesystem being analyzed.** It does not stat, +open, or otherwise access any path that appears in the records; its only I/O is +reading the database, writing stdout/stderr, and the temporary file SQLite sorts +in when the duplicate rows do not fit in memory. SQLite puts that file in +`$SQLITE_TMPDIR` or `$TMPDIR` when set, otherwise in `/var/tmp` (or `/tmp`), and +deletes it as soon as it has opened it. `report` must produce identical output +whether or not the scanned filesystem is still mounted. Processing: -- Records without a `content` hash (see "Database" above) are - excluded: their content is unknown, so they are never reported as - duplicates. -- Group the remaining records by the key - `(size, head, tail, content)`. -- Every group with two or more paths is a duplicate group. -- Within each group, sort paths lexicographically (byte order). The - first path is the group's `first`; every other path is a `dupe`. -- Order groups by size descending (biggest reclaimable space first), - tie-broken by `first` path ascending. Output must be fully - deterministic for a given database state. +- Records without a `content` hash (see "Database" above) are excluded: their + content is unknown, so they are never reported as duplicates. +- Group the remaining records by the key `(size, head, tail, content)`. +- Every group with two or more paths is a duplicate group. +- Within each group, sort paths lexicographically (byte order). The first path + is the group's `first`; every other path is a `dupe`. +- Order groups by size descending (biggest reclaimable space first), tie-broken + by `first` path ascending. Output must be fully deterministic for a given + database state. #### Report output format -TSV on stdout: a header line, then one row per duplicate file (N-1 rows -for a group of N): +TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a +group of N): ``` first dupe size @@ -428,102 +591,105 @@ first dupe size /srv/a/big.iso /srv/c/big-copy2.iso 4294967296 ``` -Summary to stderr: records read, number of duplicate groups, number of -dupe files, and total reclaimable bytes (sum of `size` over all dupe -rows) in human units. +Paths are raw bytes and may hold any byte except NUL, so the path columns +(`first` and `dupe`) are escaped to keep every row one line of tab-separated +fields: a backslash is written as `\\`, a tab as `\t`, a newline as `\n`, and a +carriage return as `\r`. Every other byte is written unchanged, including bytes +that are not valid UTF-8. Undoing those four escapes gives back the stored path. +Grouping and ordering use the stored path, not the escaped one. The warnings +`scan` prints on stderr are escaped the same way, so each warning is one line. + +Summary to stderr: records read, number of duplicate groups, number of dupe +files, and total reclaimable bytes (sum of `size` over all dupe rows) in human +units. ### `trees` mode -`trees` reads the same database as `report` (no positional arguments) -and reports **entire duplicate directory trees**: directories under -which the exact same set of relative paths exists with the exact same -file signatures. +`trees` reads the same database as `report` (no positional arguments) and +reports **entire duplicate directory trees**: directories under which the exact +same set of relative paths exists with the exact same file signatures. -**`trees` must never touch the filesystem being analyzed** — the same -rule as `report`. The directory hierarchy is reconstructed purely from -the paths in the records, split on `/`. +**`trees` must never touch the filesystem being analyzed** — the same rule as +`report`. The directory hierarchy is reconstructed purely from the paths in the +records, split on `/`. Definitions: -- A file's **signature** is `(size, head, tail, content)` — mtime is - informational and excluded. A record without a `content` hash has - unknown content: its signature is treated as unique to that file, - so a tree containing such a file never compares equal to any other - tree. -- A directory's **digest** is a SHA-256 Merkle digest computed - bottom-up: serialize the directory's child entries — for a file - child, its name and signature; for a subdirectory child, its name - and that subdirectory's digest — sort the serialized entries - byte-lexicographically, and hash the concatenation. Names are part - of the digest: two trees whose files differ only in name are *not* - duplicates. -- Two directories are **duplicate trees** when their digests are - equal. Equal digests imply equal recursive file count and equal - total byte size. +- A file's **signature** is `(size, head, tail, content)` — mtime is + informational and excluded. A record without a `content` hash has unknown + content: its signature is treated as unique to that file, so a tree containing + such a file never compares equal to any other tree. +- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up: + serialize the directory's child entries — for a file child, its name and + signature; for a subdirectory child, its name and that subdirectory's digest — + sort the serialized entries byte-lexicographically, and hash the + concatenation. Names are part of the digest: two trees whose files differ only + in name are _not_ duplicates. +- Two directories are **duplicate trees** when their digests are equal. Equal + digests imply equal recursive file count and equal total byte size. -Known limitation (accepted): hard-linked paths are reported as -duplicates by `report` and count toward duplicate trees — their -content is genuinely identical — even though they share storage, so -removing one reclaims no space. Inode identity is used during the -scan to avoid redundant reads but is not persisted in the database. +Known limitation (accepted): hard-linked paths are reported as duplicates by +`report` and count toward duplicate trees — their content is genuinely identical +— even though they share storage, so removing one reclaims no space. Inode +identity is used during the scan to avoid redundant reads but is not persisted +in the database. -Known limitation (accepted): only regular files that appear in the -database define a tree. Empty directories are invisible, and a file -skipped during the scan (e.g. permission error) in one copy but not the -other will make otherwise-identical trees compare as different. +Known limitation (accepted): only regular files that appear in the database +define a tree. Empty directories are invisible, and a file skipped during the +scan (e.g. permission error) in one copy but not the other will make +otherwise-identical trees compare as different. Processing: -- Build the hierarchy, compute every directory's digest, and group - directories by digest. Every group with two or more directories is a - duplicate-tree group. -- **Report only maximal trees.** A group is suppressed when its - members' parents are pairwise distinct directories that all share a - single digest — such a group is wholly implied by its parents' (or a - further ancestor's) group. Groups containing sibling directories, or - members whose parents differ, are always reported. -- Within each group, sort paths lexicographically (byte order); the - first path is `first`, every other path is a `dupe`. -- Order groups by total tree size descending, tie-broken by `first` - path ascending. Output must be fully deterministic for a given - input. +- Build the hierarchy, compute every directory's digest, and group directories + by digest. Every group with two or more directories is a duplicate-tree group. +- **Report only maximal trees.** A group is suppressed when its members' parents + are pairwise distinct directories that all share a single digest — such a + group is wholly implied by its parents' (or a further ancestor's) group. + Groups containing sibling directories, or members whose parents differ, are + always reported. +- Within each group, sort paths lexicographically (byte order); the first path + is `first`, every other path is a `dupe`. +- Order groups by total tree size descending, tie-broken by `first` path + ascending. Output must be fully deterministic for a given input. #### Trees output format -TSV on stdout: a header line, then one row per duplicate tree (N-1 rows -for a group of N). `files` is the recursive regular-file count of one -copy of the tree; `size` is the recursive total byte size of one copy: +TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a +group of N). `files` is the recursive regular-file count of one copy of the +tree; `size` is the recursive total byte size of one copy: ``` first dupe files size /srv/a/project /srv/backup/project 3417 104857600 ``` -Summary to stderr: records read, number of duplicate-tree groups, -number of dupe trees, and total reclaimable bytes (sum of `size` over -all dupe rows) in human units. +The `first` and `dupe` paths are escaped as described under "Report output +format". The root directory's path is `/`. + +Summary to stderr: records read, number of duplicate-tree groups, number of dupe +trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human +units. ### Progress -Use the progress-bar library for all scan progress; rendering in the -style of `pv` is the model. All progress goes to stderr. +Use the progress-bar library for all scan progress; rendering in the style of +`pv` is the model. All progress goes to stderr. -Each phase gets its own display, rendered the moment the phase -starts — a scan must never look hung. Loading the existing-record -index (`load`) and the walk have no known totals while running: show -a live count, rate, and elapsed time (spinner-style, no percentage or -ETA). The content phase's display (`content`) starts the same way, -counting the records checked while SQLite finds the files to read and -`lstat` checks them, then shows a bar once reading starts. The hash -and update phases, and the content phase's reads, have exact totals — -only files that actually need hashing appear in the hash and content -totals, so their ETAs are meaningful. Required elements for the bars -with known totals: +Each phase gets its own display, rendered the moment the phase starts — a scan +must never look hung. Loading the existing-record index (`load`) and the walk +have no known totals while running: show a live count, rate, and elapsed time +(spinner-style, no percentage or ETA). The content phase's display (`content`) +starts the same way, counting the records checked while SQLite finds the files +to read and `lstat` checks them, then shows a bar once reading starts. The hash +and update phases, and the content phase's reads, have exact totals — only files +that actually need hashing appear in the hash and content totals, so their ETAs +are meaningful. Required elements for the bars with known totals: -- elapsed time -- estimated time remaining -- a `[m/n] x%` display (items processed / total items, percent) -- current rate (items/s) +- elapsed time +- estimated time remaining +- a `[m/n] x%` display (items processed / total items, percent) +- current rate (items/s) Example shape (exact layout is flexible, content is not): @@ -533,160 +699,207 @@ hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54 Additional requirements: -- When stderr is not a TTY, do not emit ANSI redraws: print a plain - one-line progress update no more often than every 5 seconds instead. -- Progress updates are driven from the main goroutine and must be - non-blocking with respect to the worker pool. -- `report` and `trees` modes need no progress display, only their - stderr summaries. +- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI + redraws: print a plain one-line progress update the moment each phase starts, + then no more often than every 5 seconds. +- Progress updates are driven from the main goroutine and must be non-blocking + with respect to the worker pool. On a terminal the spinner-style displays also + redraw on their own several times a second, so their count and elapsed time + stay current while a phase waits for its next item. +- A warning printed during a phase always lands on a line of its own, never + inside the progress display. +- A bar whose phase stops short of its total, as an interrupted one does, is + left as last drawn rather than filled up. +- `report` and `trees` modes need no progress display, only their stderr + summaries. ### Error handling and exit codes -- `0`: success, even if individual files were skipped with warnings. -- `1`: fatal error (e.g., a `PATH` operand does not exist, the - database cannot be created/opened/read/written, a missing database - for `report`/`trees`, stdout write failure). -- `2`: usage error (including `scan` with no `PATH` operand and - `report`/`trees` with any positional argument). +- `0`: success, even if individual files were skipped with warnings. +- `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is + already running against the same database, the database cannot be + created/opened/read/written, a missing database for `report`/`trees`, stdout + write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below). +- `2`: usage error (including `scan` with no `PATH` operand, `scan` with + `--workers` below 1, and `report`/`trees` with any positional argument). + +A stdout write failure, such as a full disk, is reported in one line on stderr +and exits 1. Two cases never reach sfdupes as a failed write: + +- When the reader of a stdout pipe exits early, as in `sfdupes report | head`, + the next write ends sfdupes with `SIGPIPE`, quietly and without a summary, the + way `cat` or `sort` end. The shell reports the signal (status 141 in most + shells), not exit 1. +- When stdout is closed outright (`sfdupes report >&-`), the Go runtime opens + `/dev/null` in its place before sfdupes starts, so the output is discarded and + the run succeeds, as with `> /dev/null`. + +`scan` stops cleanly on `SIGINT` (Ctrl-C) or `SIGTERM`. Its workers stop taking +work, each finishing at most the directory listing or file it is reading; the +progress display is finished; and the records it has hashed but not yet +committed are committed, so the next scan does not hash them again. Apart from +that commit it starts no further writes or deletions: records are deleted only +after a complete walk, so those under paths an interrupted walk never reached +are kept. The database is closed and the lock released as on any other exit, the +line `scan: interrupted after N files` goes to stderr, N being the number of +files the walk reached, and the exit code is 1. The next scan skips the records +already written and converges as usual. + +After the first signal `scan` stops catching them, so a second one ends it at +once, as an uncaught signal does: the records not yet committed are lost, and +the database is left valid, as when any scan dies (see "Database"). A `SIGINT` +that `scan` inherits as ignored, as a script's background job does, stays +ignored. ## Entrypoints This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) -standard: the normalized executables in `script/` are the entrypoints -for the development workflow, and the `Makefile` targets are thin -shims that call them. Every script is POSIX `sh`, resolves the -repository root itself so it can be run from any working directory, -and may be invoked directly. The provided entrypoints are: +standard: the normalized executables in `script/` are the entrypoints for the +development workflow, and the `Makefile` targets are thin shims that call them. +Every script is POSIX `sh`, resolves the repository root itself so it can be run +from any working directory, and may be invoked directly. The provided +entrypoints are: -- `script/bootstrap` — install everything needed to build and - develop this repository, idempotently, assuming nothing is - present. `git`, `make`, and `go` come from the first of nix, apt, - brew, or apk found on the host, and are presence-checked only. - `golangci-lint` is deliberately **not** installed: it runs from a - digest-pinned image via `script/lint` and never from a host - install, so there is no host copy to drift from the pin. A missing - `docker` is warned about rather than installed or treated as - fatal — everything except linting works without it. Ends with - `go mod download`. -- `script/setup` — make a fresh clone ready for development: runs - `script/bootstrap`, then `script/install-precommit`. -- `script/projectname` — print this project's name (`sfdupes`). - Scripts that need the name call it, so they stay identical across - repositories. -- `script/test` — run the test suite with a 30-second timeout and - coverage enabled, rerunning verbosely on failure so the logs show - which test failed. -- `script/lint` — run the linter. It builds `Dockerfile.lint`, which - copies the repository into the digest-pinned - `golangci/golangci-lint` image and runs - `golangci-lint config verify` and `golangci-lint run` as build - steps, so a successful build is a clean lint. The linter is never - run on the host, which makes a working `docker` the one - prerequisite for linting — and therefore for `make check` and the - pre-commit hook. Offline machines: the gate steps themselves make - no network calls. `golangci-lint run` does not, and neither does - `golangci-lint config verify` — it validates against a schema the - pinned binary embeds, measured under `--network none` to both - pass a valid config and reject an invalid one. The build around - them does. `Dockerfile.lint` runs `go mod download` before the - gates and this module has external dependencies, so a first lint - on a machine with a cold BuildKit cache reaches the network there - (as well as pulling the pinned image); under `--network none` it - fails at that step, before any gate. That layer sits above the - gates and stays cached, so once it is warm `script/lint` — and - with it `make check` — runs entirely offline, until `go.mod` or - `go.sum` changes and the download layer goes cold again. Because - the daemon only ever sees a build context, this works when the - docker daemon is remote and bind mounts are impossible. -- `script/fmt` — format the Go sources in place (`gofmt -s -w`). - Markdown is not formatted. -- `script/fmt-check` — the read-only counterpart of `script/fmt`: - prints any unformatted file and exits non-zero instead of writing. -- `script/check` — run `script/test`, `script/lint`, and - `script/fmt-check`, in that order. Modifies nothing. Needs - `docker`, because `script/lint` does. -- `script/docker` — build the Docker image, tagged with the name - from `script/projectname`. The `Dockerfile` runs the gates as - build steps, so this is also the check a developer or reviewer - runs by hand. -- `script/cibuild` — build the Docker image untagged. This is what - the Gitea workflow runs on push; because the gates run as build - steps, a successful build implies the repository is green. -- `script/precommit` — run by the git pre-commit hook: `go mod tidy` - must be a no-op (a resulting change to `go.mod` or `go.sum` fails - the commit), then `script/check`. -- `script/install-precommit` — install the git pre-commit hook that - runs `script/precommit`. The hook is written to the common git - directory, so the main checkout and every worktree share it. -- `script/verify-lint-image-pin` — fail unless the - `golangci/golangci-lint` reference in `Dockerfile.lint` and the - one in the `Dockerfile` lint stage are the same image at the same - digest, naming both if not. The linter is pinned in those two - files and nothing else keeps them in sync, so a bump applied to - one alone would leave `make lint` and the `Dockerfile`'s - fail-fast lint stage checking the same tree against different - rulesets, both green. The guard restates neither pin — a third - copy would be the same drift one file further out — and runs as a - gate in both files, so `make lint`, `make check` and `make docker` - all catch it. +- `script/bootstrap` — install everything needed to build and develop this + repository, idempotently, assuming nothing is present. `git`, `make`, and `go` + come from the first of nix, apt, brew, or apk found on the host, and are + presence-checked only. `golangci-lint` and prettier are deliberately **not** + installed: they run in Docker (see `script/lint` and `script/fmt`) and never + from a host install, so there is no host copy to drift from the pin. A missing + `docker` is warned about rather than installed or treated as fatal — + everything except linting, formatting and `make test-race` works without it. + Ends with `go mod download`. +- `script/setup` — make a fresh clone ready for development: runs + `script/bootstrap`, then `script/install-precommit`. +- `script/projectname` — print this project's name (`sfdupes`). Scripts that + need the name call it, so they stay identical across repositories. +- `script/test` — run the test suite with a 30-second timeout and coverage + enabled, rerunning verbosely on failure so the logs show which test failed. +- `script/test-race` — run the test suite under the race detector with a + 60-second timeout. The detector needs cgo and a C compiler, which the build + never uses, so the tests run in a digest-pinned Debian `golang` image that has + `gcc`, with the checkout mounted read-only; the container is removed when it + exits. They run as the calling user, or as `nobody` when that is root, because + several tests make a file unreadable and root reads it anyway; only then must + the checkout be readable by other users. Not part of `script/check`. Every run + starts with empty caches, so it needs the network and takes minutes, and the + mount needs a local docker daemon. +- `script/lint` — run the linter. It builds `Dockerfile.lint`, which copies the + repository into the digest-pinned `golangci/golangci-lint` image and runs + `golangci-lint config verify` and `golangci-lint run` as build steps, so a + successful build is a clean lint. That exit status is all it produces, so it + runs with `--output=type=cacheonly` and writes no image; a run leaves only + build cache. The linter is never run on the host, which makes a working + `docker` the one prerequisite for linting — and therefore for `make check` and + the pre-commit hook. Offline machines: the gate steps themselves make no + network calls. `golangci-lint run` does not, and neither does + `golangci-lint config verify` — it validates against a schema the pinned + binary embeds, measured under `--network none` to both pass a valid config and + reject an invalid one. The build around them does. `Dockerfile.lint` runs + `go mod download` before the gates and this module has external dependencies, + so a first lint on a machine with a cold BuildKit cache reaches the network + there (as well as pulling the pinned image); under `--network none` it fails + at that step, before any gate. That layer sits above the gates and stays + cached, so once it is warm `script/lint` — and with it `make check` — runs + entirely offline, until `go.mod` or `go.sum` changes and the download layer + goes cold again. Because the daemon only ever sees a build context, this works + when the docker daemon is remote and bind mounts are impossible. +- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every + Markdown file with prettier, at the settings in `.prettierrc` (4-space + indents, prose wrapped at 80 columns). prettier is pinned by hash through + `package.json` and `yarn.lock` and never installed on the host: this builds + the `Dockerfile`'s `prettier` stage, a digest-pinned node image into which + `yarn install --frozen-lockfile` installs it, tagged `sfdupes-prettier`, and + runs that with the repository mounted, as the calling user. Needs `docker`, + and because of the mount, unlike `script/lint`, a local docker daemon. +- `script/fmt-check` — the read-only counterpart of `script/fmt`, with the + repository mounted read-only: prints any unformatted file and exits non-zero + instead of writing. gofmt and prettier both run every time, and each names + itself when it fails. The `Dockerfile` runs the same two checks as gates: the + gofmt check in its lint stage, prettier in its `markdown` stage. +- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in + that order. Modifies nothing. Needs `docker`, because `script/lint` and + `script/fmt-check` do. +- `script/docker` — build the Docker image, tagged with the name from + `script/projectname`. The `Dockerfile` runs the gates as build steps, so this + is also the check a developer or reviewer runs by hand. +- `script/cibuild` — build the Docker image untagged. This is what the Gitea + workflow runs on push; because the gates run as build steps, a successful + build implies the repository is green. +- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a + no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then + `script/check`. +- `script/install-precommit` — install the git pre-commit hook that runs + `script/precommit`. The hook is written to the common git directory, so the + main checkout and every worktree share it. +- `script/verify-lint-image-pin` — fail unless the `golangci/golangci-lint` + reference in `Dockerfile.lint` and the one in the `Dockerfile` lint stage are + the same image at the same digest, naming both if not. The linter is pinned in + those two files and nothing else keeps them in sync, so a bump applied to one + alone would leave `make lint` and the `Dockerfile`'s fail-fast lint stage + checking the same tree against different rulesets, both green. The guard + restates neither pin — a third copy would be the same drift one file further + out — and runs as a gate in both files, so `make lint`, `make check` and + `make docker` all catch it. -`script/verify-linter-pin` used to live here. It compared a linter -binary against a version pin in `script/bootstrap`, and both of its -subjects are gone: no linter binary is copied between build stages any -more, and bootstrap pins no version because it installs no linter. The -drift it existed to catch has moved from binary-versus-pin to -pin-versus-pin, which is what `script/verify-lint-image-pin` above -checks. +`script/verify-linter-pin` used to live here. It compared a linter binary +against a version pin in `script/bootstrap`, and both of its subjects are gone: +no linter binary is copied between build stages any more, and bootstrap pins no +version because it installs no linter. The drift it existed to catch has moved +from binary-versus-pin to pin-versus-pin, which is what +`script/verify-lint-image-pin` above checks. -`script/lint`, `script/docker` and `script/cibuild` all pass a freshly -computed `CHECK_EPOCH` build argument, and the gate steps in -`Dockerfile.lint` and `Dockerfile` reference it. Without that, an -unchanged tree lets Docker serve the gate layers from cache and the -build exits 0 having executed no tests and no lint — a green it never -earned, and one this repository has produced twice. `CHECK_EPOCH` -invalidates the gate layers on every run while leaving the pinned base -images and the dependency layers cached. `script/lint`'s value carries -the process id as well as the epoch, because two lint runs land inside -the same second easily and a bare epoch would cache the second one. +`script/lint`, `script/docker` and `script/cibuild` all pass a freshly computed +`CHECK_EPOCH` build argument, and the gate steps in `Dockerfile.lint` and +`Dockerfile` reference it. Without that, an unchanged tree lets Docker serve the +gate layers from cache and the build exits 0 having executed no tests and no +lint — a green it never earned, and one this repository has produced twice. +`CHECK_EPOCH` invalidates the gate layers on every run while leaving the pinned +base images and the dependency layers cached. Each script's value carries the +process id as well as the epoch, because two runs land inside the same second +easily and a bare epoch would cache the second one. Each `Dockerfile` stage with +gates fails when the value is empty, so a bare `docker build .` stops with +`CHECK_EPOCH is unset; build via script/cibuild or script/docker` instead of +serving the gates from cache. ## Build -The `script/` entrypoints above are where the implementations live; -the `Makefile` targets are shims onto them, except `build`, which -carries the compile recipe: +The `script/` entrypoints above are where the implementations live; the +`Makefile` targets are shims onto them, except `build`, which carries the +compile recipe: -- `make` / `make build` — build the `sfdupes` binary (cgo - disabled); building is the default target. -- `make bootstrap` — install the build and development - dependencies. -- `make setup` — prepare a fresh clone: `bootstrap` plus the - pre-commit hook. -- `make test` — run the test suite (30-second timeout; reruns with - `-v` on failure). -- `make lint` — run `golangci-lint` with the repo config, in Docker - (see `script/lint`); requires `docker`. -- `make fmt` / `make fmt-check` — format Go sources / verify - formatting without writing. -- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. - Requires `docker`, via `lint`. -- `make docker` — build the Docker image, which runs the gates as - build stages. -- `make hooks` — install the pre-commit hook. -- `make clean` — remove the binary. +- `make` / `make build` — build the `sfdupes` binary (cgo disabled); building is + the default target. +- `make bootstrap` — install the build and development dependencies. +- `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook. +- `make test` — run the test suite (30-second timeout; reruns with `-v` on + failure). +- `make test-race` — run the test suite under the race detector, in Docker (see + `script/test-race`); requires `docker`. Not part of `make check`. +- `make lint` — run `golangci-lint` with the repo config, in Docker (see + `script/lint`); requires `docker`. +- `make fmt` / `make fmt-check` — format the Go sources and the Markdown / + verify formatting without writing; requires `docker`, for prettier (see + `script/fmt`). +- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires + `docker`, via `lint` and `fmt-check`. +- `make docker` — build the Docker image, which runs the gates as build stages. +- `make hooks` — install the pre-commit hook. +- `make clean` — remove the binary. ### Definition of done All of the following, run in this directory, must pass: -1. `make check` passes (tests, lint, `gofmt`). +1. `make check` passes (tests, lint, `gofmt`, prettier). 2. `make docker` succeeds. -3. Smoke test — create a throwaway tree in a temp dir (never test - against real data): +3. Smoke test — create a throwaway tree in a temp dir (never test against real + data): ```sh d=$(mktemp -d) - export SFDUPES_DATABASE="$d/db.sqlite" + export SFDUPES_DATABASE="$(mktemp -d)/db.sqlite" mkdir -p "$d/a" "$d/b" head -c 2000 /dev/urandom > "$d/a/one.bin" cp "$d/a/one.bin" "$d/b/copy.bin" @@ -714,30 +927,30 @@ All of the following, run in this directory, must pass: ./sfdupes report ``` - (The scan database lives inside `$d` here purely for test hygiene; - scanning `$d` therefore also records the SQLite file itself, which - is harmless.) + (The database lives in a temp directory of its own: inside `$d`, the scan + would record it, and its empty lock file would join the `empty1`/`empty2` + group.) - Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` - form one group (two dupe rows, `first` is the lexicographically - smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group; - `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group; - `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and - `tiny3` appear nowhere; groups ordered by size descending. + Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one + group (two dupe rows, `first` is the lexicographically smallest path); + `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/ + `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair; + `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere; groups + ordered by size descending. - Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` - `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is - suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3` - appears nowhere (its file set differs by name). + Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2 + files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal + (implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set + differs by name). Expected from the second `report` (after the modify/delete rescan): `one.bin` has left its group (its content changed), so - `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is - gone from the database. + `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is gone from the + database. - The test suite automates this scenario (see `scan_test.go`), plus a - negative check: `report` and `trees` operate on the database alone - and never touch the scanned filesystem. + The test suite automates this scenario (see `scan_test.go`), plus a negative + check: `report` and `trees` operate on the database alone and never touch + the scanned filesystem. ## TODO @@ -745,13 +958,13 @@ Tracked in [TODO.md](TODO.md). ## Non-goals -- No byte-for-byte compare, and no deletion or linking of - duplicates. Files that match are compared by a SHA-256 of the whole - file below 50 MiB, and only by samples at 50 MiB and over. The - reports are advisory; acting on them is the user's job. -- No persistence beyond the SQLite database described above; no - export/import formats. -- No daemon or filesystem watcher; scheduling rescans is cron's job. +- No byte-for-byte compare, and no deletion or linking of duplicates. Files that + match are compared by a SHA-256 of the whole file below 50 MiB, and only by + samples at 50 MiB and over. The reports are advisory; acting on them is the + user's job. +- No persistence beyond the SQLite database described above; no export/import + formats. +- No daemon or filesystem watcher; scheduling rescans is cron's job. ## License diff --git a/TODO.md b/TODO.md index 7fd3083..185157f 100644 --- a/TODO.md +++ b/TODO.md @@ -1,453 +1,432 @@ # Workflow -- take an issue from the `1.0.0` milestone on the tracker; work not - yet on the tracker gets filed as an issue first -- branch (from `main`) +- take an issue from the `1.0.0` milestone on the tracker; work not yet on the + tracker gets filed as an issue first +- branch from `next` - do the work, with tests, in small focused commits -- record it at the top of Completed Steps (`TODO.md` changes in the - same commit as the work) -- push the branch and open a PR whose title ends with +- record it at the top of Completed Steps (`TODO.md` changes in the same commit + as the work) +- push the branch and open a PR against `next` whose title ends with ` (closes #N)` -- an independent review gates the merge; every finding is addressed +- an independent review gates each merge to `next`; every finding is addressed or explicitly rebutted on the PR -- merge to `main` once the review passes +- only the owner merges `next` to `main` # Status - pre-1.0 -- the Gitea tracker is authoritative for the pre-1.0 backlog: the - open issues under the `1.0.0` milestone are what remains before - the tag, and this file records history and process, not the queue +- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues + under the `1.0.0` milestone are what remains before the tag, and this file + records history and process, not the queue # Next Step - take the next issue from the `1.0.0` milestone on the tracker: - https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is - the source of truth for what is left before 1.0.0. Individual - issues are deliberately not restated here; a copy in this file - drifts out of date the moment the tracker moves + https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source + of truth for what is left before 1.0.0. Individual issues are deliberately not + restated here; a copy in this file drifts out of date the moment the tracker + moves # Completed Steps -- replace the 1 KiB end-window sampling with the head/tail plus - content-hash ladder (2026-09-22, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is - hashed in full and compared directly, with no end-window step — its - `head`, `tail`, and `content` all hold the whole-file hash. A file at - 10 MiB or above gets only the 64 KiB `head` and `tail` in the hash - phase; a new content phase, after the update phase, reads it for its - `content` hash — the whole file below 50 MiB, gigabyte-spaced 1 MiB - samples at or above — only when its size, `head`, and `tail` match - another record's, from the same scan or stored by an earlier one, so - a stored file gains its content hash when it gains a match. A file - that is gone or has changed since its record was written is not - read. The `content` column is part of the version 1 schema. `report` - and `trees` group by the extended signature and leave out any record - without a `content` hash, so the ladder is applied across the whole - database. README "Duplicate detection" documents every rung including - the probabilistic large-file path. +- with `-x`, an operand under another operand is walked on its own instead of + being dropped when a directory from it (from its directory, for a regular + file) up to but not including that operand is on a different filesystem than + that operand, whose walk then cannot reach it, and the records of files gone + from below it are deleted (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/106) -- remove the dead `files.dat` references from `Makefile`, `.gitignore` - and `.dockerignore` (2026-09-21, branch `next`, closes +- with `-x`, `scan` keeps the records below a directory on another filesystem + under its operand, and counts their sizes in the size census, instead of + deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99) + +- the hash and content phases open each file without blocking and check that it + is still a regular file, so one replaced by a FIFO or device node after it was + statted is warned about and skipped instead of hanging the scan (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/100) + +- the scan lock file is named from the database's path with symlinks resolved, + after `scan` creates the database file if it is missing, so a scan through a + symlink to the database fails while another scan holds the lock (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/101) + +- on a terminal, a warning for the last file of a phase no longer erases the + phase's finished bar: the bar is drawn again below the warning (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/102) + +- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus + `mtime_nsec`, and compares it at that resolution, so a same-size rewrite + within the same second is re-hashed (2026-10-07, + https://git.eeqj.de/sneak/sfdupes/issues/12) + +- cut the narration from `TODO.md` Completed Steps and from the comments in + `script/` and both Dockerfiles; §Workflow now branches from and merges to + `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49) + +- `make test-race` runs the test suite under the race detector in a cgo-enabled + container, outside `make check` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/18) + +- a bare `docker build .` fails with a message naming `script/cibuild` and + `script/docker` instead of serving the gates from cache (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/39) + +- `make fmt` and `make fmt-check` run prettier over all Markdown, in Docker, and + CI checks it; all Markdown reformatted (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/19) + +- `script/lint` writes no image, so a run no longer leaves an untagged one + behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48) + +- tests cover a missing database, `scan` keeping stdout empty, its skip warning, + the `report` and `trees` summary lines, and every subcommand going through + `runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16) + +- `.golangci.yml` replaced with the current canonical copy, which uses + `gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/26) + +- a test fails when either `hashWorker` cancellation check in `scan.go` is + removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) + +- a database path holding `?`, `#` or `%` opens exactly the file it names + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) + +- `scan` rejects `--workers` below 1 as a usage error instead of running + single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10) + +- a test fails when either walk cancellation check in `scan.go` is removed + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) + +- test that `scan` refuses a database with another schema version (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/64) + +- correct four inaccurate comments in `cancel_test.go` and rename + `walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/33) + +- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/17) + +- `scan` creates the schema in one transaction; a version-0 database with a + `files` table is refused with a clear schema-version error (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/11) + +- README documents install, Docker, a daily cron scan and how to read and check + the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) + +- the `Dockerfile` build stage keeps the Go module cache out of `builder`'s home + and copies the sources with `--chown`, so no `chown -R` walks them + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43) + +- `--version` prints `sfdupes VERSION` to stdout; README documents it and + `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) + +- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed, + deletes nothing more, exits 1 (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/5) + +- `report` and `trees` stream the records instead of holding them all in memory; + the schema gains the `files_signature` index (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/14) + +- progress prints at once on a non-terminal, uses a real terminal test, and + prints warnings through a spinner instead of racing its redraw (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/13) + +- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping + the records beneath them (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/9) + +- `scan` holds a lock on a lock file beside the database for its whole run, so a + second `scan` fails at once with exit 1 (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/53) + +- test stdout write failures in `report` and `trees`; README states that + `| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null` + (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30) + +- `report` and `trees` open the database read-only, and `scan` leaves it out of + WAL mode, so reading needs only read access (2026-10-03, closes + https://git.eeqj.de/sneak/sfdupes/issues/8) + +- escape tabs, newlines, carriage returns and backslashes in report, trees and + warning paths; the root directory's path is `/` (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/7) + +- stamp the git tag or short commit in a plain `docker build .` instead of `dev` + (2026-10-02, branch `next`, closes + https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git` + without `.git/config`; the build stage stamps the `VERSION` build argument, + else `git describe --tags --always`, and fails if the context carries `.git` + and the version is still empty, `dev` or `unknown`. CI checks out the full + history (`fetch-depth: 0`) so it stamps the same value as `make build`. + +- replace the 1 KiB end-window sampling with the head/tail plus content-hash + ladder (2026-09-22, branch `next`, closes + https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection" + documents every rung. A file under 10 MiB is hashed in full, and its `head`, + `tail` and `content` all hold that hash. A larger file gets only its 64 KiB + `head` and `tail` in the hash phase; the content phase, after the update + phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1 + MiB samples at or above) only when its size, `head` and `tail` match another + record's from this scan or an earlier one, and never reads a file gone or + changed since its record was written. `report` and `trees` leave out any + record without a `content` hash. The `content` column is part of the version 1 + schema. + +- remove the dead `files.dat` references from `Makefile`, `.gitignore` and + `.dockerignore` (2026-09-21, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/22) - fix the lint-image pin comments and `FROM` form in `Dockerfile` and `Dockerfile.lint` (2026-08-10, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/25): dropped the false - `(Debian-based)` parenthetical (v2.12.1 was Debian too) and the - redundant tag, so both pins are the policy `# image:vX.Y.Z, - YYYY-MM-DD` comment over a bare `FROM image@sha256:...`. Digest - unchanged. `script/verify-lint-image-pin` parses those `FROM` lines - and still matches the tagless form; its advice line lost the now - meaningless "tag and digest". With no tag in either reference, a - tag-only disagreement no longer exists — a one-sided tag is caught as - a plain mismatch. + https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy + `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`, + without the false `(Debian-based)` note or the tag; digest unchanged. + `script/verify-lint-image-pin` still matches the tagless form, and a tag on + one side only is caught as a plain mismatch. -- run all linting in Docker via `Dockerfile.lint` and `script/lint` - (2026-08-10, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/46): per the owner ruling, the - linter runs inside a container invoked through the `script/` - entrypoint and is never installed on a host. New root - `Dockerfile.lint` COPYs the repo into the digest-pinned - `golangci/golangci-lint:v2.12.2` image and runs - `golangci-lint config verify` and `golangci-lint run` as build - steps, so a successful build IS a clean lint; `script/lint` is - reduced to building it. `script/bootstrap` loses the `go install`, - the pin constants, the version parser and `verify_golangci_lint` - outright rather than hardening them — with nothing linting on the - host, the `$GOPATH/bin` versus `PATH` problem that motivated them has - no subject — and now warns rather than fails when `docker` is absent. - Two traps handled. A lint build on an unchanged tree returns success - in well under a second having run no linter, which is - https://git.eeqj.de/sneak/sfdupes/issues/32 and - https://git.eeqj.de/sneak/sfdupes/issues/39 again, so - `Dockerfile.lint` carries `ARG CHECK_EPOCH` referenced - inside every gate `RUN` (BuildKit hashes the expanded command, not - the declaration) and `script/lint` passes `"$(date +%s)-$$"` — the - PID matters because two lint runs land inside the same second easily. - And nothing inside an image build may shell out to docker, so the - main `Dockerfile`'s lint stage now invokes `golangci-lint` directly - instead of `make lint`, and its build stage runs `make test` and - `make fmt-check` instead of the `make check` aggregate (`make`, not - the scripts bare, because the Makefile's `export CGO_ENABLED = 0` - only reaches what it invokes). `COPY --from=lint` - `/usr/bin/golangci-lint` is replaced by - `COPY --from=lint /src/go.sum /dev/null`: the copied binary was the - only edge forcing BuildKit to finish linting before the build stage - starts, and dropping it without replacing the edge would have ended - fail-fast linting silently under a still-green build. That is - canonical `REPO_POLICIES.md:107`'s ordering edge, restored. - `ENV PATH=/home/builder/go/bin:$PATH` is gone with the `go install` - that justified it. `script/verify-linter-pin` is retired, deleted - along with its README entry, because both of its subjects ceased to - exist in the same change: it compared a linter binary against - `GOLANGCI_LINT_VERSION` in `script/bootstrap`, and there is now - neither a binary crossing between stages nor a version pin in - bootstrap. The drift it guarded has not gone away, it has moved — the - linter is still pinned twice, now as the `FROM` line of - `Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage, - with nothing syncing them, which is exactly what - https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its - replacement is one new `script/verify-lint-image-pin`, - run as a gate in both files, which compares the two references to - each other and deliberately restates neither: a hardcoded expected - digest would be a third copy and the same drift one file further out. - `golangci-lint config verify` is included per the ruling, and the - concern about its unpinned live HTTPS schema fetch was measured - rather than assumed — under `--network none` the pinned binary both - passes a valid config and rejects an invalid one with the jsonschema - error, so it validates from an embedded schema and makes no network - call of its own. The README scopes that to the gate steps rather - than to linting as a whole: `Dockerfile.lint` runs `go mod download` - above them, so a cold cache still needs the network and only a warm - one lints offline. Verified: `make lint` green with every `PATH` - directory containing a `golangci-lint` removed - (`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`; - `command -v golangci-lint` empty); two consecutive `script/lint` runs - on an untouched tree both executed the linter, 27.7s and 28.7s in the - lint step under distinct epochs with the `COPY . .` layer `CACHED` - above them, at 42.2s and 41.8s wall clock — the no-cache rule was not - weakened to shorten that. Negative control: a planted - `var unusedIssue46Sentinel = 1` failed `script/lint` with - `report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and - failed `make docker` at `[lint 9/9]` with the build stage stopped at - `[builder 3/12]` — `COPY --from=lint`, `script/bootstrap`, the test - gate and `make build` all zero occurrences — then reverted clean. The - drift guard fails on a tag-only disagreement, on a digest-only - disagreement, and on an unreadable reference, naming both sides. - `make docker` green in 5m35s with all six gates executing under one - epoch (lint 37.6s, test 25.2s reporting - `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`). - The non-root quirk still holds: in the builder image with the Go test - cache off, `--user 0:0` fails `TestScanHardlinkRunFailsTogether` - (exit 1) where the unprivileged user passes (exit 0). Noted for - follow-up, not fixed here: `golangci-lint` warns that the - `gomodguard` linter is deprecated since v2.12.0 in favour of - `gomodguard_v2`. +- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, + branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the + owner ruling the linter is never installed on a host. `Dockerfile.lint` copies + the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and + runs `golangci-lint config verify` and `golangci-lint run` as build steps; + `script/lint` builds it. `script/bootstrap` no longer installs or pins the + linter, and warns rather than fails when `docker` is absent; + `ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`. + `script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate + in both files, compares their two `FROM` lines and restates neither pin. + Traps: an unchanged tree lets a lint build pass in under a second having run + no linter, so every gate `RUN` references `ARG CHECK_EPOCH` (BuildKit hashes + the expanded command) and `script/lint` passes `"$(date +%s)-$$"`, the PID + because two runs land in the same second easily. Nothing inside an image build + may shell out to docker, so the `Dockerfile` lint stage calls `golangci-lint` + directly and the build stage runs `make test` and `make fmt-check` instead of + `make check`, through `make` because the Makefile's `export CGO_ENABLED = 0` + only reaches what it invokes. `COPY --from=lint /src/go.sum /dev/null` + replaces the copied linter binary as the only edge making the build stage wait + for lint; dropping it would end fail-fast linting under a still-green build. + `golangci-lint config verify`, included per the ruling, validates from an + embedded schema with no network call, but `go mod download` above the gates + still needs the network on a cold cache. Verified: `make lint` green with no + `golangci-lint` on `PATH`; two back-to-back `script/lint` runs on an untouched + tree both ran the linter (27.7s and 28.7s in the lint step, `COPY . .` + `CACHED` above); a planted unused variable failed `script/lint`, and failed + `make docker` at `[lint 9/9]` with the build stage stopped at + `[builder 3/12]`; the drift guard fails on a tag-only, a digest-only and an + unreadable reference, naming both sides; under `--network none` config verify + passes a valid config and rejects an invalid one; `make docker` green in 5m35s + with all six gates run under one epoch (lint 37.6s, test 25.2s reporting + `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`); in the + builder image with the Go test cache off, `--user 0:0` still fails + `TestScanHardlinkRunFailsTogether` where the unprivileged user passes. Noted + for follow-up, not fixed here: `golangci-lint` warns that `gomodguard` is + deprecated since v2.12.0 in favour of `gomodguard_v2`. -- install the Docker build stage's prerequisites by running - `script/bootstrap` instead of `apk add --no-cache make` inline - (2026-08-09, branch `dockerfile-bootstrap`, closes #42): canonical - `REPO_POLICIES.md:97` requires it, and the inline install left the - build stage maintaining its own notion of the toolchain — exactly - the divergence #24 exists to close, one layer down. The stage now - copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`, - which ends in `go mod download`, so the separate invocation of that - is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, and moves - above the bootstrap layer. It is the only edge making this stage - depend on the lint stage, so deleting it as redundant would end - fail-fast linting silently. Letting bootstrap install its own linter - here would have reintroduced the second toolchain and paid for a - from-source build of it. What makes the two stages provably one - toolchain rather than two that happen to agree is a new - `script/verify-linter-pin`, run in the build stage on the binary - that arrives from the lint stage, before bootstrap: it fails the - build naming both versions unless that binary is the version - `script/bootstrap` pins. Bootstrap's own check could not serve that - purpose — it reinstalls its pin from source and then verifies - whatever `PATH` resolves, so drift self-heals silently and a lint - stage image bumped on its own would lint at the new version while - `make check` ran at the old one, green. The linter version is pinned - in two independent places (the lint stage image digest and - `GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a - half-applied bump is now a build failure. The pin is read out of - `script/bootstrap`, which stays the single source of truth; a pin - that cannot be read is a hard failure, not a skip. The check needs - no `CHECK_EPOCH`: its only inputs are the copied binary and - `script/`, so Docker invalidates the layer exactly when a cached - result would stop being true, and it is documented with the other - entrypoints in the README. `$GOPATH/bin` joins `PATH` because - that is where bootstrap's `go install` lands and bootstrap verifies - its installs against what `PATH` resolves — nothing in the image is - shadowed by it, the directory does not exist until bootstrap runs. - Everything added sits above `ARG CHECK_EPOCH`, and the `chown` and - `USER builder` still precede `make check`. Verified: the guard fails - the build with both versions named when the lint stage's linter is - faked to a different version, and an unmodified build still passes - it; bootstrap runs clean under Alpine's `sh` and its `apk` branch, - installing `git` and `make` and finding the copied - linter already at the pin; a second build served the bootstrap and - dependency layers `CACHED` while both gates ran with a fresh epoch; - a planted `unused` finding failed the build at the lint gate in - 48.9s with the build stage's `make check` never starting; and the - suite run in the image as `--user 0:0` fails - `TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged - user is still load-bearing. That last check needs the Go test cache - disabled — the first attempt reported `ok ... (cached)` as root, - reusing the result the build-time run had left in the shared cache, - which would have read as a pass. Build wall time, on a shared host - running many concurrent builds and so noisy: 2m13s on an unchanged - tree, 2m17s and 4m29s for two builds after a source change, 5m14s - cold. Only the cold one breaches the policy ceiling, and not because - of this change — `chown -R builder:builder /src /home/builder` walks - the module cache and re-runs on every source change, and it alone - varied between 77s and 210s across those four builds, which is also - the whole spread in the totals. The same cold measurement against - `main` is 5m03s with a 209s `chown`. Filed as #43 -- bust the Docker layer cache for the gate steps, so `script/cibuild` - and `script/docker` cannot report a green they did not earn - (2026-08-09, branch `cibuild-cache-bust`, closes #32): both scripts - were bare `docker build` invocations with no cache control, and the - `Dockerfile` copies the tree before running its gates, so on an - unchanged tree Docker served those layers from cache and the build - exited 0 having executed nothing. That is not hypothetical here — - every merge this repo has done is a non-fast-forward merge of an - undiverged branch, so each merge commit's tree is byte-identical to - the branch head's and each merge CI run was almost certainly a full - cache hit; and PR #31's reviewer found `make docker` returning - success as a 17-layer cache hit, catching it only by being - suspicious. The fix is `ARG CHECK_EPOCH` with the scripts passing - `--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or break - it. `ARG` is scoped per stage and this `Dockerfile` has three gates - across two — `make fmt-check` and `make lint` in the lint stage, - `make check` in the build stage — so a single declaration would have - left one stage silently cacheable; it is declared in both. And - BuildKit hashes the expanded command, not the declaration, so a - declared-but-unreferenced `ARG` invalidates nothing: each gate `RUN` - echoes the epoch, which also puts the value in the build log as - evidence the layer really ran. Placement is below the dependency - layers on purpose — a build that goes cold every time would be a - different bug, not a fix. Verified by running each script twice back - to back on an unchanged tree under `BUILDKIT_PROGRESS=plain`: all - three gates executed on all four runs, each with a fresh epoch in - the log (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s - then 53.4s), and twelve steps were still served `CACHED` in the - steady state — both `go mod download`s, `apk add`, `adduser`, the - `chown`, every `go.mod`/`go.sum` and source copy, the linter copy - out of the lint stage, and the binary copy into the runtime stage. - The lint stage still gates the build stage: with a deliberate - `unused` finding planted in the tree, the build failed at - `make lint` in 36.1s and the build-stage `make check` never started. - The build stage also still drops to the unprivileged `builder` user - before `make check`, which the suite depends on rather than merely - prefers: forcing the same image to run the tests as root fails - `TestScanHardlinkRunFailsTogether`, because root reads straight - through the `chmod(0)` the test uses to prove hard links are read - once. This is the local fix only; propagating it to the canonical - templates is `prompts` #26 -- check the installed golangci-lint version in `script/bootstrap` - instead of only its presence (2026-08-09, branch - `bootstrap-version-check`, closes #24): `missing golangci-lint` meant - any linter already on `PATH` satisfied the check, so the pin was never - consulted and the v2.12.2 bump from #3 was inert on every host that - already had one — this host ran v2.10.1 against a v2.12.2 pin, - `make check` went green, and `make docker` then rejected the same - commit with findings the local gate never saw. The version now lives - in one place, `GOLANGCI_LINT_VERSION`, with the `go install` module - ref derived from it so a bump cannot half-apply; a - `golangci_lint_version` helper parses `golangci-lint --version` - (taking the field after the word `version` and tolerating an optional - leading `v`, which the module ref carries and the binary's output does - not), and any version that is not the pin — older, newer, absent or - unparseable — is reinstalled. The install is then verified against the - binary `PATH` actually resolves: `go install` writes into `GOBIN` (or - `GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes - first on `PATH`, so a wrong-version one sitting ahead of it — nix, - apt, brew, apk, or the `/usr/local/bin` copy the `Dockerfile` builder - stage makes — would swallow the install and leave the local gate - disagreeing with CI under an affirmative `bootstrap complete`. - Bootstrap now re-reads the effective version after installing and, on - a mismatch, prints both paths and both versions to stderr and exits - non-zero instead of claiming success; it does not reorder anyone's - `PATH` or delete their binary. The `--version` call keeps its stderr - connected, so a present-but-broken binary says why rather than - reinstalling forever in silence, and is bounded by `timeout(1)` where - that exists, so a wedged binary cannot hang bootstrap. `git`, `make` - and `go` keep their presence-only checks and now say why in a - comment: they are host package-manager tools the repo deliberately - does not pin, with `go.mod` governing the language version and the - digest-pinned images covering reproducible builds. Verified on this - host by bootstrapping from v2.10.1 to v2.12.2 and running it again to - a no-op, plus stub runs of the real script under `dash` covering a - thirteen-input parse matrix (absent, older, newer, host-style, - image-style, leading-`v`, stderr-only, empty, non-zero exit, impostor - binary, `(devel)`, trailing `version`), a shadowed install that must - exit non-zero, an install destination not on `PATH` at all, `GOBIN` - set, and a wedged binary that must hit the timeout; `make check` and - `make lint` are clean at v2.12.2, so v2.10.1 was not hiding any - findings on `main` +- install the Docker build stage's prerequisites by running `script/bootstrap` + instead of `apk add --no-cache make` inline (2026-08-09, branch + `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42): + the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`, + which ends in `go mod download`, so the separate call to it is gone. + `COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap + layer: it is the only edge making this stage depend on the lint stage, so + deleting it would end fail-fast linting silently. A new + `script/verify-linter-pin`, run in the build stage before bootstrap, fails the + build naming both versions unless that copied binary is the version + `script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip. + `$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. Everything + added sits above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still + precede `make check`. Verified: the guard fails the build with both versions + named when the lint stage's linter is faked to another version, and passes an + unmodified build; bootstrap runs clean under Alpine's `sh` and `apk`, finding + the copied linter already at the pin; a second build served the bootstrap and + dependency layers `CACHED` while both gates ran with a fresh epoch; a planted + `unused` finding failed the build at the lint gate in 48.9s with the build + stage's `make check` never starting; and the suite run in the image as + `--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the drop to the + unprivileged user is still needed. That last check needs the Go test cache + off: as root it first reported `ok ... (cached)`, reusing the build-time + result. Build times on a noisy shared host: 2m13s on an unchanged tree, 2m17s + and 4m29s after a source change, 5m14s cold, which breaches the policy + ceiling; `chown -R builder:builder /src /home/builder` walks the module cache + and alone varied from 77s to 210s across those builds, and `main` measured + 5m03s cold with a 209s `chown`. Filed as + https://git.eeqj.de/sneak/sfdupes/issues/43 +- bust the Docker layer cache for the gate steps, so `script/cibuild` and + `script/docker` cannot report a green they did not earn (2026-08-09, branch + `cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the + `Dockerfile` copies the tree before its gates, so on an unchanged tree Docker + served them from cache and the build exited 0 having run nothing. Both scripts + now pass `--build-arg CHECK_EPOCH="$(date +%s)"`. `ARG` is per stage and the + gates span two stages, so it is declared in both; BuildKit hashes the expanded + command, so each gate `RUN` echoes the epoch, which also logs it as evidence + the layer ran. It sits below the dependency layers so they stay cached. + Verified under `BUILDKIT_PROGRESS=plain`, each script run twice back to back + on an unchanged tree: all three gates ran on all four runs with a fresh epoch + (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s then 53.4s), and + thirteen steps were still served `CACHED`. With a planted `unused` finding the + build failed at `make lint` in 36.1s and the build-stage `make check` never + started. Run as root, the same image fails `TestScanHardlinkRunFailsTogether`, + because root reads through the `chmod(0)` the test relies on, so the build + stage must drop to the unprivileged `builder` user. Local fix only; + propagating it to the canonical templates is + https://git.eeqj.de/sneak/prompts/issues/26 +- check the installed golangci-lint version in `script/bootstrap` instead of + only its presence (2026-08-09, branch `bootstrap-version-check`, closes + https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in + `GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and + any installed version that is not the pin — older, newer, absent or + unparseable — is reinstalled. `go install` writes into `GOBIN` (or + `GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so + bootstrap re-reads the effective version after installing and, on a mismatch, + prints both paths and both versions and exits non-zero; it does not reorder + `PATH` or delete anyone's binary. The `--version` call keeps its stderr and is + bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep + presence-only checks. Verified by bootstrapping this host from v2.10.1 to + v2.12.2 and again to a no-op, and by stub runs of the script under `dash` + covering a thirteen-input version-parse matrix, a shadowed install that must + exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged + binary that must hit the timeout; `make check` and `make lint` are clean at + v2.12.2, so v2.10.1 was not hiding any findings on `main` - unwind the hash worker pool on the error path (2026-08-09, branch - `hash-pool-cleanup`, closes #6): `hashPhase` used to return the - moment `recordRun` failed and abandon the pool — the feeder parked - forever on a full `jobs` channel and every worker on a full - `results` channel. That only stopped being invisible when #4 landed - and `runScan` began unwinding instead of calling `os.Exit`. The - pool is now an owned, context-aware `hashPool`: every blocking send - in the feeder and the workers selects on `ctx.Done()`, `jobs` is - closed on every path out, and `hashPhase` defers `pool.stop()`, - which cancels and then drains `results` until the last goroutine - has exited — draining is what frees a worker already parked on a - send. `ctx` is threaded from `cmd.Context()` through `runScan`, - `syncScan`, both worker pools and the whole database layer (it is - the first parameter everywhere), so #5 can hand this path a signal - and needs to add nothing else. The walk pool never leaked, because - `walkPhase` always drains its events to close, but it has the same - unbounded-send shape and #5 will give it an early return, so it - gets the same treatment plus a `ctx.Err()` guard after the walk: a - cancelled walk yields a partial size census, and every file it never - reached looks vanished to the update phase. That phase's own - `BeginTx` fails on the same cancelled context before deleting - anything, so the guard is defence in depth rather than the only - barrier — but it is the one that survives #5 deciding an interrupted - scan may commit what it has. Tests drive `run(scan)` against a - database whose insert trigger aborts, and assert both that the scan - fails instead of hanging and that `runtime.NumGoroutine()` polls - back to its pre-scan baseline; a second set cancels a scan part-way - through the walk — deterministically, by counting the scan's own - consultations of `ctx.Done()` rather than racing a timer — and - asserts that it stops at the guard holding a partial census and a - still-populated record index, with every record intact. The - remaining cancellation branches of both pools are covered by direct - tests of `sendEvent`, the walk workers, `dispatchDirs`, - `feedHashJobs`, `hashWorker` and `hashPhase` -- guarantee the database is closed on every fatal exit path - (2026-08-09, branch `db-close-on-fatal`, closes #4): `fatalf` and - its `os.Exit(1)` are gone, so the deferred `db.Close()` — and with - it the SQLite WAL checkpoint — now actually runs when a subcommand - fails; `runScan`, `runReport`, `runTrees`, `loadRecords` and - `resolveRoots` return errors instead. The single exit point is `run` - in `main.go`: it maps a `fatalError` (anything a subcommand - returned) to exit 1 and cobra's own argument and flag errors to exit - 2, which keeps a runtime failure from being reported as a usage - error or printing the usage text. New `main_test.go` drives the CLI - in-process and asserts the exit codes from README §Error handling - plus the stdout/stderr split, including that a fatal error raised - after the database is open leaves no `-wal`/`-shm` sidecar behind - for `scan`, `report` or `trees` -- update golangci-lint to v2.12.2 with the canonical config - (2026-08-09, branch `golangci-v2.12.2`, merged as `38a01bd`, - closes #3): bumped the pinned linter in the `Dockerfile` lint - stage and `script/bootstrap` from v2.12.1 to v2.12.2, and replaced - `.golangci.yml` with the canonical file — the linter settings - (`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under - `linters.settings` per the v2 schema, so they are actually - applied; no new lint findings surfaced -- convert Makefile targets to scripts-to-rule-them-all `script/` - entrypoints like the other managed repos (2026-07-26, commit - `3abeacf`, closes #1): all 12 `script/` entrypoints exist - (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, - `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, - `install-precommit`) and every Makefile target is now a thin shim - over them, matching the other managed repos + `hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the + pool is now an owned, context-aware `hashPool`: every blocking send in the + feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path + out, and `hashPhase` defers `pool.stop()`, which cancels and then drains + `results` until the last goroutine has exited — draining is what frees a + worker already parked on a send. `ctx` is threaded from `cmd.Context()` + through `runScan`, `syncScan`, both worker pools and the whole database layer, + as the first parameter everywhere. The walk pool gets the same treatment plus + a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size + census, and every file it never reached looks vanished to the update phase. + That phase's own `BeginTx` also fails on the cancelled context before deleting + anything, but the guard is the barrier that still holds once an interrupted + scan may commit what it has. Tests drive `run(scan)` against a database whose + insert trigger aborts and assert that the scan fails instead of hanging and + that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others + cancel a scan part-way through the walk, deterministically, by counting its + own consultations of `ctx.Done()`, and assert that it stops at the guard + holding a partial census and a still-populated record index, with every record + intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`, + `feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation + branches of both pools +- guarantee the database is closed on every fatal exit path (2026-08-09, branch + `db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4): + `fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and + with it the SQLite WAL checkpoint — now actually runs when a subcommand fails; + `runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return + errors instead. The single exit point is `run` in `main.go`: it maps a + `fatalError` (anything a subcommand returned) to exit 1 and cobra's own + argument and flag errors to exit 2, which keeps a runtime failure from being + reported as a usage error or printing the usage text. New `main_test.go` + drives the CLI in-process and asserts the exit codes from README §Error + handling plus the stdout/stderr split, including that a fatal error raised + after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`, + `report` or `trees` +- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch + `golangci-v2.12.2`, merged as `38a01bd`, closes + https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the + `Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and + replaced `.golangci.yml` with the canonical file — the linter settings (`lll`, + `funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per + the v2 schema, so they are actually applied; no new lint findings surfaced +- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints + like the other managed repos (2026-07-26, commit `3abeacf`, closes + https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints + exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, + `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`) + and every Makefile target is now a thin shim over them - make the binary the default Make target (2026-07-24, branch - `make-default-target`): plain `make` now builds `sfdupes` - (previously it ran `check` plus `build`); `make build` remains as - an alias -- scan-wide phases, concurrent operands, batched updates (2026-07-24, - branch `scan-wide-phases`): all operands seed the shared walk pool - and every pass runs once over the whole scan, so totals and ETAs - are scan-global; the per-operand walk/hash/update cycles and their - stderr announcements are gone; the update pass commits in batched - transactions — the filesystem is authoritative and the database an - eventually-consistent reflection, so scan-level atomicity is not - required + `make-default-target`): plain `make` now builds `sfdupes` (previously it ran + `check` plus `build`); `make build` remains as an alias +- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch + `scan-wide-phases`): all operands seed the shared walk pool and every pass + runs once over the whole scan, so totals and ETAs are scan-global; the + per-operand walk/hash/update cycles and their stderr announcements are gone; + the update pass commits in batched transactions — the filesystem is + authoritative and the database an eventually-consistent reflection, so + scan-level atomicity is not required - split the stat pass back out of the walk (2026-07-24, branch - `parallel-phases`): phases are strictly sequential again — walk, - stat, hash, update per operand — with parallelism only inside each - phase; the walk enumerates paths with per-directory workers and the - stat pass lstats them with per-file workers, restoring the exact - total/ETA stat bar -- announce each operand on stderr before its passes (2026-07-24, - branch `scan-operand-progress`): with per-operand walk/hash/update - cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals - that looked like the whole run's — an operator watching operand 3 of - 14 hash 300k files concluded 20M files were being skipped -- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass - was a single goroutine and took hours at ~20M files on a busy pool - (observed: 22M files in 4h on a ZFS server); it is now a - per-directory worker-pool traversal that records size/mtime during - the walk (folding away the separate stat pass, halving metadata - I/O), and each `PATH` operand commits in its own transaction so an - interrupted scan keeps completed operands + `parallel-phases`): phases are strictly sequential again — walk, stat, hash, + update per operand — with parallelism only inside each phase; the walk + enumerates paths with per-directory workers and the stat pass lstats them with + per-file workers, restoring the exact total/ETA stat bar +- announce each operand on stderr before its passes (2026-07-24, branch + `scan-operand-progress`): with per-operand walk/hash/update cycles, a + multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the + whole run's +- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single + goroutine and took hours at ~20M files on a busy pool (observed: 22M files in + 4h on a ZFS server); it is now a per-directory worker-pool traversal that + records size/mtime during the walk (folding away the separate stat pass, + halving metadata I/O), and each `PATH` operand commits in its own transaction + so an interrupted scan keeps completed operands -- persistent scan database (2026-07-24, branch `persistent-database`): - `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure - Go, cgo stays disabled) keyed by absolute path that survives between - runs — a rescan hashes only new or changed files (by mtime/size), - deletes records for files vanished from under the scanned operands, - and leaves records outside them untouched, so `scan` can be cronned - daily; `report` and `trees` read the database (no positional - arguments) instead of a scan stream. Database at - `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; - WAL journaling plus a single-transaction update keep a report run - during a scan safe -- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag - `v0.0.1`, and push `main` plus tags (2026-07-23) +- persistent scan database (2026-07-24, branch `persistent-database`): `scan` + now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays + disabled) keyed by absolute path that survives between runs — a rescan hashes + only new or changed files (by mtime/size), deletes records for files vanished + from under the scanned operands, and leaves records outside them untouched, so + `scan` can be cronned daily; `report` and `trees` read the database (no + positional arguments) instead of a scan stream. Database at + `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL + journaling plus a single-transaction update keep a report run during a scan + safe +- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`, + and push `main` plus tags (2026-07-23) - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required - `PATH...` operands via cobra flags replacing the `/srv` `-root` - default; new `-x`/`--one-file-system` flag (GNU convention) to stop - at filesystem boundaries, which are crossed by default + `PATH...` operands via cobra flags replacing the `/srv` `-root` default; new + `-x`/`--one-file-system` flag (GNU convention) to stop at filesystem + boundaries, which are crossed by default - bring the repo into full policy compliance (2026-07-23, branch `repo-policy-compliance`; checklist below) -- `git init` with README-only first commit; code baseline committed on - `main` (2026-07-22) +- `git init` with README-only first commit; code baseline committed on `main` + (2026-07-22) - implement `scan`, `report`, and `trees` subcommands (pre-git history) # Future Steps -- possible later features (explicitly out of scope per README): - full-content verification of candidates, removal-script helpers +- possible later features (explicitly out of scope per README): full-content + verification of candidates, removal-script helpers # Repo Policy Compliance -Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing -repo checklist, and the Go styleguide. Code is already gofmt-clean, so no -standalone formatting commit is needed. +Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo +checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone +formatting commit is needed. -- [x] `.gitignore` missing — the compiled `sfdupes` binary and - `files.dat` sit untracked in the tree; needs OS/editor/Go - artifacts plus secrets patterns +- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit + untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns - [x] `.editorconfig` missing -- [x] `LICENSE` missing and README has no License section (MIT assumed - from house convention — user to confirm) +- [x] `LICENSE` missing and README has no License section (MIT assumed from + house convention — user to confirm) - [x] `REPO_POLICIES.md` missing from repo root -- [x] `.golangci.yml` missing (install canonical copy); code must then - pass `make lint` (150 findings fixed; `make lint` is clean) -- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, - `fmt-check`, `docker`, `hooks`; `check` currently depends on - `build`, which writes the binary (`make check` must not modify - files) -- [x] no tests — `go test ./...` has nothing to run; policy requires - real tests with a 30-second timeout and the conditional `-v` - rerun pattern (suite covers parsing, grouping, digests, - suppression, hashing, and the scan pipeline; 64% coverage) -- [x] `Dockerfile` missing — Go multistage with hash-pinned images: - fail-fast lint stage, build stage running `make check` +- [x] `.golangci.yml` missing (install canonical copy); code must then pass + `make lint` (150 findings fixed; `make lint` is clean) +- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`, + `docker`, `hooks`; `check` currently depends on `build`, which writes the + binary (`make check` must not modify files) +- [x] no tests — `go test ./...` has nothing to run; policy requires real tests + with a 30-second timeout and the conditional `-v` rerun pattern (suite + covers parsing, grouping, digests, suppression, hashing, and the scan + pipeline; 64% coverage) +- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast + lint stage, build stage running `make check` - [x] `.dockerignore` missing -- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, - checkout action pinned by commit SHA) +- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout + action pinned by commit SHA) - [x] README lacks required sections: Description first line - (name/purpose/category/license/author), Getting Started, - Rationale, TODO, License, Author -- [x] README non-goal "no git repository setup and no CI" is stale now - that the repo is under git with CI -- [x] pre-commit hook not installed (`make hooks` once the target - exists) + (name/purpose/category/license/author), Getting Started, Rationale, TODO, + License, Author +- [x] README non-goal "no git repository setup and no CI" is stale now that the + repo is under git with CI +- [x] pre-commit hook not installed (`make hooks` once the target exists) Accepted divergences (no action): -- flat single-package layout with `.go` files in the repo root — fine - for a small single-binary tool per the Go styleguide; the tracker - audit agrees -- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0` - (pure-Go builds) and the race detector requires cgo +- flat single-package layout with `.go` files in the repo root — fine for a + small single-binary tool per the Go styleguide; the tracker audit agrees +- `make test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go + builds) and the race detector requires cgo, so the detector runs in a separate + cgo-enabled container, `make test-race`, which is not part of `make check` diff --git a/cancel_test.go b/cancel_test.go index f8c18d6..556e150 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -4,20 +4,35 @@ import ( "context" "database/sql" "errors" + "fmt" "os" + "os/signal" "path/filepath" + "slices" "strconv" + "strings" "sync" "sync/atomic" + "syscall" "testing" "time" ) -// poolUnwind bounds how long a goroutine is given to leave a pool -// after its context is cancelled. Only a failing run ever waits this -// long: a pool that ignored its cancellation parks forever, and this -// is what turns that into a failed assertion instead of a suite that -// hangs until the test binary's own timeout. +// This file gathers the tests for scan cancellation and worker-pool +// unwinding. Everything it exercises lives in scan.go, so by the repo's +// convention of one test file per source file it would belong in +// scan_test.go. It is kept separate on purpose: cancellation behaviour +// cuts across both the walk pool and the hash pool as a single concern, +// and scan_test.go is already over 1,600 lines. That is the deliberate +// exception the convention otherwise expects to be stated. + +// poolUnwind bounds how long a test waits for a cancellation to take +// effect: for a goroutine to return or a channel to close once its +// context is cancelled, or for a signal to cancel the scan's context. +// Only a failing run waits this long, and the bound is what makes that +// failure an assertion instead of a hang. A call made without it, as +// most of this file's scans are, has no bound: a regression that parks +// it is caught only as the test binary's own timeout. const poolUnwind = 2 * time.Second // walkClock is a context whose cancellation is driven by the scan's @@ -28,9 +43,14 @@ const poolUnwind = 2 * time.Second // // The accounting behind the n chosen by each test: every blocking // channel operation in the walk selects on Done, so the walk spends -// one consultation per file event plus a couple per directory, while -// the index load that runs ahead of it spends a small fixed number -// (three) whatever the record count. +// one consultation per file event plus a couple per directory. The +// index load that runs ahead of it also consults Done, but a bounded +// number of times that does not grow with the record count. The tests +// depend on that property, not on the bound's exact value: each test +// sets n from the consultations of the walk, plus those of the hash +// phase when it cancels mid-hash, far from both ends of the phase it +// interrupts, so the cancellation lands inside that phase whatever the +// record count. type walkClock struct { n int64 seen atomic.Int64 @@ -86,11 +106,14 @@ func (c *walkClock) Value(_ any) any { // directory still queued and only the handful already in flight can // emit anything more. const ( - walkCancelDirs = 100 - walkCancelFilesPerDir = 20 - walkCancelFiles = walkCancelDirs * walkCancelFilesPerDir - walkCancelWorkers = 4 - walkCancelInFlightDirs = walkCancelWorkers * walkCancelFilesPerDir + walkCancelDirs = 100 + walkCancelFilesPerDir = 20 + walkCancelFiles = walkCancelDirs * walkCancelFilesPerDir + walkCancelWorkers = 4 + // The most files the walkCancelWorkers directories already in + // flight when the scan is cancelled can still emit, at + // walkCancelFilesPerDir each. A file count, not a directory count. + walkCancelInFlightFiles = walkCancelWorkers * walkCancelFilesPerDir ) // walkCancelAtDone is the consultation on which the fixture's context @@ -150,9 +173,13 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) { // Every one of those records would look vanished to the update phase. // The guard is what stops the scan there, and this test is what // notices if it stops doing so: deleting the guard, or making it -// unreachable, makes the scan carry its truncated view into a later -// phase and fail there instead, with a wrapped error rather than the -// bare cancellation. +// unreachable, makes the scan carry its truncated view into the update +// phase, which counts every record the walk never reached for removal. +// +// The syncScan call here is not bounded by poolUnwind: a regression +// that left a worker pool parked would hang it, and that regression is +// caught only by the test binary's own timeout, not by a quick +// assertion. // //nolint:paralleltest // counts goroutines: must not run beside others func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { @@ -182,10 +209,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { // assertWalkGuardAborted checks that the scan stopped at the post-walk // guard: with a census that is neither empty (the walk really ran) -// nor complete (it really was cut short), and with the guard's own -// bare cancellation as the error. A wrapped error means the partial -// census was carried past the guard into the hash or update phase, -// which is the failure this test exists to catch. +// nor complete (it really was cut short), and with no record counted +// for removal. A removal count means the partial census was carried +// past the guard into the update phase, which is the failure this test +// exists to catch. func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { t.Helper() @@ -194,12 +221,6 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { err, context.Canceled) } - if errors.Unwrap(err) != nil { - t.Errorf("syncScan reported %q, want the guard's bare "+ - "cancellation: a wrapped error means the truncated census "+ - "reached a later phase", err) - } - if st.unchanged == 0 { t.Fatalf("stats = %+v: the census is empty, so the walk never "+ "ran and the guard was reached for the wrong reason", st) @@ -211,10 +232,11 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { } // The workers drop every directory still queued once the scan is - // cancelled, so only the directories already in flight can add to - // the census after the fact. A census beyond that bound would mean - // the cancellation was not observed where it should have been. - limit := walkCancelAtDone + walkCancelInFlightDirs + // cancelled, so only the files in the directories already in flight + // can add to the census after the fact. A census beyond that bound + // would mean the cancellation was not observed where it should have + // been. + limit := walkCancelAtDone + walkCancelInFlightFiles if st.unchanged > limit { t.Errorf("census covers %d files, want at most %d: the walk kept "+ "taking directories off the queue after cancellation", @@ -260,6 +282,252 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) { assertRecordsIntact(t, db, before) } +// hashCancelAtDone is the consultation on which the mid-hash test's +// context cancels itself. The walk of buildWalkCancelTree spends about +// one per file and three per directory, and the hash phase then one per +// file hashed, so this lands about half way through the hash phase. +const hashCancelAtDone = walkCancelFiles + 3*walkCancelDirs + + walkCancelFiles/2 + +// TestSyncScanCancelledMidHashKeepsHashedRecords cancels a first scan +// part-way through its hash phase. The fixture holds fewer files than a +// batch, so every file hashed is still waiting to be committed: the scan +// must commit them all before it returns, and the next scan must hash +// only the rest. +func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) { + t.Parallel() + + dir := buildWalkCancelTree(t) + db := openTestDB(t) + + st, err := syncScan(newWalkClock(hashCancelAtDone), db, + []string{dir}, walkCancelWorkers, false) + if !errors.Is(err, context.Canceled) { + t.Fatalf("syncScan cancelled mid-hash = %v, want %v", + err, context.Canceled) + } + + if st.walked != walkCancelFiles || st.added == 0 || + st.added >= walkCancelFiles { + t.Fatalf("stats = %+v: want the walk complete and the hash phase "+ + "cut short", st) + } + + if got := len(dbRecords(t, db)); got != st.added { + t.Errorf("%d records after the cancelled scan, want the %d it hashed", + got, st.added) + } + + hashed := st.added + + st = syncTree(t, db, dir) + if st.added != walkCancelFiles-hashed || st.unchanged != hashed { + t.Errorf("next scan stats = %+v, want %d added %d unchanged", + st, walkCancelFiles-hashed, hashed) + } +} + +// storedPaths opens the database at path as report does, which fails +// unless it is a valid database, and returns its records' paths. +func storedPaths(t *testing.T, path string) []string { + t.Helper() + + db, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + return recordPaths(dbRecords(t, db)) +} + +// TestRunScanInterrupted calls the scan entrypoint with a context that +// is already cancelled, as when a signal arrives at once. It must return +// errInterrupted promptly with its one line on stderr and nothing on +// stdout, leave the database valid and as it was, and leave nothing in +// the way of the next scan, which must bring the database up to date. +func TestRunScanInterrupted(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + stdout := captureStdout(t) + stderr := captureStderr(t) + dir := buildSmokeTree(t) + + err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + if err != nil { + t.Fatal(err) + } + + before := storedPaths(t, path) + + // A vanished file and a new one: the interrupted scan records + // neither. + gone := filepath.Join(dir, "a", "unique.bin") + + err = os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "a/new.bin", pattern(50, 10)) + shown := len(stderr()) + done := make(chan struct{}) + + go func() { + defer close(done) + + err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, + false) + }() + + awaitReturn(t, done, "runScan") + + if !errors.Is(err, errInterrupted) { + t.Fatalf("runScan on a cancelled context = %v, want %v", + err, errInterrupted) + } + + want := "scan: interrupted after 0 files\n" + if got := stderr()[shown:]; got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + assertNoSidecars(t, path) + + if got := storedPaths(t, path); !slices.Equal(got, before) { + t.Errorf("records = %q after the interrupted scan, want %q", + got, before) + } + + err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + if err != nil { + t.Fatal(err) + } + + got := storedPaths(t, path) + if slices.Contains(got, gone) || !slices.Contains(got, added) { + t.Errorf("records = %q after the next scan, want %q gone and %q "+ + "added", got, gone, added) + } +} + +// TestRunScanInterruptedMidHash interrupts the scan entrypoint part-way +// through its hash phase, after the database is open. It must return +// errInterrupted, release the lock, end stderr with its line counting +// every file the walk reached, write nothing to stdout, close the +// database out of WAL mode, and keep the records it hashed. +func TestRunScanInterruptedMidHash(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + stdout := captureStdout(t) + stderr := captureStderr(t) + dir := buildWalkCancelTree(t) + + err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, + walkCancelWorkers, false) + if !errors.Is(err, errInterrupted) { + t.Fatalf("runScan interrupted mid-hash = %v, want %v", + err, errInterrupted) + } + + holdScanLock(t, path) + + want := fmt.Sprintf("scan: interrupted after %d files\n", walkCancelFiles) + if got := stderr(); !strings.HasSuffix(got, want) { + t.Errorf("stderr = %q, want it to end with %q", got, want) + } + + if got := stdout(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + assertNoSidecars(t, path) + + db, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + // A plain close also removes the sidecars, but leaves WAL mode on. + var mode string + + err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&mode) + if err != nil { + t.Fatal(err) + } + + if mode != "delete" { + t.Errorf("journal mode = %q after the interrupted scan, want %q", + mode, "delete") + } + + kept := len(dbRecords(t, db)) + if kept == 0 || kept >= walkCancelFiles { + t.Errorf("%d records after the interrupted scan, want those it "+ + "hashed: some but not all of the %d files", kept, walkCancelFiles) + } +} + +// TestInterruptContextCatchesSIGTERM sends SIGTERM to the test process +// while the scan's handler is installed, and checks that it cancels the +// scan's context. +// +//nolint:paralleltest // signals the whole process: must not run beside a scan +func TestInterruptContextCatchesSIGTERM(t *testing.T) { + // Caught here as well, so that a handler that misses SIGTERM fails + // this test instead of ending the test process. + caught := make(chan os.Signal, 1) + signal.Notify(caught, syscall.SIGTERM) + + defer signal.Stop(caught) + + ctx, stop := interruptContext(t.Context()) + defer stop() + + err := syscall.Kill(os.Getpid(), syscall.SIGTERM) + if err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + case <-time.After(poolUnwind): + t.Fatal("SIGTERM did not cancel the scan's context") + } +} + +// TestCommitFullBatchKeepsFailedBatch checks that a full batch whose +// commit fails, as it does once the scan is interrupted, stays in the +// batch, so that syncScan's final commit saves it. +func TestCommitFullBatchKeepsFailedBatch(t *testing.T) { + t.Parallel() + + s := &scanState{db: openTestDB(t)} + for i := range updateBatchSize { + s.batch = append(s.batch, scanRec{path: "/f" + strconv.Itoa(i)}) + } + + err := s.commitFullBatch(cancelledContext(t)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("commitFullBatch on a cancelled context = %v, want %v", + err, context.Canceled) + } + + if len(s.batch) != updateBatchSize { + t.Errorf("batch holds %d records after the failed commit, want %d", + len(s.batch), updateBatchSize) + } +} + // drainClosed counts the values received from ch until it closes, // failing the test if it does not close within poolUnwind. A pool that // ignored its cancellation leaves its channel open with its goroutines @@ -328,20 +596,49 @@ func TestSendEventAbandonsBlockedSend(t *testing.T) { awaitReturn(t, done, "sendEvent") } -// TestWalkWorkersDropQueuedDirs checks that cancelled walk workers keep -// reading jobs and drop the directories rather than stopping their -// read: the range over jobs has to run out for the pool to tear down -// and close its event stream. -func TestWalkWorkersDropQueuedDirs(t *testing.T) { +// TestWalkOneDirStopsWhenCancelled checks that a cancelled scan stops +// reading a directory instead of going through the rest of its +// entries. A walk that kept going would return the subdirectory below +// to descend into. Unlike a file event, that return is not a send the +// cancellation can abandon, so the test catches the regression every +// time. +func TestWalkOneDirStopsWhenCancelled(t *testing.T) { t.Parallel() dir := t.TempDir() - writeEmptyFiles(t, dir, walkCancelFilesPerDir) + + err := os.Mkdir(filepath.Join(dir, "sub"), 0o750) + if err != nil { + t.Fatal(err) + } + + // Unbuffered and unread: on a cancelled scan every send gives up. + events := make(chan walkEvent) + + subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events) + if len(subs) != 0 { + t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ + "want none", subs) + } +} + +// TestWalkWorkersDropQueuedDirs checks that cancelled walk workers keep +// reading jobs and drop the directories rather than stopping their +// read: the range over jobs has to run out for the pool to tear down +// and close its event stream. The queued directory does not exist, so +// a worker that walked it anyway would send a warning before +// walkOneDir's own cancellation check could stop it. On a cancelled +// scan that send delivers or gives up at random, so with 64 jobs +// queued the regression has a one in 2^64 chance of passing. +func TestWalkWorkersDropQueuedDirs(t *testing.T) { + t.Parallel() + + missing := filepath.Join(t.TempDir(), "missing") jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false) - for range 4 { - jobs <- dirJob{path: dir} + for range 64 { + jobs <- dirJob{path: missing} } close(jobs) @@ -412,7 +709,11 @@ func TestDispatchDirsClosesJobsWhenCancelled(t *testing.T) { // TestFeedHashJobsClosesJobsWhenCancelled checks that the hash feeder // abandons the runs it has not queued yet and still closes the job -// channel, which is what lets the workers' range terminate. +// channel, which is what lets the workers' range terminate. The +// receive on jobs below is not bounded: a feeder that returned without +// closing jobs would leave that receive with no sender and no close, so +// this regression is caught by the test binary's timeout rather than by +// a bounded assertion. func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) { t.Parallel() @@ -438,41 +739,84 @@ func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) { // TestHashWorkerDropsQueuedRuns checks that a cancelled hash worker // keeps reading jobs and drops the runs rather than reading files // nobody wants the hashes of — while still letting the range run out -// so the pool tears down. The queued run names a file that does not -// exist, so a worker that hashed it anyway would produce a result. +// so the pool tears down. The hash function records that it was +// called, so a worker that hashed the queued run anyway is caught +// every time. func TestHashWorkerDropsQueuedRuns(t *testing.T) { t.Parallel() done := make(chan struct{}) jobs := make(chan []fileRec, 1) - results := make(chan hashResult, 1) + results := make(chan hashResult) - run := []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} - - jobs <- run + jobs <- []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} close(jobs) + var hashed atomic.Bool + + hash := func(path string, size int64) (string, string, string, error) { + hashed.Store(true) + + return hashSignature(path, size) + } + go func() { defer close(done) - hashWorker(cancelledContext(t), jobs, results, hashSignature) + hashWorker(cancelledContext(t), jobs, results, hash) }() awaitReturn(t, done, "hashWorker") - select { - case r := <-results: - t.Errorf("cancelled hash worker produced %+v, want the run dropped", - r) - default: + if hashed.Load() { + t.Error("cancelled hash worker hashed the queued run, want it dropped") } } +// TestHashWorkerAbandonsBlockedSend checks that a hash worker with a +// result to deliver and nobody to deliver it to leaves once the scan +// is cancelled, instead of holding the pool open. The scan tests do +// not catch this: stop drains results, which frees a parked worker +// anyway. +func TestHashWorkerAbandonsBlockedSend(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + done := make(chan struct{}) + jobs := make(chan []fileRec, 1) + // Unbuffered and unread, with jobs left open: the worker's only way + // out is the cancellation case beside its send. + results := make(chan hashResult) + + jobs <- []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} + + // The scan is cancelled while the worker hashes, so the worker has + // already passed the check that drops queued runs. + hash := func(path string, size int64) (string, string, string, error) { + cancel() + + return hashSignature(path, size) + } + + go func() { + defer close(done) + + hashWorker(ctx, jobs, results, hash) + }() + + awaitReturn(t, done, "hashWorker") +} + // TestHashPhaseCancelledReturnsContextError checks the result loop's // own exit: with the pool cancelled, no result will ever arrive, and // the loop must leave through the cancellation rather than wait for a -// receive that cannot happen. +// receive that cannot happen. This call is not bounded by poolUnwind: a +// loop that dropped its cancellation case would block on that receive, +// so the regression surfaces as the test binary's timeout rather than +// as a bounded assertion. func TestHashPhaseCancelledReturnsContextError(t *testing.T) { t.Parallel() diff --git a/db.go b/db.go index f732608..e0dff42 100644 --- a/db.go +++ b/db.go @@ -6,11 +6,14 @@ import ( "errors" "fmt" "io/fs" + "net/url" "os" "path/filepath" "slices" "strconv" + "time" + "golang.org/x/sys/unix" // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo // disabled. _ "modernc.org/sqlite" @@ -32,26 +35,45 @@ const schemaVersion = 1 // scan. const dbDirPerm = 0o755 +// dbFilePerm is the mode for a database file created by scan, the mode +// SQLite gives a database file it creates. +const dbFilePerm = 0o644 + +// lockFilePerm is the mode for the scan lock file. Anyone who can open +// the file can hold the lock and keep every scan from running, so it +// is open to its owner only. +const lockFilePerm = 0o600 + // createTableSQL is the schema applied to a fresh database. Paths are -// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. +// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime +// holds whole Unix seconds and mtime_nsec the nanoseconds within that +// second. const createTableSQL = ` CREATE TABLE files ( - path BLOB PRIMARY KEY, - size INTEGER NOT NULL, - mtime INTEGER NOT NULL, - head TEXT NOT NULL, - tail TEXT NOT NULL, - content TEXT NOT NULL + path BLOB PRIMARY KEY, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL, + mtime_nsec INTEGER NOT NULL, + head TEXT NOT NULL, + tail TEXT NOT NULL, + content TEXT NOT NULL ) WITHOUT ROWID ` +// createIndexSQL indexes the records by signature, so report can have +// SQLite group them without sorting the whole table. +const createIndexSQL = ` +CREATE INDEX files_signature ON files (size, head, tail, content) +` + // upsertSQL inserts one file record, replacing any existing record for // the same path. const upsertSQL = ` -INSERT INTO files (path, size, mtime, head, tail, content) -VALUES (?, ?, ?, ?, ?, ?) +INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content) +VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (path) DO UPDATE SET size = excluded.size, mtime = excluded.mtime, + mtime_nsec = excluded.mtime_nsec, head = excluded.head, tail = excluded.tail, content = excluded.content ` @@ -64,6 +86,10 @@ var errNoDatabase = errors.New( // does not understand. var errSchemaVersion = errors.New("unsupported database schema version") +// errScanRunning reports that another scan holds the lock on the +// database. +var errScanRunning = errors.New("another scan is running") + // databasePath resolves the database location: SFDUPES_DATABASE when // set and non-empty, the compiled-in default otherwise. func databasePath() string { @@ -74,16 +100,36 @@ func databasePath() string { return defaultDatabasePath } -// openDB opens the SQLite database at path with WAL journaling and a -// busy timeout, so a report can run while a cron scan is in progress. -// It does not create or verify the schema. -func openDB(path string) (*sql.DB, error) { - dsn := "file:" + path + - "?_pragma=busy_timeout(10000)" + - "&_pragma=journal_mode(WAL)" + - "&_pragma=synchronous(NORMAL)" +// scanParams are the connection parameters for scan: read-write, with +// WAL journaling and a busy timeout, so a report can run while a cron +// scan is in progress. closeScanDatabase leaves WAL mode again. +const scanParams = "_pragma=busy_timeout(10000)" + + "&_pragma=journal_mode(WAL)" + + "&_pragma=synchronous(NORMAL)" - db, err := sql.Open("sqlite", dsn) +// reportParams are the connection parameters for report and trees: +// read-only, with the same busy timeout. They set no journal mode, +// because setting one is a write. +const reportParams = "mode=ro" + + "&_pragma=busy_timeout(10000)" + + "&_pragma=query_only(1)" + +// openDB opens the SQLite database at path with the connection +// parameters params. It does not create or verify the schema. +func openDB(path, params string) (*sql.DB, error) { + // The path is escaped into a file: URI, so ?, # and % in it stay + // part of the file name. SQLite reads what follows file:// up to + // the next / as a host name, so an absolute path goes after an + // empty host (file:///abs) and a relative path goes without one + // (file:rel). + uri := url.URL{ + Scheme: "file", + OmitHost: !filepath.IsAbs(path), + Path: path, + RawQuery: params, + } + + db, err := sql.Open("sqlite", uri.String()) if err != nil { return nil, fmt.Errorf("open database %s: %w", path, err) } @@ -96,6 +142,61 @@ func openDB(path string) (*sql.DB, error) { return db, nil } +// lockScanDatabase takes the lock that keeps a second scan off the +// database at path: an exclusive flock(2) on the file beside it named +// by the database's path with symlinks resolved and ".lock" appended, +// so every path to one database names the same lock file. If missing, +// the database's parent directory and an empty database file are +// created first, so the lock file is named the same way before the +// database is set up as after. A lock held by another scan fails at +// once instead of waiting. The lock lasts until the returned file is +// closed or the process ends. The lock file is never deleted: a scan +// that deleted it would let the next scan lock a new file while +// another still holds the old one. +func lockScanDatabase(path string) (*os.File, error) { + err := os.MkdirAll(filepath.Dir(path), dbDirPerm) + if err != nil { + return nil, fmt.Errorf("create database directory: %w", err) + } + + // Like SQLite, this follows a symlink at path and creates its + // target. SQLite opens an empty file as a new database. + //nolint:gosec // the operator chooses the database path + dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm) + if err != nil { + return nil, fmt.Errorf("create database: %w", err) + } + + _ = dbFile.Close() + + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, fmt.Errorf("resolve database path: %w", err) + } + + lockPath := resolved + ".lock" + + //nolint:gosec // the operator chooses the database path + f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm) + if err != nil { + return nil, err + } + + err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err != nil { + _ = f.Close() + + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, fmt.Errorf("%w (lock held on %s)", + errScanRunning, lockPath) + } + + return nil, fmt.Errorf("lock %s: %w", lockPath, err) + } + + return f, nil +} + // openScanDatabase opens the database for the scan subcommand, creating // the file, its parent directory, and the schema as needed. func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { @@ -104,7 +205,7 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { return nil, fmt.Errorf("create database directory: %w", err) } - db, err := openDB(path) + db, err := openDB(path, scanParams) if err != nil { return nil, err } @@ -119,6 +220,24 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { return db, nil } +// closeScanDatabase switches the database at path from WAL back to +// rollback-journal mode and closes it. Out of WAL mode the database +// file alone holds the whole database, so a reader needs no -wal or +// -shm file beside it, nor write access to create them. The switch +// fails while a report has the database open; the database then stays +// in WAL mode, still readable, until a later scan closes it. +func closeScanDatabase(ctx context.Context, db *sql.DB, path string) { + // Runs on the way out of a cancelled scan too. + _, err := db.ExecContext(context.WithoutCancel(ctx), + "PRAGMA journal_mode = DELETE") + if err != nil { + fmt.Fprintf(os.Stderr, "scan: database %s left in WAL mode: %v\n", + path, err) + } + + _ = db.Close() +} + // openReportDatabase opens an existing database for the report and // trees subcommands. A missing database file is an error directing the // user to run scan first; the schema version must match exactly. @@ -134,12 +253,18 @@ func openReportDatabase(ctx context.Context, return nil, fmt.Errorf("database: %w", err) } - db, err := openDB(path) + db, err := openDB(path, reportParams) if err != nil { return nil, err } v, err := userVersion(ctx, db) + if err == nil && v == 0 { + // An empty database passes this check and fails the version + // check below. + err = checkUnversioned(ctx, db) + } + if err != nil { _ = db.Close() @@ -166,6 +291,11 @@ func initSchema(ctx context.Context, db *sql.DB) error { switch v { case 0: + err = checkUnversioned(ctx, db) + if err != nil { + return err + } + return createSchema(ctx, db) case schemaVersion: return nil @@ -175,20 +305,65 @@ func initSchema(ctx context.Context, db *sql.DB) error { } } +// checkUnversioned checks a database at user_version 0 before it is +// taken for an empty one. createSchema creates the files table and +// sets the version together, so a files table at version 0 was made by +// something else. Adopting it could corrupt unrelated data, so that is +// a schema-version error telling the operator to remove the file and +// rescan. +func checkUnversioned(ctx context.Context, db *sql.DB) error { + var name string + + err := db.QueryRowContext(ctx, + "SELECT name FROM sqlite_master "+ + "WHERE type = 'table' AND name = 'files'").Scan(&name) + + switch { + case err == nil: + return fmt.Errorf( + "has a files table but no schema version; "+ + "remove the file and rescan: %w", errSchemaVersion) + case errors.Is(err, sql.ErrNoRows): + return nil + default: + return fmt.Errorf("check for files table: %w", err) + } +} + // createSchema applies the schema to a fresh database and stamps the -// schema version. +// schema version in one transaction, so a creation stopped partway, by +// an interrupt or an error, leaves an empty database the next scan +// sets up, never a files table at version 0, which checkUnversioned +// refuses. func createSchema(ctx context.Context, db *sql.DB) error { - _, err := db.ExecContext(ctx, createTableSQL) + tx, err := db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("create schema: %w", err) } - _, err = db.ExecContext(ctx, + defer func() { _ = tx.Rollback() }() + + _, err = tx.ExecContext(ctx, createTableSQL) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + + _, err = tx.ExecContext(ctx, createIndexSQL) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + + _, err = tx.ExecContext(ctx, "PRAGMA user_version = "+strconv.Itoa(schemaVersion)) if err != nil { return fmt.Errorf("set schema version: %w", err) } + err = tx.Commit() + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + return nil } @@ -204,51 +379,12 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) { return v, nil } -// loadFileRows reads every record from the files table. -func loadFileRows(ctx context.Context, db *sql.DB) ([]scanRec, error) { +// loadFileRows streams every record to fn in path order: byte order, +// which is the order of the primary key, so SQLite does not sort. +func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error { rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, head, tail, content FROM files") - if err != nil { - return nil, fmt.Errorf("read records: %w", err) - } - - defer func() { _ = rows.Close() }() - - var recs []scanRec - - for rows.Next() { - var ( - path []byte - r scanRec - ) - - err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, - &r.content) - if err != nil { - return nil, fmt.Errorf("read record: %w", err) - } - - r.path = string(path) - recs = append(recs, r) - } - - err = rows.Err() - if err != nil { - return nil, fmt.Errorf("read records: %w", err) - } - - return recs, nil -} - -// loadFileMeta streams every record's path, size, mtime, and whether -// it carries hashes to fn. Scan change detection needs no hash -// values, and skipping the hash columns keeps the scan's in-memory -// index small on multi-million-file databases. -func loadFileMeta(ctx context.Context, db *sql.DB, - fn func(path string, size, mtime int64, hashed bool), -) error { - rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, head <> '' FROM files") + "SELECT path, size, mtime, mtime_nsec, head, tail, content "+ + "FROM files ORDER BY path") if err != nil { return fmt.Errorf("read records: %w", err) } @@ -257,17 +393,131 @@ func loadFileMeta(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - size, mtime int64 - hashed int64 + path []byte + sec, nsec int64 + r scanRec ) - err = rows.Scan(&path, &size, &mtime, &hashed) + err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, + &r.content) if err != nil { return fmt.Errorf("read record: %w", err) } - fn(string(path), size, mtime, hashed != 0) + r.path = string(path) + r.mtime = time.Unix(sec, nsec) + fn(r) + } + + err = rows.Err() + if err != nil { + return fmt.Errorf("read records: %w", err) + } + + return nil +} + +// dupeRowsSQL selects every record in a duplicate group, with the +// group's first path. A group is the records with a content hash that +// share a size, head, tail, and content, when there are two or more of +// them. The rows come in report order: groups by size descending, then +// by first path, and each group's paths ascending. +const dupeRowsSQL = ` +SELECT g.first, f.path, f.size +FROM files AS f +JOIN ( + SELECT size, head, tail, content, MIN(path) AS first + FROM files + WHERE content <> '' + GROUP BY size, head, tail, content + HAVING COUNT(*) > 1 +) AS g USING (size, head, tail, content) +ORDER BY f.size DESC, g.first, f.path +` + +// loadDupeRows streams the rows of dupeRowsSQL to fn and returns the +// number of records in the database. The count and the rows are read +// in one transaction, so they agree while a scan is committing. An +// error from fn stops the reading and is returned as it is. +func loadDupeRows(ctx context.Context, db *sql.DB, + fn func(first, path string, size int64) error, +) (int, error) { + // Everything goes through tx: the report connection is the only + // one, so a query on db would wait for tx forever. + tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + var records int + + err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + rows, err := tx.QueryContext(ctx, dupeRowsSQL) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ( + first, path []byte + size int64 + ) + + err = rows.Scan(&first, &path, &size) + if err != nil { + return 0, fmt.Errorf("read record: %w", err) + } + + err = fn(string(first), string(path), size) + if err != nil { + return 0, err + } + } + + err = rows.Err() + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + return records, nil +} + +// loadFileMeta streams every record's path, size, mtime, and whether +// it carries hashes to fn. Scan change detection needs no hash +// values, and skipping the hash columns keeps the scan's in-memory +// index small on multi-million-file databases. +func loadFileMeta(ctx context.Context, db *sql.DB, + fn func(path string, size int64, mtime time.Time, hashed bool), +) error { + rows, err := db.QueryContext(ctx, + "SELECT path, size, mtime, mtime_nsec, head <> '' FROM files") + if err != nil { + return fmt.Errorf("read records: %w", err) + } + + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ( + path []byte + size, sec, nsec int64 + hashed int64 + ) + + err = rows.Scan(&path, &size, &sec, &nsec, &hashed) + if err != nil { + return fmt.Errorf("read record: %w", err) + } + + fn(string(path), size, time.Unix(sec, nsec), hashed != 0) } err = rows.Err() @@ -286,7 +536,7 @@ func loadFileMeta(ctx context.Context, db *sql.DB, // memory; the rows come ordered by size, head, and tail, so each // group's rows arrive together. const contentCandidatesSQL = ` -SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> '' +SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> '' FROM files AS f JOIN ( SELECT size, head, tail @@ -312,17 +562,20 @@ func loadContentCandidates(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - r scanRec - hashed int64 + path []byte + sec, nsec int64 + r scanRec + hashed int64 ) - err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed) + err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, + &hashed) if err != nil { return fmt.Errorf("read record: %w", err) } r.path = string(path) + r.mtime = time.Unix(sec, nsec) fn(r, hashed != 0) } @@ -406,8 +659,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, defer func() { _ = st.Close() }() for _, r := range upserts { - _, err = st.ExecContext(ctx, - []byte(r.path), r.size, r.mtime, r.head, r.tail, r.content) + _, err = st.ExecContext(ctx, []byte(r.path), r.size, + r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content) if err != nil { return fmt.Errorf("upsert %s: %w", r.path, err) } diff --git a/db_test.go b/db_test.go index b68bddd..e4b8600 100644 --- a/db_test.go +++ b/db_test.go @@ -5,17 +5,27 @@ import ( "database/sql" "errors" "fmt" + "os" "path/filepath" "slices" "strings" "testing" + "time" ) -// testDBPath returns a database path inside a fresh temp dir. +// testDBPath returns a database path inside a fresh temp dir, with +// symlinks resolved as the scan lock resolves them, so the lock file +// is this path plus ".lock" even where the temp dir is reached through +// a symlink. func testDBPath(t *testing.T) string { t.Helper() - return filepath.Join(t.TempDir(), "db.sqlite") + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + + return filepath.Join(dir, "db.sqlite") } // openTestDB creates a fresh scan database in a temp dir. @@ -72,9 +82,78 @@ func TestOpenScanDatabaseCreates(t *testing.T) { defer func() { _ = db.Close() }() - recs, err := loadFileRows(t.Context(), db) - if err != nil || len(recs) != 0 { - t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err) + if recs := dbRecords(t, db); len(recs) != 0 { + t.Fatalf("records = %v, want none", recs) + } +} + +func TestOpenDatabaseUnversionedForeign(t *testing.T) { + t.Parallel() + + // A database that has a files table but user_version 0, written by + // some other tool. report, trees and scan must refuse it with the + // schema-version error, not adopt it and not emit a raw SQLite + // "table files already exists". + path := testDBPath(t) + + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + + _, err = db.ExecContext(t.Context(), "CREATE TABLE files (x INTEGER)") + if err != nil { + t.Fatal(err) + } + + _ = db.Close() + + _, err = openReportDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) || + !strings.Contains(err.Error(), "remove the file and rescan") { + t.Fatalf("report: err = %v, want errSchemaVersion telling the "+ + "operator to remove the file and rescan", err) + } + + _, err = openScanDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) || + !strings.Contains(err.Error(), "remove the file and rescan") { + t.Fatalf("scan: err = %v, want errSchemaVersion telling the "+ + "operator to remove the file and rescan", err) + } +} + +func TestSchemaCreationStoppedPartway(t *testing.T) { + t.Parallel() + + // A first scan stopped while creating the schema must leave a + // database the next scan accepts. max_page_count(2) leaves room for + // the files table but not its index, so schema creation fails right + // after CREATE TABLE, a point an interrupt could also stop it at. + path := testDBPath(t) + + db, err := openDB(path, scanParams+"&_pragma=max_page_count(2)") + if err != nil { + t.Fatal(err) + } + + err = initSchema(t.Context(), db) + _ = db.Close() + + if err == nil { + t.Fatal("initSchema with no room for the index succeeded") + } + + db, err = openScanDatabase(t.Context(), path) + if err != nil { + t.Fatalf("next scan: %v", err) + } + + defer func() { _ = db.Close() }() + + v, err := userVersion(t.Context(), db) + if err != nil || v != schemaVersion { + t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion) } } @@ -87,9 +166,11 @@ func TestOpenReportDatabaseMissing(t *testing.T) { } } -func TestOpenReportDatabaseVersionMismatch(t *testing.T) { +func TestOpenDatabaseVersionMismatch(t *testing.T) { t.Parallel() + // A database stamped with a schema version other than 0 and + // schemaVersion. report, trees and scan must all refuse it. path := testDBPath(t) db, err := openScanDatabase(t.Context(), path) @@ -106,7 +187,12 @@ func TestOpenReportDatabaseVersionMismatch(t *testing.T) { _, err = openReportDatabase(t.Context(), path) if !errors.Is(err, errSchemaVersion) { - t.Fatalf("err = %v, want errSchemaVersion", err) + t.Fatalf("report: err = %v, want errSchemaVersion", err) + } + + _, err = openScanDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) { + t.Fatalf("scan: err = %v, want errSchemaVersion", err) } } @@ -130,6 +216,49 @@ func TestOpenReportDatabaseOK(t *testing.T) { _ = db.Close() } +func TestCloseScanDatabaseWhileReportOpen(t *testing.T) { + t.Parallel() + + // A report holding the database open stops scan from taking it out + // of WAL mode. The -wal and -shm files must then stay beside it, so + // that a later report still needs only read access. + path := testDBPath(t) + + scanDB, err := openScanDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + reportDB, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + closeScanDatabase(t.Context(), scanDB, path) + + _ = reportDB.Close() + + _, err = os.Stat(path + "-wal") + if err != nil { + t.Fatalf("no -wal left: the switch out of WAL mode was not "+ + "stopped: %v", err) + } + + makeReadOnly(t, path) + + reportDB, err = openReportDatabase(t.Context(), path) + if err != nil { + t.Fatalf("openReportDatabase: %v", err) + } + + defer func() { _ = reportDB.Close() }() + + err = loadFileRows(t.Context(), reportDB, func(scanRec) {}) + if err != nil { + t.Fatalf("loadFileRows: %v", err) + } +} + func TestApplyChangesRoundTrip(t *testing.T) { t.Parallel() @@ -140,10 +269,13 @@ func TestApplyChangesRoundTrip(t *testing.T) { // written. recs := []scanRec{ { - size: 2, mtime: 20, head: "h2", tail: "t2", content: "c2", - path: "/a/tab\tnew\nline", + size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2", + content: "c2", path: "/a/tab\tnew\nline", + }, + { + size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1", + content: "c1", path: "/a/x", }, - {size: 1, mtime: 10, head: "h1", tail: "t1", content: "c1", path: "/a/x"}, } err := applyChanges(t.Context(), db, recs, nil, @@ -152,15 +284,8 @@ func TestApplyChangesRoundTrip(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err := loadFileRows(t.Context(), db) - if err != nil { - t.Fatal(err) - } - - slices.SortFunc(got, func(a, b scanRec) int { - return strings.Compare(a.path, b.path) - }) - + // The records come back in path order, which is the order of recs. + got := dbRecords(t, db) if !slices.Equal(got, recs) { t.Fatalf("rows = %+v, want %+v", got, recs) } @@ -168,7 +293,8 @@ func TestApplyChangesRoundTrip(t *testing.T) { // An upsert for an existing path updates in place; a delete // removes exactly its path. upd := scanRec{ - size: 3, mtime: 30, head: "h3", tail: "t3", content: "c3", path: "/a/x", + size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3", + path: "/a/x", } err = applyChanges(t.Context(), db, []scanRec{upd}, @@ -177,11 +303,7 @@ func TestApplyChangesRoundTrip(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err = loadFileRows(t.Context(), db) - if err != nil { - t.Fatal(err) - } - + got = dbRecords(t, db) if len(got) != 1 || got[0] != upd { t.Fatalf("rows = %+v, want just %+v", got, upd) } @@ -199,7 +321,7 @@ func TestApplyChangesBatching(t *testing.T) { recs := make([]scanRec, 0, n) for i := range n { recs = append(recs, scanRec{ - size: int64(i), mtime: 1, head: "h", tail: "t", + size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t", path: fmt.Sprintf("/batch/%07d", i), }) } @@ -210,9 +332,8 @@ func TestApplyChangesBatching(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err := loadFileRows(t.Context(), db) - if err != nil || len(got) != n { - t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n) + if got := dbRecords(t, db); len(got) != n { + t.Fatalf("records = %d, want %d", len(got), n) } deletes := make([]string, 0, n) @@ -226,8 +347,7 @@ func TestApplyChangesBatching(t *testing.T) { t.Fatalf("applyChanges deletes: %v", err) } - got, err = loadFileRows(t.Context(), db) - if err != nil || len(got) != 0 { - t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err) + if got := dbRecords(t, db); len(got) != 0 { + t.Fatalf("records = %d, want 0", len(got)) } } diff --git a/go.mod b/go.mod index 9b0faaa..cf8c37c 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.25.7 require ( github.com/schollz/progressbar/v3 v3.19.1 github.com/spf13/cobra v1.10.2 + golang.org/x/sys v0.46.0 + golang.org/x/term v0.44.0 modernc.org/sqlite v1.54.0 ) @@ -18,8 +20,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/main.go b/main.go index d2160c1..6133b50 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ // sfdupes scan [--workers N] [-x] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv +// sfdupes --version // // See README.md for the complete specification. package main @@ -50,6 +51,10 @@ const ( // cobra prints for it is the whole message. var errNoSubcommand = errors.New("no subcommand") +// errWorkersBelowOne is the usage error for a scan --workers value +// below 1. +var errWorkersBelowOne = errors.New("--workers must be at least 1") + // Version is the build version, injected at link time via -ldflags // (see the Makefile); "dev" for a plain go build. // @@ -57,22 +62,27 @@ var errNoSubcommand = errors.New("no subcommand") var Version = "dev" func main() { - os.Exit(run(os.Args[1:], os.Stderr)) + // Once the reader of a stdout pipe has gone, as in "sfdupes report | + // head", the Go runtime ends the process with SIGPIPE on the next + // write instead of returning an error (README "Error handling"). + // Registering for SIGPIPE with os/signal would change that. + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } // run executes args against the command tree and returns the process // exit code. It is the program's single exit point: the subcommands // return their errors instead of exiting, so every deferred cleanup — // above all closing the database, which checkpoints the SQLite WAL — -// runs before the process ends. -func run(args []string, stderr io.Writer) int { +// runs before the process ends. The report and trees subcommands write +// their data to stdout. +func run(args []string, stdout, stderr io.Writer) int { // A nil slice makes cobra fall back to os.Args, which would let a // test binary's own flags reach the command tree. if args == nil { args = []string{} } - root := newRootCommand(stderr) + root := newRootCommand(stdout, stderr) root.SetArgs(args) err := root.Execute() @@ -82,6 +92,9 @@ func run(args []string, stderr io.Writer) int { switch { case err == nil: return exitOK + case errors.Is(err, errInterrupted): + // The interrupted scan has printed its own line. + return exitFatal case errors.As(err, &fatal): // The command ran and failed: a runtime error, reported // without the usage text that a usage error gets. @@ -89,22 +102,35 @@ func run(args []string, stderr io.Writer) int { return exitFatal default: - // A usage error: cobra has already printed the message and - // the usage text. + // A usage error, which cobra has already reported on stderr. return exitUsage } } // newRootCommand builds the command tree. Everything on stdout is -// machine-readable data; all human-facing output (help, usage, errors) -// goes to stderr. -func newRootCommand(stderr io.Writer) *cobra.Command { +// machine-readable data, the version line included; all human-facing +// output (help, usage, errors) goes to stderr. +func newRootCommand(stdout, stderr io.Writer) *cobra.Command { + var showVersion bool + + printVersion := runE(func(context.Context, []string) error { + _, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version) + if err != nil { + return fmt.Errorf("write stdout: %w", err) + } + + return nil + }) + root := &cobra.Command{ - Use: "sfdupes", - Short: "Find candidate duplicate files by size and head/tail/content SHA-256", - Version: Version, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { + Use: "sfdupes", + Short: "Find candidate duplicate files by size and head/tail/content SHA-256", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if showVersion { + return printVersion(cmd, args) + } + // A missing subcommand prints usage and exits 2: cobra // prints the usage text for the returned error, and run // maps everything that is not a fatal error to exit 2. @@ -117,6 +143,11 @@ func newRootCommand(stderr io.Writer) *cobra.Command { root.SetErr(stderr) root.CompletionOptions.DisableDefaultCmd = true + // Cobra's built-in version flag prints through the help writer, + // stderr; this one prints to stdout. + root.Flags().BoolVarP(&showVersion, "version", "v", false, + "print the version to stdout") + var ( scanWorkers int scanOneFS bool @@ -126,7 +157,13 @@ func newRootCommand(stderr io.Writer) *cobra.Command { Use: cmdScan + " [--workers N] [-x] PATH...", Short: "Walk trees and synchronize the scan database", Args: cobra.MinimumNArgs(1), + PreRunE: func(cmd *cobra.Command, _ []string) error { + return checkScanWorkers(cmd, scanWorkers) + }, RunE: runE(func(ctx context.Context, args []string) error { + ctx, stop := interruptContext(ctx) + defer stop() + return runScan(ctx, args, scanWorkers, scanOneFS) }), } @@ -140,7 +177,7 @@ func newRootCommand(stderr io.Writer) *cobra.Command { Short: "Read the scan database and print the file-level duplicates report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { - return runReport(ctx) + return runReport(ctx, stdout) }), } @@ -149,7 +186,7 @@ func newRootCommand(stderr io.Writer) *cobra.Command { Short: "Read the scan database and print the duplicate-tree report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { - return runTrees(ctx) + return runTrees(ctx, stdout) }), } @@ -158,13 +195,26 @@ func newRootCommand(stderr io.Writer) *cobra.Command { return root } -// runE adapts a subcommand implementation to cobra's RunE. Cobra -// prints the error and the command's usage text for every error RunE -// returns, but a subcommand that ran and failed has no usage problem -// to report: both are silenced here, and the error is marked fatal so -// that run reports it on stderr and exits 1 rather than 2. The command's -// context is handed to the implementation: cancelling it unwinds the -// scan's worker pools. +// checkScanWorkers rejects a scan --workers value below 1. That is a +// usage error reported in one line: cobra prints the returned message +// without the usage text, and run exits 2. +func checkScanWorkers(cmd *cobra.Command, workers int) error { + if workers >= 1 { + return nil + } + + cmd.SilenceUsage = true + + return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers) +} + +// runE adapts a subcommand implementation, or the version print, to +// cobra's RunE. Cobra prints the error and the command's usage text for +// every error RunE returns, but a subcommand that ran and failed has no +// usage problem to report: both are silenced here, and the error is +// marked fatal so that run reports it on stderr and exits 1 rather than +// 2. The command's context is handed to the implementation: cancelling +// it unwinds the scan's worker pools. func runE( fn func(ctx context.Context, args []string) error, ) func(*cobra.Command, []string) error { diff --git a/main_test.go b/main_test.go index b307a82..ca6faf7 100644 --- a/main_test.go +++ b/main_test.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -44,23 +45,81 @@ func assertNoSidecars(t *testing.T, path string) { } } -// captureStdout redirects os.Stdout to a file for the rest of the test -// and returns a function reading back everything written to it. Only -// machine-readable data belongs on stdout (README design goal 4), so -// the tests assert on it directly. -func captureStdout(t *testing.T) func() string { +// makeReadOnly takes write permission away from the database at path, +// from any WAL sidecar beside it, and from their directory, as for a +// user reading a database that a root cron scan keeps. Root ignores +// file permissions, so it skips the test when run as root. +func makeReadOnly(t *testing.T, path string) { t.Helper() - f, err := os.Create(filepath.Join(t.TempDir(), "stdout")) + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + + err := os.Chmod(path, 0o400) if err != nil { t.Fatal(err) } - saved := os.Stdout - os.Stdout = f + for _, suffix := range walSuffixes { + err = os.Chmod(path+suffix, 0o400) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + t.Fatal(err) + } + } + + dir := filepath.Dir(path) + + //nolint:gosec // reaching the database needs the search bit + err = os.Chmod(dir, 0o500) + if err != nil { + t.Fatal(err) + } + + // Runs before t.TempDir's own cleanup, which must delete the files. + t.Cleanup(func() { + //nolint:gosec // removing the directory needs its search bit back + _ = os.Chmod(dir, 0o700) + }) +} + +// captureStderr redirects os.Stderr to a file for the rest of the test +// and returns a function reading back everything written to it. scan +// writes its warnings and summary, and report and trees their +// summaries, straight to os.Stderr, not to the stderr writer run is +// given. +func captureStderr(t *testing.T) func() string { + t.Helper() + + return capture(t, &os.Stderr) +} + +// captureStdout does for os.Stdout what captureStderr does for +// os.Stderr. scan is never given run's stdout writer, so anything it +// printed would go straight to os.Stdout. The scan tests pass os.Stdout +// as run's stdout too, so the one capture sees both. +func captureStdout(t *testing.T) func() string { + t.Helper() + + return capture(t, &os.Stdout) +} + +// capture redirects *std, which is os.Stdout or os.Stderr, to a file +// for the rest of the test and returns a function reading back +// everything written to it. +func capture(t *testing.T, std **os.File) func() string { + t.Helper() + + f, err := os.Create(filepath.Join(t.TempDir(), "output")) + if err != nil { + t.Fatal(err) + } + + saved := *std + *std = f t.Cleanup(func() { - os.Stdout = saved + *std = saved _ = f.Close() }) @@ -91,13 +150,13 @@ func captureStdout(t *testing.T) func() string { // brokenDatabase writes a database that opens cleanly and passes the // schema-version check but has no files table, so the first query // fails with the database already open: a fatal error on a path that -// owns an open database. +// owns an open database. It closes the database the way scan does. func brokenDatabase(t *testing.T) string { t.Helper() path := testDBPath(t) - db, err := openDB(path) + db, err := openDB(path, scanParams) if err != nil { t.Fatal(err) } @@ -108,10 +167,7 @@ func brokenDatabase(t *testing.T) string { t.Fatal(err) } - err = db.Close() - if err != nil { - t.Fatal(err) - } + closeScanDatabase(t.Context(), db, path) return path } @@ -144,27 +200,31 @@ func TestOpenDatabaseKeepsWALWhileOpen(t *testing.T) { func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { // Every subcommand that owns an open database must close it when - // it fails: no os.Exit between the open and the return. - cases := map[string][]string{ - cmdScan: {cmdScan}, - cmdReport: {cmdReport}, - cmdTrees: {cmdTrees}, - } + // it fails: no os.Exit between the open and the return. The + // sidecar check is evidence of the close only for scan: report and + // trees only read a database that is out of WAL mode, which leaves + // nothing on disk whether they close it or not. + // + // The subcommands come from the command tree, so a new one is + // checked too: one wired with a bare RunE instead of runE reports + // its failure as a usage error, exit 2 with the usage text. + for _, cmd := range newRootCommand(io.Discard, io.Discard).Commands() { + name := cmd.Name() - for name, args := range cases { t.Run(name, func(t *testing.T) { path := brokenDatabase(t) t.Setenv(databaseEnv, path) + args := []string{name} if name == cmdScan { args = append(args, t.TempDir()) } - var stderr bytes.Buffer - stdout := captureStdout(t) - code := run(args, &stderr) + var stderr bytes.Buffer + + code := run(args, os.Stdout, &stderr) if code != exitFatal { t.Errorf("run(%v) = %d, want %d", args, code, exitFatal) } @@ -182,19 +242,19 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { } } -func TestRunMissingOperandIsFatalNotUsage(t *testing.T) { +func TestRunNonexistentPathIsFatalNotUsage(t *testing.T) { // README §Error handling: a PATH operand that does not exist is a // fatal error (1), not a usage error (2) — and a runtime failure // must not dump the usage text. t.Setenv(databaseEnv, testDBPath(t)) - var stderr bytes.Buffer - stdout := captureStdout(t) + var stderr bytes.Buffer + missing := filepath.Join(t.TempDir(), "nope") - code := run([]string{cmdScan, missing}, &stderr) + code := run([]string{cmdScan, missing}, os.Stdout, &stderr) if code != exitFatal { t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal) } @@ -223,6 +283,34 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) { } } +func TestRunMissingDatabaseIsFatal(t *testing.T) { + // README §Database: report and trees need an existing database; a + // missing one exits 1 with a message telling the user to run scan. + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + var stdout, stderr bytes.Buffer + + code := run([]string{name}, &stdout, &stderr) + if code != exitFatal { + t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) + } + + want := "sfdupes: " + path + ": no database (run \"sfdupes " + + "scan\" first, or set " + databaseEnv + ")\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout.String(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + }) + } +} + func TestRunUsageErrors(t *testing.T) { // Usage errors keep exiting 2 with cobra's own report on stderr. cases := map[string]struct { @@ -243,11 +331,9 @@ func TestRunUsageErrors(t *testing.T) { // path that does not exist. t.Setenv(databaseEnv, testDBPath(t)) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run(tc.args, &stderr) + code := run(tc.args, &stdout, &stderr) if code != exitUsage { t.Errorf("run(%v) = %d, want %d", tc.args, code, exitUsage) } @@ -256,43 +342,115 @@ func TestRunUsageErrors(t *testing.T) { t.Errorf("stderr = %q, want %q", stderr.String(), tc.want) } - if got := stdout(); got != "" { + if got := stdout.String(); got != "" { t.Errorf("stdout = %q, want nothing (data only)", got) } }) } } -// TestRunHelpAndVersionSucceed checks that the two informational flags -// exit 0 and keep their human-facing output on stderr. -// -//nolint:paralleltest // captureStdout replaces the process-wide os.Stdout -func TestRunHelpAndVersionSucceed(t *testing.T) { - assertHumanOutput(t, "--help") - assertHumanOutput(t, "--version") +func TestRunScanRejectsWorkersBelowOne(t *testing.T) { + // README §scan mode: --workers below 1 is a usage error reported in + // one line on stderr, before the scan opens the database. + for _, workers := range []string{"0", "-1"} { + t.Run(workers, func(t *testing.T) { + dbPath := testDBPath(t) + t.Setenv(databaseEnv, dbPath) + + var stdout, stderr bytes.Buffer + + args := []string{cmdScan, "--workers", workers, t.TempDir()} + + code := run(args, &stdout, &stderr) + if code != exitUsage { + t.Errorf("run(%v) = %d, want %d", args, code, exitUsage) + } + + want := "Error: --workers must be at least 1, got " + workers + + "\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout.String(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + _, err := os.Stat(dbPath) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("stat %s: %v, want the database never created", + dbPath, err) + } + }) + } } -// assertHumanOutput runs sfdupes with one informational flag and checks -// that it succeeds with its output on stderr and stdout untouched -// (README design goal 4). -func assertHumanOutput(t *testing.T, arg string) { - t.Helper() +func TestRunHelp(t *testing.T) { + t.Parallel() + // README §Subcommands: help goes to stderr, exits 0, and leaves + // stdout empty. + cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}} + + for _, args := range cases { + var stdout, stderr bytes.Buffer + + code := run(args, &stdout, &stderr) + if code != exitOK { + t.Errorf("run(%v) = %d, want %d", args, code, exitOK) + } + + if !strings.Contains(stderr.String(), usageMarker) { + t.Errorf("run(%v) stderr = %q, want the help text", + args, stderr.String()) + } + + if got := stdout.String(); got != "" { + t.Errorf("run(%v) stdout = %q, want nothing (data only)", + args, got) + } + } +} + +func TestRunVersion(t *testing.T) { + t.Parallel() + + // README §Subcommands: the version is one line on stdout, with + // nothing on stderr, and exits 0. + for _, arg := range []string{"--version", "-v"} { + var stdout, stderr bytes.Buffer + + code := run([]string{arg}, &stdout, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) + } + + want := "sfdupes " + Version + "\n" + if got := stdout.String(); got != want { + t.Errorf("run(%s) stdout = %q, want %q", arg, got, want) + } + + if got := stderr.String(); got != "" { + t.Errorf("run(%s) stderr = %q, want nothing", arg, got) + } + } +} + +func TestRunVersionWriteFailureIsFatal(t *testing.T) { + t.Parallel() + + // README §Error handling: a stdout write failure exits 1, reported + // in one line on stderr. var stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{arg}, &stderr) - if code != exitOK { - t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) + code := run([]string{"--version"}, failingWriter{}, &stderr) + if code != exitFatal { + t.Errorf("run(--version) = %d, want %d", code, exitFatal) } - if stderr.Len() == 0 { - t.Errorf("run(%s) wrote nothing to stderr", arg) - } - - if got := stdout(); got != "" { - t.Errorf("stdout = %q, want nothing (data only)", got) + want := "sfdupes: write stdout: " + errWriteFailed.Error() + "\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) } } @@ -320,52 +478,195 @@ func scanFixture(t *testing.T) []string { t.Fatal(err) } - var stderr bytes.Buffer + scanOK(t, dir) + + return dupes +} + +// scanOK runs scan over operands, fails the test unless it exits 0 with +// nothing on stdout, and returns everything it printed to stderr. +func scanOK(t *testing.T, operands ...string) string { + t.Helper() stdout := captureStdout(t) + stderr := captureStderr(t) - code := run([]string{cmdScan, dir}, &stderr) + code := run(append([]string{cmdScan}, operands...), os.Stdout, os.Stderr) if code != exitOK { - t.Fatalf("run(scan) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + t.Fatalf("run(scan %q) = %d, want %d; stderr: %s", + operands, code, exitOK, stderr()) } if got := stdout(); got != "" { t.Errorf("scan stdout = %q, want nothing (data only)", got) } - return dupes + return stderr() } func TestRunScanSucceedsDespiteWarnings(t *testing.T) { + // README §Error handling: a scan that skips a file it cannot read + // warns, counts the skip in its summary, and still exits 0, which + // scanOK checks along with the empty stdout. + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + path := testDBPath(t) t.Setenv(databaseEnv, path) - scanFixture(t) + dir := t.TempDir() + writeFile(t, dir, "a.bin", pattern(1, 300)) + + // Same size as a.bin, so the scan reads it, and the read fails. + unreadable := writeFile(t, dir, "unreadable.bin", pattern(2, 300)) + + err := os.Chmod(unreadable, 0) + if err != nil { + t.Fatal(err) + } + + stderr := scanOK(t, dir) + + warning := "hash " + unreadable + ": open " + unreadable + + ": permission denied\n" + if !strings.Contains(stderr, warning) { + t.Errorf("stderr = %q, want %q", stderr, warning) + } + + summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " + + "0 unchanged), 1 skipped\n" + if !strings.Contains(stderr, summary) { + t.Errorf("stderr = %q, want %q", stderr, summary) + } + assertNoSidecars(t, path) } +func TestRunScanSkipsSymlinkOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + writeFile(t, dir, "target/sub/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(filepath.Join(dir, "target"), link) + if err != nil { + t.Fatal(err) + } + + // Scanning a directory through the symlink stores a record beneath + // the symlink's own path for a file beneath its target. + scanOK(t, filepath.Join(link, "sub")) + + assertOperandSkipped(t, path, link, "symlink", + filepath.Join(link, "sub", "f")) +} + +func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + writeFile(t, dir, "target/sub/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(filepath.Join(dir, "target"), link) + if err != nil { + t.Fatal(err) + } + + // link is dropped as a symlink, but link/sub must still be scanned, + // not dropped as lying under link. + scanOK(t, link, filepath.Join(link, "sub")) + + db, err := openDB(path, reportParams) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f")) +} + +func TestRunScanSkipsZFSOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + zfs := filepath.Join(t.TempDir(), ".zfs") + snapshot := filepath.Join(zfs, "snapshot", "hourly") + f := writeFile(t, snapshot, "f", pattern(1, 10)) + + // An operand beneath a .zfs directory is walked, because it is not + // itself named .zfs. + scanOK(t, snapshot) + + assertOperandSkipped(t, path, zfs, ".zfs directory", f) +} + +// assertOperandSkipped scans operand alone and checks that it is skipped +// as kind: a warning naming it, one skip in the summary, exit 0, and the +// record for kept, which an earlier scan stored beneath operand, still +// in the database at dbPath. +func assertOperandSkipped(t *testing.T, dbPath, operand, kind, + kept string, +) { + t.Helper() + + stderr := scanOK(t, operand) + + warning := "walk " + operand + ": skipping " + kind + " operand\n" + if !strings.Contains(stderr, warning) { + t.Errorf("stderr = %q, want %q", stderr, warning) + } + + summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " + + "0 unchanged), 1 skipped\n" + if !strings.Contains(stderr, summary) { + t.Errorf("stderr = %q, want %q", stderr, summary) + } + + db, err := openDB(dbPath, reportParams) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + recordByPath(t, dbRecords(t, db), kept) +} + func TestRunReportSucceeds(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) dupes := scanFixture(t) - var stderr bytes.Buffer + var stdout bytes.Buffer - stdout := captureStdout(t) + stderr := captureStderr(t) - code := run([]string{cmdReport}, &stderr) + code := run([]string{cmdReport}, &stdout, os.Stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + code, exitOK, stderr()) } want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } + want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " + + "300 B reclaimable\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + assertNoSidecars(t, path) } @@ -375,23 +676,375 @@ func TestRunTreesSucceeds(t *testing.T) { dupes := scanFixture(t) - var stderr bytes.Buffer + var stdout bytes.Buffer - stdout := captureStdout(t) + stderr := captureStderr(t) - code := run([]string{cmdTrees}, &stderr) + code := run([]string{cmdTrees}, &stdout, os.Stderr) if code != exitOK { t.Fatalf("run(trees) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + code, exitOK, stderr()) } // The two directories holding the duplicate pair are duplicate // trees of each other. want := "first\tdupe\tfiles\tsize\n" + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } + want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " + + "300 B reclaimable\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + assertNoSidecars(t, path) } + +func TestRunReportsNeedOnlyReadAccess(t *testing.T) { + // README §Database: report and trees need only read access to the + // database file. With its directory read-only as well, SQLite + // cannot create any file beside it. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dupes := scanFixture(t) + assertNoSidecars(t, path) + makeReadOnly(t, path) + + cases := map[string]string{ + cmdReport: "first\tdupe\tsize\n" + + dupes[0] + "\t" + dupes[1] + "\t300\n", + cmdTrees: "first\tdupe\tfiles\tsize\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + + "\t1\t300\n", + } + + for name, want := range cases { + var stdout, stderr bytes.Buffer + + code := run([]string{name}, &stdout, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + + continue + } + + if got := stdout.String(); got != want { + t.Errorf("%s stdout = %q, want %q", name, got, want) + } + } +} + +// assertRunsUseDatabase runs scan, then report and trees, against the +// database that SFDUPES_DATABASE names, the file name in dir. It fails +// unless the reports find the duplicate pair the scan recorded and dir +// then holds only that file and its lock file: nothing was created +// under a shortened name. +func assertRunsUseDatabase(t *testing.T, dir, name string) { + t.Helper() + + dupes := scanFixture(t) + + want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n" + if got := runStdout(t, cmdReport); got != want { + t.Errorf("report stdout = %q, want %q", got, want) + } + + want = "first\tdupe\tfiles\tsize\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" + if got := runStdout(t, cmdTrees); got != want { + t.Errorf("trees stdout = %q, want %q", got, want) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + got := make([]string, 0, len(entries)) + for _, e := range entries { + got = append(got, e.Name()) + } + + if wantFiles := []string{name, name + ".lock"}; !slices.Equal(got, wantFiles) { + t.Errorf("%s holds %q, want %q", dir, got, wantFiles) + } +} + +func TestRunDatabasePathUsedAsGiven(t *testing.T) { + // README §Database: the path names the database file exactly. In + // SQLite's connection string an unescaped ? or # would end the file + // name and % would start an escape, and a path starting with // + // could be read as a host name. %25 is a valid escape, so unescaped + // this name opens a file named a without any error. + const name = "a?b#c%25d e.sqlite" + + t.Run("absolute", func(t *testing.T) { + dir := t.TempDir() + t.Setenv(databaseEnv, filepath.Join(dir, name)) + + assertRunsUseDatabase(t, dir, name) + }) + + t.Run("leading double slash", func(t *testing.T) { + dir := t.TempDir() + t.Setenv(databaseEnv, "/"+filepath.Join(dir, name)) + + assertRunsUseDatabase(t, dir, name) + }) + + t.Run("relative", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + t.Setenv(databaseEnv, name) + + assertRunsUseDatabase(t, dir, name) + }) +} + +// holdScanLock takes the lock on the database at path, as a running +// scan does, and holds it until the test ends. It fails the test when +// the lock is already held. +func holdScanLock(t *testing.T, path string) { + t.Helper() + + lock, err := lockScanDatabase(path) + if err != nil { + t.Fatalf("lock %s: %v", path, err) + } + + t.Cleanup(func() { _ = lock.Close() }) +} + +func TestRunSecondScanFails(t *testing.T) { + // README §Database: while one scan holds the lock, a second scan + // fails at once, naming the lock file, without opening the + // database. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + holdScanLock(t, path) + + stdout := captureStdout(t) + + var stderr bytes.Buffer + + code := run([]string{cmdScan, t.TempDir()}, os.Stdout, &stderr) + if code != exitFatal { + t.Errorf("run(scan) = %d, want %d", code, exitFatal) + } + + want := "sfdupes: another scan is running (lock held on " + + path + ".lock)\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + // Taking the lock created the database file, empty; opening it as + // a database would have written to it. + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + if info.Size() != 0 { + t.Errorf("database size = %d, want 0 (not opened)", info.Size()) + } +} + +// assertSecondScanFails runs a scan against the database +// SFDUPES_DATABASE names while the test holds the lock on it. It fails +// the test unless that scan fails at once, naming the lock file beside +// path, the database's real path. +func assertSecondScanFails(t *testing.T, path string) { + t.Helper() + + var stderr bytes.Buffer + + code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr) + if code != exitFatal { + t.Errorf("run(scan) = %d, want %d", code, exitFatal) + } + + want := "sfdupes: another scan is running (lock held on " + + path + ".lock)\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } +} + +func TestRunSecondScanThroughSymlinkFails(t *testing.T) { + // README §Database: every path to one database names the same lock + // file, so while a scan holds it through one path, a scan through + // another fails, naming the lock file beside the database's real + // path. + t.Run("symlink to the database", func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + + link := filepath.Join(t.TempDir(), "link.sqlite") + + err := os.Symlink(path, link) + if err != nil { + t.Fatal(err) + } + + holdScanLock(t, path) + t.Setenv(databaseEnv, link) + assertSecondScanFails(t, path) + }) + + t.Run("symlink to its directory, no database yet", func(t *testing.T) { + path := testDBPath(t) + link := filepath.Join(t.TempDir(), "link") + + err := os.Symlink(filepath.Dir(path), link) + if err != nil { + t.Fatal(err) + } + + holdScanLock(t, path) + t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path))) + assertSecondScanFails(t, path) + }) + + t.Run("symlink whose target does not exist yet", func(t *testing.T) { + path := testDBPath(t) + link := filepath.Join(t.TempDir(), "link.sqlite") + + err := os.Symlink(path, link) + if err != nil { + t.Fatal(err) + } + + // The first scan takes the lock through the link before the + // database exists, then creates the database through it. + holdScanLock(t, link) + + db, err := openScanDatabase(t.Context(), link) + if err != nil { + t.Fatal(err) + } + + closeScanDatabase(t.Context(), db, link) + + t.Setenv(databaseEnv, link) + assertSecondScanFails(t, path) + }) +} + +func TestRunScanReleasesLock(t *testing.T) { + // README §Database: a scan releases the lock however it ends. + t.Run("success", func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + holdScanLock(t, path) + }) + + t.Run("fatal error", func(t *testing.T) { + path := brokenDatabase(t) + t.Setenv(databaseEnv, path) + + code := run([]string{cmdScan, t.TempDir()}, io.Discard, io.Discard) + if code != exitFatal { + t.Fatalf("run(scan) = %d, want %d", code, exitFatal) + } + + holdScanLock(t, path) + }) +} + +func TestRunReportsDuringScan(t *testing.T) { + // README §Database: report and trees never take the lock, so they + // run while a scan holds it. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + holdScanLock(t, path) + + for _, name := range []string{cmdReport, cmdTrees} { + var stderr bytes.Buffer + + code := run([]string{name}, io.Discard, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + } + } +} + +func TestRunStdoutClosedIsFatal(t *testing.T) { + // README §Error handling: a stdout write failure exits 1, reported + // in one line on stderr. + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + t.Setenv(databaseEnv, testDBPath(t)) + + scanFixture(t) + + stdout, err := os.Create(filepath.Join(t.TempDir(), "stdout")) + if err != nil { + t.Fatal(err) + } + + err = stdout.Close() + if err != nil { + t.Fatal(err) + } + + var stderr bytes.Buffer + + code := run([]string{name}, stdout, &stderr) + if code != exitFatal { + t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) + } + + got := stderr.String() + if !strings.HasPrefix(got, "sfdupes: write stdout: ") || + !strings.Contains(got, os.ErrClosed.Error()) || + strings.Count(got, "\n") != 1 { + t.Errorf("stderr = %q, want one line reporting the "+ + "failed stdout write", got) + } + }) + } +} + +// errWriteFailed is the error failingWriter returns. +var errWriteFailed = errors.New("write failed") + +// failingWriter is a stdout that fails every write. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errWriteFailed } + +func TestStdoutWriteErrorPropagates(t *testing.T) { + t.Setenv(databaseEnv, testDBPath(t)) + + scanFixture(t) + + cases := map[string]func(context.Context, io.Writer) error{ + cmdReport: runReport, + cmdTrees: runTrees, + } + + for name, fn := range cases { + err := fn(t.Context(), failingWriter{}) + if !errors.Is(err, errWriteFailed) { + t.Errorf("%s: error = %v, want %v", name, err, errWriteFailed) + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc05cde --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/progress.go b/progress.go index ecb0585..79b05bb 100644 --- a/progress.go +++ b/progress.go @@ -6,6 +6,7 @@ import ( "time" "github.com/schollz/progressbar/v3" + "golang.org/x/term" ) // plainInterval is the minimum time between progress lines when stderr @@ -24,24 +25,23 @@ const percentScale = 100 // stderrIsTTY reports whether stderr is attached to a terminal. func stderrIsTTY() bool { - fi, err := os.Stderr.Stat() - if err != nil { - return false - } - - return fi.Mode()&os.ModeCharDevice != 0 + return term.IsTerminal(int(os.Stderr.Fd())) } // progress renders one scan pass's progress on stderr. On a TTY it // delegates to the progressbar library (spinner style when the total is // unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When // stderr is not a TTY it emits no ANSI redraws: it prints a plain -// one-line update no more often than every plainInterval. +// one-line update as the pass starts, then no more often than every +// plainInterval. // -// All methods must be called from the main goroutine only. A nil -// *progress is a valid no-display receiver: every method is a no-op, -// so batched database flushes during the streaming pass can reuse the -// update-pass helpers without rendering anything. +// All methods must be called from the main goroutine only. On a TTY +// the library also redraws a spinner from its own goroutine, several +// times a second, so its count and elapsed time stay current while a +// pass waits for its next item. A nil *progress is a valid +// no-display receiver: every method is a no-op, so batched database +// flushes during the streaming pass can reuse the update-pass helpers +// without rendering anything. type progress struct { label string total int64 // -1 when unknown (walk pass) @@ -53,10 +53,22 @@ type progress struct { func newProgress(label string, total int64) *progress { p := &progress{label: label, total: total, start: time.Now()} - if !stderrIsTTY() { + if stderrIsTTY() { + p.bar = newBar(label, total) + return p } + // Print the zero state at once: the first item may take minutes, + // and a pass must never look hung. + p.last = p.start + fmt.Fprintln(os.Stderr, p.plainLine()) + + return p +} + +// newBar builds the TTY display for newProgress. +func newBar(label string, total int64) *progressbar.ProgressBar { opts := []progressbar.Option{ progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetDescription(label), @@ -81,9 +93,7 @@ func newProgress(label string, total int64) *progress { ) } - p.bar = progressbar.NewOptions64(total, opts...) - - return p + return progressbar.NewOptions64(total, opts...) } // increment records one completed item and refreshes the display. @@ -106,26 +116,52 @@ func (p *progress) increment() { } // warnf prints a one-line warning to stderr without corrupting the bar. +// The whole message is escaped like a report's path columns, so a path +// holding a newline cannot split the warning. func (p *progress) warnf(format string, args ...any) { if p == nil { return } + msg := escapePath(fmt.Sprintf(format, args...)) + + if p.bar != nil && p.total < 0 { + // The library also redraws a spinner from its own goroutine, so + // a direct write could land inside a redraw. The bar prints the + // warning itself, just before its next redraw. + _, _ = progressbar.Bprintln(p.bar, msg) + + return + } + if p.bar != nil { _ = p.bar.Clear() } - fmt.Fprintf(os.Stderr, format+"\n", args...) + fmt.Fprintln(os.Stderr, msg) + + // The library never draws a finished bar again, so the warning for + // a pass's last item would erase it for good: draw its last frame + // again below the warning. + if p.bar != nil && p.bar.IsFinished() { + fmt.Fprint(os.Stderr, p.bar.String()) + } } -// finish terminates the pass's display. +// finish terminates the pass's display. A bar whose pass stopped short +// of its total, as an interrupted one does, is left as last drawn; the +// library's Finish would fill it up. func (p *progress) finish() { if p == nil { return } if p.bar != nil { - _ = p.bar.Finish() + if p.total >= 0 && p.count < p.total { + _ = p.bar.Exit() + } else { + _ = p.bar.Finish() + } fmt.Fprintln(os.Stderr) diff --git a/progress_test.go b/progress_test.go new file mode 100644 index 0000000..453fa94 --- /dev/null +++ b/progress_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// spinnerIdle comfortably outlasts the 100ms interval at which the +// progressbar library redraws a spinner from its own goroutine. +const spinnerIdle = 500 * time.Millisecond + +//nolint:paralleltest // replaces the process-wide os.Stderr +func TestStderrIsTTYFalseForNonTerminals(t *testing.T) { + r, pipe, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + regular, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + if err != nil { + t.Fatal(err) + } + + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + + saved := os.Stderr + + t.Cleanup(func() { + os.Stderr = saved + + for _, f := range []*os.File{r, pipe, regular, devNull} { + _ = f.Close() + } + }) + + cases := map[string]*os.File{ + "a pipe": pipe, + "a regular file": regular, + os.DevNull: devNull, + } + + for name, f := range cases { + os.Stderr = f + + if stderrIsTTY() { + t.Errorf("stderrIsTTY() = true with stderr on %s", name) + } + } +} + +// TestNewProgressPrintsBeforeFirstItem checks that each pass shows its +// zero state the moment it starts when stderr is not a terminal, and +// that the next line still waits for plainInterval. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestNewProgressPrintsBeforeFirstItem(t *testing.T) { + stderr := captureStderr(t) + + newProgress("walk", -1).increment() + newProgress("hash", 10).increment() + + want := "walk: 0 files, elapsed 0s\n" + + "hash: [0/10] 0% 0 files/s elapsed 0s eta ?\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } +} + +// newWalkSpinner returns the walk pass's terminal display, writing to +// os.Stderr whether or not it is a terminal, and stops the library's +// redraws when the test ends. +func newWalkSpinner(t *testing.T) *progress { + t.Helper() + + p := &progress{ + label: "walk", total: -1, start: time.Now(), + bar: newBar("walk", -1), + } + t.Cleanup(p.finish) + + return p +} + +// TestProgressWarningsOnOwnLines drives the terminal display of the walk +// pass through a run of warnings with no items between them, as when the +// walk meets many unreadable paths, for several of the spinner's +// redraws: every warning must land on a line of its own, never inside a +// redraw. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestProgressWarningsOnOwnLines(t *testing.T) { + stderr := captureStderr(t) + p := newWalkSpinner(t) + + // No pause between warnings: one written straight to stderr is + // garbled only if a redraw lands while it is being written. + issued := 0 + for start := time.Now(); time.Since(start) < spinnerIdle; issued++ { + p.warnf("warning") + } + + // The spinner prints the warnings at its next redraw. + time.Sleep(spinnerIdle) + + // A terminal shows each line as the text after its last carriage + // return. + shown := 0 + + for line := range strings.SplitSeq(stderr(), "\n") { + if !strings.Contains(line, "warning") { + continue + } + + shown++ + + if text := line[strings.LastIndex(line, "\r")+1:]; text != "warning" { + t.Errorf("terminal shows %q, want %q", text, "warning") + } + } + + if shown != issued { + t.Errorf("%d warning lines, want %d", shown, issued) + } +} + +// TestSpinnerShowsCountAfterBurst checks that once a burst of items +// faster than the redraw limit is over, the walk display shows every +// item completed while it waits for the next one. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestSpinnerShowsCountAfterBurst(t *testing.T) { + stderr := captureStderr(t) + p := newWalkSpinner(t) + + for range 50 { + p.increment() + } + + time.Sleep(spinnerIdle) + + if shown := lastFrame(stderr()); !strings.Contains(shown, "(50/-,") { + t.Errorf("terminal shows %q, want a count of 50", shown) + } +} + +// lastFrame returns what a terminal shows of the frames a bar drew: the +// last one. The library starts each frame with a carriage return and +// erases the previous one with spaces first. +func lastFrame(out string) string { + var shown string + + for frame := range strings.SplitSeq(out, "\r") { + if strings.TrimSpace(frame) != "" { + shown = frame + } + } + + return shown +} + +// TestBarStoppedShortKeepsCount checks that the terminal display of a +// pass that stops before its total, as an interrupted one does, is left +// as last drawn instead of being filled up. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestBarStoppedShortKeepsCount(t *testing.T) { + stderr := captureStderr(t) + p := &progress{ + label: "hash", total: 10, start: time.Now(), + bar: newBar("hash", 10), + } + + p.increment() + + // Past the redraw limit, so the bar draws the next count. + time.Sleep(2 * barThrottle) + p.increment() + p.finish() + + if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/10,") { + t.Errorf("terminal shows %q, want a count of 2 of 10", shown) + } +} + +// TestWarningAfterLastItemKeepsBar checks that a warning printed after +// a pass's last item, as for a failed read of the last file, leaves the +// pass's finished bar on the terminal below the warning. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestWarningAfterLastItemKeepsBar(t *testing.T) { + stderr := captureStderr(t) + p := &progress{ + label: "hash", total: 2, start: time.Now(), + bar: newBar("hash", 2), + } + + p.increment() + p.increment() + p.warnf("warning") + p.finish() + + if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") { + t.Errorf("terminal shows %q, want a count of 2 of 2", shown) + } +} diff --git a/report.go b/report.go index c5761b5..425f412 100644 --- a/report.go +++ b/report.go @@ -4,9 +4,10 @@ import ( "bufio" "context" "fmt" + "io" "os" - "slices" "strings" + "time" ) // ioBufSize is the buffer size for the buffered stdout writers. @@ -21,80 +22,66 @@ const minGroupSize = 2 // only and used by scan for change detection. type scanRec struct { size int64 - mtime int64 + mtime time.Time head string tail string content string path string } -// loadRecords opens the database and reads every file record for the -// report and trees subcommands. Any database problem — including a -// missing database — is fatal. The error is returned rather than -// exiting, so that the deferred close — which checkpoints the SQLite -// WAL — always runs; the database is closed before the caller formats -// its output, so it stays closed even if that output fails. -func loadRecords(ctx context.Context) ([]scanRec, error) { +// runReport implements the report subcommand: it prints the file-level +// duplicates report as TSV on stdout. SQLite groups and orders the +// records, and each row is written as it is read, so no group is held +// in memory. It never touches the scanned filesystem; its only I/O is +// the database (with SQLite's temporary sort file), stdout, and stderr. +// Any database problem, including a missing database, is fatal. +func runReport(ctx context.Context, stdout io.Writer) error { dbPath := databasePath() db, err := openReportDatabase(ctx, dbPath) if err != nil { - return nil, err + return err } defer func() { _ = db.Close() }() - recs, err := loadFileRows(ctx, db) - if err != nil { - return nil, fmt.Errorf("database %s: %w", dbPath, err) - } - - return recs, nil -} - -// dupeGroup is one set of candidate-duplicate files: identical size, -// head hash, tail hash, and content hash. paths is sorted -// lexicographically; the first entry is the group's "first", the rest -// are dupes. -type dupeGroup struct { - size int64 - paths []string -} - -// runReport implements the report subcommand: it reads every record -// from the database and prints the file-level duplicates report as TSV -// on stdout. It never touches the scanned filesystem; its only I/O is -// the database, stdout, and stderr. -func runReport(ctx context.Context) error { - recs, err := loadRecords(ctx) - if err != nil { - return err - } - - dupes := collectDupeGroups(recs) - - out := bufio.NewWriterSize(os.Stdout, ioBufSize) + out := bufio.NewWriterSize(stdout, ioBufSize) _, err = fmt.Fprintln(out, "first\tdupe\tsize") if err != nil { return fmt.Errorf("write stdout: %w", err) } - dupeFiles := 0 + var ( + groups, dupeFiles int + reclaimable int64 + writeErr error + ) - var reclaimable int64 + records, err := loadDupeRows(ctx, db, + func(first, path string, size int64) error { + // A group's first path is its first row; every other + // path is a dupe. + if path == first { + groups++ - for _, g := range dupes { - for _, p := range g.paths[1:] { - _, err = fmt.Fprintf(out, "%s\t%s\t%d\n", - g.paths[0], p, g.size) - if err != nil { - return fmt.Errorf("write stdout: %w", err) + return nil } + _, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n", + escapePath(first), escapePath(path), size) dupeFiles++ - reclaimable += g.size - } + reclaimable += size + + return writeErr + }) + + if writeErr != nil { + return fmt.Errorf("write stdout: %w", writeErr) + } + + if err != nil { + return fmt.Errorf("database %s: %w", dbPath, err) } err = out.Flush() @@ -105,55 +92,24 @@ func runReport(ctx context.Context) error { fmt.Fprintf(os.Stderr, "report: %d records read, %d duplicate groups, %d dupe files, "+ "%s reclaimable\n", - len(recs), len(dupes), dupeFiles, humanBytes(reclaimable)) + records, groups, dupeFiles, humanBytes(reclaimable)) return nil } -// collectDupeGroups groups records by signature and returns every group -// with two or more paths, each group's paths sorted lexicographically, -// groups ordered by size descending then by first path ascending. -func collectDupeGroups(recs []scanRec) []dupeGroup { - groups := make(map[fileSig][]string) - - for _, r := range recs { - // A record without a content hash has unknown content and is - // never reported as a duplicate (README "Database"). - if r.content == "" { - continue - } - - k := fileSig{ - size: r.size, head: r.head, tail: r.tail, content: r.content, - } - groups[k] = append(groups[k], r.path) +// escapePath returns a path as it is written in a report column (README +// "Report output format"): a backslash, tab, newline or carriage return +// becomes \\, \t, \n or \r, and every other byte is kept as it is. +// Grouping and sorting use the raw path, never this form. +func escapePath(p string) string { + // Most paths need no escaping; skip building a replacer for them. + if !strings.ContainsAny(p, "\\\t\n\r") { + return p } - var dupes []dupeGroup - - for k, paths := range groups { - if len(paths) < minGroupSize { - continue - } - - slices.Sort(paths) - dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) - } - - // Biggest reclaimable space first; ties broken by first path. - slices.SortFunc(dupes, func(a, b dupeGroup) int { - if a.size != b.size { - if a.size > b.size { - return -1 - } - - return 1 - } - - return strings.Compare(a.paths[0], b.paths[0]) - }) - - return dupes + return strings.NewReplacer( + `\`, `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`, + ).Replace(p) } // humanBytes formats a byte count in human units (binary prefixes). diff --git a/report_test.go b/report_test.go index 1aa7cc5..82f8b03 100644 --- a/report_test.go +++ b/report_test.go @@ -1,11 +1,255 @@ package main import ( + "bytes" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" "slices" + "strings" "testing" + "time" ) -func TestCollectDupeGroups(t *testing.T) { +// awkwardDir is a directory name holding every byte the reports escape. +const awkwardDir = "/d/\tone\ntwo\rthree\\four" + +// awkwardPairRecs is a duplicate pair in sibling directories /d/A and +// awkwardDir. A raw tab sorts before "A" but its escaped form `\t` +// sorts after it, so awkwardDir coming first shows that sorting uses +// the raw path. +func awkwardPairRecs() []scanRec { + return []scanRec{ + {size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"}, + {size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"}, + } +} + +// seedDatabase writes recs into a fresh database and returns its path. +func seedDatabase(t *testing.T, recs []scanRec) string { + t.Helper() + + path := testDBPath(t) + + db, err := openScanDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + err = applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + err = db.Close() + if err != nil { + t.Fatal(err) + } + + return path +} + +// dupeGroup is one duplicate group as report reads it: the size, and +// the paths in report order, first path first. +type dupeGroup struct { + size int64 + paths []string +} + +// dupeGroups returns the duplicate groups report reads from db, in +// report order. +func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup { + t.Helper() + + var groups []dupeGroup + + _, err := loadDupeRows(t.Context(), db, + func(first, path string, size int64) error { + if path == first { + groups = append(groups, dupeGroup{size: size}) + } + + g := &groups[len(groups)-1] + g.paths = append(g.paths, path) + + return nil + }) + if err != nil { + t.Fatal(err) + } + + return groups +} + +// dupeGroupsOf writes recs into a fresh database and returns the +// duplicate groups report reads from it. +func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup { + t.Helper() + + db := openTestDB(t) + + err := applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + return dupeGroups(t, db) +} + +func TestRunReportEscapesPaths(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) + + var stdout, stderr bytes.Buffer + + code := run([]string{cmdReport}, &stdout, &stderr) + if code != exitOK { + t.Fatalf("run(report) = %d, want %d; stderr: %s", + code, exitOK, stderr.String()) + } + + want := "first\tdupe\tsize\n" + + `/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n" + if got := stdout.String(); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + +func TestReportStdoutFailsWhileReading(t *testing.T) { + // Each row holds two paths longer than dir, so the report is more + // than twice the stdout buffer and stdout fails while rows are + // still being read, not at the final flush. + dir := "/" + strings.Repeat("d", 4096) + + recs := make([]scanRec, ioBufSize/len(dir)) + for i := range recs { + recs[i] = scanRec{ + size: 1, head: "h", tail: "t", content: "c", + path: fmt.Sprintf("%s/%d", dir, i), + } + } + + t.Setenv(databaseEnv, seedDatabase(t, recs)) + + err := runReport(t.Context(), failingWriter{}) + if !errors.Is(err, errWriteFailed) || + !strings.HasPrefix(err.Error(), "write stdout: ") { + t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed) + } +} + +func TestRunReportsIgnoreInsertionOrder(t *testing.T) { + // README §Constraints: identical database contents give identical + // output, whatever order the records were inserted in. + recs := append(smokeTreeRecs(), awkwardPairRecs()...) + recs = append(recs, + scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"}, + scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"}, + scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"}, + scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"}, + scanRec{size: 50, path: "/x/unhashed"}, + ) + + reversed := slices.Clone(recs) + slices.Reverse(reversed) + + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, recs)) + + forward := runStdout(t, name) + + t.Setenv(databaseEnv, seedDatabase(t, reversed)) + + backward := runStdout(t, name) + + if strings.Count(forward, "\n") < 3 { + t.Errorf("stdout = %q, want at least two rows", forward) + } + + if forward != backward { + t.Errorf("stdout depends on insertion order: %q vs %q", + forward, backward) + } + }) + } +} + +// runStdout runs the subcommand name and returns its stdout, failing +// the test unless it succeeds. +func runStdout(t *testing.T, name string) string { + t.Helper() + + var stdout, stderr bytes.Buffer + + code := run([]string{name}, &stdout, &stderr) + if code != exitOK { + t.Fatalf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + } + + return stdout.String() +} + +func TestEscapePath(t *testing.T) { + t.Parallel() + + cases := map[string]string{ + "/srv/plain": "/srv/plain", + "/a\tb": `/a\tb`, + "/a\nb": `/a\nb`, + "/a\rb": `/a\rb`, + `/a\b`: `/a\\b`, + `/a\tb`: `/a\\tb`, + "/not-utf8\xff": "/not-utf8\xff", + } + for in, want := range cases { + if got := escapePath(in); got != want { + t.Errorf("escapePath(%q) = %q, want %q", in, got, want) + } + } +} + +// TestWarnfEscapes checks that a warning naming a path that holds a +// newline is still one line. +// +//nolint:paralleltest // replaces the process-wide os.Stderr +func TestWarnfEscapes(t *testing.T) { + f, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + if err != nil { + t.Fatal(err) + } + + saved := os.Stderr + os.Stderr = f + + t.Cleanup(func() { + os.Stderr = saved + + _ = f.Close() + }) + + (&progress{}).warnf("stat %s: %s", "/d/a\nb", "gone") + + _, err = f.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + want := `stat /d/a\nb: gone` + "\n" + if string(got) != want { + t.Errorf("warning = %q, want %q", got, want) + } +} + +func TestDupeGroups(t *testing.T) { t.Parallel() recs := []scanRec{ @@ -20,7 +264,7 @@ func TestCollectDupeGroups(t *testing.T) { {size: 7, head: "u", tail: "u", content: "u", path: "/lonely"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } @@ -38,7 +282,7 @@ func TestCollectDupeGroups(t *testing.T) { } } -func TestCollectDupeGroupsContentSeparates(t *testing.T) { +func TestDupeGroupsContentSeparates(t *testing.T) { t.Parallel() // Same size, head, and tail, but different content hashes: the final @@ -53,7 +297,7 @@ func TestCollectDupeGroupsContentSeparates(t *testing.T) { {size: 100, head: "h", tail: "t", path: "/e"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1 (only the matching content)", len(groups)) @@ -64,33 +308,41 @@ func TestCollectDupeGroupsContentSeparates(t *testing.T) { } } -func TestCollectDupeGroupsMtimeExcluded(t *testing.T) { +func TestDupeGroupsMtimeExcluded(t *testing.T) { t.Parallel() // mtime is informational only; records differing only in mtime // still group together. recs := []scanRec{ - {size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"}, - {size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"}, + { + size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", + content: "c", path: "/m/1", + }, + { + size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t", + content: "c", path: "/m/2", + }, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1", len(groups)) } } -func TestCollectDupeGroupsTieBreak(t *testing.T) { +func TestDupeGroupsTieBreak(t *testing.T) { t.Parallel() + // The hashes sort opposite to the first paths, so ordering the + // groups by hash instead of by first path fails this test. recs := []scanRec{ - {size: 50, head: "b", tail: "b", content: "b", path: "/beta/2"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/beta/1"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/alpha/2"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/alpha/1"}, + {size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"}, + {size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"}, + {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"}, + {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } @@ -102,7 +354,7 @@ func TestCollectDupeGroupsTieBreak(t *testing.T) { } } -func TestCollectDupeGroupsDeterministic(t *testing.T) { +func TestDupeGroupsDeterministic(t *testing.T) { t.Parallel() recs := []scanRec{ @@ -112,12 +364,12 @@ func TestCollectDupeGroupsDeterministic(t *testing.T) { {size: 2, head: "b", tail: "b", content: "b", path: "/q/2"}, } - forward := collectDupeGroups(recs) + forward := dupeGroupsOf(t, recs) reversed := slices.Clone(recs) slices.Reverse(reversed) - backward := collectDupeGroups(reversed) + backward := dupeGroupsOf(t, reversed) if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool { return a.size == b.size && slices.Equal(a.paths, b.paths) }) { diff --git a/scan.go b/scan.go index 6d7aaf4..0f70378 100644 --- a/scan.go +++ b/scan.go @@ -11,11 +11,13 @@ import ( "io" "io/fs" "os" + "os/signal" "path/filepath" "slices" "strings" "sync" "syscall" + "time" ) // The duplicate ladder (see hashSignature and README "Duplicate @@ -57,13 +59,17 @@ const sampleWindow = 1024 * 1024 // and hash worker pools. const workQueueDepth = 1024 +// errInterrupted reports a scan stopped by SIGINT or SIGTERM. runScan +// has already printed its line, so run prints nothing more. +var errInterrupted = errors.New("scan interrupted") + // fileRec carries one statted file between the scan phases. dev and // ino identify the underlying inode so hard-linked paths can share // one read; both are zero when the platform exposes no inode. type fileRec struct { path string size int64 - mtime int64 + mtime time.Time dev uint64 ino uint64 } @@ -74,7 +80,7 @@ type fileRec struct { // they would dominate the scan's memory. type fileMeta struct { size int64 - mtime int64 + mtime time.Time hashed bool } @@ -85,17 +91,19 @@ type fileMeta struct { // least one other file shares are ever hashed: a size-unique file // cannot be a duplicate. A file of headTailMin or more gets its content // hash only when its size, head, and tail match another file's. Flag -// parsing and the at-least-one-operand check are done by cobra. Errors -// are returned rather than exiting, so that the deferred close — which -// checkpoints the SQLite WAL — always runs. Cancelling ctx unwinds the -// worker pools and aborts the scan with the context's error. +// parsing and the at-least-one-operand check are done by cobra. The +// scan holds the lock on the database for its whole run, so a second +// scan fails before it walks the filesystem or opens the database. +// Errors are returned rather than exiting, so that the deferred close — +// which takes the database out of WAL mode — always runs, and the lock +// is released after it. When ctx is cancelled, as by the SIGINT or +// SIGTERM that interruptContext catches, the scan keeps what it has +// hashed (see syncScan), prints how many files its walk reached, and +// returns errInterrupted. workers must be at least 1; the scan command +// rejects anything less. func runScan(ctx context.Context, roots []string, workers int, oneFS bool, ) error { - if workers < 1 { - workers = 1 - } - roots, err := resolveRoots(roots) if err != nil { return err @@ -103,14 +111,31 @@ func runScan(ctx context.Context, roots []string, workers int, dbPath := databasePath() - db, err := openScanDatabase(ctx, dbPath) + lock, err := lockScanDatabase(dbPath) if err != nil { return err } - defer func() { _ = db.Close() }() + defer func() { _ = lock.Close() }() + + db, err := openScanDatabase(ctx, dbPath) + if err != nil && ctx.Err() != nil { + // Interrupted while opening; SQLite may report that with an + // error of its own rather than the context's. + return interrupted(0) + } + + if err != nil { + return err + } + + defer closeScanDatabase(ctx, db, dbPath) st, err := syncScan(ctx, db, roots, workers, oneFS) + if errors.Is(err, context.Canceled) { + return interrupted(st.walked) + } + if err != nil { return fmt.Errorf("update database %s: %w", dbPath, err) } @@ -124,6 +149,34 @@ func runScan(ctx context.Context, roots []string, workers int, return nil } +// interruptContext returns a copy of ctx that the first SIGINT or +// SIGTERM cancels; the scan command runs the scan under it. stop +// releases the signals. +func interruptContext(ctx context.Context) (context.Context, func()) { + // A SIGINT ignored from the start, as by a script's background job, + // stays ignored. + signals := []os.Signal{syscall.SIGTERM} + if !signal.Ignored(syscall.SIGINT) { + signals = append(signals, syscall.SIGINT) + } + + ctx, stop := signal.NotifyContext(ctx, signals...) + + // Stopping restores the default handling, so a second signal ends + // the process at once. + context.AfterFunc(ctx, stop) + + return ctx, stop +} + +// interrupted prints the line for a scan stopped by a signal after its +// walk reached walked files, and returns errInterrupted. +func interrupted(walked int) error { + fmt.Fprintf(os.Stderr, "scan: interrupted after %d files\n", walked) + + return errInterrupted +} + // resolveRoots converts each PATH operand to an absolute, lexically // cleaned path (symlinks are not resolved) and verifies that it // exists. Database records are keyed by absolute path, so scan results @@ -150,10 +203,16 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. Every remaining -// file is then reachable through exactly one root, so walked paths are -// unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string) []string { +// duplicates and any operand lying under another one. With -x, an +// operand under another one is kept when the walk of the deepest kept +// operand above it cannot reach it: when device puts any directory from +// it (from its directory, for a regular file) up to but not including +// that operand on a different filesystem than that operand. Every +// remaining file is then reachable through exactly one root, so walked +// paths are unique without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, + device func(path string) (uint64, bool), +) []string { // Shorter-first ordering guarantees an ancestor is kept before any // operand under it is considered. sorted := slices.Clone(roots) @@ -168,17 +227,62 @@ func pruneRoots(roots []string) []string { kept := make([]string, 0, len(sorted)) for _, r := range sorted { - if !underAnyRoot(r, kept) { + above := deepestRoot(r, kept) + if above == "" { kept = append(kept, r) + + continue + } + + // Without -x, and always for a duplicate, r is dropped. + if !oneFS || above == r { + continue + } + + aboveDev, aboveOK := device(above) + if !aboveOK { + continue + } + + // The walk of above enters a directory only when it is on + // above's filesystem, and then reaches every regular file in it + // whatever device the file reports, so for a regular-file + // operand the directories checked start at its directory. + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + dev, ok := device(dir) + if ok && dev != aboveDev { + kept = append(kept, r) + + break + } } } return kept } +// deviceOfPath returns the filesystem device ID of the file at path, +// when lstat succeeds and the platform exposes one. +func deviceOfPath(path string) (uint64, bool) { + fi, err := os.Lstat(path) + if err != nil { + return 0, false + } + + return deviceOfInfo(fi) +} + // scanStats summarizes one scan's database synchronization for the -// final stderr summary. +// final stderr summary, or for the line an interrupted scan prints. type scanStats struct { + walked int // files the walk reached added int updated int removed int @@ -200,7 +304,30 @@ type scanState struct { st scanStats } -// syncScan synchronizes the database with the filesystem under roots +// syncScan synchronizes the database with the filesystem under roots; +// see runPhases. When ctx is cancelled, as by an interrupt, it commits +// the hashed records still waiting in the batch, starts no other write +// or deletion, and returns the cancellation. +func syncScan(ctx context.Context, db *sql.DB, roots []string, + workers int, oneFS bool, +) (scanStats, error) { + s := &scanState{db: db} + + err := s.runPhases(ctx, roots, workers, oneFS) + if err == nil || ctx.Err() == nil { + return s.st, err + } + + // The one write made after the cancellation, so it cannot use ctx. + err = applyChanges(context.WithoutCancel(ctx), db, s.batch, nil, nil) + if err != nil { + return s.st, err + } + + return s.st, ctx.Err() +} + +// runPhases synchronizes the database with the filesystem under roots // in four sequential phases: walk (enumerate and stat every file, // building a complete size census), hash (read only the new or // changed — or previously unhashed — files whose size at least one @@ -210,47 +337,74 @@ type scanState struct { // in the content hash of every record of headTailMin or more whose // size, head, and tail match another record's). Records outside the // roots are never touched, except that the content phase fills in -// their content hash. -func syncScan(ctx context.Context, db *sql.DB, roots []string, +// their content hash. Operands the walk cannot start from are dropped +// first, so the records beneath them are treated as if those operands +// had not been given. +func (s *scanState) runPhases(ctx context.Context, roots []string, workers int, oneFS bool, -) (scanStats, error) { - roots = pruneRoots(roots) - - s := &scanState{db: db} +) error { + // Types are checked before pruning so that an operand under a + // dropped one is still scanned, not dropped as lying under it. + roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { - return s.st, err + return err } - changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers)) + changed, unhashed := s.walkPhase( + startWalk(ctx, roots, oneFS, workers), roots) // A cancelled walk stops early, so its size census covers only part - // of the roots, and every file it never reached looks vanished to - // the update phase. Defence in depth rather than the only barrier: - // that phase would today fail on its first BeginTx with the same - // cancelled context before deleting anything. But it is the barrier - // that survives a later decision to let an interrupted scan commit - // what it has, and it turns a confusing failure deep in the update - // phase into a clean abort at the phase boundary. + // of the roots, and every file it never reached would look vanished + // to the update phase. Stop before anything is written or deleted. err = ctx.Err() if err != nil { - return s.st, err + return err } s.partition(changed, unhashed) err = s.hashPhase(ctx, workers) if err != nil { - return s.st, err + return err } err = s.updatePhase(ctx) if err != nil { - return s.st, err + return err } - return s.st, s.contentPhase(ctx, workers) + return s.contentPhase(ctx, workers) +} + +// walkableRoots returns the operands the walk can start from: regular +// files, and directories not named .zfs. Every other operand is warned +// about, counted as skipped, and dropped. A dropped operand is no +// longer a root, so the records stored beneath it are treated as if it +// had not been given: those under no other root count as outside the +// roots and are not deleted as unverified. An operand that fails lstat +// here is kept, and the walk warns about it. +func (s *scanState) walkableRoots(roots []string) []string { + kept := make([]string, 0, len(roots)) + + for _, root := range roots { + fi, err := os.Lstat(root) + if err == nil { + warn := operandWarning(root, fi) + if warn != "" { + s.st.skipped++ + + fmt.Fprintln(os.Stderr, escapePath(warn)) + + continue + } + } + + kept = append(kept, root) + } + + return kept } // loadIndex indexes the database records under the scan roots for @@ -267,7 +421,7 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { s.existing = make(map[string]fileMeta) return loadFileMeta(ctx, s.db, - func(path string, size, mtime int64, hashed bool) { + func(path string, size int64, mtime time.Time, hashed bool) { prog.increment() if underAnyRoot(path, roots) { @@ -288,15 +442,27 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. +// their sizes are shared. A record below a directory -x did not enter +// counts as outside the roots: it is not deleted, and its size joins +// the census. That does not hold below a root at or inside that +// directory, since that root's own walk verifies the records below it. func (s *scanState) walkPhase( - events <-chan walkEvent, + events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { prog := newProgress("walk", -1) - var changed, unhashed []fileRec + var ( + changed, unhashed []fileRec + otherFSDirs []string + ) for ev := range events { + if ev.otherFSDir != "" { + otherFSDirs = append(otherFSDirs, ev.otherFSDir) + + continue + } + if ev.fail { s.st.skipped++ @@ -306,13 +472,14 @@ func (s *scanState) walkPhase( } s.sizes = append(s.sizes, ev.rec.size) + s.st.walked++ prog.increment() old, ok := s.existing[ev.rec.path] switch { - case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime: + case !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime): changed = append(changed, ev.rec) case old.hashed: delete(s.existing, ev.rec.path) @@ -325,6 +492,19 @@ func (s *scanState) walkPhase( prog.finish() + for path, old := range s.existing { + // Kept only when a directory -x did not enter lies below the + // deepest root above the record. A root at or below that + // directory was walked on its own and did not find the file. + // Both contain the record, so the longer path is the deeper. + otherFSDir := deepestRoot(path, otherFSDirs) + if len(otherFSDir) > len(deepestRoot(path, roots)) { + delete(s.existing, path) + + s.sizes = append(s.sizes, old.size) + } + } + return changed, unhashed } @@ -509,16 +689,22 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error { } // commitFullBatch commits the running batch once it holds -// updateBatchSize records. +// updateBatchSize records. A batch that fails to commit is kept: the +// commit fails when the scan is interrupted, and syncScan then commits +// the batch itself. func (s *scanState) commitFullBatch(ctx context.Context) error { if len(s.batch) < updateBatchSize { return nil } err := applyBatch(ctx, s.db, s.batch, nil, nil) + if err != nil { + return err + } + s.batch = s.batch[:0] - return err + return nil } // updatePhase writes the scan's tail under one progress display: the @@ -698,7 +884,7 @@ func unchangedFile(r scanRec) (fileRec, bool, error) { } if !fi.Mode().IsRegular() || fi.Size() != r.size || - fi.ModTime().Unix() > r.mtime { + mtimeAfter(fi.ModTime(), r.mtime) { return fileRec{}, false, nil } @@ -709,6 +895,16 @@ func unchangedFile(r scanRec) (fileRec, bool, error) { }, true, nil } +// mtimeAfter reports whether mtime a is later than mtime b. +// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it. +func mtimeAfter(a, b time.Time) bool { + if a.Unix() != b.Unix() { + return a.Unix() > b.Unix() + } + + return a.Nanosecond() > b.Nanosecond() +} + // underAnyRoot reports whether path is any of the roots or lies under // one of them. func underAnyRoot(path string, roots []string) bool { @@ -736,6 +932,20 @@ func underRoot(path, root string) bool { return strings.HasPrefix(path, prefix) } +// deepestRoot returns the longest of the roots that path is or lies +// under, or "" when it lies under none. +func deepestRoot(path string, roots []string) string { + deepest := "" + + for _, root := range roots { + if underRoot(path, root) && len(root) > len(deepest) { + deepest = root + } + } + + return deepest +} + // dirJob is one directory awaiting traversal by the walk workers. It // carries its operand's filesystem device so -x can stop at // filesystem boundaries. @@ -746,19 +956,23 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, or a warning when fail is set. +// file's statted record, a warning when fail is set, or, when +// otherFSDir is set, a directory -x did not enter because it is on +// another filesystem than its operand. type walkEvent struct { - rec fileRec - warn string - fail bool + rec fileRec + warn string + fail bool + otherFSDir string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error. The channel is closed when the walk -// completes, and also when ctx is cancelled — every goroutine in the -// pool abandons its blocking send in that case, so the consumer sees a -// truncated but properly terminated stream instead of a stalled one. +// event per per-path error, and one event per directory -x did not +// enter. The channel is closed when the walk completes, and also when +// ctx is cancelled — every goroutine in the pool abandons its blocking +// send in that case, so the consumer sees a truncated but properly +// terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, workers int, ) <-chan walkEvent { @@ -789,11 +1003,44 @@ func sendEvent(ctx context.Context, events chan<- walkEvent, } } +// operandWarning returns the one-line warning for an operand the walk +// does not start from, naming the path and what it is, or "" for one it +// does: a regular file, or a directory not named .zfs. Symlinks are +// never followed, including as operands. +func operandWarning(root string, fi fs.FileInfo) string { + var kind string + + switch mode := fi.Mode(); { + case mode.IsRegular(): + return "" + case mode.IsDir(): + if filepath.Base(root) != ".zfs" { + return "" + } + + kind = ".zfs directory" + case mode&fs.ModeSymlink != 0: + kind = "symlink" + case mode&fs.ModeSocket != 0: + kind = "socket" + case mode&fs.ModeNamedPipe != 0: + kind = "FIFO" + case mode&fs.ModeDevice != 0: + kind = "device node" + default: + kind = "non-regular file" + } + + return fmt.Sprintf("walk %s: skipping %s operand", root, kind) +} + // seedRoot turns one PATH operand into the walk's starting state: a -// regular-file operand is statted and emitted directly, a directory -// operand becomes an initial job, and a symlink or other non-regular -// operand yields nothing (symlinks are never followed, including as -// operands). +// regular-file operand is statted and emitted directly, and a directory +// operand becomes an initial job. walkableRoots has already dropped +// every other operand. One that has changed into something else since +// is warned about and skipped here; it is still a root, so the records +// stored beneath it, other than those under another root, are deleted +// as unverified. func seedRoot(ctx context.Context, root string, events chan<- walkEvent, ) []dirJob { @@ -807,30 +1054,30 @@ func seedRoot(ctx context.Context, root string, return nil } - switch { - case fi.IsDir(): - if filepath.Base(root) == ".zfs" { - return nil - } + warn := operandWarning(root, fi) + if warn != "" { + sendEvent(ctx, events, walkEvent{warn: warn, fail: true}) + return nil + } + + if fi.IsDir() { dev, ok := deviceOfInfo(fi) return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}} - case fi.Mode().IsRegular(): - dev, ino := inodeOfInfo(fi) - - sendEvent(ctx, events, walkEvent{rec: fileRec{ - path: root, - size: fi.Size(), - mtime: fi.ModTime().Unix(), - dev: dev, - ino: ino, - }}) - - return nil - default: - return nil } + + dev, ino := inodeOfInfo(fi) + + sendEvent(ctx, events, walkEvent{rec: fileRec{ + path: root, + size: fi.Size(), + mtime: fi.ModTime(), + dev: dev, + ino: ino, + }}) + + return nil } // startWalkWorkers starts the walk worker pool. Each worker processes @@ -929,6 +1176,12 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, var subs []dirJob for _, e := range entries { + // A cancelled scan wants nothing more from this directory: stop + // rather than lstat the rest of a large one. + if ctx.Err() != nil { + return nil + } + p := filepath.Join(job.path, e.Name()) if e.IsDir() { @@ -977,7 +1230,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, sendEvent(ctx, events, walkEvent{rec: fileRec{ path: p, size: info.Size(), - mtime: info.ModTime().Unix(), + mtime: info.ModTime(), dev: dev, ino: ino, }}) @@ -986,7 +1239,9 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per // snapshot), and with -x never enter a directory on a different -// filesystem than its operand. +// filesystem than its operand, sending an event that names it so the +// scan keeps the records below it, except those below an operand at or +// inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, events chan<- walkEvent, ) (dirJob, bool) { @@ -1010,6 +1265,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, } if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { + sendEvent(ctx, events, walkEvent{otherFSDir: p}) + return dirJob{}, false } @@ -1151,6 +1408,38 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec, } } +// errNotRegular reports a path that is no longer a regular file when it +// is opened to be read. +var errNotRegular = errors.New("not a regular file") + +// openRegularFile opens path for reading and returns it only if it is +// still a regular file, since the path may have been replaced after it +// was statted. O_NONBLOCK keeps the open of a FIFO or a device node +// from waiting forever, and fstat then rejects it. Reads of a regular +// file ignore O_NONBLOCK. +func openRegularFile(path string) (*os.File, error) { + //nolint:gosec // hashing operator-supplied paths is the tool's purpose + f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + + fi, err := f.Stat() + if err != nil { + _ = f.Close() + + return nil, err + } + + if !fi.Mode().IsRegular() { + _ = f.Close() + + return nil, errNotRegular + } + + return f, nil +} + // emptyHash is the lowercase-hex SHA-256 of the empty input: the head, // tail, and content hash of every zero-length file. const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + @@ -1173,8 +1462,7 @@ func hashSignature(path string, size int64) (string, string, string, error) { return emptyHash, emptyHash, emptyHash, nil } - //nolint:gosec // hashing operator-supplied paths is the tool's purpose - f, err := os.Open(path) + f, err := openRegularFile(path) if err != nil { return "", "", "", err } @@ -1205,8 +1493,7 @@ func hashSignature(path string, size int64) (string, string, string, error) { // tail are returned empty, because the content phase keeps the ones its // records already hold. func hashContentOnly(path string, size int64) (string, string, string, error) { - //nolint:gosec // hashing operator-supplied paths is the tool's purpose - f, err := os.Open(path) + f, err := openRegularFile(path) if err != nil { return "", "", "", err } diff --git a/scan_test.go b/scan_test.go index f47005d..778fe64 100644 --- a/scan_test.go +++ b/scan_test.go @@ -6,15 +6,22 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" + "io" + "io/fs" "os" + "os/signal" "path/filepath" "runtime" "slices" "strconv" "strings" + "syscall" "testing" "time" + + "golang.org/x/sys/unix" ) // writeFile creates a file with the given content and returns its path. @@ -399,7 +406,7 @@ func TestScanContentGate(t *testing.T) { } } - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, same) { t.Fatalf("groups = %+v, want only the identical pair %q", groups, same) @@ -434,7 +441,7 @@ func TestScanContentAcrossOperands(t *testing.T) { want := []string{a, b} slices.Sort(want) - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q", groups, want) } @@ -455,13 +462,13 @@ func TestScanContentWithinOperand(t *testing.T) { added := sparseFile(t, dir, "d2", headTailMin) st := syncTree(t, db, dir) - if st != (scanStats{added: 1, unchanged: 2}) { + if st != (scanStats{walked: 3, added: 1, unchanged: 2}) { t.Fatalf("rescan stats = %+v, want 1 added 2 unchanged", st) } want := []string{stored, added} - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q", groups, want) } @@ -501,7 +508,7 @@ func TestScanContentStalePartners(t *testing.T) { sparseFile(t, dirB, "changed-copy", headTailMin+1) st := syncTree(t, db, dirB) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Errorf("stats = %+v, want 2 added and nothing skipped", st) } @@ -519,7 +526,7 @@ func TestScanContentStalePartners(t *testing.T) { } } - if groups := collectDupeGroups(recs); len(groups) != 0 { + if groups := dupeGroups(t, db); len(groups) != 0 { t.Errorf("groups = %+v, want none", groups) } } @@ -558,7 +565,7 @@ func TestScanContentHashedStalePartners(t *testing.T) { b := sparseFile(t, t.TempDir(), "copy", headTailMin) st := syncTree(t, db, filepath.Dir(b)) - if st != (scanStats{added: 1}) { + if st != (scanStats{walked: 1, added: 1}) { t.Errorf("stats = %+v, want 1 added and nothing skipped", st) } @@ -570,12 +577,71 @@ func TestScanContentHashedStalePartners(t *testing.T) { // The stored records lie outside the operand and are left as they // are, so they still group with each other, but not with the copy. - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, stored) { t.Errorf("groups = %+v, want only the stored pair %q", groups, stored) } } +// TestScanContentSameSecondRewrite is TestScanContentStalePartners for +// a stored file rewritten in place at the same size with an mtime later +// in the same second than recorded: the file counts as changed, so +// neither it nor its match inside the operand is read. +func TestScanContentSameSecondRewrite(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + dirA := t.TempDir() + changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(changed, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, dirA) + + before := dbRecords(t, db) + + // Rewrite one byte in place, keeping the size. + pokeAt(t, changed, headTailMin/2, []byte{1}) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(changed, later, later) + if err != nil { + t.Fatal(err) + } + + dirB := t.TempDir() + sparseFile(t, dirB, "changed-copy", headTailMin) + + st := syncTree(t, db, dirB) + if st != (scanStats{walked: 1, added: 1}) { + t.Errorf("stats = %+v, want 1 added and nothing skipped", st) + } + + recs := dbRecords(t, db) + for _, r := range recs { + if r.content != "" { + t.Errorf("%s: content = %q, want none: its only match is stale", + r.path, r.content) + } + } + + for _, old := range before { + if r := recordByPath(t, recs, old.path); r != old { + t.Errorf("record = %+v, want it left as %+v", r, old) + } + } + + if groups := dupeGroups(t, db); len(groups) != 0 { + t.Errorf("groups = %+v, want none", groups) + } +} + // TestScanContentReadFailure checks that a failed content read is // counted as skipped and leaves the record without a content hash, and // that a later scan tries the read again. @@ -598,7 +664,7 @@ func TestScanContentReadFailure(t *testing.T) { b := sparseFile(t, dirB, "b", headTailMin) st := syncTree(t, db, dirB) - if st != (scanStats{added: 1, skipped: 1}) { + if st != (scanStats{walked: 1, added: 1, skipped: 1}) { t.Fatalf("stats = %+v, want 1 added 1 skipped", st) } @@ -612,14 +678,14 @@ func TestScanContentReadFailure(t *testing.T) { } st = syncTree(t, db, dirB) - if st != (scanStats{unchanged: 1}) { + if st != (scanStats{walked: 1, unchanged: 1}) { t.Fatalf("rescan stats = %+v, want 1 unchanged", st) } want := []string{a, b} slices.Sort(want) - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q after the retry", groups, want) @@ -658,7 +724,7 @@ func TestScanContentCheckError(t *testing.T) { b := sparseFile(t, t.TempDir(), "b", headTailMin) st := syncTree(t, db, filepath.Dir(b)) - if st != (scanStats{added: 1, skipped: 1}) { + if st != (scanStats{walked: 1, added: 1, skipped: 1}) { t.Fatalf("stats = %+v, want 1 added 1 skipped", st) } @@ -686,7 +752,7 @@ func TestScanContentHardlinks(t *testing.T) { c := sparseFile(t, dir, "copy", headTailMin) st := syncTree(t, db, dir) - if st != (scanStats{added: 3}) { + if st != (scanStats{walked: 3, added: 3}) { t.Fatalf("stats = %+v, want 3 added", st) } @@ -778,8 +844,8 @@ func TestWalk(t *testing.T) { t.Errorf("%s: size = %d, want 1..3", r.path, r.size) } - if r.mtime <= 0 { - t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime) + if r.mtime.Unix() <= 0 { + t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime) } } } @@ -856,9 +922,11 @@ func TestWalkFileAndSymlinkOperands(t *testing.T) { t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs) } - // A symlink operand is not followed and yields nothing. + // A symlink operand that reaches the walk (it became one after + // walkableRoots checked it) is not followed: it yields a warning and + // no records. recs, errs = collectWalk(t, []string{link}, false, 2) - if errs != 0 || len(recs) != 0 { + if errs != 1 || len(recs) != 0 { t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs) } } @@ -907,6 +975,456 @@ func TestDeviceOfInfo(t *testing.T) { } } +// dirEntryFor returns the fs.DirEntry for name within dir, obtained via +// the same os.ReadDir the walk uses, so it carries a real Info(). +func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry { + t.Helper() + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + for _, e := range entries { + if e.Name() == name { + return e + } + } + + t.Fatalf("entry %q not found in %q", name, dir) + + return nil +} + +// callSubdirJob runs subdirJob against e under parent, collecting any +// events it emits (subdirJob emits at most one). +func callSubdirJob(t *testing.T, p string, e fs.DirEntry, + parent dirJob, oneFS bool, +) (dirJob, bool, []walkEvent) { + t.Helper() + + events := make(chan walkEvent, 1) + job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events) + close(events) + + var evs []walkEvent + for ev := range events { + evs = append(evs, ev) + } + + return job, ok, evs +} + +// TestSubdirJobOneFilesystem exercises the -x boundary check in +// subdirJob directly, so no second real filesystem is needed. The +// subdirectory's real device is compared against a fabricated operand +// device. +func TestSubdirJobOneFilesystem(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + sub := filepath.Join(dir, "sub") + + err := os.Mkdir(sub, 0o750) + if err != nil { + t.Fatal(err) + } + + info, err := os.Lstat(sub) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + // A device the subdirectory is not on, standing in for an operand + // rooted on a different filesystem. + otherDev := dev + 1 + e := dirEntryFor(t, dir, "sub") + + cases := []struct { + name string + oneFS bool + parent dirJob + wantOK bool + }{ + // -x on, subdirectory on a different device than its operand: + // descent is refused. + {"reject across boundary", true, + dirJob{rootDev: otherDev, rootDevOK: true}, false}, + // -x on but the operand's own device is unknown: the boundary + // check is bypassed and descent proceeds. + {"bypass when root device unknown", true, + dirJob{rootDev: otherDev, rootDevOK: false}, true}, + // Default (no -x): boundaries are crossed even onto a different + // device. + {"cross by default", false, + dirJob{rootDev: otherDev, rootDevOK: true}, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + job, ok, evs := callSubdirJob(t, sub, e, tc.parent, tc.oneFS) + if ok != tc.wantOK { + t.Fatalf("accepted = %v, want %v", ok, tc.wantOK) + } + + // A refused descent names the directory, so the scan keeps + // the records below it. + var wantEvs []walkEvent + if !ok { + wantEvs = []walkEvent{{otherFSDir: sub}} + } + + if !slices.Equal(evs, wantEvs) { + t.Fatalf("events = %+v, want %+v", evs, wantEvs) + } + + // The accepted job must carry the operand's device down, or -x + // stops checking below the first level. + want := dirJob{ + path: sub, + rootDev: tc.parent.rootDev, + rootDevOK: tc.parent.rootDevOK, + } + if ok && job != want { + t.Fatalf("job = %+v, want %+v", job, want) + } + }) + } +} + +// errInfoUnavailable is returned by errDirEntry.Info(). +var errInfoUnavailable = errors.New("info unavailable") + +// errDirEntry is a directory entry whose Info() always fails, driving +// subdirJob's stat-error branch deterministically. +type errDirEntry struct{ name string } + +func (e errDirEntry) Name() string { return e.name } +func (errDirEntry) IsDir() bool { return true } +func (errDirEntry) Type() fs.FileMode { + return fs.ModeDir +} + +func (errDirEntry) Info() (fs.FileInfo, error) { + return nil, errInfoUnavailable +} + +// TestSubdirJobStatError asserts that when a subdirectory's Info() +// fails under -x, subdirJob warns and refuses descent. +func TestSubdirJobStatError(t *testing.T) { + t.Parallel() + + p := "/does/not/matter/sub" + + _, ok, evs := callSubdirJob(t, p, errDirEntry{name: "sub"}, + dirJob{rootDev: 1, rootDevOK: true}, true) + if ok { + t.Fatal("descent accepted after stat error, want refused") + } + + if len(evs) != 1 || !evs[0].fail || !strings.Contains(evs[0].warn, p) { + t.Fatalf("want one warning naming the path, got %+v", evs) + } +} + +// syncOneFilesystem scans roots into db with -x and returns the scan +// stats. Every device pruneRoots looks up, and the device each +// operand's walk compares its subdirectories with, comes from devices +// instead of from lstat, so a test can put a path on a filesystem that +// no directory here is on. Unlike runPhases, it skips walkableRoots, +// the check for a cancelled walk, and the content phase. +func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, + devices map[string]uint64, +) scanStats { + t.Helper() + + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + + roots = pruneRoots(roots, true, device) + + ctx := t.Context() + s := &scanState{db: db} + + err := s.loadIndex(ctx, roots) + if err != nil { + t.Fatal(err) + } + + jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + + // seedRoot emits a regular-file operand itself and returns a job + // only for a directory operand. + var initial []dirJob + + for _, root := range roots { + for _, job := range seedRoot(ctx, root, events) { + job.rootDev = devices[root] + initial = append(initial, job) + } + } + + dispatchDirs(ctx, initial, jobs, subdirs) + + changed, unhashed := s.walkPhase(events, roots) + s.partition(changed, unhashed) + + err = s.hashPhase(ctx, 2) + if err != nil { + t.Fatal(err) + } + + err = s.updatePhase(ctx) + if err != nil { + t.Fatal(err) + } + + return s.st +} + +// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a +// record below a directory the walk does not enter, because it is on +// another filesystem than its operand, counts as outside the operands: +// it is not deleted, and its size joins the size census. The record of +// a gone file that is not below that directory is still deleted, even +// when its path begins with the directory's name. The operand is given +// a device that no directory here is on, so no second real filesystem +// is needed. +func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records the file below mnt, and mnt2, which is + // beside mnt, not below it. + mounted := writeFile(t, dir, "mnt/f", pattern(1, 100)) + gone := writeFile(t, dir, "mnt2", pattern(3, 50)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + // The same size as mnt/f, so it is hashed only if mnt/f's size is + // in the census. + added := writeFile(t, dir, "added", pattern(2, 100)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1}) + + recs := dbRecords(t, db) + recordByPath(t, recs, mounted) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, mounted) + } + + if slices.Contains(recordPaths(recs), gone) { + t.Errorf("record for %s kept: the file is gone and not below mnt", + gone) + } +} + +// TestScanOneFilesystemWalksNestedOperand checks that with -x an +// operand under another one, on a different filesystem than it, is +// walked on its own: a file added below it is recorded, and the record +// of a file gone from below it is deleted. A record below a directory +// that its own walk does not enter is still kept. Each operand is given +// a device that no directory here is on, so neither walk enters a +// subdirectory. +func TestScanOneFilesystemWalksNestedOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + syncOneFilesystem(t, db, []string{dir, mnt}, + map[string]uint64{dir: dev + 1, mnt: dev + 2}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, mnt) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, mnt, filepath.Dir(kept)) + } +} + +// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that +// with -x an operand on the same filesystem as the operand covering it +// is walked on its own when a directory between them is on another +// filesystem, since the walk of the covering operand does not enter +// that directory: a file added below the nested operand is recorded, +// and the record of a file gone from below it is deleted. A record +// below that directory but outside the nested operand is still kept. +// Both operands are given one device that no directory here is on, +// while mnt keeps its own. +func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/back/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + back := filepath.Join(mnt, "back") + syncOneFilesystem(t, db, []string{dir, back}, + map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, back) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, dir, mnt) + } +} + +// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a +// regular-file operand under another operand is dropped when the file +// reports a different device than that operand but its directory does +// not, as a file bind-mounted into the tree does: the walk of the other +// operand enters the directory and reaches the file, so the file is +// recorded once. +func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev, filepath.Dir(file): dev, file: dev + 1, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + +// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a +// regular-file operand under another operand is kept when its directory +// is on a different filesystem than that operand: the walk of the other +// operand does not enter the directory, so the file is recorded once, +// as its own operand. The other operand is given a device that no +// directory here is on. +func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev + 1, filepath.Dir(file): dev, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { @@ -953,11 +1471,16 @@ func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { return st } -// dbRecords returns every record currently in the database. +// dbRecords returns every record currently in the database, in path +// order. func dbRecords(t *testing.T, db *sql.DB) []scanRec { t.Helper() - recs, err := loadFileRows(t.Context(), db) + var recs []scanRec + + err := loadFileRows(t.Context(), db, func(r scanRec) { + recs = append(recs, r) + }) if err != nil { t.Fatal(err) } @@ -994,10 +1517,10 @@ func recordPaths(recs []scanRec) []string { // assertSmokeDupeGroups checks the file-level duplicate groups for the // smoke tree rooted at dir. -func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { +func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) { t.Helper() - groups := collectDupeGroups(parsed) + groups := dupeGroups(t, db) if len(groups) != 5 { t.Fatalf("len(groups) = %d, want 5", len(groups)) } @@ -1022,11 +1545,10 @@ func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { // assertSmokeTreeGroups checks the duplicate-tree groups for the smoke // tree rooted at dir. -func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) { +func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) { t.Helper() - super, dirs := buildHierarchy(parsed) - super.compute() + super, dirs := dbTree(t, db) tg := collectTreeGroups(dirs, super) if len(tg) != 1 { @@ -1051,7 +1573,7 @@ func TestScanPipeline(t *testing.T) { db := openTestDB(t) st := syncTree(t, db, dir) - if st != (scanStats{added: smokeTreeFiles}) { + if st != (scanStats{walked: smokeTreeFiles, added: smokeTreeFiles}) { t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles) } @@ -1060,8 +1582,8 @@ func TestScanPipeline(t *testing.T) { t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles) } - assertSmokeDupeGroups(t, dir, parsed) - assertSmokeTreeGroups(t, dir, parsed) + assertSmokeDupeGroups(t, dir, db) + assertSmokeTreeGroups(t, dir, db) } func TestSyncScanUnchangedReuse(t *testing.T) { @@ -1074,7 +1596,7 @@ func TestSyncScanUnchangedReuse(t *testing.T) { writeFile(t, dir, "b.bin", pattern(2, 600)) st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("first scan stats = %+v, want 2 added", st) } @@ -1088,7 +1610,7 @@ func TestSyncScanUnchangedReuse(t *testing.T) { } st = syncTree(t, db, dir) - if st != (scanStats{unchanged: 2}) { + if st != (scanStats{walked: 2, unchanged: 2}) { t.Fatalf("rescan stats = %+v, want 2 unchanged", st) } @@ -1117,15 +1639,176 @@ func TestSyncScanMtimeBump(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{updated: 1}) { + if st != (scanStats{walked: 1, updated: 1}) { t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) } - if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() { - t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix()) + if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) { + t.Fatalf("mtime = %v, want %v", r.mtime, future) } } +// assertWholeFileHashed fails unless the record for path holds the +// whole-file hash of data as its head, tail, and content. +func assertWholeFileHashed(t *testing.T, db *sql.DB, path string, + data []byte, +) { + t.Helper() + + r := recordByPath(t, dbRecords(t, db), path) + if want := hexSum(data); r.head != want || r.tail != want || + r.content != want { + t.Fatalf("head, tail, content = %q, %q, %q, want %q for each", + r.head, r.tail, r.content, want) + } +} + +// TestSyncScanSameSecondRewrite rewrites a file in place at the same +// size with an mtime later in the same second as the recorded one: the +// next scan must notice the change and re-hash the file. +func TestSyncScanSameSecondRewrite(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + writeFile(t, dir, "b.bin", pattern(2, 500)) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(a, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, dir) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(a, later, later) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + +// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite +// for files given to scan as operands, which scan stats without reading +// their directory. +func TestSyncScanOperandSameSecondRewrite(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + b := writeFile(t, dir, "b.bin", pattern(2, 500)) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(a, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, a, b) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(a, later, later) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, a, b) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + +// TestSyncScanRewriteAfter2262 runs assertLateRewriteRehashed with an +// mtime after 2262, a time too late to count in nanoseconds in an int64. +func TestSyncScanRewriteAfter2262(t *testing.T) { + t.Parallel() + + assertLateRewriteRehashed(t, time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC)) +} + +// TestSyncScanRewritePastTimeLimit runs assertLateRewriteRehashed with an +// mtime one second past the latest a time.Time holds without wrapping it +// to a time far in the past. +func TestSyncScanRewritePastTimeLimit(t *testing.T) { + t.Parallel() + + assertLateRewriteRehashed(t, time.Unix(9223371974719179008, 0)) +} + +// assertLateRewriteRehashed scans a directory, rewrites a file in it in +// place at the same size, sets its mtime to late, and fails unless the +// next scan re-hashes the file. It skips where late does not fit the +// platform's timespec or the filesystem does not store it. +func assertLateRewriteRehashed(t *testing.T, late time.Time) { + t.Helper() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + writeFile(t, dir, "b.bin", pattern(2, 500)) + + syncTree(t, db, dir) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + // os.Chtimes cannot set such a time: it converts through UnixNano. + ts, err := unix.TimeToTimespec(late) + if err != nil { + t.Skipf("an mtime %d seconds after 1970 does not fit this platform's "+ + "timespec: %v", late.Unix(), err) + } + + err = unix.UtimesNano(a, []unix.Timespec{ts, ts}) + if err != nil { + t.Fatal(err) + } + + fi, err := os.Lstat(a) + if err != nil { + t.Fatal(err) + } + + if fi.ModTime().Unix() != late.Unix() { + t.Skipf("the filesystem stored the mtime as %d seconds after 1970, "+ + "not %d", fi.ModTime().Unix(), late.Unix()) + } + + st := syncTree(t, db, dir) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + func TestSyncScanAddRemove(t *testing.T) { t.Parallel() @@ -1145,7 +1828,7 @@ func TestSyncScanAddRemove(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{added: 1, removed: 1, unchanged: 1}) { + if st != (scanStats{walked: 2, added: 1, removed: 1, unchanged: 1}) { t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged", st) } @@ -1171,9 +1854,7 @@ func TestSyncScanSizeChange(t *testing.T) { writeFile(t, dir, "f", pattern(1, 200)) - mt := time.Unix(old.mtime, 0) - - err := os.Chtimes(p, mt, mt) + err := os.Chtimes(p, old.mtime, old.mtime) if err != nil { t.Fatal(err) } @@ -1262,7 +1943,7 @@ func TestSyncScanOverlappingRoots(t *testing.T) { // A file reachable via two overlapping operands is deduplicated // by path in the shared walk and processed once. st := syncTree(t, db, dir, filepath.Join(dir, "sub")) - if st != (scanStats{added: 1}) { + if st != (scanStats{walked: 1, added: 1}) { t.Fatalf("stats = %+v, want 1 added", st) } @@ -1283,7 +1964,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { // Neither size is shared, so neither file is read: both records // are written without hashes and no duplicates are reported. st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("stats = %+v, want 2 added", st) } @@ -1295,7 +1976,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { } } - if groups := collectDupeGroups(recs); len(groups) != 0 { + if groups := dupeGroups(t, db); len(groups) != 0 { t.Fatalf("groups = %+v, want none from unhashed records", groups) } @@ -1305,12 +1986,12 @@ func TestScanSkipsUniqueSizes(t *testing.T) { c := writeFile(t, dir, "c.bin", pattern(1, 500)) st = syncTree(t, db, dir) - if st != (scanStats{added: 1, updated: 1, unchanged: 1}) { + if st != (scanStats{walked: 3, added: 1, updated: 1, unchanged: 1}) { t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged", st) } - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 { t.Fatalf("groups = %+v, want the a/c pair", groups) } @@ -1346,8 +2027,7 @@ func TestTreesUnhashedNeverEqual(t *testing.T) { } for name, unknown := range cases { - super, dirs := buildHierarchy(append(slices.Clone(shared), unknown...)) - super.compute() + super, dirs := treeOf(t, append(slices.Clone(shared), unknown...)) if tg := collectTreeGroups(dirs, super); len(tg) != 0 { t.Errorf("%s: tree groups = %d, want 0 (the files may differ)", @@ -1370,7 +2050,7 @@ func TestScanHardlinksReadOnce(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("stats = %+v, want 2 added", st) } @@ -1384,7 +2064,7 @@ func TestScanHardlinksReadOnce(t *testing.T) { t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb) } - if groups := collectDupeGroups(recs); len(groups) != 1 { + if groups := dupeGroups(t, db); len(groups) != 1 { t.Fatalf("groups = %+v, want the hardlink pair", groups) } } @@ -1414,6 +2094,54 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) { } } +// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file +// with a FIFO before the hash phase reads it. Opening a FIFO that has +// no writer waits for one, so a blocking open would hang the phase. The +// path must be warned about once and skipped, with no record written. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + p := writeFile(t, dir, "a.bin", pattern(1, 300)) + + walked, _ := collectWalk(t, []string{dir}, false, 1) + + err := os.Remove(p) + if err != nil { + t.Fatal(err) + } + + err = unix.Mkfifo(p, 0o600) + if err != nil { + t.Fatal(err) + } + + s := &scanState{db: openTestDB(t), toHash: walked} + + err = s.hashPhase(t.Context(), 1) + if err != nil { + t.Fatalf("hashPhase = %v, want nil", err) + } + + if s.st.skipped != 1 || len(s.batch) != 0 { + t.Errorf("skipped = %d, records = %d, want 1 skipped and none", + s.st.skipped, len(s.batch)) + } + + warning := "hash " + p + ": not a regular file\n" + if got := stderr(); strings.Count(got, warning) != 1 { + t.Errorf("stderr = %q, want %q once", got, warning) + } + + // The content phase opens files the same way. + _, _, content, err := hashContentOnly(p, headTailMin) + if content != "" || !errors.Is(err, errNotRegular) { + t.Errorf("hashContentOnly on a FIFO = %q, %v; want no hash and %v", + content, err, errNotRegular) + } +} + // injectedWriteFailure is the message the injected database trigger // aborts with, so the test can recognize its own failure in the error // the scan reports. @@ -1492,6 +2220,13 @@ func injectWriteFailure(t *testing.T, path string) { func baselineGoroutines(t *testing.T) int { t.Helper() + // The first scan command in a process starts os/signal's goroutine, + // which never exits. Start it now, so the baseline counts it + // instead of the scan seeming to leave it behind. + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT) + signal.Stop(ch) + deadline := time.Now().Add(goroutineSettle) last := runtime.NumGoroutine() @@ -1548,7 +2283,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) { code := run([]string{ cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir, - }, &stderr) + }, io.Discard, &stderr) if code != exitFatal { t.Fatalf("run(scan) = %d, want %d; stderr: %s", code, exitFatal, stderr.String()) @@ -1600,14 +2335,43 @@ func TestHashRuns(t *testing.T) { func TestPruneRoots(t *testing.T) { t.Parallel() + const pqRoot = "/p/q" + + // /p/q is on another filesystem than /p; /p/q/r is on the same one as + // /p/q, /p/s as /p, and /p/t has no known device. + devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1} + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). - got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}) + got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, + false, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { t.Fatalf("pruneRoots = %q, want %q", got, want) } + + nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} + + got = pruneRoots(nested, false, device) + + want = []string{"/p"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots without -x = %q, want %q", got, want) + } + + // With -x, /p/q, on another filesystem than /p, is kept as well. + got = pruneRoots(nested, true, device) + + want = []string{"/p", pqRoot} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with -x = %q, want %q", got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) { @@ -1625,10 +2389,8 @@ func TestReportsNeverTouchFilesystem(t *testing.T) { t.Fatal(err) } - recs := dbRecords(t, db) - - assertSmokeDupeGroups(t, dir, recs) - assertSmokeTreeGroups(t, dir, recs) + assertSmokeDupeGroups(t, dir, db) + assertSmokeTreeGroups(t, dir, db) } func TestUnderRoot(t *testing.T) { diff --git a/script/bootstrap b/script/bootstrap index 936ea8b..3e4c106 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -1,12 +1,9 @@ #!/bin/sh # script/bootstrap: install all dependencies needed to build and develop -# this repo. Idempotent: every install is guarded by a check so already -# installed tools are skipped. Base tooling comes from nix, apt, brew, -# or apk (detected in that order); assumes nothing is present (not git, -# make, or go). The linter is NOT installed: golangci-lint runs via -# docker only (script/lint), pinned by image digest, so the only lint -# prerequisite is a working docker — which is warned about, not -# installed, because everything except linting works without it. +# this repo. Idempotent; assumes nothing is present (not git, make, or +# go). Base tooling comes from nix, apt, brew, or apk (detected in that +# order). golangci-lint and prettier are never installed: they run via +# docker only (script/lint, script/fmt, script/fmt-check). set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -61,25 +58,19 @@ missing() { main() { cd "$ROOT" - # System tooling, deliberately unpinned: these come from the host - # package manager and whatever version it ships is what the host - # gets, so a presence check is the right check. The repo pins no - # system toolchain versions — the Go language version is governed by - # go.mod, and builds that must be reproducible run in the Docker - # image, whose base images are pinned by digest. + # Deliberately unpinned, so presence is the whole check: go.mod + # governs the Go version, and reproducible builds run in the + # digest-pinned Docker images. if missing git; then pkg_install git git git git; fi if missing make; then pkg_install gnumake make make make; fi if missing go; then pkg_install go golang go go; fi - # Linting runs via docker only (script/lint), so docker is a lint - # prerequisite rather than something bootstrap installs. Warn, do - # not fail: everything except `make lint` — and, through it, - # `make check`, `make docker` and the pre-commit hook — works - # without it. + # Warn, do not fail: only the targets named below, and the + # pre-commit hook, need docker. if missing docker; then - echo "bootstrap: WARNING: docker not found; make lint, make check" >&2 - echo "bootstrap: and make docker require it. Install docker to" >&2 - echo "bootstrap: run the linter." >&2 + echo "bootstrap: WARNING: docker not found; make lint, make fmt," >&2 + echo "bootstrap: make fmt-check, make check, make docker and" >&2 + echo "bootstrap: make test-race require it." >&2 fi go mod download diff --git a/script/cibuild b/script/cibuild index 91fd8b2..5eca17f 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,34 +1,19 @@ #!/bin/sh # script/cibuild: run the CI build. The Gitea workflow runs this on -# push. +# push. The Dockerfile runs every gate make check runs, as build steps, +# so a successful build means the repo is green. # -# The Dockerfile runs the gates individually as build steps, not the -# make check aggregate: the lint stage runs make fmt-check, -# script/verify-lint-image-pin, golangci-lint config verify and -# golangci-lint run; the build stage, dropped to an unprivileged user, -# runs make test and make fmt-check. Neither make lint nor make check -# appears, because both reach script/lint, which is itself a docker -# build, and a docker build cannot run inside one. Lint is not skipped -# by that — the linter is invoked directly in the lint stage, and the -# build stage's COPY --from=lint makes that stage a prerequisite, so -# BuildKit must finish it first. Between the two stages everything -# make check would run has run, which is why a successful build here -# implies the repo is green. -# -# That implication holds only because of CHECK_EPOCH. A COPY layer is -# invalidated by changed content, and a merge commit's tree is -# byte-identical to the branch head it merges, so without a fresh value -# here Docker serves the gate layers from cache and the build reports a -# green it never earned. Passing the current epoch invalidates the gate -# layers on every run while leaving the pinned base images and -# go mod download cached; see the Dockerfile for the placement. +# Without a fresh CHECK_EPOCH, a rebuild of an unchanged checkout serves +# the gate layers from cache and passes having run none of them. The +# process id goes in with the epoch so two runs started in the same +# second still differ. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - docker build --build-arg CHECK_EPOCH="$(date +%s)" . + docker build --build-arg CHECK_EPOCH="$(date +%s)-$$" . } main "$@" diff --git a/script/docker b/script/docker index 9f52294..7ebc548 100755 --- a/script/docker +++ b/script/docker @@ -1,14 +1,9 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. -# The tag comes from script/projectname. -# -# CHECK_EPOCH is passed for the same reason script/cibuild passes it: -# without it Docker serves the Dockerfile's gate layers from cache on an -# unchanged tree and this exits 0 having run neither the lint stage's -# gates nor the builder stage's test and fmt-check gates. This is the -# set of gates a developer or reviewer runs by hand, so a cached pass -# here is the most misleading result the repo can produce. Dependency -# layers sit above the ARG and stay cached. +# The tag comes from script/projectname. CHECK_EPOCH is passed for the +# same reason script/cibuild passes it: without a fresh value an +# unchanged tree is served from cache and this exits 0 having run no +# gate. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -17,7 +12,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" docker build \ - --build-arg CHECK_EPOCH="$(date +%s)" \ + --build-arg CHECK_EPOCH="$(date +%s)-$$" \ -t "$("$SCRIPT_DIR/projectname")" \ . } diff --git a/script/fmt b/script/fmt index 28af13b..8994c8b 100755 --- a/script/fmt +++ b/script/fmt @@ -1,12 +1,22 @@ #!/bin/sh -# script/fmt: format all files (writes). +# script/fmt: format all files (writes): the Go sources with gofmt, the +# Markdown with prettier. prettier is never installed on the host: it +# runs from the Dockerfile's prettier stage with the repository mounted, +# as the calling user so the files it rewrites keep their owner. The tag +# makes each build replace the previous image instead of leaving another +# one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" gofmt -s -w . + image="$("$SCRIPT_DIR/projectname")-prettier" + docker build -q --target prettier -t "$image" . >/dev/null + docker run --rm --user "$(id -u):$(id -g)" -v "$ROOT:/src" "$image" \ + prettier --write '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 96fffa0..788f585 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -1,18 +1,39 @@ #!/bin/sh # script/fmt-check: check formatting (read-only). Same scope as -# script/fmt, but fails instead of writing. +# script/fmt, but fails instead of writing. gofmt and prettier both run +# every time and each reports its own failure, so the output says which +# one failed. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - files="$(gofmt -s -l .)" + status=0 + + # Under set -e a bare assignment would end the script when gofmt + # fails (a Go file it cannot parse), and prettier would never run. + if ! files="$(gofmt -s -l .)"; then + echo "gofmt: failed; see its errors above" >&2 + status=1 + fi if [ -n "$files" ]; then echo "gofmt: files not formatted:" >&2 echo "$files" >&2 - exit 1 + status=1 fi + + # Same image as script/fmt; see there. + image="$("$SCRIPT_DIR/projectname")-prettier" + docker build -q --target prettier -t "$image" . >/dev/null + if ! docker run --rm -v "$ROOT:/src:ro" "$image" \ + prettier --check '**/*.md' --tab-width 4 --prose-wrap always; then + echo "prettier: Markdown not formatted; run make fmt" >&2 + status=1 + fi + + exit "$status" } main "$@" diff --git a/script/lint b/script/lint index d402d8a..d942a1c 100755 --- a/script/lint +++ b/script/lint @@ -1,19 +1,17 @@ #!/bin/sh # script/lint: run the linter. golangci-lint is never installed on a -# host: it runs via docker only, one way, everywhere — this builds -# Dockerfile.lint, which COPYs the repo into the digest-pinned -# golangci-lint image and lints as a build step, so a successful build -# is a clean lint. The only prerequisite is a working docker. The gate -# steps make no network calls of their own, but Dockerfile.lint runs -# `go mod download` above them, so a cold cache does reach the network -# (as does pulling the pinned image); that layer stays cached, and once -# it is warm this runs offline until go.mod or go.sum changes. +# host: this builds Dockerfile.lint, which copies the repo into the +# digest-pinned golangci-lint image and lints as a build step, so a +# successful build is a clean lint. A cold cache needs the network to +# pull the image and for `go mod download`; once warm this runs offline +# until go.mod or go.sum changes. # -# CHECK_EPOCH is what makes the result mean anything. Without it docker -# serves the gate layers from cache on an unchanged tree and this exits -# 0 in well under a second having run no linter. The PID is in the value -# as well as the epoch because two lint runs land inside the same second -# easily, and `date +%s` alone would cache the second one. +# Without a fresh CHECK_EPOCH docker serves the gate layers from cache +# on an unchanged tree and this exits 0 having run no linter. The PID is +# in the value because two lint runs land inside the same second easily. +# +# The image is never used, so --output=type=cacheonly writes none; +# without it every run leaves an untagged image behind. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -22,6 +20,7 @@ main() { cd "$ROOT" docker build \ --build-arg CHECK_EPOCH="$(date +%s)-$$" \ + --output=type=cacheonly \ -f Dockerfile.lint \ . } diff --git a/script/test-race b/script/test-race new file mode 100755 index 0000000..8e85862 --- /dev/null +++ b/script/test-race @@ -0,0 +1,40 @@ +#!/bin/sh +# script/test-race: run the test suite under the race detector. Not part +# of script/check. +# +# The race detector needs cgo and a C compiler, which the host build +# never uses, so the tests run in a golang image that has gcc. The +# checkout is mounted read-only, so the docker daemon must be local. The +# container starts with empty caches every time: each run downloads the +# dependencies and compiles them with the detector, which needs the +# network and takes minutes. +# +# The tests run as the calling user, never as root: several of them make +# a file unreadable and expect reading it to fail, and root reads it +# anyway. When the caller is root they run as nobody, and then the +# checkout must be readable by other users. Neither user has a home +# directory in the image, so HOME is /tmp, where Go puts its build cache. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# golang:1.25-trixie, 2026-10-04. Debian rather than the Alpine image the +# Dockerfile builds with, because this one includes gcc. +IMAGE="golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73" + +main() { + user="$(id -u):$(id -g)" + if [ "$(id -u)" -eq 0 ]; then + user=65534:65534 + fi + docker run --rm \ + --user "$user" \ + --env HOME=/tmp \ + --env CGO_ENABLED=1 \ + --volume "$ROOT:/src:ro" \ + --workdir /src \ + "$IMAGE" \ + go test -race -timeout 60s ./... +} + +main "$@" diff --git a/script/verify-lint-image-pin b/script/verify-lint-image-pin index fa30722..1a9fb69 100755 --- a/script/verify-lint-image-pin +++ b/script/verify-lint-image-pin @@ -2,23 +2,16 @@ # script/verify-lint-image-pin: fail unless the golangci-lint image # referenced by Dockerfile.lint and the one referenced by the main # Dockerfile's lint stage are the same image at the same digest. Our own -# extension to scripts-to-rule-them-all, not one of its entrypoints. +# extension to scripts-to-rule-them-all, not one of its entrypoints; run +# as a gate in both files. Nothing else keeps the two pins in sync, and +# a bump applied to one alone would lint the same tree against different +# rulesets, both green. # -# The linter version is pinned in two independent files. That is the -# shape #42 turned into a build failure rather than tolerate: nothing -# else keeps the two in sync, and a bump applied to one file alone would -# leave `make lint` and the fail-fast lint stage of `make docker` -# linting the same tree against different rulesets, both green. This is -# the single guard that stops it, run as a gate in both files. +# Do not hardcode the expected digest here: that is a third copy to keep +# in sync. # -# It deliberately restates neither pin. A hardcoded expected digest here -# would be a third copy — one more thing to bump, and the same drift one -# file further out. It compares the two files to each other and knows -# nothing about which version is correct. -# -# A reference that cannot be read is a hard failure, not a skip: a -# comparison of two empty strings succeeds, which would turn this guard -# into exactly the unearned green it exists to prevent. +# A reference that cannot be read is a hard failure, not a skip: two +# empty strings compare equal. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/trees.go b/trees.go index f5337fc..5c1bbf7 100644 --- a/trees.go +++ b/trees.go @@ -5,49 +5,59 @@ import ( "context" "crypto/sha256" "fmt" + "io" "os" "slices" "strconv" "strings" ) -// fileSig is a file's duplicate signature; mtime is excluded. -type fileSig struct { - size int64 - head string - tail string - content string -} - -// treeNode is one directory reconstructed from the scan stream. +// treeNode is one directory reconstructed from the record paths. type treeNode struct { - path string - parent *treeNode - dirs map[string]*treeNode - files map[string]fileSig + path string + parent *treeNode + // entries holds the serialized child entries until the digest is + // computed from them, and is then dropped. + entries []string digest [sha256.Size]byte fileCount int64 totalSize int64 } // runTrees implements the trees subcommand: it reads every record from -// the database, reconstructs the directory hierarchy from the record -// paths, computes a Merkle-style digest per directory, and prints -// maximal duplicate-tree groups as TSV on stdout. It never touches the -// scanned filesystem; its only I/O is the database, stdout, and -// stderr. -func runTrees(ctx context.Context) error { - recs, err := loadRecords(ctx) +// the database in path order, reconstructs the directory hierarchy from +// the record paths, computes a Merkle-style digest per directory, and +// prints maximal duplicate-tree groups as TSV on stdout. It never +// touches the scanned filesystem; its only I/O is the database, stdout, +// and stderr. Any database problem, including a missing database, is +// fatal. +func runTrees(ctx context.Context, stdout io.Writer) error { + dbPath := databasePath() + + db, err := openReportDatabase(ctx, dbPath) if err != nil { return err } - super, allDirs := buildHierarchy(recs) - super.compute() + defer func() { _ = db.Close() }() + + records := 0 + tree := newTreeBuilder() + + err = loadFileRows(ctx, db, func(r scanRec) { + records++ + + tree.add(r) + }) + if err != nil { + return fmt.Errorf("database %s: %w", dbPath, err) + } + + super, allDirs := tree.finish() dupes := collectTreeGroups(allDirs, super) - out := bufio.NewWriterSize(os.Stdout, ioBufSize) + out := bufio.NewWriterSize(stdout, ioBufSize) _, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize") if err != nil { @@ -62,7 +72,8 @@ func runTrees(ctx context.Context) error { first := g[0] for _, n := range g[1:] { _, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", - first.path, n.path, first.fileCount, first.totalSize) + escapePath(first.path), escapePath(n.path), + first.fileCount, first.totalSize) if err != nil { return fmt.Errorf("write stdout: %w", err) } @@ -80,65 +91,128 @@ func runTrees(ctx context.Context) error { fmt.Fprintf(os.Stderr, "trees: %d records read, %d duplicate tree groups, %d dupe trees, "+ "%s reclaimable\n", - len(recs), len(dupes), dupeTrees, humanBytes(reclaimable)) + records, len(dupes), dupeTrees, humanBytes(reclaimable)) return nil } -// buildHierarchy reconstructs the directory hierarchy from the record -// paths under a synthetic super-root. Paths are split on "/"; for -// absolute paths the first component is empty, which simply becomes a -// top-level node representing "/". It returns the super-root and every -// directory node created. -func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) { +// treeBuilder reconstructs the directory hierarchy from records added +// in path order, under a synthetic super-root. Paths are split on "/"; +// for absolute paths the first component is empty, which becomes the +// top-level directory with path "/". In path order all the paths under +// one directory come together, so a directory is complete once a path +// outside it is added: its digest is computed then and its entries are +// dropped. Only the directories holding the latest path keep entries. +type treeBuilder struct { + super *treeNode + // open lists the directories holding the latest path, outermost + // first, starting with the super-root; names[i] is open[i]'s name. + open []*treeNode + names []string + // dirs lists every completed directory. + dirs []*treeNode +} + +func newTreeBuilder() *treeBuilder { super := &treeNode{} - var allDirs []*treeNode + return &treeBuilder{ + super: super, + open: []*treeNode{super}, + names: []string{""}, + } +} - for _, r := range recs { - comps := strings.Split(r.path, "/") +// add adds one record. Each record must come after the previous one in +// path order (byte order); otherwise a completed directory would be +// started again as a second directory with the same path. +func (b *treeBuilder) add(r scanRec) { + comps := strings.Split(r.path, "/") + dirNames, name := comps[:len(comps)-1], comps[len(comps)-1] - node := super - for _, c := range comps[:len(comps)-1] { - child := node.dirs[c] - if child == nil { - childPath := c - if node != super { - childPath = node.path + "/" + c - } - - child = &treeNode{path: childPath, parent: node} - if node.dirs == nil { - node.dirs = make(map[string]*treeNode) - } - - node.dirs[c] = child - allDirs = append(allDirs, child) - } - - node = child - } - - if node.files == nil { - node.files = make(map[string]fileSig) - } - - sig := fileSig{ - size: r.size, head: r.head, tail: r.tail, content: r.content, - } - - // A record without a content hash has unknown content (README - // "Database"): give it a signature no other file can share, so - // trees containing it never compare equal. Real hashes are - // hex, so the NUL-prefixed form cannot collide. - if sig.content == "" { - sig.content = "unhashed\x00" + r.path - } - - node.files[comps[len(comps)-1]] = sig + // Keep the open directories that hold this path; complete the rest. + depth := 1 + for depth < len(b.open) && depth <= len(dirNames) && + b.names[depth] == dirNames[depth-1] { + depth++ } - return super, allDirs + b.closeTo(depth) + + for _, c := range dirNames[depth-1:] { + b.openDir(c) + } + + dir := b.open[len(b.open)-1] + dir.entries = append(dir.entries, fileEntry(name, r)) + dir.fileCount++ + dir.totalSize += r.size +} + +// openDir starts the directory called name inside the innermost open +// one. +func (b *treeBuilder) openDir(name string) { + parent := b.open[len(b.open)-1] + path := parent.path + "/" + name + + // The root directory's path is "/", not empty, and its children's + // paths start with one slash, not two. + switch { + case parent == b.super && name == "": + path = "/" + case parent == b.super: + path = name + case parent.path == "/": + path = "/" + name + } + + b.open = append(b.open, &treeNode{path: path, parent: parent}) + b.names = append(b.names, name) +} + +// closeTo completes the open directories after the first n, innermost +// first: each one's digest is computed and entered in its parent along +// with its totals. +func (b *treeBuilder) closeTo(n int) { + for len(b.open) > n { + last := len(b.open) - 1 + dir, name := b.open[last], b.names[last] + b.open, b.names = b.open[:last], b.names[:last] + + dir.computeDigest() + + dir.parent.entries = append(dir.parent.entries, + "d\x00"+name+"\x00"+string(dir.digest[:])) + dir.parent.fileCount += dir.fileCount + dir.parent.totalSize += dir.totalSize + b.dirs = append(b.dirs, dir) + } +} + +// finish completes every open directory and returns the super-root and +// every directory. +func (b *treeBuilder) finish() (*treeNode, []*treeNode) { + b.closeTo(1) + + return b.super, b.dirs +} + +// fileEntry serializes a file child for its directory's digest: its +// name and its signature (size, head, tail, content); mtime is +// excluded. +func fileEntry(name string, r scanRec) string { + content := r.content + + // A record without a content hash has unknown content (README + // "Database"): give it a signature no other file can share, so + // trees containing it never compare equal. Real hashes are hex, so + // the NUL-prefixed form cannot collide. + if content == "" { + content = "unhashed\x00" + r.path + } + + return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) + + "\x00" + r.head + "\x00" + r.tail + "\x00" + content } // collectTreeGroups groups directories by digest and returns every @@ -180,38 +254,22 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode { return dupes } -// compute fills in digest, fileCount, and totalSize for n and all of -// its descendants. A directory's digest is the SHA-256 of its child -// entries — files serialized with name and signature, subdirectories -// with name and recursive digest — sorted byte-lexicographically. -// Filenames cannot contain NUL or "/", so NUL delimiters are -// unambiguous. -func (n *treeNode) compute() { - entries := make([]string, 0, len(n.dirs)+len(n.files)) - for name, sig := range n.files { - entries = append(entries, - "f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+ - "\x00"+sig.head+"\x00"+sig.tail+"\x00"+sig.content) - n.fileCount++ - n.totalSize += sig.size - } - - for name, child := range n.dirs { - child.compute() - entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) - n.fileCount += child.fileCount - n.totalSize += child.totalSize - } - - slices.Sort(entries) +// computeDigest sets n's digest and drops its entries. A directory's +// digest is the SHA-256 of its child entries — files serialized with +// name and signature, subdirectories with name and recursive digest — +// sorted byte-lexicographically. Filenames cannot contain NUL or "/", +// so NUL delimiters are unambiguous. +func (n *treeNode) computeDigest() { + slices.Sort(n.entries) h := sha256.New() - for _, e := range entries { + for _, e := range n.entries { h.Write([]byte(e)) h.Write([]byte{0}) } copy(n.digest[:], h.Sum(nil)) + n.entries = nil } // suppressed reports whether a duplicate-tree group is non-maximal: its diff --git a/trees_test.go b/trees_test.go index d0995ae..969c438 100644 --- a/trees_test.go +++ b/trees_test.go @@ -1,6 +1,8 @@ package main import ( + "bytes" + "database/sql" "slices" "testing" ) @@ -29,6 +31,36 @@ func smokeTreeRecs() []scanRec { } } +// dbTree builds the directory hierarchy from the records in db the way +// trees does, and returns the super-root and every directory. +func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) { + t.Helper() + + tree := newTreeBuilder() + + err := loadFileRows(t.Context(), db, tree.add) + if err != nil { + t.Fatal(err) + } + + return tree.finish() +} + +// treeOf writes recs into a fresh database and builds the directory +// hierarchy from it the way trees does. +func treeOf(t *testing.T, recs []scanRec) (*treeNode, []*treeNode) { + t.Helper() + + db := openTestDB(t) + + err := applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + return dbTree(t, db) +} + // nodeByPath finds the directory node with the given path. func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode { t.Helper() @@ -59,11 +91,10 @@ func groupPaths(groups [][]*treeNode) [][]string { return out } -func TestBuildHierarchyCounts(t *testing.T) { +func TestTreeCounts(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + _, dirs := treeOf(t, smokeTreeRecs()) d := nodeByPath(t, dirs, "/d") if d.fileCount != 6 || d.totalSize != 9300 { @@ -84,11 +115,98 @@ func TestBuildHierarchyCounts(t *testing.T) { } } +func TestTreeRootPath(t *testing.T) { + t.Parallel() + + // The root directory's path is "/", never empty, and its + // children's paths start with a single slash. + _, dirs := treeOf(t, []scanRec{{path: "/f"}, {path: "/srv/g"}}) + + got := make([]string, 0, len(dirs)) + for _, d := range dirs { + got = append(got, d.path) + } + + slices.Sort(got) + + want := []string{"/", "/srv"} + if !slices.Equal(got, want) { + t.Fatalf("directory paths = %q, want %q", got, want) + } +} + +func TestTreeNamesSortingBeforeSlash(t *testing.T) { + t.Parallel() + + // In path order "/a/b-x/f" and "/a/b.txt" come between the file + // "/a/b" and "/a/b/f", because "-" and "." sort before "/". Each + // directory must still be built once, whole, so /a matches /c. + recs := make([]scanRec, 0, 8) + + for _, top := range []string{"/a", "/c"} { + for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} { + content := "c" + if p == "/b-x/f" { + content = "other" + } + + recs = append(recs, scanRec{ + size: 1, head: "h", tail: "t", content: content, path: top + p, + }) + } + } + + super, dirs := treeOf(t, recs) + + got := make([]string, 0, len(dirs)) + for _, d := range dirs { + got = append(got, d.path) + } + + slices.Sort(got) + + want := []string{"/", "/a", "/a/b", "/a/b-x", "/c", "/c/b", "/c/b-x"} + if !slices.Equal(got, want) { + t.Fatalf("directory paths = %q, want %q", got, want) + } + + groups := collectTreeGroups(dirs, super) + + gotGroups := groupPaths(groups) + wantGroups := [][]string{{"/a", "/c"}} + + if !slices.EqualFunc(gotGroups, wantGroups, slices.Equal) { + t.Fatalf("groups = %v, want %v", gotGroups, wantGroups) + } + + if groups[0][0].fileCount != 4 || groups[0][0].totalSize != 4 { + t.Errorf("group totals: %d files %d bytes, want 4 4", + groups[0][0].fileCount, groups[0][0].totalSize) + } +} + +func TestRunTreesEscapesPaths(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) + + var stdout, stderr bytes.Buffer + + code := run([]string{cmdTrees}, &stdout, &stderr) + if code != exitOK { + t.Fatalf("run(trees) = %d, want %d; stderr: %s", + code, exitOK, stderr.String()) + } + + want := "first\tdupe\tfiles\tsize\n" + + `/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n" + if got := stdout.String(); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + func TestTreeDigests(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + _, dirs := treeOf(t, smokeTreeRecs()) t1 := nodeByPath(t, dirs, "/d/t1") t2 := nodeByPath(t, dirs, "/d/t2") @@ -121,8 +239,7 @@ func TestTreeDigestContentSensitivity(t *testing.T) { {size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + _, dirs := treeOf(t, recs) a := nodeByPath(t, dirs, "/r/a") b := nodeByPath(t, dirs, "/r/b") @@ -135,8 +252,7 @@ func TestTreeDigestContentSensitivity(t *testing.T) { func TestCollectTreeGroupsMaximal(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + super, dirs := treeOf(t, smokeTreeRecs()) groups := collectTreeGroups(dirs, super) @@ -160,16 +276,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) { recs := smokeTreeRecs() - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) forward := groupPaths(collectTreeGroups(dirs, super)) reversed := slices.Clone(recs) slices.Reverse(reversed) - superR, dirsR := buildHierarchy(reversed) - superR.compute() + superR, dirsR := treeOf(t, reversed) backward := groupPaths(collectTreeGroups(dirsR, superR)) if !slices.EqualFunc(forward, backward, slices.Equal) { @@ -188,8 +302,7 @@ func TestCollectTreeGroupsSiblings(t *testing.T) { {size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) got := groupPaths(collectTreeGroups(dirs, super)) @@ -211,8 +324,7 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) { {size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) got := groupPaths(collectTreeGroups(dirs, super)) diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==