Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
282146a409 | ||
|
|
2dd4b8c401 | ||
|
|
337b319542 |
@@ -3,6 +3,7 @@
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
sfdupes
|
sfdupes
|
||||||
files.dat
|
files.dat
|
||||||
|
node_modules
|
||||||
*.log
|
*.log
|
||||||
*.out
|
*.out
|
||||||
*.test
|
*.test
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
node_modules/
|
||||||
|
yarn.lock
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"tabWidth": 4,
|
||||||
|
"proseWrap": "always"
|
||||||
|
}
|
||||||
+13
-8
@@ -1,6 +1,6 @@
|
|||||||
# Lint stage — fast feedback on formatting and lint issues
|
# Lint stage — fast feedback on formatting and lint issues
|
||||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
# golangci/golangci-lint:v2.12.2, 2026-08-07
|
||||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
@@ -27,9 +27,12 @@ ARG CHECK_EPOCH
|
|||||||
# target now runs `docker build -f Dockerfile.lint`, and a docker build
|
# target now runs `docker build -f Dockerfile.lint`, and a docker build
|
||||||
# cannot run a docker build: routing the gate through make would mean
|
# cannot run a docker build: routing the gate through make would mean
|
||||||
# nesting docker inside this image. Same reason `make check` is gone
|
# nesting docker inside this image. Same reason `make check` is gone
|
||||||
# from the build stage below. `make fmt-check` stays as it is — it is a
|
# from the build stage below.
|
||||||
# gate, not the aggregate, and it shells out to nothing.
|
#
|
||||||
RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check
|
# `make fmt-check` is not run in this stage: it now also runs prettier
|
||||||
|
# over Markdown, and this golangci-lint image has no node. The gate runs
|
||||||
|
# in the build stage below, where script/bootstrap installs node and
|
||||||
|
# prettier.
|
||||||
|
|
||||||
# The FROM above and the one in Dockerfile.lint pin the same linter
|
# The FROM above and the one in Dockerfile.lint pin the same linter
|
||||||
# twice, and nothing else keeps them in sync; this fails the build when
|
# twice, and nothing else keeps them in sync; this fails the build when
|
||||||
@@ -77,10 +80,12 @@ COPY --from=lint /src/go.sum /dev/null
|
|||||||
# rather than duplicating the installs inline. Only script/ and the
|
# rather than duplicating the installs inline. Only script/ and the
|
||||||
# dependency manifests are copied first, nothing else, so this layer
|
# dependency manifests are copied first, nothing else, so this layer
|
||||||
# stays cached until the scripts or the dependencies change — bootstrap
|
# stays cached until the scripts or the dependencies change — bootstrap
|
||||||
# ends in `go mod download`, which is why there is no separate
|
# runs `go mod download` and `yarn install`, which is why there is no
|
||||||
# invocation of it here.
|
# separate invocation of either here. The JS manifests (package.json,
|
||||||
|
# yarn.lock) are copied too so the yarn install layer caches alongside
|
||||||
|
# the Go one.
|
||||||
COPY script/ script/
|
COPY script/ script/
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum package.json yarn.lock ./
|
||||||
RUN script/bootstrap
|
RUN script/bootstrap
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|||||||
+2
-2
@@ -9,8 +9,8 @@
|
|||||||
# stage of the main Dockerfile because script/lint must not depend on
|
# stage of the main Dockerfile because script/lint must not depend on
|
||||||
# the rest of that build; the two FROM lines are kept identical by
|
# the rest of that build; the two FROM lines are kept identical by
|
||||||
# script/verify-lint-image-pin, run as a gate below.
|
# script/verify-lint-image-pin, run as a gate below.
|
||||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
# golangci/golangci-lint:v2.12.2, 2026-08-07
|
||||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
|
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
|
|||||||
@@ -2,19 +2,18 @@
|
|||||||
|
|
||||||
## Description
|
## Description
|
||||||
|
|
||||||
`sfdupes` is an MIT-licensed Go CLI tool by
|
`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that
|
||||||
[@sneak](https://sneak.berlin) that quickly identifies *candidate*
|
quickly identifies _candidate_ duplicate files — and, ultimately, entire
|
||||||
duplicate files — and, ultimately, entire duplicate directory trees —
|
duplicate directory trees — across very large filesystems without reading full
|
||||||
across very large filesystems without reading full file contents. Files
|
file contents. Files are considered duplicates when they have identical size,
|
||||||
are considered duplicates when they have identical size, identical
|
identical SHA-256 of their first 1024 bytes, and identical SHA-256 of their last
|
||||||
SHA-256 of their first 1024 bytes, and identical SHA-256 of their last
|
1024 bytes. This is a strong candidate signal, not proof of identical content
|
||||||
1024 bytes. This is a strong candidate signal, not proof of identical
|
(the middle of the file is never read); the intended use is finding duplicate
|
||||||
content (the middle of the file is never read); the intended use is
|
downloads and duplicated directory trees on multi-terabyte ZFS servers where
|
||||||
finding duplicate downloads and duplicated directory trees on
|
reading every byte is prohibitively expensive. `scan` maintains a persistent
|
||||||
multi-terabyte ZFS servers where reading every byte is prohibitively
|
SQLite database of file signatures that survives between runs, so it can be run
|
||||||
expensive. `scan` maintains a persistent SQLite database of file
|
from cron and the reports can be generated at any time from the most recent
|
||||||
signatures that survives between runs, so it can be run from cron and
|
scan.
|
||||||
the reports can be generated at any time from the most recent scan.
|
|
||||||
|
|
||||||
This README is the complete and authoritative specification.
|
This README is the complete and authoritative specification.
|
||||||
|
|
||||||
@@ -28,91 +27,81 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
|
|||||||
./sfdupes trees > dupetrees.tsv
|
./sfdupes trees > dupetrees.tsv
|
||||||
```
|
```
|
||||||
|
|
||||||
`scan` walks one or more filesystem trees and maintains one database
|
`scan` walks one or more filesystem trees and maintains one database record per
|
||||||
record per regular file (path, size, mtime, head hash, tail hash). The
|
regular file (path, size, mtime, head hash, tail hash). The database persists
|
||||||
database persists between runs; a rescan only hashes files that are new
|
between runs; a rescan only hashes files that are new or changed, and removes
|
||||||
or changed, and removes records for files that no longer exist.
|
records for files that no longer exist. `report` reads the database and prints
|
||||||
`report` reads the database and prints the file-level duplicates
|
the file-level duplicates report. `trees` reads the same database and prints the
|
||||||
report. `trees` reads the same database and prints the duplicate-tree
|
duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation
|
||||||
report. A missing/invalid subcommand — or a `scan` invocation with no
|
with no `PATH` operand — prints a usage message and exits 2.
|
||||||
`PATH` operand — prints a usage message and exits 2.
|
|
||||||
|
|
||||||
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed
|
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
||||||
anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a
|
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
||||||
daily `sfdupes scan` cron job, with the reporting commands run
|
cron job, with the reporting commands run interactively whenever needed; their
|
||||||
interactively whenever needed; their results are as fresh as the last
|
results are as fresh as the last completed scan.
|
||||||
completed scan.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
Duplicate finders that hash entire files do not scale to the target
|
Duplicate finders that hash entire files do not scale to the target environment:
|
||||||
environment: ~10 million files and ~150 TB on possibly slow or busy
|
~10 million files and ~150 TB on possibly slow or busy disks (a ZFS pool under
|
||||||
disks (a ZFS pool under resilver). Reading at most 2 KiB per file — and
|
resilver). Reading at most 2 KiB per file — and only from files whose size at
|
||||||
only from files whose size at least one other file shares, since a
|
least one other file shares, since a size-unique file cannot be a duplicate —
|
||||||
size-unique file cannot be a duplicate — makes a full-filesystem sweep
|
makes a full-filesystem sweep tractable, and the signatures are kept in a
|
||||||
tractable, and the signatures are kept in a persistent database, so
|
persistent database, so the expensive filesystem pass is incremental: a rescan
|
||||||
the expensive filesystem pass is incremental: a rescan re-hashes only
|
re-hashes only files whose recorded mtime or size changed, and all analysis
|
||||||
files whose recorded mtime or size changed, and all analysis happens
|
happens offline from the database alone. The end goal is not individual files
|
||||||
offline from the database alone. The end goal is
|
but whole duplicated trees — duplicate extractions, duplicate downloads, copied
|
||||||
not individual files but whole duplicated trees — duplicate
|
project trees — which an operator can consider removing as a unit.
|
||||||
extractions, duplicate downloads, copied project trees — which an
|
|
||||||
operator can consider removing as a unit.
|
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
Goals, in order:
|
Goals, in order:
|
||||||
|
|
||||||
1. **Find whole duplicate trees, not just files.** The end goal is to
|
1. **Find whole duplicate trees, not just files.** The end goal is to identify
|
||||||
identify places where the exact same set of files and directories
|
places where the exact same set of files and directories exists at two or
|
||||||
exists at two or more paths (duplicate extractions, duplicate
|
more paths (duplicate extractions, duplicate downloads, copied project
|
||||||
downloads, copied project trees), so the operator can consider
|
trees), so the operator can consider removing an entire subtree at once.
|
||||||
removing an entire subtree at once. File-level duplicate detection is
|
File-level duplicate detection is the foundation; tree-level detection is
|
||||||
the foundation; tree-level detection is built on top of it.
|
built on top of it.
|
||||||
2. **Never read full file contents.** At most 2 KiB is read per file
|
2. **Never read full file contents.** At most 2 KiB is read per file (first and
|
||||||
(first and last 1024 bytes), and only files whose size at least
|
last 1024 bytes), and only files whose size at least one other file shares
|
||||||
one other file shares are read at all — a size-unique file cannot
|
are read at all — a size-unique file cannot be a duplicate. Scale target:
|
||||||
be a duplicate. Scale target: tens of millions of files, ~150 TB
|
tens of millions of files, ~150 TB filesystem, possibly slow or busy disks
|
||||||
filesystem, possibly slow or busy disks (ZFS pool under resilver).
|
(ZFS pool under resilver). Holding one small record (path, size, mtime) per
|
||||||
Holding one small record (path, size, mtime) per file in memory
|
file in memory during a scan is acceptable; holding every file's hashes is
|
||||||
during a scan is acceptable; holding every file's hashes is not
|
not (they stay in the database).
|
||||||
(they stay in the database).
|
3. **Scan incrementally, analyze offline.** The expensive filesystem scan
|
||||||
3. **Scan incrementally, analyze offline.** The expensive filesystem
|
maintains a persistent database; an unchanged file is never read again on a
|
||||||
scan maintains a persistent database; an unchanged file is never
|
rescan. All analysis (`report`, `trees`) works from the database alone and
|
||||||
read again on a rescan. All analysis (`report`, `trees`) works from
|
must never touch the scanned filesystem again. `scan` is designed to be
|
||||||
the database alone and must never touch the scanned filesystem
|
cronned; the reports run at any time against the last completed scan.
|
||||||
again. `scan` is designed to be cronned; the reports run at any
|
4. **Clean stream separation.** Everything on stdout is machine-readable data.
|
||||||
time against the last completed scan.
|
All progress, warnings, and summaries go to stderr. Never mix them.
|
||||||
4. **Clean stream separation.** Everything on stdout is machine-readable
|
|
||||||
data. All progress, warnings, and summaries go to stderr. Never mix
|
|
||||||
them.
|
|
||||||
|
|
||||||
### Constraints
|
### Constraints
|
||||||
|
|
||||||
- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name:
|
- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: `sfdupes`.
|
||||||
`sfdupes`.
|
- Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one
|
||||||
- Dependencies: standard library, `github.com/spf13/cobra` for the
|
progress-bar library** (`github.com/schollz/progressbar/v3`), and **one SQLite
|
||||||
CLI, **one progress-bar library**
|
driver** (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled).
|
||||||
(`github.com/schollz/progressbar/v3`), and **one SQLite driver**
|
`github.com/spf13/viper` is permitted if configuration-file support is ever
|
||||||
(`modernc.org/sqlite`, pure Go, so builds keep cgo disabled).
|
needed, but is not currently used. No other third-party deps.
|
||||||
`github.com/spf13/viper` is permitted if configuration-file support
|
|
||||||
is ever needed, but is not currently used. No other third-party
|
|
||||||
deps.
|
|
||||||
- Cross-compilation is not a concern. Builds run with cgo disabled (the
|
- Cross-compilation is not a concern. Builds run with cgo disabled (the
|
||||||
`Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go.
|
`Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go.
|
||||||
- Analysis modes (`report`, `trees`) must be deterministic: identical
|
- Analysis modes (`report`, `trees`) must be deterministic: identical database
|
||||||
database contents, identical output, regardless of the order in
|
contents, identical output, regardless of the order in which records were
|
||||||
which records were inserted.
|
inserted.
|
||||||
|
|
||||||
### Subcommands
|
### Subcommands
|
||||||
|
|
||||||
Three subcommands, all implemented:
|
Three subcommands, all implemented:
|
||||||
|
|
||||||
1. `scan` — walk the filesystem and synchronize the database: one
|
1. `scan` — walk the filesystem and synchronize the database: one signature
|
||||||
signature record per regular file.
|
record per regular file.
|
||||||
2. `report` — file-level duplicate report from the database.
|
2. `report` — file-level duplicate report from the database.
|
||||||
3. `trees` — tree-level duplicate report: reconstruct the directory
|
3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy
|
||||||
hierarchy from the database records, compute a Merkle-style digest
|
from the database records, compute a Merkle-style digest per directory, and
|
||||||
per directory, and report maximal groups of identical trees.
|
report maximal groups of identical trees.
|
||||||
|
|
||||||
```
|
```
|
||||||
sfdupes scan [--workers N] [-x] PATH...
|
sfdupes scan [--workers N] [-x] PATH...
|
||||||
@@ -124,25 +113,22 @@ sfdupes trees > dupetrees.tsv
|
|||||||
|
|
||||||
All three subcommands operate on a single SQLite database file:
|
All three subcommands operate on a single SQLite database file:
|
||||||
|
|
||||||
- Location: the value of the `SFDUPES_DATABASE` environment variable
|
- Location: the value of the `SFDUPES_DATABASE` environment variable when set
|
||||||
when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`.
|
and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no
|
||||||
There is no command-line flag.
|
command-line flag.
|
||||||
- `scan` creates the database (and its parent directory) on first
|
- `scan` creates the database (and its parent directory) on first use. `report`
|
||||||
use. `report` and `trees` require an existing database; a missing
|
and `trees` require an existing database; a missing database file is a fatal
|
||||||
database file is a fatal error (exit 1) telling the user to run
|
error (exit 1) telling the user to run `scan` first.
|
||||||
`scan` first.
|
- The database uses WAL journal mode and a busy timeout, so running a report
|
||||||
- The database uses WAL journal mode and a busy timeout, so running a
|
while a cron `scan` is in progress is safe. The filesystem is authoritative;
|
||||||
report while a cron `scan` is in progress is safe. The filesystem
|
the database is an eventually-consistent reflection of it. Hashed records are
|
||||||
is authoritative; the database is an eventually-consistent
|
committed in batched transactions while the scan is still running (keeping the
|
||||||
reflection of it. Hashed records are committed in batched
|
WAL small and letting concurrent reports observe progress), so a report may
|
||||||
transactions while the scan is still running (keeping the WAL
|
see a scan's changes partially applied, and a scan that dies partway leaves a
|
||||||
small and letting concurrent reports observe progress), so a
|
valid database holding everything hashed so far; the next scan skips those
|
||||||
report may see a scan's changes partially applied, and a scan
|
records and converges toward the filesystem.
|
||||||
that dies partway leaves a valid database holding everything
|
- Schema (`PRAGMA user_version` is the schema version, currently 1; a database
|
||||||
hashed so far; the next scan skips those records and converges
|
with any other version is a fatal error):
|
||||||
toward the filesystem.
|
|
||||||
- Schema (`PRAGMA user_version` is the schema version, currently 1; a
|
|
||||||
database with any other version is a fatal error):
|
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
CREATE TABLE files (
|
CREATE TABLE files (
|
||||||
@@ -154,167 +140,147 @@ All three subcommands operate on a single SQLite database file:
|
|||||||
) WITHOUT ROWID;
|
) WITHOUT ROWID;
|
||||||
```
|
```
|
||||||
|
|
||||||
Paths are stored as BLOBs because Unix paths are raw bytes, not
|
Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
|
||||||
guaranteed UTF-8. `mtime` is used only for change detection; it is
|
UTF-8. `mtime` is used only for change detection; it is not part of the
|
||||||
not part of the duplicate key. `head` and `tail` are empty strings
|
duplicate key. `head` and `tail` are empty strings when the file has never
|
||||||
when the file has never been hashed because its size was unique as
|
been hashed because its size was unique as of the last scan that covered it;
|
||||||
of the last scan that covered it; such records still define the
|
such records still define the file for tree reconstruction but never
|
||||||
file for tree reconstruction but never participate in duplicate
|
participate in duplicate groups.
|
||||||
groups.
|
|
||||||
|
|
||||||
### `scan` mode
|
### `scan` mode
|
||||||
|
|
||||||
`scan` requires one or more `PATH` operands naming the trees to scan.
|
`scan` requires one or more `PATH` operands naming the trees to scan. There is
|
||||||
There is no default path; invoking `scan` with no operand is a usage
|
no default path; invoking `scan` with no operand is a usage error (usage message
|
||||||
error (usage message on stderr, exit 2). An operand may be a directory
|
on stderr, exit 2). An operand may be a directory or a regular file; an operand
|
||||||
or a regular file; an operand that does not exist is a fatal error
|
that does not exist is a fatal error (exit 1). Because database records persist
|
||||||
(exit 1). Because database records persist between runs and are keyed
|
between runs and are keyed by absolute path, each operand is resolved to an
|
||||||
by absolute path, each operand is resolved to an absolute, lexically
|
absolute, lexically cleaned path (symlinks are not resolved) before walking, so
|
||||||
cleaned path (symlinks are not resolved) before walking, so results do
|
results do not depend on the working directory. All operands belong to a single
|
||||||
not depend on the working directory. All operands belong to a single
|
scan and are enumerated concurrently: every operand seeds the shared walk worker
|
||||||
scan and are enumerated concurrently: every operand seeds the shared
|
pool. Overlapping operands are harmless — an operand that duplicates another or
|
||||||
walk worker pool. Overlapping operands are harmless — an operand that
|
lies under another is dropped before walking, so every file is reached exactly
|
||||||
duplicates another or lies under another is dropped before walking,
|
once and produces one database record.
|
||||||
so every file is reached exactly once and produces one database
|
|
||||||
record.
|
|
||||||
|
|
||||||
`scan` synchronizes the database with the filesystem state under the
|
`scan` synchronizes the database with the filesystem state under the scanned
|
||||||
scanned operands:
|
operands:
|
||||||
|
|
||||||
- Only a file whose size at least one other file shares is ever
|
- Only a file whose size at least one other file shares is ever read: a
|
||||||
read: a size-unique file cannot be a duplicate, so it is recorded
|
size-unique file cannot be a duplicate, so it is recorded without hashes
|
||||||
without hashes (`head` and `tail` empty). The size census covers
|
(`head` and `tail` empty). The size census covers every file walked this scan
|
||||||
every file walked this scan plus every database record outside
|
plus every database record outside the scanned operands, so a possible
|
||||||
the scanned operands, so a possible duplicate of a separately
|
duplicate of a separately scanned tree is still recognized.
|
||||||
scanned tree is still recognized.
|
- A file not yet in the database is inserted: hashed when its size is shared,
|
||||||
- A file not yet in the database is inserted: hashed when its size
|
without hashes otherwise.
|
||||||
is shared, without hashes otherwise.
|
- A file already in the database is **skipped without reading its contents**
|
||||||
- A file already in the database is **skipped without reading its
|
when its lstat size equals the recorded size and its lstat mtime is not newer
|
||||||
contents** when its lstat size equals the recorded size and its
|
than the recorded mtime. This is what makes a daily rescan cheap. Exception:
|
||||||
lstat mtime is not newer than the recorded mtime. This is what
|
an unchanged file whose record lacks hashes is hashed — and its record updated
|
||||||
makes a daily rescan cheap. Exception: an unchanged file whose
|
— once its size becomes shared, so hashing deferred by size-uniqueness happens
|
||||||
record lacks hashes is hashed — and its record updated — once its
|
as soon as it could matter.
|
||||||
size becomes shared, so hashing deferred by size-uniqueness
|
- A file whose mtime is newer than recorded, or whose size differs, is processed
|
||||||
happens as soon as it could matter.
|
as if new: re-hashed, or recorded without hashes, per the shared-size rule.
|
||||||
- A file whose mtime is newer than recorded, or whose size differs,
|
- A database record whose path lies under one of the scanned operands but was
|
||||||
is processed as if new: re-hashed, or recorded without hashes,
|
not successfully processed this run is deleted. This removes records for
|
||||||
per the shared-size rule.
|
deleted files. It also removes records for paths that failed to stat or hash
|
||||||
- A database record whose path lies under one of the scanned operands
|
this run: the database only ever contains signatures verified by the most
|
||||||
but was not successfully processed this run is deleted. This
|
recent scan that covered them (a subsequent successful scan re-adds such
|
||||||
removes records for deleted files. It also removes records for
|
files).
|
||||||
paths that failed to stat or hash this run: the database only ever
|
- Database records outside the scanned operands are untouched, so disjoint trees
|
||||||
contains signatures verified by the most recent scan that covered
|
can be scanned on different schedules into the same database.
|
||||||
them (a subsequent successful scan re-adds such files).
|
|
||||||
- Database records outside the scanned operands are untouched, so
|
|
||||||
disjoint trees can be scanned on different schedules into the same
|
|
||||||
database.
|
|
||||||
|
|
||||||
`scan` runs **three sequential phases over the whole scan**.
|
`scan` runs **three sequential phases over the whole scan**. Parallelism lives
|
||||||
Parallelism lives inside each phase; batched database writes begin
|
inside each phase; batched database writes begin during the hash phase:
|
||||||
during the hash phase:
|
|
||||||
|
|
||||||
1. **walk + stat** — enumerate the trees under all `PATH` operands
|
1. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
|
||||||
concurrently with the walk worker pool: every operand seeds the
|
with the walk worker pool: every operand seeds the shared queue, and each
|
||||||
shared queue, and each worker reads one directory at a time,
|
worker reads one directory at a time, handing discovered subdirectories back
|
||||||
handing discovered subdirectories back to the queue and running
|
to the queue and running `lstat` on each regular file as it is discovered
|
||||||
`lstat` on each regular file as it is discovered (while the
|
(while the directory's metadata is still hot). Sequential directory
|
||||||
directory's metadata is still hot). Sequential directory
|
enumeration is metadata-latency-bound and takes hours at tens of millions of
|
||||||
enumeration is metadata-latency-bound and takes hours at tens of
|
files; per-directory parallelism is what makes the walk tractable on large
|
||||||
millions of files; per-directory parallelism is what makes the
|
or busy pools. The walk builds the size census and resolves unchanged
|
||||||
walk tractable on large or busy pools. The walk builds the size
|
already-hashed files on the fly; every other file is carried to the hash
|
||||||
census and resolves unchanged already-hashed files on the fly;
|
phase as a (path, size, mtime) record.
|
||||||
every other file is carried to the hash phase as a (path, size,
|
2. **hash** — with the census complete, each carried file's size decides its
|
||||||
mtime) record.
|
fate. Size-unique files are never read: new or changed ones are recorded
|
||||||
2. **hash** — with the census complete, each carried file's size
|
without hashes in the update phase, unchanged unhashed ones simply keep
|
||||||
decides its fate. Size-unique files are never read: new or
|
their records. Every file with a shared size is hashed by the worker pool:
|
||||||
changed ones are recorded without hashes in the update phase,
|
read the first `min(1024, size)` bytes and the last `min(1024, size)` bytes
|
||||||
unchanged unhashed ones simply keep their records. Every file
|
(one read when `size <= 1024`, since the two windows coincide) and compute
|
||||||
with a shared size is hashed by the worker pool: read the first
|
the SHA-256 of each. Zero-length files have constant hashes and are never
|
||||||
`min(1024, size)` bytes and the last `min(1024, size)` bytes
|
opened. Files are hashed in **inode order** (minimizing seeks on spinning
|
||||||
(one read when `size <= 1024`, since the two windows coincide)
|
disks), and paths that are hard links to the same inode are **read once**,
|
||||||
and compute the SHA-256 of each. Zero-length files have constant
|
all sharing the one result — a hard-link backup farm costs one read per
|
||||||
hashes and are never opened. Files are hashed in **inode order**
|
inode, not per path. The phase total counts actual reads, so progress and
|
||||||
(minimizing seeks on spinning disks), and paths that are hard
|
ETA are meaningful. Completed records are committed in batched transactions
|
||||||
links to the same inode are **read once**, all sharing the one
|
**while hashing runs**, so a scan interrupted after hours keeps everything
|
||||||
result — a hard-link backup farm costs one read per inode, not
|
hashed so far and the next scan resumes cheaply, skipping records already
|
||||||
per path. The phase total counts actual reads, so progress and
|
written.
|
||||||
ETA are meaningful. Completed records are committed in batched
|
3. **update** — commit the final partial batch, the hash-less records for
|
||||||
transactions **while hashing runs**, so a scan interrupted after
|
size-unique new and changed files, and the deletions for records the scan
|
||||||
hours keeps everything hashed so far and the next scan resumes
|
did not verify (vanished files, plus paths that failed to stat or hash).
|
||||||
cheaply, skipping records already written.
|
|
||||||
3. **update** — commit the final partial batch, the hash-less
|
|
||||||
records for size-unique new and changed files, and the deletions
|
|
||||||
for records the scan did not verify (vanished files, plus paths
|
|
||||||
that failed to stat or hash).
|
|
||||||
|
|
||||||
Rules for the walk:
|
Rules for the walk:
|
||||||
|
|
||||||
- Only regular files. Skip directories, symlinks (do not follow,
|
- Only regular files. Skip directories, symlinks (do not follow, including
|
||||||
including symlink operands), sockets, FIFOs, and device nodes.
|
symlink operands), sockets, FIFOs, and device nodes.
|
||||||
- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs;
|
- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking
|
||||||
walking them would list every file once per snapshot).
|
them would list every file once per snapshot).
|
||||||
- Filesystem boundaries are crossed by default. With `-x`
|
- Filesystem boundaries are crossed by default. With `-x` (long form
|
||||||
(long form `--one-file-system`, following the GNU `du`/`rsync`
|
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
||||||
convention), never descend into a directory on a different
|
into a directory on a different filesystem than its `PATH` operand; each
|
||||||
filesystem than its `PATH` operand; each operand is bounded by its
|
operand is bounded by its own filesystem.
|
||||||
own filesystem.
|
- On any per-path error (permission denied, file vanished between passes,
|
||||||
- On any per-path error (permission denied, file vanished between
|
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
||||||
passes, unreadable): print a one-line warning to stderr, skip the
|
Per-file errors never abort the run; the final summary reports how many were
|
||||||
path, and continue. Per-file errors never abort the run; the final
|
skipped. As specified above, a skipped path that has a database record from an
|
||||||
summary reports how many were skipped. As specified above, a
|
earlier scan loses that record; an unreadable directory subtree likewise loses
|
||||||
skipped path that has a database record from an earlier scan loses
|
its records (accepted: the database mirrors what the latest scan could
|
||||||
that record; an unreadable directory subtree likewise loses its
|
|
||||||
records (accepted: the database mirrors what the latest scan could
|
|
||||||
actually verify).
|
actually verify).
|
||||||
|
|
||||||
Concurrency: the walk phase (which also stats files) and the hash
|
Concurrency: the walk phase (which also stats files) and the hash phase each use
|
||||||
phase each use a worker pool of `--workers` workers (default
|
a worker pool of `--workers` workers (default `runtime.NumCPU()`); the walk
|
||||||
`runtime.NumCPU()`); the walk parallelizes across directories,
|
parallelizes across directories, hashing across files. Both phases are
|
||||||
hashing across files. Both phases are seek-bound on spinning disks,
|
seek-bound on spinning disks, so raising `--workers` well past the core count
|
||||||
so raising `--workers` well past the core count can help on pools
|
can help on pools with many spindles. The main goroutine owns partitioning,
|
||||||
with many spindles. The main goroutine owns partitioning, database
|
database writes, and progress rendering; progress display must never block the
|
||||||
writes, and progress rendering; progress display must never block
|
workers.
|
||||||
the workers.
|
|
||||||
|
|
||||||
`scan` writes nothing to stdout. The summary line on stderr reports the
|
`scan` writes nothing to stdout. The summary line on stderr reports the files
|
||||||
files seen this run broken down by disposition, plus skips:
|
seen this run broken down by disposition, plus skips:
|
||||||
|
|
||||||
```
|
```
|
||||||
scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped
|
scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped
|
||||||
```
|
```
|
||||||
|
|
||||||
(`removed` counts deleted database records, which are not part of the
|
(`removed` counts deleted database records, which are not part of the files-seen
|
||||||
files-seen total.)
|
total.)
|
||||||
|
|
||||||
### `report` mode
|
### `report` mode
|
||||||
|
|
||||||
`report` reads every record from the database and takes no positional
|
`report` reads every record from the database and takes no positional arguments.
|
||||||
arguments.
|
|
||||||
|
|
||||||
**`report` must never touch the filesystem being analyzed.** It does not
|
**`report` must never touch the filesystem being analyzed.** It does not stat,
|
||||||
stat, open, or otherwise access any path that appears in the records; its
|
open, or otherwise access any path that appears in the records; its only I/O is
|
||||||
only I/O is reading the database and writing stdout/stderr. It must
|
reading the database and writing stdout/stderr. It must produce identical output
|
||||||
produce identical output whether or not the scanned filesystem is still
|
whether or not the scanned filesystem is still mounted.
|
||||||
mounted.
|
|
||||||
|
|
||||||
Processing:
|
Processing:
|
||||||
|
|
||||||
- Records without hashes (size-unique when last scanned) are
|
- Records without hashes (size-unique when last scanned) are excluded: their
|
||||||
excluded: their content is unknown, so they are never reported as
|
content is unknown, so they are never reported as duplicates.
|
||||||
duplicates.
|
- Group the remaining records by the key `(size, head_hash, tail_hash)`.
|
||||||
- Group the remaining records by the key
|
|
||||||
`(size, head_hash, tail_hash)`.
|
|
||||||
- Every group with two or more paths is a duplicate group.
|
- Every group with two or more paths is a duplicate group.
|
||||||
- Within each group, sort paths lexicographically (byte order). The
|
- Within each group, sort paths lexicographically (byte order). The first path
|
||||||
first path is the group's `first`; every other path is a `dupe`.
|
is the group's `first`; every other path is a `dupe`.
|
||||||
- Order groups by size descending (biggest reclaimable space first),
|
- Order groups by size descending (biggest reclaimable space first), tie-broken
|
||||||
tie-broken by `first` path ascending. Output must be fully
|
by `first` path ascending. Output must be fully deterministic for a given
|
||||||
deterministic for a given database state.
|
database state.
|
||||||
|
|
||||||
#### Report output format
|
#### Report output format
|
||||||
|
|
||||||
TSV on stdout: a header line, then one row per duplicate file (N-1 rows
|
TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a
|
||||||
for a group of N):
|
group of N):
|
||||||
|
|
||||||
```
|
```
|
||||||
first dupe size
|
first dupe size
|
||||||
@@ -322,94 +288,86 @@ first dupe size
|
|||||||
/srv/a/big.iso /srv/c/big-copy2.iso 4294967296
|
/srv/a/big.iso /srv/c/big-copy2.iso 4294967296
|
||||||
```
|
```
|
||||||
|
|
||||||
Summary to stderr: records read, number of duplicate groups, number of
|
Summary to stderr: records read, number of duplicate groups, number of dupe
|
||||||
dupe files, and total reclaimable bytes (sum of `size` over all dupe
|
files, and total reclaimable bytes (sum of `size` over all dupe rows) in human
|
||||||
rows) in human units.
|
units.
|
||||||
|
|
||||||
### `trees` mode
|
### `trees` mode
|
||||||
|
|
||||||
`trees` reads the same database as `report` (no positional arguments)
|
`trees` reads the same database as `report` (no positional arguments) and
|
||||||
and reports **entire duplicate directory trees**: directories under
|
reports **entire duplicate directory trees**: directories under which the exact
|
||||||
which the exact same set of relative paths exists with the exact same
|
same set of relative paths exists with the exact same file signatures.
|
||||||
file signatures.
|
|
||||||
|
|
||||||
**`trees` must never touch the filesystem being analyzed** — the same
|
**`trees` must never touch the filesystem being analyzed** — the same rule as
|
||||||
rule as `report`. The directory hierarchy is reconstructed purely from
|
`report`. The directory hierarchy is reconstructed purely from the paths in the
|
||||||
the paths in the records, split on `/`.
|
records, split on `/`.
|
||||||
|
|
||||||
Definitions:
|
Definitions:
|
||||||
|
|
||||||
- A file's **signature** is `(size, head_hash, tail_hash)` — mtime is
|
- A file's **signature** is `(size, head_hash, tail_hash)` — mtime is
|
||||||
informational and excluded. An unhashed record (empty hashes) has
|
informational and excluded. An unhashed record (empty hashes) has unknown
|
||||||
unknown content: its signature is treated as unique to that file,
|
content: its signature is treated as unique to that file, so a tree containing
|
||||||
so a tree containing an unhashed file never compares equal to any
|
an unhashed file never compares equal to any other tree.
|
||||||
other tree.
|
- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up:
|
||||||
- A directory's **digest** is a SHA-256 Merkle digest computed
|
serialize the directory's child entries — for a file child, its name and
|
||||||
bottom-up: serialize the directory's child entries — for a file
|
signature; for a subdirectory child, its name and that subdirectory's digest —
|
||||||
child, its name and signature; for a subdirectory child, its name
|
sort the serialized entries byte-lexicographically, and hash the
|
||||||
and that subdirectory's digest — sort the serialized entries
|
concatenation. Names are part of the digest: two trees whose files differ only
|
||||||
byte-lexicographically, and hash the concatenation. Names are part
|
in name are _not_ duplicates.
|
||||||
of the digest: two trees whose files differ only in name are *not*
|
- Two directories are **duplicate trees** when their digests are equal. Equal
|
||||||
duplicates.
|
digests imply equal recursive file count and equal total byte size.
|
||||||
- Two directories are **duplicate trees** when their digests are
|
|
||||||
equal. Equal digests imply equal recursive file count and equal
|
|
||||||
total byte size.
|
|
||||||
|
|
||||||
Known limitation (accepted): hard-linked paths are reported as
|
Known limitation (accepted): hard-linked paths are reported as duplicates by
|
||||||
duplicates by `report` and count toward duplicate trees — their
|
`report` and count toward duplicate trees — their content is genuinely identical
|
||||||
content is genuinely identical — even though they share storage, so
|
— even though they share storage, so removing one reclaims no space. Inode
|
||||||
removing one reclaims no space. Inode identity is used during the
|
identity is used during the scan to avoid redundant reads but is not persisted
|
||||||
scan to avoid redundant reads but is not persisted in the database.
|
in the database.
|
||||||
|
|
||||||
Known limitation (accepted): only regular files that appear in the
|
Known limitation (accepted): only regular files that appear in the database
|
||||||
database define a tree. Empty directories are invisible, and a file
|
define a tree. Empty directories are invisible, and a file skipped during the
|
||||||
skipped during the scan (e.g. permission error) in one copy but not the
|
scan (e.g. permission error) in one copy but not the other will make
|
||||||
other will make otherwise-identical trees compare as different.
|
otherwise-identical trees compare as different.
|
||||||
|
|
||||||
Processing:
|
Processing:
|
||||||
|
|
||||||
- Build the hierarchy, compute every directory's digest, and group
|
- Build the hierarchy, compute every directory's digest, and group directories
|
||||||
directories by digest. Every group with two or more directories is a
|
by digest. Every group with two or more directories is a duplicate-tree group.
|
||||||
duplicate-tree group.
|
- **Report only maximal trees.** A group is suppressed when its members' parents
|
||||||
- **Report only maximal trees.** A group is suppressed when its
|
are pairwise distinct directories that all share a single digest — such a
|
||||||
members' parents are pairwise distinct directories that all share a
|
group is wholly implied by its parents' (or a further ancestor's) group.
|
||||||
single digest — such a group is wholly implied by its parents' (or a
|
Groups containing sibling directories, or members whose parents differ, are
|
||||||
further ancestor's) group. Groups containing sibling directories, or
|
always reported.
|
||||||
members whose parents differ, are always reported.
|
- Within each group, sort paths lexicographically (byte order); the first path
|
||||||
- Within each group, sort paths lexicographically (byte order); the
|
is `first`, every other path is a `dupe`.
|
||||||
first path is `first`, every other path is a `dupe`.
|
- Order groups by total tree size descending, tie-broken by `first` path
|
||||||
- Order groups by total tree size descending, tie-broken by `first`
|
ascending. Output must be fully deterministic for a given input.
|
||||||
path ascending. Output must be fully deterministic for a given
|
|
||||||
input.
|
|
||||||
|
|
||||||
#### Trees output format
|
#### Trees output format
|
||||||
|
|
||||||
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows
|
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
|
||||||
for a group of N). `files` is the recursive regular-file count of one
|
group of N). `files` is the recursive regular-file count of one copy of the
|
||||||
copy of the tree; `size` is the recursive total byte size of one copy:
|
tree; `size` is the recursive total byte size of one copy:
|
||||||
|
|
||||||
```
|
```
|
||||||
first dupe files size
|
first dupe files size
|
||||||
/srv/a/project /srv/backup/project 3417 104857600
|
/srv/a/project /srv/backup/project 3417 104857600
|
||||||
```
|
```
|
||||||
|
|
||||||
Summary to stderr: records read, number of duplicate-tree groups,
|
Summary to stderr: records read, number of duplicate-tree groups, number of dupe
|
||||||
number of dupe trees, and total reclaimable bytes (sum of `size` over
|
trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human
|
||||||
all dupe rows) in human units.
|
units.
|
||||||
|
|
||||||
### Progress
|
### Progress
|
||||||
|
|
||||||
Use the progress-bar library for all scan progress; rendering in the
|
Use the progress-bar library for all scan progress; rendering in the style of
|
||||||
style of `pv` is the model. All progress goes to stderr.
|
`pv` is the model. All progress goes to stderr.
|
||||||
|
|
||||||
Each phase gets its own display, rendered the moment the phase
|
Each phase gets its own display, rendered the moment the phase starts — a scan
|
||||||
starts — a scan must never look hung. Loading the existing-record
|
must never look hung. Loading the existing-record index (`load`) and the walk
|
||||||
index (`load`) and the walk have no known totals while running: show
|
have no known totals while running: show a live count, rate, and elapsed time
|
||||||
a live count, rate, and elapsed time (spinner-style, no percentage or
|
(spinner-style, no percentage or ETA). The hash and update phases have exact
|
||||||
ETA). The hash and update phases
|
totals — only files that actually need hashing appear in the hash total, so its
|
||||||
have exact totals — only files that actually need hashing appear in
|
ETA is meaningful. Required elements for the bars with known totals:
|
||||||
the hash total, so its ETA is meaningful. Required elements for the
|
|
||||||
bars with known totals:
|
|
||||||
|
|
||||||
- elapsed time
|
- elapsed time
|
||||||
- estimated time remaining
|
- estimated time remaining
|
||||||
@@ -424,145 +382,136 @@ hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54
|
|||||||
|
|
||||||
Additional requirements:
|
Additional requirements:
|
||||||
|
|
||||||
- When stderr is not a TTY, do not emit ANSI redraws: print a plain
|
- When stderr is not a TTY, do not emit ANSI redraws: print a plain one-line
|
||||||
one-line progress update no more often than every 5 seconds instead.
|
progress update no more often than every 5 seconds instead.
|
||||||
- Progress updates are driven from the main goroutine and must be
|
- Progress updates are driven from the main goroutine and must be non-blocking
|
||||||
non-blocking with respect to the worker pool.
|
with respect to the worker pool.
|
||||||
- `report` and `trees` modes need no progress display, only their
|
- `report` and `trees` modes need no progress display, only their stderr
|
||||||
stderr summaries.
|
summaries.
|
||||||
|
|
||||||
### Error handling and exit codes
|
### Error handling and exit codes
|
||||||
|
|
||||||
- `0`: success, even if individual files were skipped with warnings.
|
- `0`: success, even if individual files were skipped with warnings.
|
||||||
- `1`: fatal error (e.g., a `PATH` operand does not exist, the
|
- `1`: fatal error (e.g., a `PATH` operand does not exist, the database cannot
|
||||||
database cannot be created/opened/read/written, a missing database
|
be created/opened/read/written, a missing database for `report`/`trees`,
|
||||||
for `report`/`trees`, stdout write failure).
|
stdout write failure).
|
||||||
- `2`: usage error (including `scan` with no `PATH` operand and
|
- `2`: usage error (including `scan` with no `PATH` operand and `report`/`trees`
|
||||||
`report`/`trees` with any positional argument).
|
with any positional argument).
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|
||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: the normalized executables in `script/` are the entrypoints
|
standard: the normalized executables in `script/` are the entrypoints for the
|
||||||
for the development workflow, and the `Makefile` targets are thin
|
development workflow, and the `Makefile` targets are thin shims that call them.
|
||||||
shims that call them. Every script is POSIX `sh`, resolves the
|
Every script is POSIX `sh`, resolves the repository root itself so it can be run
|
||||||
repository root itself so it can be run from any working directory,
|
from any working directory, and may be invoked directly. The provided
|
||||||
and may be invoked directly. The provided entrypoints are:
|
entrypoints are:
|
||||||
|
|
||||||
- `script/bootstrap` — install everything needed to build and
|
- `script/bootstrap` — install everything needed to build and develop this
|
||||||
develop this repository, idempotently, assuming nothing is
|
repository, idempotently, assuming nothing is present. `git`, `make`, `go`,
|
||||||
present. `git`, `make`, and `go` come from the first of nix, apt,
|
and `node` come from the first of nix, apt, brew, or apk found on the host,
|
||||||
brew, or apk found on the host, and are presence-checked only.
|
and are presence-checked only; `node` is an unpinned host runtime like the
|
||||||
`golangci-lint` is deliberately **not** installed: it runs from a
|
rest, because nvm's prebuilt node is glibc-linked and does not run on this
|
||||||
digest-pinned image via `script/lint` and never from a host
|
repo's musl/Alpine build image. The Markdown formatter itself — `prettier` —
|
||||||
install, so there is no host copy to drift from the pin. A missing
|
is pinned by `yarn.lock`'s integrity hash and installed with
|
||||||
`docker` is warned about rather than installed or treated as
|
`yarn install --frozen-lockfile`. `golangci-lint` is deliberately **not**
|
||||||
fatal — everything except linting works without it. Ends with
|
installed: it runs from a digest-pinned image via `script/lint` and never from
|
||||||
`go mod download`.
|
a host install, so there is no host copy to drift from the pin. A missing
|
||||||
|
`docker` is warned about rather than installed or treated as fatal —
|
||||||
|
everything except linting works without it. Ends with `go mod download` and
|
||||||
|
the `yarn` install.
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`.
|
`script/bootstrap`, then `script/install-precommit`.
|
||||||
- `script/projectname` — print this project's name (`sfdupes`).
|
- `script/projectname` — print this project's name (`sfdupes`). Scripts that
|
||||||
Scripts that need the name call it, so they stay identical across
|
need the name call it, so they stay identical across repositories.
|
||||||
repositories.
|
- `script/test` — run the test suite with a 30-second timeout and coverage
|
||||||
- `script/test` — run the test suite with a 30-second timeout and
|
enabled, rerunning verbosely on failure so the logs show which test failed.
|
||||||
coverage enabled, rerunning verbosely on failure so the logs show
|
- `script/lint` — run the linter. It builds `Dockerfile.lint`, which copies the
|
||||||
which test failed.
|
repository into the digest-pinned `golangci/golangci-lint` image and runs
|
||||||
- `script/lint` — run the linter. It builds `Dockerfile.lint`, which
|
`golangci-lint config verify` and `golangci-lint run` as build steps, so a
|
||||||
copies the repository into the digest-pinned
|
successful build is a clean lint. The linter is never run on the host, which
|
||||||
`golangci/golangci-lint` image and runs
|
makes a working `docker` the one prerequisite for linting — and therefore for
|
||||||
`golangci-lint config verify` and `golangci-lint run` as build
|
`make check` and the pre-commit hook. Offline machines: the gate steps
|
||||||
steps, so a successful build is a clean lint. The linter is never
|
themselves make no network calls. `golangci-lint run` does not, and neither
|
||||||
run on the host, which makes a working `docker` the one
|
does `golangci-lint config verify` — it validates against a schema the pinned
|
||||||
prerequisite for linting — and therefore for `make check` and the
|
binary embeds, measured under `--network none` to both pass a valid config and
|
||||||
pre-commit hook. Offline machines: the gate steps themselves make
|
reject an invalid one. The build around them does. `Dockerfile.lint` runs
|
||||||
no network calls. `golangci-lint run` does not, and neither does
|
`go mod download` before the gates and this module has external dependencies,
|
||||||
`golangci-lint config verify` — it validates against a schema the
|
so a first lint on a machine with a cold BuildKit cache reaches the network
|
||||||
pinned binary embeds, measured under `--network none` to both
|
there (as well as pulling the pinned image); under `--network none` it fails
|
||||||
pass a valid config and reject an invalid one. The build around
|
at that step, before any gate. That layer sits above the gates and stays
|
||||||
them does. `Dockerfile.lint` runs `go mod download` before the
|
cached, so once it is warm `script/lint` — and with it `make check` — runs
|
||||||
gates and this module has external dependencies, so a first lint
|
entirely offline, until `go.mod` or `go.sum` changes and the download layer
|
||||||
on a machine with a cold BuildKit cache reaches the network there
|
goes cold again. Because the daemon only ever sees a build context, this works
|
||||||
(as well as pulling the pinned image); under `--network none` it
|
when the docker daemon is remote and bind mounts are impossible.
|
||||||
fails at that step, before any gate. That layer sits above the
|
- `script/fmt` — format in place: `gofmt -s -w` for Go sources and `prettier`
|
||||||
gates and stays cached, so once it is warm `script/lint` — and
|
for Markdown (`--tab-width 4 --prose-wrap always`, the house settings, also
|
||||||
with it `make check` — runs entirely offline, until `go.mod` or
|
carried in `.prettierrc`). prettier is the pinned devDependency in
|
||||||
`go.sum` changes and the download layer goes cold again. Because
|
`package.json`/`yarn.lock`, installed by `script/bootstrap`.
|
||||||
the daemon only ever sees a build context, this works when the
|
- `script/fmt-check` — the read-only counterpart of `script/fmt`: runs both
|
||||||
docker daemon is remote and bind mounts are impossible.
|
checks, reports each independently so it is clear which failed, and exits
|
||||||
- `script/fmt` — format the Go sources in place (`gofmt -s -w`).
|
non-zero if either found unformatted files instead of writing.
|
||||||
Markdown is not formatted.
|
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in
|
||||||
- `script/fmt-check` — the read-only counterpart of `script/fmt`:
|
that order. Modifies nothing. Needs `docker`, because `script/lint` does.
|
||||||
prints any unformatted file and exits non-zero instead of writing.
|
- `script/docker` — build the Docker image, tagged with the name from
|
||||||
- `script/check` — run `script/test`, `script/lint`, and
|
`script/projectname`. The `Dockerfile` runs the gates as build steps, so this
|
||||||
`script/fmt-check`, in that order. Modifies nothing. Needs
|
is also the check a developer or reviewer runs by hand.
|
||||||
`docker`, because `script/lint` does.
|
- `script/cibuild` — build the Docker image untagged. This is what the Gitea
|
||||||
- `script/docker` — build the Docker image, tagged with the name
|
workflow runs on push; because the gates run as build steps, a successful
|
||||||
from `script/projectname`. The `Dockerfile` runs the gates as
|
build implies the repository is green.
|
||||||
build steps, so this is also the check a developer or reviewer
|
- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a
|
||||||
runs by hand.
|
no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then
|
||||||
- `script/cibuild` — build the Docker image untagged. This is what
|
`script/check`.
|
||||||
the Gitea workflow runs on push; because the gates run as build
|
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||||
steps, a successful build implies the repository is green.
|
`script/precommit`. The hook is written to the common git directory, so the
|
||||||
- `script/precommit` — run by the git pre-commit hook: `go mod tidy`
|
main checkout and every worktree share it.
|
||||||
must be a no-op (a resulting change to `go.mod` or `go.sum` fails
|
- `script/verify-lint-image-pin` — fail unless the `golangci/golangci-lint`
|
||||||
the commit), then `script/check`.
|
reference in `Dockerfile.lint` and the one in the `Dockerfile` lint stage are
|
||||||
- `script/install-precommit` — install the git pre-commit hook that
|
the same image at the same digest, naming both if not. The linter is pinned in
|
||||||
runs `script/precommit`. The hook is written to the common git
|
those two files and nothing else keeps them in sync, so a bump applied to one
|
||||||
directory, so the main checkout and every worktree share it.
|
alone would leave `make lint` and the `Dockerfile`'s fail-fast lint stage
|
||||||
- `script/verify-lint-image-pin` — fail unless the
|
checking the same tree against different rulesets, both green. The guard
|
||||||
`golangci/golangci-lint` reference in `Dockerfile.lint` and the
|
restates neither pin — a third copy would be the same drift one file further
|
||||||
one in the `Dockerfile` lint stage are the same image at the same
|
out — and runs as a gate in both files, so `make lint`, `make check` and
|
||||||
digest, naming both if not. The linter is pinned in those two
|
`make docker` all catch it.
|
||||||
files and nothing else keeps them in sync, so a bump applied to
|
|
||||||
one alone would leave `make lint` and the `Dockerfile`'s
|
|
||||||
fail-fast lint stage checking the same tree against different
|
|
||||||
rulesets, both green. The guard restates neither pin — a third
|
|
||||||
copy would be the same drift one file further out — and runs as a
|
|
||||||
gate in both files, so `make lint`, `make check` and `make docker`
|
|
||||||
all catch it.
|
|
||||||
|
|
||||||
`script/verify-linter-pin` used to live here. It compared a linter
|
`script/verify-linter-pin` used to live here. It compared a linter binary
|
||||||
binary against a version pin in `script/bootstrap`, and both of its
|
against a version pin in `script/bootstrap`, and both of its subjects are gone:
|
||||||
subjects are gone: no linter binary is copied between build stages any
|
no linter binary is copied between build stages any more, and bootstrap pins no
|
||||||
more, and bootstrap pins no version because it installs no linter. The
|
version because it installs no linter. The drift it existed to catch has moved
|
||||||
drift it existed to catch has moved from binary-versus-pin to
|
from binary-versus-pin to pin-versus-pin, which is what
|
||||||
pin-versus-pin, which is what `script/verify-lint-image-pin` above
|
`script/verify-lint-image-pin` above checks.
|
||||||
checks.
|
|
||||||
|
|
||||||
`script/lint`, `script/docker` and `script/cibuild` all pass a freshly
|
`script/lint`, `script/docker` and `script/cibuild` all pass a freshly computed
|
||||||
computed `CHECK_EPOCH` build argument, and the gate steps in
|
`CHECK_EPOCH` build argument, and the gate steps in `Dockerfile.lint` and
|
||||||
`Dockerfile.lint` and `Dockerfile` reference it. Without that, an
|
`Dockerfile` reference it. Without that, an unchanged tree lets Docker serve the
|
||||||
unchanged tree lets Docker serve the gate layers from cache and the
|
gate layers from cache and the build exits 0 having executed no tests and no
|
||||||
build exits 0 having executed no tests and no lint — a green it never
|
lint — a green it never earned, and one this repository has produced twice.
|
||||||
earned, and one this repository has produced twice. `CHECK_EPOCH`
|
`CHECK_EPOCH` invalidates the gate layers on every run while leaving the pinned
|
||||||
invalidates the gate layers on every run while leaving the pinned base
|
base images and the dependency layers cached. `script/lint`'s value carries the
|
||||||
images and the dependency layers cached. `script/lint`'s value carries
|
process id as well as the epoch, because two lint runs land inside the same
|
||||||
the process id as well as the epoch, because two lint runs land inside
|
second easily and a bare epoch would cache the second one.
|
||||||
the same second easily and a bare epoch would cache the second one.
|
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
The `script/` entrypoints above are where the implementations live;
|
The `script/` entrypoints above are where the implementations live; the
|
||||||
the `Makefile` targets are shims onto them, except `build`, which
|
`Makefile` targets are shims onto them, except `build`, which carries the
|
||||||
carries the compile recipe:
|
compile recipe:
|
||||||
|
|
||||||
- `make` / `make build` — build the `sfdupes` binary (cgo
|
- `make` / `make build` — build the `sfdupes` binary (cgo disabled); building is
|
||||||
disabled); building is the default target.
|
the default target.
|
||||||
- `make bootstrap` — install the build and development
|
- `make bootstrap` — install the build and development dependencies.
|
||||||
dependencies.
|
- `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook.
|
||||||
- `make setup` — prepare a fresh clone: `bootstrap` plus the
|
- `make test` — run the test suite (30-second timeout; reruns with `-v` on
|
||||||
pre-commit hook.
|
failure).
|
||||||
- `make test` — run the test suite (30-second timeout; reruns with
|
- `make lint` — run `golangci-lint` with the repo config, in Docker (see
|
||||||
`-v` on failure).
|
`script/lint`); requires `docker`.
|
||||||
- `make lint` — run `golangci-lint` with the repo config, in Docker
|
- `make fmt` / `make fmt-check` — format Go and Markdown sources / verify both
|
||||||
(see `script/lint`); requires `docker`.
|
without writing.
|
||||||
- `make fmt` / `make fmt-check` — format Go sources / verify
|
- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires
|
||||||
formatting without writing.
|
`docker`, via `lint`.
|
||||||
- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing.
|
- `make docker` — build the Docker image, which runs the gates as build stages.
|
||||||
Requires `docker`, via `lint`.
|
|
||||||
- `make docker` — build the Docker image, which runs the gates as
|
|
||||||
build stages.
|
|
||||||
- `make hooks` — install the pre-commit hook.
|
- `make hooks` — install the pre-commit hook.
|
||||||
- `make clean` — remove the binary.
|
- `make clean` — remove the binary.
|
||||||
|
|
||||||
@@ -572,8 +521,8 @@ All of the following, run in this directory, must pass:
|
|||||||
|
|
||||||
1. `make check` passes (tests, lint, `gofmt`).
|
1. `make check` passes (tests, lint, `gofmt`).
|
||||||
2. `make docker` succeeds.
|
2. `make docker` succeeds.
|
||||||
3. Smoke test — create a throwaway tree in a temp dir (never test
|
3. Smoke test — create a throwaway tree in a temp dir (never test against real
|
||||||
against real data):
|
data):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
d=$(mktemp -d)
|
d=$(mktemp -d)
|
||||||
@@ -605,30 +554,29 @@ All of the following, run in this directory, must pass:
|
|||||||
./sfdupes report
|
./sfdupes report
|
||||||
```
|
```
|
||||||
|
|
||||||
(The scan database lives inside `$d` here purely for test hygiene;
|
(The scan database lives inside `$d` here purely for test hygiene; scanning
|
||||||
scanning `$d` therefore also records the SQLite file itself, which
|
`$d` therefore also records the SQLite file itself, which is harmless.)
|
||||||
is harmless.)
|
|
||||||
|
|
||||||
Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin`
|
Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
|
||||||
form one group (two dupe rows, `first` is the lexicographically
|
group (two dupe rows, `first` is the lexicographically smallest path);
|
||||||
smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group;
|
`t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
|
||||||
`t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group;
|
`t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair;
|
||||||
`tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and
|
`empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere; groups
|
||||||
`tiny3` appear nowhere; groups ordered by size descending.
|
ordered by size descending.
|
||||||
|
|
||||||
Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe`
|
Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2
|
||||||
`$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is
|
files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal
|
||||||
suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3`
|
(implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set
|
||||||
appears nowhere (its file set differs by name).
|
differs by name).
|
||||||
|
|
||||||
Expected from the second `report` (after the modify/delete rescan):
|
Expected from the second `report` (after the modify/delete rescan):
|
||||||
`one.bin` has left its group (its content changed), so
|
`one.bin` has left its group (its content changed), so
|
||||||
`copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is
|
`copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is gone from the
|
||||||
gone from the database.
|
database.
|
||||||
|
|
||||||
The test suite automates this scenario (see `scan_test.go`), plus a
|
The test suite automates this scenario (see `scan_test.go`), plus a negative
|
||||||
negative check: `report` and `trees` operate on the database alone
|
check: `report` and `trees` operate on the database alone and never touch
|
||||||
and never touch the scanned filesystem.
|
the scanned filesystem.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
@@ -636,11 +584,10 @@ Tracked in [TODO.md](TODO.md).
|
|||||||
|
|
||||||
## Non-goals
|
## Non-goals
|
||||||
|
|
||||||
- No full-content verification, no byte-for-byte compare, no deletion
|
- No full-content verification, no byte-for-byte compare, no deletion or linking
|
||||||
or linking of duplicates. The reports are advisory; acting on them is
|
of duplicates. The reports are advisory; acting on them is the user's job.
|
||||||
the user's job.
|
- No persistence beyond the SQLite database described above; no export/import
|
||||||
- No persistence beyond the SQLite database described above; no
|
formats.
|
||||||
export/import formats.
|
|
||||||
- No daemon or filesystem watcher; scheduling rescans is cron's job.
|
- No daemon or filesystem watcher; scheduling rescans is cron's job.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
@@ -1,419 +1,386 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
- take an issue from the `1.0.0` milestone on the tracker; work not
|
- take an issue from the `1.0.0` milestone on the tracker; work not yet on the
|
||||||
yet on the tracker gets filed as an issue first
|
tracker gets filed as an issue first
|
||||||
- branch (from `main`)
|
- branch (from `main`)
|
||||||
- do the work, with tests, in small focused commits
|
- do the work, with tests, in small focused commits
|
||||||
- record it at the top of Completed Steps (`TODO.md` changes in the
|
- record it at the top of Completed Steps (`TODO.md` changes in the same commit
|
||||||
same commit as the work)
|
as the work)
|
||||||
- push the branch and open a PR whose title ends with
|
- push the branch and open a PR whose title ends with ` (closes #N)`
|
||||||
` (closes #N)`
|
- an independent review gates the merge; every finding is addressed or
|
||||||
- an independent review gates the merge; every finding is addressed
|
explicitly rebutted on the PR
|
||||||
or explicitly rebutted on the PR
|
|
||||||
- merge to `main` once the review passes
|
- merge to `main` once the review passes
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
- pre-1.0
|
- pre-1.0
|
||||||
- the Gitea tracker is authoritative for the pre-1.0 backlog: the
|
- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues
|
||||||
open issues under the `1.0.0` milestone are what remains before
|
under the `1.0.0` milestone are what remains before the tag, and this file
|
||||||
the tag, and this file records history and process, not the queue
|
records history and process, not the queue
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
- take the next issue from the `1.0.0` milestone on the tracker:
|
- take the next issue from the `1.0.0` milestone on the tracker:
|
||||||
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is
|
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source
|
||||||
the source of truth for what is left before 1.0.0. Individual
|
of truth for what is left before 1.0.0. Individual issues are deliberately not
|
||||||
issues are deliberately not restated here; a copy in this file
|
restated here; a copy in this file drifts out of date the moment the tracker
|
||||||
drifts out of date the moment the tracker moves
|
moves
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- run all linting in Docker via `Dockerfile.lint` and `script/lint`
|
- restore Markdown formatting in `script/fmt`/`fmt-check` and reformat all
|
||||||
(2026-08-10, branch `next`, closes
|
Markdown to the house prettier settings (2026-09-21, closes
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/46): per the owner ruling, the
|
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
||||||
linter runs inside a container invoked through the `script/`
|
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and
|
||||||
entrypoint and is never installed on a host. New root
|
`Dockerfile.lint` (2026-08-10, branch `next`, closes
|
||||||
`Dockerfile.lint` COPYs the repo into the digest-pinned
|
https://git.eeqj.de/sneak/sfdupes/issues/25): dropped the false
|
||||||
`golangci/golangci-lint:v2.12.2` image and runs
|
`(Debian-based)` parenthetical (v2.12.1 was Debian too) and the redundant tag,
|
||||||
`golangci-lint config verify` and `golangci-lint run` as build
|
so both pins are the policy `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare
|
||||||
steps, so a successful build IS a clean lint; `script/lint` is
|
`FROM image@sha256:...`. Digest unchanged. `script/verify-lint-image-pin`
|
||||||
reduced to building it. `script/bootstrap` loses the `go install`,
|
parses those `FROM` lines and still matches the tagless form; its advice line
|
||||||
the pin constants, the version parser and `verify_golangci_lint`
|
lost the now meaningless "tag and digest". With no tag in either reference, a
|
||||||
outright rather than hardening them — with nothing linting on the
|
tag-only disagreement no longer exists — a one-sided tag is caught as a plain
|
||||||
host, the `$GOPATH/bin` versus `PATH` problem that motivated them has
|
mismatch.
|
||||||
no subject — and now warns rather than fails when `docker` is absent.
|
|
||||||
Two traps handled. A lint build on an unchanged tree returns success
|
|
||||||
in well under a second having run no linter, which is
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/32 and
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/39 again, so
|
|
||||||
`Dockerfile.lint` carries `ARG CHECK_EPOCH` referenced
|
|
||||||
inside every gate `RUN` (BuildKit hashes the expanded command, not
|
|
||||||
the declaration) and `script/lint` passes `"$(date +%s)-$$"` — the
|
|
||||||
PID matters because two lint runs land inside the same second easily.
|
|
||||||
And nothing inside an image build may shell out to docker, so the
|
|
||||||
main `Dockerfile`'s lint stage now invokes `golangci-lint` directly
|
|
||||||
instead of `make lint`, and its build stage runs `make test` and
|
|
||||||
`make fmt-check` instead of the `make check` aggregate (`make`, not
|
|
||||||
the scripts bare, because the Makefile's `export CGO_ENABLED = 0`
|
|
||||||
only reaches what it invokes). `COPY --from=lint`
|
|
||||||
`/usr/bin/golangci-lint` is replaced by
|
|
||||||
`COPY --from=lint /src/go.sum /dev/null`: the copied binary was the
|
|
||||||
only edge forcing BuildKit to finish linting before the build stage
|
|
||||||
starts, and dropping it without replacing the edge would have ended
|
|
||||||
fail-fast linting silently under a still-green build. That is
|
|
||||||
canonical `REPO_POLICIES.md:107`'s ordering edge, restored.
|
|
||||||
`ENV PATH=/home/builder/go/bin:$PATH` is gone with the `go install`
|
|
||||||
that justified it. `script/verify-linter-pin` is retired, deleted
|
|
||||||
along with its README entry, because both of its subjects ceased to
|
|
||||||
exist in the same change: it compared a linter binary against
|
|
||||||
`GOLANGCI_LINT_VERSION` in `script/bootstrap`, and there is now
|
|
||||||
neither a binary crossing between stages nor a version pin in
|
|
||||||
bootstrap. The drift it guarded has not gone away, it has moved — the
|
|
||||||
linter is still pinned twice, now as the `FROM` line of
|
|
||||||
`Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage,
|
|
||||||
with nothing syncing them, which is exactly what
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its
|
|
||||||
replacement is one new `script/verify-lint-image-pin`,
|
|
||||||
run as a gate in both files, which compares the two references to
|
|
||||||
each other and deliberately restates neither: a hardcoded expected
|
|
||||||
digest would be a third copy and the same drift one file further out.
|
|
||||||
`golangci-lint config verify` is included per the ruling, and the
|
|
||||||
concern about its unpinned live HTTPS schema fetch was measured
|
|
||||||
rather than assumed — under `--network none` the pinned binary both
|
|
||||||
passes a valid config and rejects an invalid one with the jsonschema
|
|
||||||
error, so it validates from an embedded schema and makes no network
|
|
||||||
call of its own. The README scopes that to the gate steps rather
|
|
||||||
than to linting as a whole: `Dockerfile.lint` runs `go mod download`
|
|
||||||
above them, so a cold cache still needs the network and only a warm
|
|
||||||
one lints offline. Verified: `make lint` green with every `PATH`
|
|
||||||
directory containing a `golangci-lint` removed
|
|
||||||
(`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`;
|
|
||||||
`command -v golangci-lint` empty); two consecutive `script/lint` runs
|
|
||||||
on an untouched tree both executed the linter, 27.7s and 28.7s in the
|
|
||||||
lint step under distinct epochs with the `COPY . .` layer `CACHED`
|
|
||||||
above them, at 42.2s and 41.8s wall clock — the no-cache rule was not
|
|
||||||
weakened to shorten that. Negative control: a planted
|
|
||||||
`var unusedIssue46Sentinel = 1` failed `script/lint` with
|
|
||||||
`report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and
|
|
||||||
failed `make docker` at `[lint 9/9]` with the build stage stopped at
|
|
||||||
`[builder 3/12]` — `COPY --from=lint`, `script/bootstrap`, the test
|
|
||||||
gate and `make build` all zero occurrences — then reverted clean. The
|
|
||||||
drift guard fails on a tag-only disagreement, on a digest-only
|
|
||||||
disagreement, and on an unreadable reference, naming both sides.
|
|
||||||
`make docker` green in 5m35s with all six gates executing under one
|
|
||||||
epoch (lint 37.6s, test 25.2s reporting
|
|
||||||
`ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`).
|
|
||||||
The non-root quirk still holds: in the builder image with the Go test
|
|
||||||
cache off, `--user 0:0` fails `TestScanHardlinkRunFailsTogether`
|
|
||||||
(exit 1) where the unprivileged user passes (exit 0). Noted for
|
|
||||||
follow-up, not fixed here: `golangci-lint` warns that the
|
|
||||||
`gomodguard` linter is deprecated since v2.12.0 in favour of
|
|
||||||
`gomodguard_v2`.
|
|
||||||
|
|
||||||
- install the Docker build stage's prerequisites by running
|
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
|
||||||
`script/bootstrap` instead of `apk add --no-cache make` inline
|
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
|
||||||
(2026-08-09, branch `dockerfile-bootstrap`, closes #42): canonical
|
owner ruling, the linter runs inside a container invoked through the `script/`
|
||||||
`REPO_POLICIES.md:97` requires it, and the inline install left the
|
entrypoint and is never installed on a host. New root `Dockerfile.lint` COPYs
|
||||||
build stage maintaining its own notion of the toolchain — exactly
|
the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and
|
||||||
the divergence #24 exists to close, one layer down. The stage now
|
runs `golangci-lint config verify` and `golangci-lint run` as build steps, so
|
||||||
copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
|
a successful build IS a clean lint; `script/lint` is reduced to building it.
|
||||||
which ends in `go mod download`, so the separate invocation of that
|
`script/bootstrap` loses the `go install`, the pin constants, the version
|
||||||
is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, and moves
|
parser and `verify_golangci_lint` outright rather than hardening them — with
|
||||||
above the bootstrap layer. It is the only edge making this stage
|
nothing linting on the host, the `$GOPATH/bin` versus `PATH` problem that
|
||||||
depend on the lint stage, so deleting it as redundant would end
|
motivated them has no subject — and now warns rather than fails when `docker`
|
||||||
fail-fast linting silently. Letting bootstrap install its own linter
|
is absent. Two traps handled. A lint build on an unchanged tree returns
|
||||||
here would have reintroduced the second toolchain and paid for a
|
success in well under a second having run no linter, which is
|
||||||
from-source build of it. What makes the two stages provably one
|
https://git.eeqj.de/sneak/sfdupes/issues/32 and
|
||||||
toolchain rather than two that happen to agree is a new
|
https://git.eeqj.de/sneak/sfdupes/issues/39 again, so `Dockerfile.lint`
|
||||||
`script/verify-linter-pin`, run in the build stage on the binary
|
carries `ARG CHECK_EPOCH` referenced inside every gate `RUN` (BuildKit hashes
|
||||||
that arrives from the lint stage, before bootstrap: it fails the
|
the expanded command, not the declaration) and `script/lint` passes
|
||||||
build naming both versions unless that binary is the version
|
`"$(date +%s)-$$"` — the PID matters because two lint runs land inside the
|
||||||
`script/bootstrap` pins. Bootstrap's own check could not serve that
|
same second easily. And nothing inside an image build may shell out to docker,
|
||||||
purpose — it reinstalls its pin from source and then verifies
|
so the main `Dockerfile`'s lint stage now invokes `golangci-lint` directly
|
||||||
whatever `PATH` resolves, so drift self-heals silently and a lint
|
instead of `make lint`, and its build stage runs `make test` and
|
||||||
stage image bumped on its own would lint at the new version while
|
`make fmt-check` instead of the `make check` aggregate (`make`, not the
|
||||||
`make check` ran at the old one, green. The linter version is pinned
|
scripts bare, because the Makefile's `export CGO_ENABLED = 0` only reaches
|
||||||
in two independent places (the lint stage image digest and
|
what it invokes). `COPY --from=lint` `/usr/bin/golangci-lint` is replaced by
|
||||||
|
`COPY --from=lint /src/go.sum /dev/null`: the copied binary was the only edge
|
||||||
|
forcing BuildKit to finish linting before the build stage starts, and dropping
|
||||||
|
it without replacing the edge would have ended fail-fast linting silently
|
||||||
|
under a still-green build. That is canonical `REPO_POLICIES.md:107`'s ordering
|
||||||
|
edge, restored. `ENV PATH=/home/builder/go/bin:$PATH` is gone with the
|
||||||
|
`go install` that justified it. `script/verify-linter-pin` is retired, deleted
|
||||||
|
along with its README entry, because both of its subjects ceased to exist in
|
||||||
|
the same change: it compared a linter binary against `GOLANGCI_LINT_VERSION`
|
||||||
|
in `script/bootstrap`, and there is now neither a binary crossing between
|
||||||
|
stages nor a version pin in bootstrap. The drift it guarded has not gone away,
|
||||||
|
it has moved — the linter is still pinned twice, now as the `FROM` line of
|
||||||
|
`Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage, with
|
||||||
|
nothing syncing them, which is exactly what
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its
|
||||||
|
replacement is one new `script/verify-lint-image-pin`, run as a gate in both
|
||||||
|
files, which compares the two references to each other and deliberately
|
||||||
|
restates neither: a hardcoded expected digest would be a third copy and the
|
||||||
|
same drift one file further out. `golangci-lint config verify` is included per
|
||||||
|
the ruling, and the concern about its unpinned live HTTPS schema fetch was
|
||||||
|
measured rather than assumed — under `--network none` the pinned binary both
|
||||||
|
passes a valid config and rejects an invalid one with the jsonschema error, so
|
||||||
|
it validates from an embedded schema and makes no network call of its own. The
|
||||||
|
README scopes that to the gate steps rather than to linting as a whole:
|
||||||
|
`Dockerfile.lint` runs `go mod download` above them, so a cold cache still
|
||||||
|
needs the network and only a warm one lints offline. Verified: `make lint`
|
||||||
|
green with every `PATH` directory containing a `golangci-lint` removed
|
||||||
|
(`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`;
|
||||||
|
`command -v golangci-lint` empty); two consecutive `script/lint` runs on an
|
||||||
|
untouched tree both executed the linter, 27.7s and 28.7s in the lint step
|
||||||
|
under distinct epochs with the `COPY . .` layer `CACHED` above them, at 42.2s
|
||||||
|
and 41.8s wall clock — the no-cache rule was not weakened to shorten that.
|
||||||
|
Negative control: a planted `var unusedIssue46Sentinel = 1` failed
|
||||||
|
`script/lint` with
|
||||||
|
`report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and failed
|
||||||
|
`make docker` at `[lint 9/9]` with the build stage stopped at `[builder 3/12]`
|
||||||
|
— `COPY --from=lint`, `script/bootstrap`, the test gate and `make build` all
|
||||||
|
zero occurrences — then reverted clean. The drift guard fails on a tag-only
|
||||||
|
disagreement, on a digest-only disagreement, and on an unreadable reference,
|
||||||
|
naming both sides. `make docker` green in 5m35s with all six gates executing
|
||||||
|
under one epoch (lint 37.6s, test 25.2s reporting
|
||||||
|
`ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`). The
|
||||||
|
non-root quirk still holds: in the builder image with the Go test cache off,
|
||||||
|
`--user 0:0` fails `TestScanHardlinkRunFailsTogether` (exit 1) where the
|
||||||
|
unprivileged user passes (exit 0). Noted for follow-up, not fixed here:
|
||||||
|
`golangci-lint` warns that the `gomodguard` linter is deprecated since v2.12.0
|
||||||
|
in favour of `gomodguard_v2`.
|
||||||
|
|
||||||
|
- install the Docker build stage's prerequisites by running `script/bootstrap`
|
||||||
|
instead of `apk add --no-cache make` inline (2026-08-09, branch
|
||||||
|
`dockerfile-bootstrap`, closes #42): canonical `REPO_POLICIES.md:97` requires
|
||||||
|
it, and the inline install left the build stage maintaining its own notion of
|
||||||
|
the toolchain — exactly the divergence #24 exists to close, one layer down.
|
||||||
|
The stage now copies `script/` plus `go.mod`/`go.sum` and runs
|
||||||
|
`script/bootstrap`, which ends in `go mod download`, so the separate
|
||||||
|
invocation of that is gone. `COPY --from=lint /usr/bin/golangci-lint` stays,
|
||||||
|
and moves above the bootstrap layer. It is the only edge making this stage
|
||||||
|
depend on the lint stage, so deleting it as redundant would end fail-fast
|
||||||
|
linting silently. Letting bootstrap install its own linter here would have
|
||||||
|
reintroduced the second toolchain and paid for a from-source build of it. What
|
||||||
|
makes the two stages provably one toolchain rather than two that happen to
|
||||||
|
agree is a new `script/verify-linter-pin`, run in the build stage on the
|
||||||
|
binary that arrives from the lint stage, before bootstrap: it fails the build
|
||||||
|
naming both versions unless that binary is the version `script/bootstrap`
|
||||||
|
pins. Bootstrap's own check could not serve that purpose — it reinstalls its
|
||||||
|
pin from source and then verifies whatever `PATH` resolves, so drift
|
||||||
|
self-heals silently and a lint stage image bumped on its own would lint at the
|
||||||
|
new version while `make check` ran at the old one, green. The linter version
|
||||||
|
is pinned in two independent places (the lint stage image digest and
|
||||||
`GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a
|
`GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a
|
||||||
half-applied bump is now a build failure. The pin is read out of
|
half-applied bump is now a build failure. The pin is read out of
|
||||||
`script/bootstrap`, which stays the single source of truth; a pin
|
`script/bootstrap`, which stays the single source of truth; a pin that cannot
|
||||||
that cannot be read is a hard failure, not a skip. The check needs
|
be read is a hard failure, not a skip. The check needs no `CHECK_EPOCH`: its
|
||||||
no `CHECK_EPOCH`: its only inputs are the copied binary and
|
only inputs are the copied binary and `script/`, so Docker invalidates the
|
||||||
`script/`, so Docker invalidates the layer exactly when a cached
|
layer exactly when a cached result would stop being true, and it is documented
|
||||||
result would stop being true, and it is documented with the other
|
with the other entrypoints in the README. `$GOPATH/bin` joins `PATH` because
|
||||||
entrypoints in the README. `$GOPATH/bin` joins `PATH` because
|
that is where bootstrap's `go install` lands and bootstrap verifies its
|
||||||
that is where bootstrap's `go install` lands and bootstrap verifies
|
installs against what `PATH` resolves — nothing in the image is shadowed by
|
||||||
its installs against what `PATH` resolves — nothing in the image is
|
it, the directory does not exist until bootstrap runs. Everything added sits
|
||||||
shadowed by it, the directory does not exist until bootstrap runs.
|
above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still precede
|
||||||
Everything added sits above `ARG CHECK_EPOCH`, and the `chown` and
|
`make check`. Verified: the guard fails the build with both versions named
|
||||||
`USER builder` still precede `make check`. Verified: the guard fails
|
when the lint stage's linter is faked to a different version, and an
|
||||||
the build with both versions named when the lint stage's linter is
|
unmodified build still passes it; bootstrap runs clean under Alpine's `sh` and
|
||||||
faked to a different version, and an unmodified build still passes
|
its `apk` branch, installing `git` and `make` and finding the copied linter
|
||||||
it; bootstrap runs clean under Alpine's `sh` and its `apk` branch,
|
already at the pin; a second build served the bootstrap and dependency layers
|
||||||
installing `git` and `make` and finding the copied
|
`CACHED` while both gates ran with a fresh epoch; a planted `unused` finding
|
||||||
linter already at the pin; a second build served the bootstrap and
|
failed the build at the lint gate in 48.9s with the build stage's `make check`
|
||||||
dependency layers `CACHED` while both gates ran with a fresh epoch;
|
never starting; and the suite run in the image as `--user 0:0` fails
|
||||||
a planted `unused` finding failed the build at the lint gate in
|
`TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged user is
|
||||||
48.9s with the build stage's `make check` never starting; and the
|
still load-bearing. That last check needs the Go test cache disabled — the
|
||||||
suite run in the image as `--user 0:0` fails
|
first attempt reported `ok ... (cached)` as root, reusing the result the
|
||||||
`TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged
|
build-time run had left in the shared cache, which would have read as a pass.
|
||||||
user is still load-bearing. That last check needs the Go test cache
|
Build wall time, on a shared host running many concurrent builds and so noisy:
|
||||||
disabled — the first attempt reported `ok ... (cached)` as root,
|
2m13s on an unchanged tree, 2m17s and 4m29s for two builds after a source
|
||||||
reusing the result the build-time run had left in the shared cache,
|
change, 5m14s cold. Only the cold one breaches the policy ceiling, and not
|
||||||
which would have read as a pass. Build wall time, on a shared host
|
because of this change — `chown -R builder:builder /src /home/builder` walks
|
||||||
running many concurrent builds and so noisy: 2m13s on an unchanged
|
the module cache and re-runs on every source change, and it alone varied
|
||||||
tree, 2m17s and 4m29s for two builds after a source change, 5m14s
|
between 77s and 210s across those four builds, which is also the whole spread
|
||||||
cold. Only the cold one breaches the policy ceiling, and not because
|
in the totals. The same cold measurement against `main` is 5m03s with a 209s
|
||||||
of this change — `chown -R builder:builder /src /home/builder` walks
|
`chown`. Filed as #43
|
||||||
the module cache and re-runs on every source change, and it alone
|
- bust the Docker layer cache for the gate steps, so `script/cibuild` and
|
||||||
varied between 77s and 210s across those four builds, which is also
|
`script/docker` cannot report a green they did not earn (2026-08-09, branch
|
||||||
the whole spread in the totals. The same cold measurement against
|
`cibuild-cache-bust`, closes #32): both scripts were bare `docker build`
|
||||||
`main` is 5m03s with a 209s `chown`. Filed as #43
|
invocations with no cache control, and the `Dockerfile` copies the tree before
|
||||||
- bust the Docker layer cache for the gate steps, so `script/cibuild`
|
running its gates, so on an unchanged tree Docker served those layers from
|
||||||
and `script/docker` cannot report a green they did not earn
|
cache and the build exited 0 having executed nothing. That is not hypothetical
|
||||||
(2026-08-09, branch `cibuild-cache-bust`, closes #32): both scripts
|
here — every merge this repo has done is a non-fast-forward merge of an
|
||||||
were bare `docker build` invocations with no cache control, and the
|
undiverged branch, so each merge commit's tree is byte-identical to the branch
|
||||||
`Dockerfile` copies the tree before running its gates, so on an
|
head's and each merge CI run was almost certainly a full cache hit; and PR
|
||||||
unchanged tree Docker served those layers from cache and the build
|
#31's reviewer found `make docker` returning success as a 17-layer cache hit,
|
||||||
exited 0 having executed nothing. That is not hypothetical here —
|
catching it only by being suspicious. The fix is `ARG CHECK_EPOCH` with the
|
||||||
every merge this repo has done is a non-fast-forward merge of an
|
scripts passing `--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or
|
||||||
undiverged branch, so each merge commit's tree is byte-identical to
|
break it. `ARG` is scoped per stage and this `Dockerfile` has three gates
|
||||||
the branch head's and each merge CI run was almost certainly a full
|
across two — `make fmt-check` and `make lint` in the lint stage, `make check`
|
||||||
cache hit; and PR #31's reviewer found `make docker` returning
|
in the build stage — so a single declaration would have left one stage
|
||||||
success as a 17-layer cache hit, catching it only by being
|
silently cacheable; it is declared in both. And BuildKit hashes the expanded
|
||||||
suspicious. The fix is `ARG CHECK_EPOCH` with the scripts passing
|
command, not the declaration, so a declared-but-unreferenced `ARG` invalidates
|
||||||
`--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or break
|
nothing: each gate `RUN` echoes the epoch, which also puts the value in the
|
||||||
it. `ARG` is scoped per stage and this `Dockerfile` has three gates
|
build log as evidence the layer really ran. Placement is below the dependency
|
||||||
across two — `make fmt-check` and `make lint` in the lint stage,
|
layers on purpose — a build that goes cold every time would be a different
|
||||||
`make check` in the build stage — so a single declaration would have
|
bug, not a fix. Verified by running each script twice back to back on an
|
||||||
left one stage silently cacheable; it is declared in both. And
|
unchanged tree under `BUILDKIT_PROGRESS=plain`: all three gates executed on
|
||||||
BuildKit hashes the expanded command, not the declaration, so a
|
all four runs, each with a fresh epoch in the log (`script/cibuild` 78.8s then
|
||||||
declared-but-unreferenced `ARG` invalidates nothing: each gate `RUN`
|
61.1s; `script/docker` 61.1s then 53.4s), and twelve steps were still served
|
||||||
echoes the epoch, which also puts the value in the build log as
|
`CACHED` in the steady state — both `go mod download`s, `apk add`, `adduser`,
|
||||||
evidence the layer really ran. Placement is below the dependency
|
the `chown`, every `go.mod`/`go.sum` and source copy, the linter copy out of
|
||||||
layers on purpose — a build that goes cold every time would be a
|
the lint stage, and the binary copy into the runtime stage. The lint stage
|
||||||
different bug, not a fix. Verified by running each script twice back
|
still gates the build stage: with a deliberate `unused` finding planted in the
|
||||||
to back on an unchanged tree under `BUILDKIT_PROGRESS=plain`: all
|
tree, the build failed at `make lint` in 36.1s and the build-stage
|
||||||
three gates executed on all four runs, each with a fresh epoch in
|
`make check` never started. The build stage also still drops to the
|
||||||
the log (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s
|
unprivileged `builder` user before `make check`, which the suite depends on
|
||||||
then 53.4s), and twelve steps were still served `CACHED` in the
|
rather than merely prefers: forcing the same image to run the tests as root
|
||||||
steady state — both `go mod download`s, `apk add`, `adduser`, the
|
fails `TestScanHardlinkRunFailsTogether`, because root reads straight through
|
||||||
`chown`, every `go.mod`/`go.sum` and source copy, the linter copy
|
the `chmod(0)` the test uses to prove hard links are read once. This is the
|
||||||
out of the lint stage, and the binary copy into the runtime stage.
|
local fix only; propagating it to the canonical templates is `prompts` #26
|
||||||
The lint stage still gates the build stage: with a deliberate
|
- check the installed golangci-lint version in `script/bootstrap` instead of
|
||||||
`unused` finding planted in the tree, the build failed at
|
only its presence (2026-08-09, branch `bootstrap-version-check`, closes #24):
|
||||||
`make lint` in 36.1s and the build-stage `make check` never started.
|
`missing golangci-lint` meant any linter already on `PATH` satisfied the
|
||||||
The build stage also still drops to the unprivileged `builder` user
|
check, so the pin was never consulted and the v2.12.2 bump from #3 was inert
|
||||||
before `make check`, which the suite depends on rather than merely
|
on every host that already had one — this host ran v2.10.1 against a v2.12.2
|
||||||
prefers: forcing the same image to run the tests as root fails
|
pin, `make check` went green, and `make docker` then rejected the same commit
|
||||||
`TestScanHardlinkRunFailsTogether`, because root reads straight
|
with findings the local gate never saw. The version now lives in one place,
|
||||||
through the `chmod(0)` the test uses to prove hard links are read
|
`GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it so a
|
||||||
once. This is the local fix only; propagating it to the canonical
|
bump cannot half-apply; a `golangci_lint_version` helper parses
|
||||||
templates is `prompts` #26
|
`golangci-lint --version` (taking the field after the word `version` and
|
||||||
- check the installed golangci-lint version in `script/bootstrap`
|
tolerating an optional leading `v`, which the module ref carries and the
|
||||||
instead of only its presence (2026-08-09, branch
|
binary's output does not), and any version that is not the pin — older, newer,
|
||||||
`bootstrap-version-check`, closes #24): `missing golangci-lint` meant
|
absent or unparseable — is reinstalled. The install is then verified against
|
||||||
any linter already on `PATH` satisfied the check, so the pin was never
|
the binary `PATH` actually resolves: `go install` writes into `GOBIN` (or
|
||||||
consulted and the v2.12.2 bump from #3 was inert on every host that
|
`GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes first on
|
||||||
already had one — this host ran v2.10.1 against a v2.12.2 pin,
|
`PATH`, so a wrong-version one sitting ahead of it — nix, apt, brew, apk, or
|
||||||
`make check` went green, and `make docker` then rejected the same
|
the `/usr/local/bin` copy the `Dockerfile` builder stage makes — would swallow
|
||||||
commit with findings the local gate never saw. The version now lives
|
the install and leave the local gate disagreeing with CI under an affirmative
|
||||||
in one place, `GOLANGCI_LINT_VERSION`, with the `go install` module
|
`bootstrap complete`. Bootstrap now re-reads the effective version after
|
||||||
ref derived from it so a bump cannot half-apply; a
|
installing and, on a mismatch, prints both paths and both versions to stderr
|
||||||
`golangci_lint_version` helper parses `golangci-lint --version`
|
and exits non-zero instead of claiming success; it does not reorder anyone's
|
||||||
(taking the field after the word `version` and tolerating an optional
|
|
||||||
leading `v`, which the module ref carries and the binary's output does
|
|
||||||
not), and any version that is not the pin — older, newer, absent or
|
|
||||||
unparseable — is reinstalled. The install is then verified against the
|
|
||||||
binary `PATH` actually resolves: `go install` writes into `GOBIN` (or
|
|
||||||
`GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes
|
|
||||||
first on `PATH`, so a wrong-version one sitting ahead of it — nix,
|
|
||||||
apt, brew, apk, or the `/usr/local/bin` copy the `Dockerfile` builder
|
|
||||||
stage makes — would swallow the install and leave the local gate
|
|
||||||
disagreeing with CI under an affirmative `bootstrap complete`.
|
|
||||||
Bootstrap now re-reads the effective version after installing and, on
|
|
||||||
a mismatch, prints both paths and both versions to stderr and exits
|
|
||||||
non-zero instead of claiming success; it does not reorder anyone's
|
|
||||||
`PATH` or delete their binary. The `--version` call keeps its stderr
|
`PATH` or delete their binary. The `--version` call keeps its stderr
|
||||||
connected, so a present-but-broken binary says why rather than
|
connected, so a present-but-broken binary says why rather than reinstalling
|
||||||
reinstalling forever in silence, and is bounded by `timeout(1)` where
|
forever in silence, and is bounded by `timeout(1)` where that exists, so a
|
||||||
that exists, so a wedged binary cannot hang bootstrap. `git`, `make`
|
wedged binary cannot hang bootstrap. `git`, `make` and `go` keep their
|
||||||
and `go` keep their presence-only checks and now say why in a
|
presence-only checks and now say why in a comment: they are host
|
||||||
comment: they are host package-manager tools the repo deliberately
|
package-manager tools the repo deliberately does not pin, with `go.mod`
|
||||||
does not pin, with `go.mod` governing the language version and the
|
governing the language version and the digest-pinned images covering
|
||||||
digest-pinned images covering reproducible builds. Verified on this
|
reproducible builds. Verified on this host by bootstrapping from v2.10.1 to
|
||||||
host by bootstrapping from v2.10.1 to v2.12.2 and running it again to
|
v2.12.2 and running it again to a no-op, plus stub runs of the real script
|
||||||
a no-op, plus stub runs of the real script under `dash` covering a
|
under `dash` covering a thirteen-input parse matrix (absent, older, newer,
|
||||||
thirteen-input parse matrix (absent, older, newer, host-style,
|
host-style, image-style, leading-`v`, stderr-only, empty, non-zero exit,
|
||||||
image-style, leading-`v`, stderr-only, empty, non-zero exit, impostor
|
impostor binary, `(devel)`, trailing `version`), a shadowed install that must
|
||||||
binary, `(devel)`, trailing `version`), a shadowed install that must
|
exit non-zero, an install destination not on `PATH` at all, `GOBIN` set, and a
|
||||||
exit non-zero, an install destination not on `PATH` at all, `GOBIN`
|
wedged binary that must hit the timeout; `make check` and `make lint` are
|
||||||
set, and a wedged binary that must hit the timeout; `make check` and
|
clean at v2.12.2, so v2.10.1 was not hiding any findings on `main`
|
||||||
`make lint` are clean at v2.12.2, so v2.10.1 was not hiding any
|
|
||||||
findings on `main`
|
|
||||||
- unwind the hash worker pool on the error path (2026-08-09, branch
|
- unwind the hash worker pool on the error path (2026-08-09, branch
|
||||||
`hash-pool-cleanup`, closes #6): `hashPhase` used to return the
|
`hash-pool-cleanup`, closes #6): `hashPhase` used to return the moment
|
||||||
moment `recordRun` failed and abandon the pool — the feeder parked
|
`recordRun` failed and abandon the pool — the feeder parked forever on a full
|
||||||
forever on a full `jobs` channel and every worker on a full
|
`jobs` channel and every worker on a full `results` channel. That only stopped
|
||||||
`results` channel. That only stopped being invisible when #4 landed
|
being invisible when #4 landed and `runScan` began unwinding instead of
|
||||||
and `runScan` began unwinding instead of calling `os.Exit`. The
|
calling `os.Exit`. The pool is now an owned, context-aware `hashPool`: every
|
||||||
pool is now an owned, context-aware `hashPool`: every blocking send
|
blocking send in the feeder and the workers selects on `ctx.Done()`, `jobs` is
|
||||||
in the feeder and the workers selects on `ctx.Done()`, `jobs` is
|
closed on every path out, and `hashPhase` defers `pool.stop()`, which cancels
|
||||||
closed on every path out, and `hashPhase` defers `pool.stop()`,
|
and then drains `results` until the last goroutine has exited — draining is
|
||||||
which cancels and then drains `results` until the last goroutine
|
what frees a worker already parked on a send. `ctx` is threaded from
|
||||||
has exited — draining is what frees a worker already parked on a
|
`cmd.Context()` through `runScan`, `syncScan`, both worker pools and the whole
|
||||||
send. `ctx` is threaded from `cmd.Context()` through `runScan`,
|
database layer (it is the first parameter everywhere), so #5 can hand this
|
||||||
`syncScan`, both worker pools and the whole database layer (it is
|
path a signal and needs to add nothing else. The walk pool never leaked,
|
||||||
the first parameter everywhere), so #5 can hand this path a signal
|
because `walkPhase` always drains its events to close, but it has the same
|
||||||
and needs to add nothing else. The walk pool never leaked, because
|
unbounded-send shape and #5 will give it an early return, so it gets the same
|
||||||
`walkPhase` always drains its events to close, but it has the same
|
treatment plus a `ctx.Err()` guard after the walk: a cancelled walk yields a
|
||||||
unbounded-send shape and #5 will give it an early return, so it
|
partial size census, and every file it never reached looks vanished to the
|
||||||
gets the same treatment plus a `ctx.Err()` guard after the walk: a
|
update phase. That phase's own `BeginTx` fails on the same cancelled context
|
||||||
cancelled walk yields a partial size census, and every file it never
|
before deleting anything, so the guard is defence in depth rather than the
|
||||||
reached looks vanished to the update phase. That phase's own
|
only barrier — but it is the one that survives #5 deciding an interrupted scan
|
||||||
`BeginTx` fails on the same cancelled context before deleting
|
may commit what it has. Tests drive `run(scan)` against a database whose
|
||||||
anything, so the guard is defence in depth rather than the only
|
insert trigger aborts, and assert both that the scan fails instead of hanging
|
||||||
barrier — but it is the one that survives #5 deciding an interrupted
|
and that `runtime.NumGoroutine()` polls back to its pre-scan baseline; a
|
||||||
scan may commit what it has. Tests drive `run(scan)` against a
|
second set cancels a scan part-way through the walk — deterministically, by
|
||||||
database whose insert trigger aborts, and assert both that the scan
|
counting the scan's own consultations of `ctx.Done()` rather than racing a
|
||||||
fails instead of hanging and that `runtime.NumGoroutine()` polls
|
timer — and asserts that it stops at the guard holding a partial census and a
|
||||||
back to its pre-scan baseline; a second set cancels a scan part-way
|
still-populated record index, with every record intact. The remaining
|
||||||
through the walk — deterministically, by counting the scan's own
|
cancellation branches of both pools are covered by direct tests of
|
||||||
consultations of `ctx.Done()` rather than racing a timer — and
|
`sendEvent`, the walk workers, `dispatchDirs`, `feedHashJobs`, `hashWorker`
|
||||||
asserts that it stops at the guard holding a partial census and a
|
and `hashPhase`
|
||||||
still-populated record index, with every record intact. The
|
- guarantee the database is closed on every fatal exit path (2026-08-09, branch
|
||||||
remaining cancellation branches of both pools are covered by direct
|
`db-close-on-fatal`, closes #4): `fatalf` and its `os.Exit(1)` are gone, so
|
||||||
tests of `sendEvent`, the walk workers, `dispatchDirs`,
|
the deferred `db.Close()` — and with it the SQLite WAL checkpoint — now
|
||||||
`feedHashJobs`, `hashWorker` and `hashPhase`
|
actually runs when a subcommand fails; `runScan`, `runReport`, `runTrees`,
|
||||||
- guarantee the database is closed on every fatal exit path
|
`loadRecords` and `resolveRoots` return errors instead. The single exit point
|
||||||
(2026-08-09, branch `db-close-on-fatal`, closes #4): `fatalf` and
|
is `run` in `main.go`: it maps a `fatalError` (anything a subcommand returned)
|
||||||
its `os.Exit(1)` are gone, so the deferred `db.Close()` — and with
|
to exit 1 and cobra's own argument and flag errors to exit 2, which keeps a
|
||||||
it the SQLite WAL checkpoint — now actually runs when a subcommand
|
runtime failure from being reported as a usage error or printing the usage
|
||||||
fails; `runScan`, `runReport`, `runTrees`, `loadRecords` and
|
text. New `main_test.go` drives the CLI in-process and asserts the exit codes
|
||||||
`resolveRoots` return errors instead. The single exit point is `run`
|
from README §Error handling plus the stdout/stderr split, including that a
|
||||||
in `main.go`: it maps a `fatalError` (anything a subcommand
|
fatal error raised after the database is open leaves no `-wal`/`-shm` sidecar
|
||||||
returned) to exit 1 and cobra's own argument and flag errors to exit
|
behind for `scan`, `report` or `trees`
|
||||||
2, which keeps a runtime failure from being reported as a usage
|
- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch
|
||||||
error or printing the usage text. New `main_test.go` drives the CLI
|
`golangci-v2.12.2`, merged as `38a01bd`, closes #3): bumped the pinned linter
|
||||||
in-process and asserts the exit codes from README §Error handling
|
in the `Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2,
|
||||||
plus the stdout/stderr split, including that a fatal error raised
|
and replaced `.golangci.yml` with the canonical file — the linter settings
|
||||||
after the database is open leaves no `-wal`/`-shm` sidecar behind
|
|
||||||
for `scan`, `report` or `trees`
|
|
||||||
- update golangci-lint to v2.12.2 with the canonical config
|
|
||||||
(2026-08-09, branch `golangci-v2.12.2`, merged as `38a01bd`,
|
|
||||||
closes #3): bumped the pinned linter in the `Dockerfile` lint
|
|
||||||
stage and `script/bootstrap` from v2.12.1 to v2.12.2, and replaced
|
|
||||||
`.golangci.yml` with the canonical file — the linter settings
|
|
||||||
(`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under
|
(`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under
|
||||||
`linters.settings` per the v2 schema, so they are actually
|
`linters.settings` per the v2 schema, so they are actually applied; no new
|
||||||
applied; no new lint findings surfaced
|
lint findings surfaced
|
||||||
- convert Makefile targets to scripts-to-rule-them-all `script/`
|
- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints
|
||||||
entrypoints like the other managed repos (2026-07-26, commit
|
like the other managed repos (2026-07-26, commit `3abeacf`, closes #1): all 12
|
||||||
`3abeacf`, closes #1): all 12 `script/` entrypoints exist
|
`script/` entrypoints exist (`bootstrap`, `setup`, `projectname`, `test`,
|
||||||
(`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
|
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
||||||
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
`install-precommit`) and every Makefile target is now a thin shim over them,
|
||||||
`install-precommit`) and every Makefile target is now a thin shim
|
matching the other managed repos
|
||||||
over them, matching the other managed repos
|
|
||||||
- make the binary the default Make target (2026-07-24, branch
|
- make the binary the default Make target (2026-07-24, branch
|
||||||
`make-default-target`): plain `make` now builds `sfdupes`
|
`make-default-target`): plain `make` now builds `sfdupes` (previously it ran
|
||||||
(previously it ran `check` plus `build`); `make build` remains as
|
`check` plus `build`); `make build` remains as an alias
|
||||||
an alias
|
- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch
|
||||||
- scan-wide phases, concurrent operands, batched updates (2026-07-24,
|
`scan-wide-phases`): all operands seed the shared walk pool and every pass
|
||||||
branch `scan-wide-phases`): all operands seed the shared walk pool
|
runs once over the whole scan, so totals and ETAs are scan-global; the
|
||||||
and every pass runs once over the whole scan, so totals and ETAs
|
per-operand walk/hash/update cycles and their stderr announcements are gone;
|
||||||
are scan-global; the per-operand walk/hash/update cycles and their
|
the update pass commits in batched transactions — the filesystem is
|
||||||
stderr announcements are gone; the update pass commits in batched
|
authoritative and the database an eventually-consistent reflection, so
|
||||||
transactions — the filesystem is authoritative and the database an
|
scan-level atomicity is not required
|
||||||
eventually-consistent reflection, so scan-level atomicity is not
|
|
||||||
required
|
|
||||||
- split the stat pass back out of the walk (2026-07-24, branch
|
- split the stat pass back out of the walk (2026-07-24, branch
|
||||||
`parallel-phases`): phases are strictly sequential again — walk,
|
`parallel-phases`): phases are strictly sequential again — walk, stat, hash,
|
||||||
stat, hash, update per operand — with parallelism only inside each
|
update per operand — with parallelism only inside each phase; the walk
|
||||||
phase; the walk enumerates paths with per-directory workers and the
|
enumerates paths with per-directory workers and the stat pass lstats them with
|
||||||
stat pass lstats them with per-file workers, restoring the exact
|
per-file workers, restoring the exact total/ETA stat bar
|
||||||
total/ETA stat bar
|
- announce each operand on stderr before its passes (2026-07-24, branch
|
||||||
- announce each operand on stderr before its passes (2026-07-24,
|
`scan-operand-progress`): with per-operand walk/hash/update cycles, a
|
||||||
branch `scan-operand-progress`): with per-operand walk/hash/update
|
multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the
|
||||||
cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals
|
whole run's — an operator watching operand 3 of 14 hash 300k files concluded
|
||||||
that looked like the whole run's — an operator watching operand 3 of
|
20M files were being skipped
|
||||||
14 hash 300k files concluded 20M files were being skipped
|
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single
|
||||||
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass
|
goroutine and took hours at ~20M files on a busy pool (observed: 22M files in
|
||||||
was a single goroutine and took hours at ~20M files on a busy pool
|
4h on a ZFS server); it is now a per-directory worker-pool traversal that
|
||||||
(observed: 22M files in 4h on a ZFS server); it is now a
|
records size/mtime during the walk (folding away the separate stat pass,
|
||||||
per-directory worker-pool traversal that records size/mtime during
|
halving metadata I/O), and each `PATH` operand commits in its own transaction
|
||||||
the walk (folding away the separate stat pass, halving metadata
|
so an interrupted scan keeps completed operands
|
||||||
I/O), and each `PATH` operand commits in its own transaction so an
|
|
||||||
interrupted scan keeps completed operands
|
|
||||||
|
|
||||||
- persistent scan database (2026-07-24, branch `persistent-database`):
|
- persistent scan database (2026-07-24, branch `persistent-database`): `scan`
|
||||||
`scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
|
now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays
|
||||||
Go, cgo stays disabled) keyed by absolute path that survives between
|
disabled) keyed by absolute path that survives between runs — a rescan hashes
|
||||||
runs — a rescan hashes only new or changed files (by mtime/size),
|
only new or changed files (by mtime/size), deletes records for files vanished
|
||||||
deletes records for files vanished from under the scanned operands,
|
from under the scanned operands, and leaves records outside them untouched, so
|
||||||
and leaves records outside them untouched, so `scan` can be cronned
|
`scan` can be cronned daily; `report` and `trees` read the database (no
|
||||||
daily; `report` and `trees` read the database (no positional
|
positional arguments) instead of a scan stream. Database at
|
||||||
arguments) instead of a scan stream. Database at
|
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL
|
||||||
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
|
journaling plus a single-transaction update keep a report run during a scan
|
||||||
WAL journaling plus a single-transaction update keep a report run
|
safe
|
||||||
during a scan safe
|
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`,
|
||||||
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
|
and push `main` plus tags (2026-07-23)
|
||||||
`v0.0.1`, and push `main` plus tags (2026-07-23)
|
|
||||||
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
|
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
|
||||||
`PATH...` operands via cobra flags replacing the `/srv` `-root`
|
`PATH...` operands via cobra flags replacing the `/srv` `-root` default; new
|
||||||
default; new `-x`/`--one-file-system` flag (GNU convention) to stop
|
`-x`/`--one-file-system` flag (GNU convention) to stop at filesystem
|
||||||
at filesystem boundaries, which are crossed by default
|
boundaries, which are crossed by default
|
||||||
- bring the repo into full policy compliance (2026-07-23, branch
|
- bring the repo into full policy compliance (2026-07-23, branch
|
||||||
`repo-policy-compliance`; checklist below)
|
`repo-policy-compliance`; checklist below)
|
||||||
- `git init` with README-only first commit; code baseline committed on
|
- `git init` with README-only first commit; code baseline committed on `main`
|
||||||
`main` (2026-07-22)
|
(2026-07-22)
|
||||||
- implement `scan`, `report`, and `trees` subcommands (pre-git history)
|
- implement `scan`, `report`, and `trees` subcommands (pre-git history)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- possible later features (explicitly out of scope per README):
|
- possible later features (explicitly out of scope per README): full-content
|
||||||
full-content verification of candidates, removal-script helpers
|
verification of candidates, removal-script helpers
|
||||||
|
|
||||||
# Repo Policy Compliance
|
# Repo Policy Compliance
|
||||||
|
|
||||||
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing
|
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo
|
||||||
repo checklist, and the Go styleguide. Code is already gofmt-clean, so no
|
checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone
|
||||||
standalone formatting commit is needed.
|
formatting commit is needed.
|
||||||
|
|
||||||
- [x] `.gitignore` missing — the compiled `sfdupes` binary and
|
- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit
|
||||||
`files.dat` sit untracked in the tree; needs OS/editor/Go
|
untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns
|
||||||
artifacts plus secrets patterns
|
|
||||||
- [x] `.editorconfig` missing
|
- [x] `.editorconfig` missing
|
||||||
- [x] `LICENSE` missing and README has no License section (MIT assumed
|
- [x] `LICENSE` missing and README has no License section (MIT assumed from
|
||||||
from house convention — user to confirm)
|
house convention — user to confirm)
|
||||||
- [x] `REPO_POLICIES.md` missing from repo root
|
- [x] `REPO_POLICIES.md` missing from repo root
|
||||||
- [x] `.golangci.yml` missing (install canonical copy); code must then
|
- [x] `.golangci.yml` missing (install canonical copy); code must then pass
|
||||||
pass `make lint` (150 findings fixed; `make lint` is clean)
|
`make lint` (150 findings fixed; `make lint` is clean)
|
||||||
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`,
|
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`,
|
||||||
`fmt-check`, `docker`, `hooks`; `check` currently depends on
|
`docker`, `hooks`; `check` currently depends on `build`, which writes the
|
||||||
`build`, which writes the binary (`make check` must not modify
|
binary (`make check` must not modify files)
|
||||||
files)
|
- [x] no tests — `go test ./...` has nothing to run; policy requires real tests
|
||||||
- [x] no tests — `go test ./...` has nothing to run; policy requires
|
with a 30-second timeout and the conditional `-v` rerun pattern (suite
|
||||||
real tests with a 30-second timeout and the conditional `-v`
|
covers parsing, grouping, digests, suppression, hashing, and the scan
|
||||||
rerun pattern (suite covers parsing, grouping, digests,
|
pipeline; 64% coverage)
|
||||||
suppression, hashing, and the scan pipeline; 64% coverage)
|
- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast
|
||||||
- [x] `Dockerfile` missing — Go multistage with hash-pinned images:
|
lint stage, build stage running `make check`
|
||||||
fail-fast lint stage, build stage running `make check`
|
|
||||||
- [x] `.dockerignore` missing
|
- [x] `.dockerignore` missing
|
||||||
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push,
|
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout
|
||||||
checkout action pinned by commit SHA)
|
action pinned by commit SHA)
|
||||||
- [x] README lacks required sections: Description first line
|
- [x] README lacks required sections: Description first line
|
||||||
(name/purpose/category/license/author), Getting Started,
|
(name/purpose/category/license/author), Getting Started, Rationale, TODO,
|
||||||
Rationale, TODO, License, Author
|
License, Author
|
||||||
- [x] README non-goal "no git repository setup and no CI" is stale now
|
- [x] README non-goal "no git repository setup and no CI" is stale now that the
|
||||||
that the repo is under git with CI
|
repo is under git with CI
|
||||||
- [x] pre-commit hook not installed (`make hooks` once the target
|
- [x] pre-commit hook not installed (`make hooks` once the target exists)
|
||||||
exists)
|
|
||||||
|
|
||||||
Accepted divergences (no action):
|
Accepted divergences (no action):
|
||||||
|
|
||||||
- flat single-package layout with `.go` files in the repo root — fine
|
- flat single-package layout with `.go` files in the repo root — fine for a
|
||||||
for a small single-binary tool per the Go styleguide; the tracker
|
small single-binary tool per the Go styleguide; the tracker audit agrees
|
||||||
audit agrees
|
- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go
|
||||||
- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0`
|
builds) and the race detector requires cgo
|
||||||
(pure-Go builds) and the race detector requires cgo
|
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"devDependencies": {
|
||||||
|
"prettier": "3.8.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,12 @@ set -eu
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# yarn provides prettier, which formats Markdown. yarn is a tool, like
|
||||||
|
# node/git/make/go below; the reference that governs formatting output is
|
||||||
|
# prettier, pinned by yarn.lock's integrity hash and installed by
|
||||||
|
# `yarn install --frozen-lockfile`.
|
||||||
|
YARN_VERSION="1.22.22"
|
||||||
|
|
||||||
PKGMGR=""
|
PKGMGR=""
|
||||||
SUDO=""
|
SUDO=""
|
||||||
APT_UPDATED=""
|
APT_UPDATED=""
|
||||||
@@ -58,6 +64,21 @@ missing() {
|
|||||||
! command -v "$1" >/dev/null 2>&1
|
! command -v "$1" >/dev/null 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ensure_node() {
|
||||||
|
if ! missing node; then return 0; fi
|
||||||
|
pkg_install nodejs nodejs node nodejs
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_yarn() {
|
||||||
|
if ! missing yarn; then return 0; fi
|
||||||
|
if ! missing corepack; then
|
||||||
|
corepack enable >/dev/null 2>&1 || true
|
||||||
|
corepack prepare "yarn@$YARN_VERSION" --activate
|
||||||
|
else
|
||||||
|
pkg_install yarn yarn yarn yarn
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
@@ -71,6 +92,16 @@ main() {
|
|||||||
if missing make; then pkg_install gnumake make make make; fi
|
if missing make; then pkg_install gnumake make make make; fi
|
||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
|
|
||||||
|
# node runs prettier and is an unpinned host tool for the same reason
|
||||||
|
# git/make/go are: it comes from the host package manager, whatever
|
||||||
|
# version it ships. It is not installed via nvm the way the canonical
|
||||||
|
# template does, because nvm's prebuilt node is glibc-linked and does
|
||||||
|
# not run on this repo's musl/Alpine build image. prettier — the tool
|
||||||
|
# whose version affects formatting output — is pinned by yarn.lock.
|
||||||
|
ensure_node
|
||||||
|
ensure_yarn
|
||||||
|
yarn install --frozen-lockfile
|
||||||
|
|
||||||
# Linting runs via docker only (script/lint), so docker is a lint
|
# Linting runs via docker only (script/lint), so docker is a lint
|
||||||
# prerequisite rather than something bootstrap installs. Warn, do
|
# prerequisite rather than something bootstrap installs. Warn, do
|
||||||
# not fail: everything except `make lint` — and, through it,
|
# not fail: everything except `make lint` — and, through it,
|
||||||
|
|||||||
+12
-1
@@ -1,12 +1,23 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/fmt: format all files (writes).
|
# script/fmt: format all files (writes). gofmt for Go, prettier for
|
||||||
|
# Markdown. prettier is the pinned devDependency in package.json/
|
||||||
|
# yarn.lock; script/bootstrap installs it (see run_prettier).
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
run_prettier() {
|
||||||
|
if ! command -v yarn >/dev/null 2>&1; then
|
||||||
|
echo "fmt: yarn not found; run script/bootstrap first" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
yarn run prettier "$@"
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
gofmt -s -w .
|
gofmt -s -w .
|
||||||
|
run_prettier --write '**/*.md' --tab-width 4 --prose-wrap always
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+21
-2
@@ -1,18 +1,37 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/fmt-check: check formatting (read-only). Same scope as
|
# script/fmt-check: check formatting (read-only). Same scope as
|
||||||
# script/fmt, but fails instead of writing.
|
# script/fmt: gofmt for Go, prettier for Markdown. Both run every time
|
||||||
|
# and each reports independently, so a failure names which formatter is
|
||||||
|
# unhappy; the script exits non-zero if either found unformatted files.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
run_prettier() {
|
||||||
|
if ! command -v yarn >/dev/null 2>&1; then
|
||||||
|
echo "fmt-check: yarn not found; run script/bootstrap first" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
yarn run prettier "$@"
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
rc=0
|
||||||
|
|
||||||
files="$(gofmt -s -l .)"
|
files="$(gofmt -s -l .)"
|
||||||
if [ -n "$files" ]; then
|
if [ -n "$files" ]; then
|
||||||
echo "gofmt: files not formatted:" >&2
|
echo "gofmt: files not formatted:" >&2
|
||||||
echo "$files" >&2
|
echo "$files" >&2
|
||||||
exit 1
|
rc=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if ! run_prettier --check '**/*.md' --tab-width 4 --prose-wrap always; then
|
||||||
|
echo "prettier: Markdown not formatted; run make fmt" >&2
|
||||||
|
rc=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit "$rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -71,9 +71,9 @@ main() {
|
|||||||
"the two pins disagree:" >&2
|
"the two pins disagree:" >&2
|
||||||
echo "verify-lint-image-pin: $LINT_DOCKERFILE: $lint_ref" >&2
|
echo "verify-lint-image-pin: $LINT_DOCKERFILE: $lint_ref" >&2
|
||||||
echo "verify-lint-image-pin: $MAIN_DOCKERFILE: $main_ref" >&2
|
echo "verify-lint-image-pin: $MAIN_DOCKERFILE: $main_ref" >&2
|
||||||
echo "verify-lint-image-pin: bump both FROM lines together, tag and" \
|
echo "verify-lint-image-pin: bump both FROM lines together so" \
|
||||||
"digest, so script/lint and the Dockerfile lint stage keep" \
|
"script/lint and the Dockerfile lint stage keep running the" \
|
||||||
"running the same linter" >&2
|
"same linter" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||||
|
# yarn lockfile v1
|
||||||
|
|
||||||
|
|
||||||
|
prettier@3.8.1:
|
||||||
|
version "3.8.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173"
|
||||||
|
integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==
|
||||||
Reference in New Issue
Block a user