Compare commits
1
Commits
next
..
18aa3a383d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18aa3a383d |
+11
-19
@@ -1,23 +1,16 @@
|
|||||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
||||||
# system packages and download Go's modules, or right after FROM in a
|
# on every build, which reruns every RUN step after it. The steps that
|
||||||
# stage with neither, and every step that lints, tests or compiles
|
# compile mount Go's build cache, which survives from build to build.
|
||||||
# comes after it. script/ passes a new value on every build, which
|
|
||||||
# reruns every RUN step after it. Docker keeps the layers above it
|
|
||||||
# until the base image, their own lines or go.mod and go.sum change.
|
|
||||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
|
||||||
# also mounts golangci-lint's; both survive from build to build. The
|
|
||||||
# test phase runs as nobody and so mounts Go's build cache at
|
|
||||||
# /tmp/.cache/go-build; its own comment says why.
|
|
||||||
|
|
||||||
# Lint phase, built alone by script/lint. The tools are invoked directly
|
# Lint phase, built alone by script/lint. The tools are invoked directly
|
||||||
# rather than through `make lint`, which runs docker itself and so cannot
|
# rather than through `make lint`, which runs docker itself and so cannot
|
||||||
# run inside a build step.
|
# run inside a build step.
|
||||||
# golangci/golangci-lint:v2.14.0, 2026-10-07
|
# golangci/golangci-lint:v2.14.0, 2026-10-07
|
||||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
|
ARG CACHEBUST
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a
|
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a
|
||||||
@@ -31,7 +24,6 @@ RUN files="$(gofmt -s -l .)" && \
|
|||||||
# Validates .golangci.yml against the schema the pinned binary embeds.
|
# Validates .golangci.yml against the schema the pinned binary embeds.
|
||||||
RUN golangci-lint config verify --config .golangci.yml
|
RUN golangci-lint config verify --config .golangci.yml
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
|
||||||
golangci-lint run --config .golangci.yml ./...
|
golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Test phase, built alone by script/test. -race needs cgo and so a C
|
# Test phase, built alone by script/test. -race needs cgo and so a C
|
||||||
@@ -46,12 +38,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
# nobody (65534) so the tests can write to it.
|
# nobody (65534) so the tests can write to it.
|
||||||
# golang:1.25-trixie, 2026-10-04
|
# golang:1.25-trixie, 2026-10-04
|
||||||
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
|
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
|
||||||
|
ARG CACHEBUST
|
||||||
USER nobody
|
USER nobody
|
||||||
ENV HOME=/tmp
|
ENV HOME=/tmp
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
|
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||||
@@ -59,19 +51,19 @@ RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
|
|||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies are
|
# Build stage. Nothing is wanted from either phase above; the copies are
|
||||||
# what make BuildKit build them first, so nothing after them can run
|
# what make BuildKit build them first, so this stage cannot run unless
|
||||||
# unless lint and test passed.
|
# lint and test passed.
|
||||||
# golang:1.25-alpine, 2026-07-23
|
# golang:1.25-alpine, 2026-07-23
|
||||||
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
|
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
|
||||||
RUN apk add --no-cache git make
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
ARG CACHEBUST
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
COPY --from=test /src/go.sum /dev/null
|
||||||
|
RUN apk add --no-cache git make
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||||
RUN git config --system --add safe.directory /src
|
RUN git config --system --add safe.directory /src
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# The version stamped into the binary: the VERSION build argument when
|
# The version stamped into the binary: the VERSION build argument when
|
||||||
|
|||||||
@@ -3,22 +3,22 @@
|
|||||||
## Description
|
## Description
|
||||||
|
|
||||||
`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that
|
`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that
|
||||||
quickly finds _candidate_ duplicate files, and ultimately whole duplicate
|
quickly identifies _candidate_ duplicate files — and, ultimately, entire
|
||||||
directory trees, on very large filesystems. It does this without reading every
|
duplicate directory trees — across very large filesystems without reading every
|
||||||
byte of every file.
|
byte of every file. Files are considered duplicates when their sizes are equal
|
||||||
|
and they agree on a short ladder of hashes. A file under 10 MiB is hashed in
|
||||||
Two files are duplicates when they have the same size and the same three SHA-256
|
full and compared directly. A larger file is gated first on the SHA-256 of its
|
||||||
hashes. Two of the hashes cover the first and the last 64 KiB of the file. The
|
first 64 KiB and of its last 64 KiB, and only when its size and both of those
|
||||||
third, the content hash, covers the whole file when it is under 100 MiB, so a
|
match another file's is it read for a content hash to compare — the SHA-256 of
|
||||||
match proves identical content. At 100 MiB and above it covers only 1 MiB at
|
the whole file when it is under 50 MiB, or of gigabyte-spaced 1 MiB samples when
|
||||||
each gigabyte, so such a file is never read in full. A match there is a strong
|
it is 50 MiB or larger. Below 50 MiB the content hash is proof of identical
|
||||||
candidate rather than proof.
|
content; at or above 50 MiB it is a strong candidate signal rather than proof,
|
||||||
|
because the gaps between samples are never read. The intended use is finding
|
||||||
The intended use is finding duplicate downloads and duplicated directory trees
|
duplicate downloads and duplicated directory trees on multi-terabyte ZFS servers
|
||||||
on multi-terabyte ZFS servers, where reading every byte of every file is
|
where reading every byte of every file is prohibitively expensive. `scan`
|
||||||
prohibitively expensive. `scan` keeps a persistent SQLite database of file
|
maintains a persistent SQLite database of file signatures that survives between
|
||||||
signatures between runs, so it can run from cron. The reports can be generated
|
runs, so it can be run from cron and the reports can be generated at any time
|
||||||
at any time from the most recent scan.
|
from the most recent scan.
|
||||||
|
|
||||||
This README is the complete and authoritative specification.
|
This README is the complete and authoritative specification.
|
||||||
|
|
||||||
@@ -34,13 +34,13 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
|
|||||||
|
|
||||||
`scan` walks one or more filesystem trees and maintains one database record per
|
`scan` walks one or more filesystem trees and maintains one database record per
|
||||||
regular file (path, size, mtime, head hash, tail hash, content hash). The
|
regular file (path, size, mtime, head hash, tail hash, content hash). The
|
||||||
database persists between runs. A rescan hashes only the files that are new or
|
database persists between runs; a rescan only hashes files that are new or
|
||||||
changed, and removes the records of files that no longer exist. `report` reads
|
changed, or that may have gained a duplicate since the last scan, and removes
|
||||||
the database and prints the file-level duplicates report. `trees` reads the same
|
records for files that no longer exist. `report` reads the database and prints
|
||||||
database and prints the duplicate-tree report. `info` prints the database's
|
the file-level duplicates report. `trees` reads the same database and prints the
|
||||||
details: its path, the sizes of its files and the number of its records. A
|
duplicate-tree report. `info` prints the database's details: its path, the sizes
|
||||||
missing/invalid subcommand — or a `scan` invocation with no `PATH` operand —
|
of its files and the number of its records. A missing/invalid subcommand — or a
|
||||||
prints a usage message and exits 2.
|
`scan` invocation with no `PATH` operand — prints a usage message and exits 2.
|
||||||
|
|
||||||
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
||||||
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
||||||
@@ -49,40 +49,15 @@ results are as fresh as the last completed scan.
|
|||||||
|
|
||||||
### Install
|
### Install
|
||||||
|
|
||||||
With Go installed, this builds and installs the newest release:
|
With Go installed, this builds and installs the current `main` branch:
|
||||||
|
|
||||||
```sh
|
|
||||||
go install sneak.berlin/go/sfdupes@latest
|
|
||||||
```
|
|
||||||
|
|
||||||
Until 1.0, releases are `v0.0.N` tags on the tip of the `main` branch: each time
|
|
||||||
`main` moves and its tip passes `script/cibuild`, that tip is tagged with the
|
|
||||||
next `v0.0.N`. So `@latest` installs `main`'s tip once it has passed the build.
|
|
||||||
`@main` installs the tip before it is tagged:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go install sneak.berlin/go/sfdupes@main
|
go install sneak.berlin/go/sfdupes@main
|
||||||
```
|
```
|
||||||
|
|
||||||
`v0.0.1` is an early build without a database. It does not work as this README
|
The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. A
|
||||||
describes, and `go.mod` retracts it.
|
binary installed this way reports its version as `dev`; one built from a clone
|
||||||
|
or into the Docker image carries the git tag or commit it was built from.
|
||||||
The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. To
|
|
||||||
see which version an installed binary is:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sfdupes --version
|
|
||||||
go version -m "$(command -v sfdupes)"
|
|
||||||
```
|
|
||||||
|
|
||||||
`sfdupes --version` prints `sfdupes VERSION`. For a binary from `go install`,
|
|
||||||
`VERSION` is the version Go recorded in it: the release tag, such as `v0.0.3`,
|
|
||||||
or for `@main` a pseudo-version ending in the first 12 characters of the
|
|
||||||
commit's hash. A binary built from a clone or into the Docker image carries the
|
|
||||||
git tag or commit it was built from instead, and `VERSION` is `dev` only when
|
|
||||||
the binary carries no version at all. `go version -m` prints what Go recorded in
|
|
||||||
the binary; its `mod` line names the module's version. Binaries from `v0.0.2`
|
|
||||||
and earlier print `dev`, so check those with `go version -m`.
|
|
||||||
|
|
||||||
From a clone, `make build` writes the binary to `./sfdupes`:
|
From a clone, `make build` writes the binary to `./sfdupes`:
|
||||||
|
|
||||||
@@ -122,19 +97,14 @@ As a file `/etc/cron.d/sfdupes`:
|
|||||||
the first scan. To keep it elsewhere, set
|
the first scan. To keep it elsewhere, set
|
||||||
`SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line.
|
`SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line.
|
||||||
- `scan` writes nothing to stdout. Its stderr, appended here to
|
- `scan` writes nothing to stdout. Its stderr, appended here to
|
||||||
`/var/log/sfdupes.log`, holds these lines:
|
`/var/log/sfdupes.log`, holds a first line naming the database (see
|
||||||
- a first line naming the database (see "Database");
|
"Database"), plain progress lines naming each phase and its place among the
|
||||||
- plain progress lines naming each phase and its place among the four, as
|
five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it
|
||||||
`[2/4] walk`: one as the phase starts, one every 5 seconds while it runs,
|
runs, and one as it ends), a warning for each path it skips, a line for each
|
||||||
and one as it ends;
|
default exclude its walk does not enter and for each path it leaves out
|
||||||
- a warning for each path it skips;
|
because it is not on a persistent filesystem (see "Rules for the walk"), and
|
||||||
- a line for each default exclude its walk does not enter, and for each path
|
the summary line (see "Progress" and "`scan` mode"). The log grows with every
|
||||||
it leaves out because it is not on a persistent filesystem (see "Rules for
|
scan; rotate it like any other.
|
||||||
the walk");
|
|
||||||
- the summary line (see "Progress" and "`scan` mode").
|
|
||||||
|
|
||||||
The log grows with every scan; rotate it like any other.
|
|
||||||
|
|
||||||
- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A
|
- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A
|
||||||
scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the
|
scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the
|
||||||
reason among the last lines of the log; `tail` prints them, and cron mails
|
reason among the last lines of the log; `tail` prints them, and cron mails
|
||||||
@@ -158,7 +128,7 @@ A row is a candidate, not proof:
|
|||||||
|
|
||||||
- The reports read only the database, so they show the files as of the last
|
- The reports read only the database, so they show the files as of the last
|
||||||
scan; a file may have changed or gone since.
|
scan; a file may have changed or gone since.
|
||||||
- A file of 100 MiB or more is compared only on samples of its content (see
|
- A file of 50 MiB or more is compared only on samples of its content (see
|
||||||
"Duplicate detection").
|
"Duplicate detection").
|
||||||
- Paths that are hard links to one file are listed as duplicates, but they share
|
- Paths that are hard links to one file are listed as duplicates, but they share
|
||||||
their data, so removing one frees nothing.
|
their data, so removing one frees nothing.
|
||||||
@@ -193,23 +163,22 @@ or a symlink, which `trees` does not see.
|
|||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
Duplicate finders that hash every file in full do not scale to the target
|
Duplicate finders that hash entire files do not scale to the target environment:
|
||||||
environment. It holds about 10 million files and 150 TB, on disks that may be
|
~10 million files and ~150 TB on possibly slow or busy disks (a ZFS pool under
|
||||||
slow or busy, such as a ZFS pool under resilver.
|
resilver). sfdupes spends disk I/O only on files whose size at least one other
|
||||||
|
file shares, since a size-unique file cannot be a duplicate. Of those, a file
|
||||||
sfdupes reads a file under 100 MiB in full, and a larger one only at its two
|
under 10 MiB is read in full; a larger one has its cheap end windows read first,
|
||||||
ends and at 1 MiB per gigabyte. A full read of a small file is cheap, and it
|
and is read for a content hash only when its size and both end windows match
|
||||||
makes the comparison exact. A full read of every large file would take
|
another file's — the whole file below 50 MiB, but only gigabyte-spaced samples
|
||||||
unreasonably long. Sampling cuts a 150 GiB file to about 150 MiB of reads, at
|
at or above 50 MiB, so the largest files are never read in full. This keeps a
|
||||||
the price of a comparison that is not exact.
|
full-filesystem sweep tractable, and the signatures are kept in a persistent
|
||||||
|
database, so the expensive filesystem pass is incremental: a rescan re-hashes
|
||||||
The hashes are kept in a persistent database, so the expensive filesystem pass
|
only files whose recorded mtime or size changed, plus — for its content hash — a
|
||||||
is incremental. A rescan reads only the files that are new or whose size or
|
file of 10 MiB or more whose size and end windows have come to match another
|
||||||
mtime changed. All analysis then works from the database alone.
|
file's. All analysis happens offline from the database alone. The end goal is
|
||||||
|
not individual files but whole duplicated trees — duplicate extractions,
|
||||||
The end goal is not individual files but whole duplicated trees, such as a
|
duplicate downloads, copied project trees — which an operator can consider
|
||||||
download extracted twice. An operator can consider removing such a tree as a
|
removing as a unit.
|
||||||
unit.
|
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
@@ -221,10 +190,14 @@ Goals, in order:
|
|||||||
trees), so the operator can consider removing an entire subtree at once.
|
trees), so the operator can consider removing an entire subtree at once.
|
||||||
File-level duplicate detection is the foundation; tree-level detection is
|
File-level duplicate detection is the foundation; tree-level detection is
|
||||||
built on top of it.
|
built on top of it.
|
||||||
2. **Bound the I/O per file.** A file is read only when it is new or has
|
2. **Spend I/O in proportion to duplicate likelihood.** Only files whose size
|
||||||
changed. A file under 100 MiB is then read in full, and a larger one only at
|
at least one other file shares are read at all — a size-unique file cannot
|
||||||
its ends and at 1 MiB per gigabyte (see "Duplicate detection" below).
|
be a duplicate. Those are compared by the ladder in "Duplicate detection"
|
||||||
Therefore no file of 100 MiB or more is read in full. Scale target: tens of
|
below: a file under 10 MiB is hashed in full, while a larger file is gated
|
||||||
|
on cheap 64 KiB end windows first, and gets a content hash only when its
|
||||||
|
size and both end windows match another file's. That hash reads the whole
|
||||||
|
file below 50 MiB but only gigabyte-spaced 1 MiB samples at or above it, so
|
||||||
|
the very largest files are still never read in full. Scale target: tens of
|
||||||
millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool
|
millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool
|
||||||
under resilver). Holding one small record (path, size, mtime) per file in
|
under resilver). Holding one small record (path, size, mtime) per file in
|
||||||
memory during a scan is acceptable; holding every file's hashes is not (they
|
memory during a scan is acceptable; holding every file's hashes is not (they
|
||||||
@@ -237,10 +210,12 @@ Goals, in order:
|
|||||||
being read, so its memory grows with the number of directories and with the
|
being read, so its memory grows with the number of directories and with the
|
||||||
size of the largest directory.
|
size of the largest directory.
|
||||||
3. **Scan incrementally, analyze offline.** The expensive filesystem scan
|
3. **Scan incrementally, analyze offline.** The expensive filesystem scan
|
||||||
maintains a persistent database, and an unchanged file is never read again
|
maintains a persistent database; an unchanged file is never read again on a
|
||||||
on a rescan. All analysis (`report`, `trees`) works from the database alone
|
rescan, except to compute its content hash once a file of 10 MiB or more
|
||||||
and must never touch the scanned filesystem again. `scan` is designed to be
|
comes to match another on size and both end windows. All analysis (`report`,
|
||||||
cronned; the reports run at any time against the last completed scan.
|
`trees`) works from the database alone and must never touch the scanned
|
||||||
|
filesystem again. `scan` is designed to be cronned; the reports run at any
|
||||||
|
time against the last completed scan.
|
||||||
4. **Clean stream separation.** Everything on stdout is machine-readable data.
|
4. **Clean stream separation.** Everything on stdout is machine-readable data.
|
||||||
All progress, warnings, summaries, and help and usage text go to stderr.
|
All progress, warnings, summaries, and help and usage text go to stderr.
|
||||||
Never mix them.
|
Never mix them.
|
||||||
@@ -272,7 +247,7 @@ Four subcommands, all implemented:
|
|||||||
from the database records, compute a Merkle-style digest per directory, and
|
from the database records, compute a Merkle-style digest per directory, and
|
||||||
report maximal groups of identical trees.
|
report maximal groups of identical trees.
|
||||||
4. `info` — the database's details: its path, the sizes of its files, its
|
4. `info` — the database's details: its path, the sizes of its files, its
|
||||||
schema version, the number of its records and their total size.
|
schema version and the counts of its records.
|
||||||
|
|
||||||
```
|
```
|
||||||
sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||||
@@ -287,10 +262,10 @@ sfdupes [command] --help
|
|||||||
CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that
|
CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that
|
||||||
operand's filesystem, and `--no-default-excludes` lets the walk enter the
|
operand's filesystem, and `--no-default-excludes` lets the walk enter the
|
||||||
default excludes, a list of paths; all three are described under "`scan` mode".
|
default excludes, a list of paths; all three are described under "`scan` mode".
|
||||||
`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION` (see
|
`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and
|
||||||
"Install"), to stdout and exits 0, writing nothing to stderr. `-h` or `--help`,
|
exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a
|
||||||
alone or after a subcommand, prints the help text to stderr and exits 0, writing
|
subcommand, prints the help text to stderr and exits 0, writing nothing to
|
||||||
nothing to stdout.
|
stdout.
|
||||||
|
|
||||||
### Database
|
### Database
|
||||||
|
|
||||||
@@ -368,7 +343,9 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
- While it loads the existing records (`load`) and walks: no records. A
|
- While it loads the existing records (`load`) and walks: no records. A
|
||||||
first scan's database holds none, however long its walk has run.
|
first scan's database holds none, however long its walk has run.
|
||||||
- During the hash phase: the first records, committed 10,000 at a time.
|
- During the hash phase: the first records, committed 10,000 at a time.
|
||||||
- In the update phase: the rest of the hashed records, and the deletions.
|
- In the update phase: the rest of the hashed records, the records for
|
||||||
|
size-unique files, and the deletions.
|
||||||
|
- In the content phase: content hashes, again committed 10,000 at a time.
|
||||||
- At the end, as it closes the database: the `-wal` and `-shm` files go
|
- At the end, as it closes the database: the `-wal` and `-shm` files go
|
||||||
away, unless a report still has the database open (see above).
|
away, unless a report still has the database open (see above).
|
||||||
- `report`, `trees` and `info` open the database read-only and need only read
|
- `report`, `trees` and `info` open the database read-only and need only read
|
||||||
@@ -388,9 +365,9 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
size INTEGER NOT NULL, -- bytes, from lstat
|
size INTEGER NOT NULL, -- bytes, from lstat
|
||||||
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat
|
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat
|
||||||
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
||||||
head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB
|
head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB
|
||||||
tail BLOB NOT NULL, -- raw 32-byte SHA-256 of the last 64 KiB
|
tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB
|
||||||
content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples
|
content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples
|
||||||
) WITHOUT ROWID;
|
) WITHOUT ROWID;
|
||||||
CREATE INDEX files_signature ON files (size, head, tail, content);
|
CREATE INDEX files_signature ON files (size, head, tail, content);
|
||||||
```
|
```
|
||||||
@@ -401,46 +378,57 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
nanoseconds past that second. Split this way they hold any mtime a
|
nanoseconds past that second. Split this way they hold any mtime a
|
||||||
filesystem can record, one before 1678 or after 2262 included, which a
|
filesystem can record, one before 1678 or after 2262 included, which a
|
||||||
single 64-bit count of nanoseconds cannot. They are used only for change
|
single 64-bit count of nanoseconds cannot. They are used only for change
|
||||||
detection and are not part of the duplicate key. Every record holds the
|
detection and are not part of the duplicate key. For a file under 10 MiB
|
||||||
three hashes that "Duplicate detection" below describes. The
|
`head`, `tail`, and `content` all hold the whole-file hash (that range is
|
||||||
`files_signature` index lets `report` read the records in signature order
|
hashed in full, with no end windows); for a larger file `head` and `tail`
|
||||||
without sorting the table.
|
hold the first- and last-64 KiB hashes and `content` the whole-file or
|
||||||
|
sampled hash. All three are empty strings when the file has never been
|
||||||
|
hashed because its size was unique as of the last scan that covered it. For
|
||||||
|
a file of 10 MiB or more, `content` stays empty until the content phase of a
|
||||||
|
scan (see "`scan` mode" below) has read the file. A record with an empty
|
||||||
|
`content` is never part of a duplicate group, though it still defines the
|
||||||
|
file for tree reconstruction. The `files_signature` index lets `report` read
|
||||||
|
the records in signature order without sorting the table.
|
||||||
|
|
||||||
### Duplicate detection
|
### Duplicate detection
|
||||||
|
|
||||||
Two files are duplicates when they have the same size and the same three hashes.
|
Two files are duplicates only when they agree on every rung of this ladder; a
|
||||||
`scan` computes the hashes and stores them in the database. `report` and `trees`
|
mismatch at any rung means they are not duplicates. `scan` stores each file's
|
||||||
group the files whose size and hashes all match, whichever scan recorded them.
|
hashes, and `report` and `trees` group files by the whole signature — size,
|
||||||
|
`head`, `tail`, and `content` — so the grouping is exactly this ladder applied
|
||||||
|
across everything scanned into the database, even across separate scans.
|
||||||
|
|
||||||
`scan` gives every file it records three SHA-256 hashes:
|
1. **Size.** Files of different sizes are never compared. Only files whose size
|
||||||
|
at least one other file shares are hashed at all.
|
||||||
|
2. **Under 10 MiB: whole file.** A file smaller than 10 MiB is hashed in full
|
||||||
|
and compared directly, with no separate end-window step — small files are
|
||||||
|
cheap to read to the last byte, and doing so makes the comparison exact.
|
||||||
|
`head`, `tail`, and `content` all hold this whole-file SHA-256, so such a
|
||||||
|
file's signature is decided entirely by its size and its content.
|
||||||
|
3. **10 MiB and above: head and tail.** For a larger file, the SHA-256 of the
|
||||||
|
first 64 KiB (`head`) and of the last 64 KiB (`tail`) are a cheap gate that
|
||||||
|
eliminates most same-size pairs before any bulk reading: the content hash of
|
||||||
|
the next two rungs is computed only for a file whose size, `head`, and
|
||||||
|
`tail` match another file's, whether that file is scanned in the same run or
|
||||||
|
stored by an earlier scan. A stored file that first gains such a match in a
|
||||||
|
later scan gets its content hash then; until it has one, its `content` is
|
||||||
|
empty and it is not a duplicate. At 10 MiB and above the two windows never
|
||||||
|
overlap.
|
||||||
|
4. **10 MiB and above, content below 50 MiB.** The SHA-256 of the entire file.
|
||||||
|
Agreement here is proof of identical content (barring a SHA-256 collision).
|
||||||
|
5. **10 MiB and above, content 50 MiB and above.** A sampled SHA-256: the 1 MiB
|
||||||
|
window at each gigabyte-aligned offset (0, 1 GiB, 2 GiB, … while inside the
|
||||||
|
file, the final window truncated at end of file) is fed, in order, into one
|
||||||
|
hash. This is **deliberately probabilistic** — the gaps between samples are
|
||||||
|
never read, so two large files that agree on every sample are reported as
|
||||||
|
duplicates without being read in full. It is the price of never reading a
|
||||||
|
150 GB file end to end. Because size is already part of the signature, only
|
||||||
|
equal-size files reach this rung, so their sample boundaries always align.
|
||||||
|
|
||||||
- `head`, the hash of the file's first 64 KiB.
|
`head`, `tail`, and `content` are one column each. A file below 10 MiB and one
|
||||||
- `tail`, the hash of the file's last 64 KiB.
|
at or above it never share a size, and neither do a file below 50 MiB and one at
|
||||||
- `content`, the content hash described below.
|
or above it, so a stored value is never ambiguous between the whole-file,
|
||||||
|
end-window, and sampled forms.
|
||||||
In a file of 64 KiB or less, the first and the last 64 KiB are both the whole
|
|
||||||
file, so all three hashes are the hash of the whole file.
|
|
||||||
|
|
||||||
The content hash depends on the file's size:
|
|
||||||
|
|
||||||
- Under 100 MiB, it is the SHA-256 of the whole file. Agreement on it is proof
|
|
||||||
of identical content, barring a SHA-256 collision.
|
|
||||||
- At 100 MiB and above, it is the SHA-256 of samples, so the file is never read
|
|
||||||
in full. A sample is the 1 MiB at each gigabyte-aligned offset: 0, 1 GiB, 2
|
|
||||||
GiB and so on. The samples are fed into one hash in order. A sample that would
|
|
||||||
run past the end of the file stops there.
|
|
||||||
|
|
||||||
The sampled hash is **deliberately probabilistic**. Bytes outside the samples
|
|
||||||
and outside the first and last 64 KiB are never read. Two large files that
|
|
||||||
differ only in such bytes are reported as duplicates. That is the price of never
|
|
||||||
reading a 150 GB file end to end.
|
|
||||||
|
|
||||||
Only files of the same size are compared, so their samples always lie at the
|
|
||||||
same offsets. A file under 100 MiB and one of 100 MiB or more never share a
|
|
||||||
size, so a whole-file hash is never compared with a sampled one.
|
|
||||||
|
|
||||||
`scan` reads a file only to hash it, and only when the file is new or has
|
|
||||||
changed since its record was written (see "`scan` mode").
|
|
||||||
|
|
||||||
### `scan` mode
|
### `scan` mode
|
||||||
|
|
||||||
@@ -466,22 +454,35 @@ FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
|||||||
a one-line warning naming the path and what it is, counts it as skipped, and
|
a one-line warning naming the path and what it is, counts it as skipped, and
|
||||||
drops it from the scanned operands before reading the database. Another operand
|
drops it from the scanned operands before reading the database. Another operand
|
||||||
beneath it is still scanned. The records stored beneath it are not deleted: they
|
beneath it is still scanned. The records stored beneath it are not deleted: they
|
||||||
are treated like any other record outside the scanned operands. If it lies under
|
are treated like any other record outside the scanned operands, including the
|
||||||
another operand, they are under that operand instead, and are treated like any
|
content-phase exception below. If it lies under another operand, they are under
|
||||||
other record there. This is not an error: a scan whose every operand is dropped
|
that operand instead, and are treated like any other record there. This is not
|
||||||
walks nothing and exits 0. An operand that is not on a persistent filesystem
|
an error: a scan whose every operand is dropped walks nothing and exits 0. An
|
||||||
(see "Rules for the walk") is dropped the same way. The line `scan` prints for
|
operand that is not on a persistent filesystem (see "Rules for the walk") is
|
||||||
it is not a warning, though, and it is not counted as skipped.
|
dropped the same way, except that the line `scan` prints for it is not a
|
||||||
|
warning, and it is not counted as skipped.
|
||||||
|
|
||||||
`scan` synchronizes the database with the filesystem state under the scanned
|
`scan` synchronizes the database with the filesystem state under the scanned
|
||||||
operands:
|
operands:
|
||||||
|
|
||||||
- A file not yet in the database is hashed and inserted.
|
- Only a file whose size at least one other file shares is ever read: a
|
||||||
|
size-unique file cannot be a duplicate, so it is recorded without hashes
|
||||||
|
(`head`, `tail`, and `content` empty). The size census covers every file
|
||||||
|
walked this scan plus every database record outside the scanned operands, so a
|
||||||
|
possible duplicate of a separately scanned tree is still recognized.
|
||||||
|
- A file not yet in the database is inserted: hashed when its size is shared,
|
||||||
|
without hashes otherwise.
|
||||||
- A file already in the database is **skipped without reading its contents**
|
- A file already in the database is **skipped without reading its contents**
|
||||||
when its lstat size equals the recorded size and its lstat mtime is not newer
|
when its lstat size equals the recorded size and its lstat mtime is not newer
|
||||||
than the recorded mtime. This is what makes a daily rescan cheap.
|
than the recorded mtime. This is what makes a daily rescan cheap. Exception:
|
||||||
- A file whose mtime is newer than recorded, or whose size differs, is hashed
|
an unchanged file whose record lacks hashes is hashed — and its record updated
|
||||||
again. Change detection compares the mtime to the nanosecond, as finely as the
|
— once its size becomes shared, so hashing deferred by size-uniqueness happens
|
||||||
|
as soon as it could matter. Likewise, an unchanged file of 10 MiB or more
|
||||||
|
whose record has no `content` hash is read for one by the content phase below
|
||||||
|
once its size, `head`, and `tail` match another record's.
|
||||||
|
- A file whose mtime is newer than recorded, or whose size differs, is processed
|
||||||
|
as if new: re-hashed, or recorded without hashes, per the shared-size rule.
|
||||||
|
Change detection compares the mtime to the nanosecond, as finely as the
|
||||||
filesystem records it, so a same-size rewrite counts as a change whenever the
|
filesystem records it, so a same-size rewrite counts as a change whenever the
|
||||||
filesystem gives it a later mtime than recorded, even within the same second.
|
filesystem gives it a later mtime than recorded, even within the same second.
|
||||||
- A database record whose path lies under one of the scanned operands but was
|
- A database record whose path lies under one of the scanned operands but was
|
||||||
@@ -489,26 +490,30 @@ operands:
|
|||||||
deleted files. It also removes records for paths that failed to stat or hash
|
deleted files. It also removes records for paths that failed to stat or hash
|
||||||
this run: the database only ever contains signatures verified by the most
|
this run: the database only ever contains signatures verified by the most
|
||||||
recent scan that covered them (a subsequent successful scan re-adds such
|
recent scan that covered them (a subsequent successful scan re-adds such
|
||||||
files).
|
files). A failure in the content phase below removes nothing: the record is
|
||||||
|
left as it is.
|
||||||
- Database records outside the scanned operands are untouched, so disjoint trees
|
- Database records outside the scanned operands are untouched, so disjoint trees
|
||||||
can be scanned on different schedules into the same database.
|
can be scanned on different schedules into the same database. The one
|
||||||
- A record at or below a path the walk left out counts as outside the scanned
|
exception is the content phase below: a stored file of 10 MiB or more without
|
||||||
operands, so it is not deleted. The walk leaves out a path that:
|
a `content` hash is read for one, wherever it lies, once its size, `head`, and
|
||||||
- is a default exclude;
|
`tail` match another record's. If that file is gone or has changed since its
|
||||||
- is not on a persistent filesystem;
|
record was written, the record is left as it is.
|
||||||
- with `-x`, is a directory on another filesystem than its operand.
|
- A record at or below a path the walk left out, because that path is a default
|
||||||
|
exclude, is not on a persistent filesystem, or, with `-x`, is a directory on
|
||||||
|
another filesystem than its operand, counts as outside the scanned operands:
|
||||||
|
it is not deleted, and its size joins the size census. A filesystem mounted
|
||||||
|
under an operand can therefore be scanned on its own schedule into the same
|
||||||
|
database. Records below an operand at or inside such a path are not kept this
|
||||||
|
way, since that operand's own walk verifies them.
|
||||||
|
|
||||||
A filesystem mounted under an operand can therefore be scanned on its own
|
`scan` runs **five sequential phases over the whole scan**. Parallelism lives
|
||||||
schedule into the same database. Records below an operand at or inside such
|
inside the walk, hash and content phases; batched database writes begin during
|
||||||
a path are not kept this way, since that operand's own walk verifies them.
|
the hash phase:
|
||||||
|
|
||||||
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
|
||||||
inside the walk and hash phases, and batched database writes begin during the
|
|
||||||
hash phase:
|
|
||||||
|
|
||||||
1. **load** — read the existing records. Each record under the operands is kept
|
1. **load** — read the existing records. Each record under the operands is kept
|
||||||
in memory, by path, with its size and its mtime, for the walk to compare
|
in memory, by path, with its size, its mtime and whether it has hashes, for
|
||||||
each file with.
|
the walk to compare each file with; the size of every other record joins the
|
||||||
|
size census.
|
||||||
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
|
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
|
||||||
with the walk worker pool: every operand seeds the shared queue, and each
|
with the walk worker pool: every operand seeds the shared queue, and each
|
||||||
worker reads one directory at a time, handing discovered subdirectories back
|
worker reads one directory at a time, handing discovered subdirectories back
|
||||||
@@ -516,23 +521,48 @@ hash phase:
|
|||||||
(while the directory's metadata is still hot). Sequential directory
|
(while the directory's metadata is still hot). Sequential directory
|
||||||
enumeration is metadata-latency-bound and takes hours at tens of millions of
|
enumeration is metadata-latency-bound and takes hours at tens of millions of
|
||||||
files; per-directory parallelism is what makes the walk tractable on large
|
files; per-directory parallelism is what makes the walk tractable on large
|
||||||
or busy pools. The walk compares each file with its record as it goes, and
|
or busy pools. The walk builds the size census and resolves unchanged
|
||||||
an unchanged file needs nothing more. Every new or changed file is carried
|
already-hashed files on the fly; every other file is carried to the hash
|
||||||
to the hash phase as a (path, size, mtime) record.
|
phase as a (path, size, mtime) record.
|
||||||
3. **hash** — the worker pool hashes every carried file as described in
|
3. **hash** — with the census complete, each carried file's size decides its
|
||||||
"Duplicate detection" above, which gives the file its `head`, `tail` and
|
fate. Size-unique files are never read: new or changed ones are recorded
|
||||||
`content`. Zero-length files have constant hashes and are never opened.
|
without hashes in the update phase, unchanged unhashed ones simply keep
|
||||||
Files are hashed in **inode order**, which minimizes seeks on spinning
|
their records. Every file with a shared size is hashed by the worker pool as
|
||||||
disks. Paths that are hard links to the same inode are **read once**, and
|
described in "Duplicate detection" above: a file under 10 MiB in full, which
|
||||||
all share the one result. A hard-link backup farm therefore costs one read
|
gives its `head`, `tail`, and `content` alike, and a larger file only in its
|
||||||
per inode, not per path. The phase total counts actual reads, so its
|
end windows, which give its `head` and `tail`; its content hash is left to
|
||||||
progress and its time remaining are meaningful. Completed records are
|
the content phase. Zero-length files have constant hashes and are never
|
||||||
committed in batched transactions **while hashing runs**. A scan interrupted
|
opened. Files are hashed in **inode order** (minimizing seeks on spinning
|
||||||
after hours therefore keeps everything hashed so far, and the next scan
|
disks), and paths that are hard links to the same inode are **read once**,
|
||||||
resumes cheaply, skipping records already written.
|
all sharing the one result — a hard-link backup farm costs one read per
|
||||||
4. **update** — commit the final partial batch of hashed records. Then delete
|
inode, not per path. The phase total counts actual reads, so progress and
|
||||||
the records the scan did not verify: those of vanished files, and of paths
|
ETA are meaningful. Completed records are committed in batched transactions
|
||||||
that failed to stat or hash.
|
**while hashing runs**, so a scan interrupted after hours keeps everything
|
||||||
|
hashed so far and the next scan resumes cheaply, skipping records already
|
||||||
|
written.
|
||||||
|
4. **update** — commit the final partial batch, the hash-less records for
|
||||||
|
size-unique new and changed files, and the deletions for records the scan
|
||||||
|
did not verify (vanished files, plus paths that failed to stat or hash).
|
||||||
|
5. **content** — find every record of 10 MiB or more without a `content` hash
|
||||||
|
whose size, `head`, and `tail` equal another record's, anywhere in the
|
||||||
|
database: records from this scan and records stored by earlier scans, inside
|
||||||
|
or outside the scanned operands. SQLite finds them, so only the records to
|
||||||
|
be read are kept in memory, never every file's hashes. Every record sharing
|
||||||
|
their size, `head`, and `tail`, including one that already has a `content`
|
||||||
|
hash, has its file checked with `lstat` first. A file that is gone, is no
|
||||||
|
longer a regular file, or has changed (a different size, or an mtime newer
|
||||||
|
than recorded) keeps its record as it is and does not count as a match for
|
||||||
|
the others. Any other `lstat` error is warned about and counted as skipped,
|
||||||
|
with the same result. If such a record has no `content` hash, it stays out
|
||||||
|
of duplicate groups; if it has one, it is still reported until a scan
|
||||||
|
covering its own tree updates or removes it. The files that pass and have no
|
||||||
|
`content` hash are read only if at least two of those records pass, so a
|
||||||
|
file whose only matches are stale costs no read; a file that already has a
|
||||||
|
`content` hash is never read again. They are read by a worker pool as in the
|
||||||
|
hash phase, in inode order and once per inode, and their content hashes are
|
||||||
|
committed in batches. A failed read is warned about and counted as skipped;
|
||||||
|
its record keeps an empty `content`, so it is not a duplicate, and a later
|
||||||
|
scan tries again.
|
||||||
|
|
||||||
Rules for the walk:
|
Rules for the walk:
|
||||||
|
|
||||||
@@ -543,22 +573,13 @@ Rules for the walk:
|
|||||||
- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking
|
- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking
|
||||||
them would list every file once per snapshot), not even when it is an operand;
|
them would list every file once per snapshot), not even when it is an operand;
|
||||||
such an operand is dropped the same way.
|
such an operand is dropped the same way.
|
||||||
- Never record the scan's own database. Once the database and its `.lock` file
|
|
||||||
are open, `scan` notes the device and inode of the database file, its `-wal`
|
|
||||||
and `-shm` files and its `.lock` file, and the walk passes over a file with
|
|
||||||
one of them, however it reaches it: as an operand, below one, or through a
|
|
||||||
symlinked directory. It prints no line for such a file and does not count it
|
|
||||||
as skipped. A record that an earlier scan left for one of them is deleted,
|
|
||||||
like the record of any file the scan no longer verifies. There is no
|
|
||||||
`-journal` file during the walk, because the database is in WAL mode then (see
|
|
||||||
"Database").
|
|
||||||
- Filesystem boundaries are crossed by default. With `-x` (long form
|
- Filesystem boundaries are crossed by default. With `-x` (long form
|
||||||
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
||||||
into a directory on a different filesystem than its `PATH` operand; each
|
into a directory on a different filesystem than its `PATH` operand; each
|
||||||
operand is bounded by its own filesystem. The records below such a directory
|
operand is bounded by its own filesystem. The records below such a directory,
|
||||||
count as outside the scanned operands, as described in "`scan` mode" above:
|
unless they lie below an operand at or inside that directory, count as outside
|
||||||
they are not deleted. That does not hold below an operand at or inside that
|
the scanned operands, as described in "`scan` mode" above: they are not
|
||||||
directory.
|
deleted, and their sizes join the size census.
|
||||||
- Never record anything that is not on a persistent filesystem. A filesystem is
|
- Never record anything that is not on a persistent filesystem. A filesystem is
|
||||||
not persistent when the kernel makes it up or keeps it in memory: one of type
|
not persistent when the kernel makes it up or keeps it in memory: one of type
|
||||||
`proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`,
|
`proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`,
|
||||||
@@ -568,26 +589,25 @@ Rules for the walk:
|
|||||||
it names the database and before it takes the lock, `scan` lists the mounted
|
it names the database and before it takes the lock, `scan` lists the mounted
|
||||||
filesystems, from `/proc/self/mountinfo` on Linux and from the system's list
|
filesystems, from `/proc/self/mountinfo` on Linux and from the system's list
|
||||||
of mounted filesystems on macOS (on any other system it lists none), and notes
|
of mounted filesystems on macOS (on any other system it lists none), and notes
|
||||||
the device ID of each one of those types. On Linux that is the device ID the
|
the device ID of the mount point of each one of those types, read with
|
||||||
list gives, so a filesystem that a later mount hides still gives its own. On
|
`lstat`. Where one filesystem is mounted over another, the one mounted last
|
||||||
macOS it is read with `lstat` on the mount point, and where one filesystem is
|
counts. If `scan` cannot list them, it fails (exit 1). The walk then never
|
||||||
mounted over another on the same mount point, the one mounted last counts. If
|
enters a directory, and never records a file, on one of those devices, however
|
||||||
`scan` cannot list them, it fails (exit 1). The walk then never enters a
|
it reaches it: as an operand, below one, or through a bind mount, which keeps
|
||||||
directory, and never records a file, on one of those devices, however it
|
the device ID of what it binds. No flag turns this off. For each operand,
|
||||||
reaches it: as an operand, below one, or through a bind mount, which keeps the
|
|
||||||
device ID of what it binds. No flag turns this off. For each operand,
|
|
||||||
directory or file it leaves out this way, `scan` prints a line on stderr
|
directory or file it leaves out this way, `scan` prints a line on stderr
|
||||||
naming it and the type of its filesystem. It prints that line as it prints a
|
naming it and the type of its filesystem, as it prints a warning, so it never
|
||||||
warning, so the line never lands inside a progress display:
|
lands inside a progress display:
|
||||||
|
|
||||||
```
|
```
|
||||||
scan: skipped /dev/shm (tmpfs, not a persistent filesystem)
|
scan: skipped /dev/shm (tmpfs, not a persistent filesystem)
|
||||||
```
|
```
|
||||||
|
|
||||||
That line is not a warning, and the path is not counted as skipped. An
|
That line is not a warning, and the path is not counted as skipped. Records
|
||||||
earlier scan's records at or below it count as outside the scanned operands,
|
that an earlier scan left at or below it, unless they lie below an operand
|
||||||
as described in "`scan` mode" above: they are not deleted. That does not
|
at or inside it, count as outside the scanned operands, as described in
|
||||||
hold below an operand at or inside it.
|
"`scan` mode" above: they are not deleted, and their sizes join the size
|
||||||
|
census.
|
||||||
|
|
||||||
- Below an operand, never descend into a default exclude: a path that holds the
|
- Below an operand, never descend into a default exclude: a path that holds the
|
||||||
kernel's view of the running system, or temporary files, so that
|
kernel's view of the running system, or temporary files, so that
|
||||||
@@ -619,35 +639,32 @@ Rules for the walk:
|
|||||||
|
|
||||||
That line is not a warning, and the directory is not counted as skipped. A
|
That line is not a warning, and the directory is not counted as skipped. A
|
||||||
default exclude that is not on a persistent filesystem, as `/proc` is, gets
|
default exclude that is not on a persistent filesystem, as `/proc` is, gets
|
||||||
the line for its filesystem instead. One that is itself an operand gets no
|
the line for its filesystem instead, and one that is itself an operand gets
|
||||||
line, since it is walked as that operand. When the walk does not enter a
|
no line, since it is walked as that operand. The records below a default
|
||||||
default exclude, the records below it count as outside the scanned operands,
|
exclude the walk does not enter, unless they lie below an operand at or
|
||||||
as described in "`scan` mode" above: they are not deleted. That does not
|
inside it, count as outside the scanned operands, as described in "`scan`
|
||||||
hold below an operand at or inside it.
|
mode" above: they are not deleted, and their sizes join the size census.
|
||||||
|
|
||||||
- On a per-path error, such as permission denied or an unreadable file: print a
|
- On any per-path error (permission denied, file vanished between passes,
|
||||||
one-line warning to stderr, skip the path, and continue. Per-file errors never
|
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
||||||
abort the run; the final summary reports how many were skipped. As specified
|
Per-file errors never abort the run; the final summary reports how many were
|
||||||
above, a skipped path that has a database record from an earlier scan loses
|
skipped. As specified above, a skipped path that has a database record from an
|
||||||
that record. An unreadable directory subtree likewise loses its records. This
|
earlier scan loses that record, unless it failed only in the content phase, or
|
||||||
is accepted: the database mirrors what the latest scan could actually verify.
|
is an operand dropped before the database was read whose record counts as
|
||||||
The exception is an operand dropped before the database was read, whose record
|
outside the scanned operands; an unreadable directory subtree likewise loses
|
||||||
counts as outside the scanned operands.
|
its records (accepted: the database mirrors what the latest scan could
|
||||||
- A file or directory removed while the scan runs is not an error. The scan
|
actually verify).
|
||||||
prints no line for it and does not count it as skipped. A path gone when the
|
|
||||||
walk reaches it gets no record. A file gone when the hash phase opens it gets
|
|
||||||
none either. The records an earlier scan left at or below such a path are
|
|
||||||
deleted.
|
|
||||||
|
|
||||||
Concurrency: the walk phase, which also stats files, and the hash phase each use
|
Concurrency: the walk phase (which also stats files), the hash phase, and the
|
||||||
a worker pool of `--workers` workers (default `runtime.NumCPU()`). The walk
|
content phase each use a worker pool of `--workers` workers (default
|
||||||
parallelizes across directories, and hashing across files. `--workers` must be
|
`runtime.NumCPU()`); the walk parallelizes across directories, hashing across
|
||||||
at least 1: a smaller value is a usage error, reported in one line on stderr
|
files. `--workers` must be at least 1: a smaller value is a usage error,
|
||||||
with exit 2 before anything is scanned. Both phases are seek-bound on spinning
|
reported in one line on stderr with exit 2 before anything is scanned. All three
|
||||||
disks, so raising `--workers` well past the core count can help on pools with
|
phases are seek-bound on spinning disks, so raising `--workers` well past the
|
||||||
many spindles. The main goroutine owns the database writes and the progress
|
core count can help on pools with many spindles. The main goroutine owns
|
||||||
counts. Each progress display draws the counts from a goroutine of its own, and
|
partitioning, database writes, and the progress counts, which each progress
|
||||||
must never block the workers.
|
display draws from a goroutine of its own; progress display must never block the
|
||||||
|
workers.
|
||||||
|
|
||||||
`scan` writes nothing to stdout. The summary line on stderr reports the files
|
`scan` writes nothing to stdout. The summary line on stderr reports the files
|
||||||
seen this run broken down by disposition, plus skips:
|
seen this run broken down by disposition, plus skips:
|
||||||
@@ -670,7 +687,9 @@ output whether or not the scanned filesystem is still mounted.
|
|||||||
|
|
||||||
Processing:
|
Processing:
|
||||||
|
|
||||||
- Group the records by the key `(size, head, tail, content)`.
|
- Records without a `content` hash (see "Database" above) are excluded: their
|
||||||
|
content is unknown, so they are never reported as duplicates.
|
||||||
|
- Group the remaining records by the key `(size, head, tail, content)`.
|
||||||
- Every group with two or more paths is a duplicate group.
|
- Every group with two or more paths is a duplicate group.
|
||||||
- Within each group, sort paths lexicographically (byte order). The first path
|
- Within each group, sort paths lexicographically (byte order). The first path
|
||||||
is the group's `first`; every other path is a `dupe`.
|
is the group's `first`; every other path is a `dupe`.
|
||||||
@@ -724,7 +743,9 @@ records, split on `/`.
|
|||||||
Definitions:
|
Definitions:
|
||||||
|
|
||||||
- A file's **signature** is `(size, head, tail, content)` — mtime is
|
- A file's **signature** is `(size, head, tail, content)` — mtime is
|
||||||
informational and excluded.
|
informational and excluded. A record without a `content` hash has unknown
|
||||||
|
content: its signature is treated as unique to that file, so a tree containing
|
||||||
|
such a file never compares equal to any other tree.
|
||||||
- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up:
|
- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up:
|
||||||
serialize the directory's child entries — for a file child, its name and
|
serialize the directory's child entries — for a file child, its name and
|
||||||
signature; for a subdirectory child, its name and that subdirectory's digest —
|
signature; for a subdirectory child, its name and that subdirectory's digest —
|
||||||
@@ -798,6 +819,8 @@ database file size 2147483648 (2.0 GiB)
|
|||||||
modified 2026-10-16T12:02:50Z
|
modified 2026-10-16T12:02:50Z
|
||||||
schema version 1
|
schema version 1
|
||||||
records 12345678
|
records 12345678
|
||||||
|
records with head and tail hashes 1234567
|
||||||
|
records with content hash 123456
|
||||||
size of recorded files 150000000000000 (136.4 TiB)
|
size of recorded files 150000000000000 (136.4 TiB)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -810,8 +833,10 @@ size of recorded files 150000000000000 (136.4 TiB)
|
|||||||
database is in WAL mode, the newest changes may be in the `-wal` file alone.
|
database is in WAL mode, the newest changes may be in the `-wal` file alone.
|
||||||
- `schema version` is the database's `PRAGMA user_version`. A database with
|
- `schema version` is the database's `PRAGMA user_version`. A database with
|
||||||
another version than this build's is refused, as by `report`.
|
another version than this build's is refused, as by `report`.
|
||||||
- `records` is the number of records. `size of recorded files` adds up the sizes
|
- `records` is the number of records. `records with head and tail hashes` and
|
||||||
of all records, in bytes, then in human units.
|
`records with content hash` count the records that have those hashes (see
|
||||||
|
"Database"), and `size of recorded files` adds up the sizes of all records, in
|
||||||
|
bytes, then in human units.
|
||||||
|
|
||||||
`info` runs in two phases, each with its own progress display on stderr (see
|
`info` runs in two phases, each with its own progress display on stderr (see
|
||||||
"Progress"): `open` opens the database; `count` reads the schema version and
|
"Progress"): `open` opens the database; `count` reads the schema version and
|
||||||
@@ -827,21 +852,28 @@ Use the progress-bar library for all progress of `scan`, `report`, `trees` and
|
|||||||
stderr.
|
stderr.
|
||||||
|
|
||||||
`scan`, `report`, `trees` and `info` must always show that they are working and
|
`scan`, `report`, `trees` and `info` must always show that they are working and
|
||||||
not hung. Each phase gets its own display from the moment it starts. The display
|
not hung. Each phase gets its own display from the moment it starts, naming the
|
||||||
names the phase and its place among the command's phases. It shows the phase's
|
phase and its place among the command's phases and showing its progress, its
|
||||||
progress, its elapsed time and its time remaining.
|
elapsed time and its estimated time remaining (ETA).
|
||||||
|
|
||||||
A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
A scan has five phases (`[1/5] load`, `[2/5] walk`, `[3/5] hash`,
|
||||||
`[4/4] update`):
|
`[4/5] update`, `[5/5] content`):
|
||||||
|
|
||||||
- `load` counts the records first, then reads them against that count. While the
|
- `load` counts the records first, then reads them against that count. While the
|
||||||
count runs, the display says it is counting the records.
|
count runs, the display says it is counting the records.
|
||||||
- `walk` has no exact total. Its estimate is the number of records the last scan
|
- `walk` has no exact total. Its estimate is the number of records the last scan
|
||||||
left under the operands, known once `load` is done and shown with a `~`. The
|
left under the operands, known once `load` is done and shown with a `~`. Once
|
||||||
time remaining is unknown once the walk passes the estimate, and when there
|
the walk passes the estimate, or when there are no such records, as on a first
|
||||||
are no such records, as on a first scan.
|
scan, the ETA is unknown.
|
||||||
- `hash` and `update` have exact totals. Only files that actually need hashing
|
- `hash` and `update` have exact totals. Only files that actually need hashing
|
||||||
appear in the hash total, so its time remaining is meaningful.
|
appear in the hash total, so its ETA is meaningful.
|
||||||
|
- `content` first counts the records whose files it checks with `lstat`, which
|
||||||
|
SQLite does by adding up the number of records with each matching size,
|
||||||
|
`head`, and `tail`, without fetching the records. It then says it is searching
|
||||||
|
the database until SQLite returns the first record; the checks, and their rate
|
||||||
|
and ETA, start there. The files to read then get a display of their own, with
|
||||||
|
their count as its total, and so does the writing of the last of their records
|
||||||
|
to the database, with the count of those records as its total.
|
||||||
|
|
||||||
`trees` has four phases (`[1/4] open`, `[2/4] read`, `[3/4] group`,
|
`trees` has four phases (`[1/4] open`, `[2/4] read`, `[3/4] group`,
|
||||||
`[4/4] write`, see "`trees` mode"):
|
`[4/4] write`, see "`trees` mode"):
|
||||||
@@ -866,8 +898,8 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
|||||||
- `open` has no total. While it opens the database, waiting on a busy one
|
- `open` has no total. While it opens the database, waiting on a busy one
|
||||||
included, the display says it is opening the database.
|
included, the display says it is opening the database.
|
||||||
- `count` has no total either, because one query counts everything: the display
|
- `count` has no total either, because one query counts everything: the display
|
||||||
says it is counting the records until the query returns, and its time
|
says it is counting the records until the query returns, and its ETA is
|
||||||
remaining is unknown.
|
unknown.
|
||||||
|
|
||||||
Each display is one line holding:
|
Each display is one line holding:
|
||||||
|
|
||||||
@@ -880,69 +912,56 @@ Each display is one line holding:
|
|||||||
while nothing completes, and a burst of items after a wait shows as a burst
|
while nothing completes, and a burst of items after a wait shows as a burst
|
||||||
rather than as a steady rate. Once no item has completed for 10 seconds, the
|
rather than as a steady rate. Once no item has completed for 10 seconds, the
|
||||||
word `stalled` and where the phase waits take the rate's place (see below).
|
word `stalled` and where the phase waits take the rate's place (see below).
|
||||||
- elapsed time, labelled `elapsed`
|
- elapsed time
|
||||||
- time remaining, labelled `remain`: a duration such as `12m9s`, or `unknown`.
|
- estimated time remaining, or `unknown`. It comes from the phase's average rate
|
||||||
It comes from the phase's average rate since it started counting, so that it
|
since it started counting, so that it does not swing with each burst.
|
||||||
does not swing with each burst.
|
|
||||||
|
|
||||||
Example lines, as printed when stderr is not a terminal:
|
Example shape (exact layout is flexible, content is not):
|
||||||
|
|
||||||
```
|
```
|
||||||
[3/4] hash: [12345/98765] 12%, 92 files/s, elapsed 2m32s, remain 17m44s
|
[3/5] hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54
|
||||||
[2/4] walk: [1234567/~1200000] 103%, 5120 files/s, elapsed 4m1s, remain unknown
|
[2/5] walk: [1234567/~1200000] 103%, 5120 files/s, elapsed 4m1s, eta unknown
|
||||||
[3/4] hash: [12345/45678] 27%, stalled disk/db, elapsed 4m30s, remain 12m9s
|
[3/5] hash: [12345/45678] 27%, stalled disk/db, elapsed 4m30s, eta 12m9s
|
||||||
```
|
```
|
||||||
|
|
||||||
When no item has completed for 10 seconds, as while one large file is read, a
|
When no item has completed for 10 seconds, as while one large file is read, a
|
||||||
disk is slow or the database is busy, the line says `stalled` and where the
|
disk is slow or the database is busy, the line says `stalled` and where the
|
||||||
phase waits in place of the rate, which would read 0: `disk` for files, `db` for
|
phase waits in place of the rate, which would read 0: `disk` for files, `db` for
|
||||||
the database, `out` for stdout, and two of them, as `disk/db`, for a phase that
|
the database, `out` for stdout, and two of them, as `disk/db`, for a phase that
|
||||||
may wait on either. `walk` says `disk`. `hash` says `disk/db`, as it reads files
|
may wait on either. `walk` says `disk`. `hash` and the content phase's reads say
|
||||||
and writes their records to the database between them. `load`, `update` and the
|
`disk/db`, as they read files and write their records to the database between
|
||||||
`read` phase of `trees` say `db`. `trees`' `write` says `out`, and its `group`,
|
them, and so do the content phase's checks, which `lstat` each record's file as
|
||||||
which works in memory, says `cpu`. The `read` phase of `report` writes the
|
the database search returns it. `load`, `update`, the content phase's last write
|
||||||
report while it reads the records, so it says `db/out`. A phase still waiting
|
and the `read` phase of `trees` say `db`. `trees`' `write` says `out`, and its
|
||||||
for its total already says what it waits for, and says nothing more.
|
`group`, which works in memory, says `cpu`. The `read` phase of `report` writes
|
||||||
|
the report while it reads the records, so it says `db/out`. A phase still
|
||||||
|
waiting for its total already says what it waits for, and says nothing more.
|
||||||
|
|
||||||
Additional requirements:
|
Additional requirements:
|
||||||
|
|
||||||
- On a terminal, every display starts with the progress-bar library's spinner in
|
- On a terminal, a phase with an exact total (every phase but `scan`'s `walk`,
|
||||||
its first column, followed by its line. Each display is drawn again several
|
the `open` phase of `report` and `trees`, and both phases of `info`, once it
|
||||||
times a second, so its spinner turns and its elapsed time keeps moving while a
|
has counted what it will do) is drawn as its line followed by the progress-bar
|
||||||
phase waits for its next item.
|
library's bar, 10 columns wide, which shows the percent in place of the line's
|
||||||
- On a terminal, a phase with an exact total ends its line with its percent and
|
own. Any other phase, and a phase whose bar has no room, is drawn as the
|
||||||
a bar 10 columns wide. This percent takes the place of the line's own. The bar
|
library's spinner followed by its line. Each display is drawn again several
|
||||||
stays at the terminal's right edge, so it does not move as the line grows or
|
times a second, so its elapsed time keeps moving while a phase waits for its
|
||||||
shrinks. A phase with nothing to do shows no bar. Every phase has an exact
|
next item.
|
||||||
total once it has counted what it will do, except these:
|
|
||||||
- `scan`'s `walk`;
|
|
||||||
- the `open` phase of `report` and `trees`;
|
|
||||||
- both phases of `info`.
|
|
||||||
|
|
||||||
On a terminal 100 columns wide:
|
|
||||||
|
|
||||||
```
|
|
||||||
⠹ [3/4] hash: [12345/98765], 92 files/s, elapsed 2m32s, remain 17m44s 12% |█ |
|
|
||||||
```
|
|
||||||
|
|
||||||
- On a terminal no line reaches the terminal's last column, however narrow the
|
- On a terminal no line reaches the terminal's last column, however narrow the
|
||||||
terminal is. A line too wide for the terminal first loses whichever of these
|
terminal is. A line too wide for the terminal first loses its bar, which gives
|
||||||
it shows: its rate, the words of a stalled phase, or what the phase is waiting
|
way to the spinner, then its rate, the words of a stalled phase, or what the
|
||||||
for. The other four parts are kept whole:
|
phase is waiting for. The phase and its place, the progress, the elapsed time
|
||||||
- the phase and its place;
|
and the ETA are kept whole. On an 80-column terminal, at counts like those of
|
||||||
- the progress;
|
the stalled line in the example above, a stalled line keeps its words whenever
|
||||||
- the elapsed time;
|
the elapsed time and the ETA are both in minutes, except in the content
|
||||||
- the time remaining.
|
phase's checks and reads. Their lines are the widest, and at most times in
|
||||||
|
minutes they lose the words. Where the words and the ETA do not both fit, as
|
||||||
The line loses its bar only when those four and the bar do not fit. It is
|
at millions of files and times in hours, the words give way. Only when those
|
||||||
cut short at the terminal's edge only when those four alone are too wide,
|
four alone are too wide, as on a terminal narrower than about 70 columns hours
|
||||||
and then the end of the time remaining is lost first. That happens hours
|
into a scan of millions of files, is the line cut short at the terminal's
|
||||||
into a scan of millions of files, on a terminal narrower than about 75
|
edge, so the end of the ETA is lost first. A terminal made narrower while a
|
||||||
columns. On a terminal 100 columns wide, the stalled line of the example
|
phase runs gets the same: the next draw erases the line within the new width
|
||||||
above keeps its words and its bar. The next draw fits the line to a terminal
|
and fits the line to it, and the display goes on redrawing in place.
|
||||||
made narrower while a phase runs. It erases the old line within the new
|
|
||||||
width, and the display goes on redrawing in place.
|
|
||||||
|
|
||||||
- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI
|
- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI
|
||||||
redraws: print the line the moment each phase starts, every 5 seconds while it
|
redraws: print the line the moment each phase starts, every 5 seconds while it
|
||||||
runs (whether or not an item has completed, and never more often), and once
|
runs (whether or not an item has completed, and never more often), and once
|
||||||
@@ -1067,22 +1086,13 @@ entrypoints are:
|
|||||||
- `script/install-precommit` — install the git pre-commit hook, as
|
- `script/install-precommit` — install the git pre-commit hook, as
|
||||||
`.git/hooks/pre-commit`, that runs `script/precommit`.
|
`.git/hooks/pre-commit`, that runs `script/precommit`.
|
||||||
|
|
||||||
Every `docker build` in `script/` passes a new `CACHEBUST` build argument. In
|
Every `docker build` in `script/` passes a new `CACHEBUST` build argument, which
|
||||||
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
|
every stage of the `Dockerfile` declares first, so every `RUN` step runs again
|
||||||
that install system packages and download the Go modules, or right after `FROM`
|
each time. On an unchanged tree Docker would otherwise serve the gate steps from
|
||||||
in a stage with neither, and every step that lints, tests or compiles comes
|
its cache, and the build would pass having run no test and no linter. Go's build
|
||||||
after it, so every `RUN` step after it runs again each time. On an unchanged
|
cache is kept between builds on the steps that compile, so only what changed is
|
||||||
tree Docker would otherwise serve the gate steps from its cache, and the build
|
compiled again; the tests run with `-count=1`, so none reports a stored result.
|
||||||
would pass having run no test and no linter. The layers above that line stay in
|
Every run downloads the Go modules again and needs the network.
|
||||||
Docker's build cache until the base image, their own lines, `go.mod` or `go.sum`
|
|
||||||
change, or until that cache is cleared; only then are the packages installed and
|
|
||||||
the modules downloaded again. Go's build cache is kept between builds on the
|
|
||||||
steps that compile, so only what changed is compiled again; the tests run with
|
|
||||||
`-count=1`, so none reports a stored result. `golangci-lint run` also keeps
|
|
||||||
golangci-lint's cache, and takes a package's findings from it only when the
|
|
||||||
package's files and those of the packages it imports, `go.mod`, the linters and
|
|
||||||
their settings in `.golangci.yml`, and the golangci-lint version are all
|
|
||||||
unchanged.
|
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
@@ -1146,6 +1156,10 @@ All of the following, run in this directory, must pass:
|
|||||||
./sfdupes report
|
./sfdupes report
|
||||||
```
|
```
|
||||||
|
|
||||||
|
(The database lives in a temp directory of its own: inside `$d`, the scan
|
||||||
|
would record it, and its empty lock file would join the `empty1`/`empty2`
|
||||||
|
group.)
|
||||||
|
|
||||||
Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
|
Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
|
||||||
group (two dupe rows, `first` is the lexicographically smallest path);
|
group (two dupe rows, `first` is the lexicographically smallest path);
|
||||||
`t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
|
`t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
|
||||||
@@ -1174,8 +1188,8 @@ Tracked in [TODO.md](TODO.md).
|
|||||||
## Non-goals
|
## Non-goals
|
||||||
|
|
||||||
- No byte-for-byte compare, and no deletion or linking of duplicates. Files that
|
- No byte-for-byte compare, and no deletion or linking of duplicates. Files that
|
||||||
match are compared by a SHA-256 of the whole file below 100 MiB, and only by
|
match are compared by a SHA-256 of the whole file below 50 MiB, and only by
|
||||||
samples at 100 MiB and over. The reports are advisory; acting on them is the
|
samples at 50 MiB and over. The reports are advisory; acting on them is the
|
||||||
user's job.
|
user's job.
|
||||||
- No persistence beyond the SQLite database described above; no export/import
|
- No persistence beyond the SQLite database described above; no export/import
|
||||||
formats.
|
formats.
|
||||||
|
|||||||
+34
-67
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-18
|
last_modified: 2026-10-15
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -159,11 +159,8 @@ style conventions are in separate documents:
|
|||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||||
org, in both directions. A container has its own `TMPDIR`, so the lock is
|
org, in both directions. A container has its own golangci-lint cache, its
|
||||||
out of reach. The lint phase does keep golangci-lint's cache between check
|
own `TMPDIR` and a digest-pinned binary, so neither is reachable.
|
||||||
builds (below), but it lints every checkout at `/src`, and a finding is
|
|
||||||
taken from that cache only for the same files, settings and golangci-lint
|
|
||||||
version.
|
|
||||||
|
|
||||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
||||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
invalidates a `COPY` layer only when the copied content changes, so on an
|
||||||
@@ -173,21 +170,15 @@ style conventions are in separate documents:
|
|||||||
id, so no two builds share a value, not even the three in one
|
id, so no two builds share a value, not even the three in one
|
||||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
||||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
||||||
two gate phases and `script/fmt-check`, and builds no image of its own. In
|
two gate phases and `script/fmt-check`, and builds no image of its own. Every
|
||||||
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
|
stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
|
||||||
that install system packages and download Go's modules, or right after `FROM`
|
`FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
|
||||||
in a stage with neither, and every step that lints, tests or compiles comes
|
are compared by content and need nothing. A stage without the `ARG` line is
|
||||||
after it. A step that installs anything else as well, such as a
|
served from cache on an unchanged tree, so a repository re-vendors the scripts
|
||||||
`script/bootstrap` that also installs node packages, comes after it too. A new
|
and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
|
||||||
value reruns every `RUN` step after that line; `COPY` steps are compared by
|
it reruns every step too, but it also empties every cache mount, and Go's
|
||||||
content and need nothing. Docker keeps the layers above the line until the
|
build cache, which the canonical Go `Dockerfile` below mounts so that compiled
|
||||||
base image, their own lines or the files they copy change. A stage without the
|
packages survive between check builds, would start empty every time. A bare
|
||||||
`ARG` line is served from cache on an unchanged tree, so a repository
|
|
||||||
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
|
|
||||||
not pass `--no-cache`: it reruns every step too, but it also builds the layers
|
|
||||||
above the line again and empties every cache mount, so Go's build cache and
|
|
||||||
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
|
|
||||||
survive between check builds, would start empty every time. A bare
|
|
||||||
`docker build .` is not evidence that anything ran: a sub-second build
|
`docker build .` is not evidence that anything ran: a sub-second build
|
||||||
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
||||||
`docker builder prune` and friends destroy a build cache shared with every
|
`docker builder prune` and friends destroy a build cache shared with every
|
||||||
@@ -203,25 +194,19 @@ style conventions are in separate documents:
|
|||||||
`Dockerfile`:
|
`Dockerfile`:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
||||||
# system packages and download Go's modules, or right after FROM in a
|
# on every build, which reruns every RUN step after it. The steps that
|
||||||
# stage with neither, and every step that lints, tests or compiles
|
# compile mount Go's build cache, which survives from build to build.
|
||||||
# comes after it. script/ passes a new value on every build, which
|
|
||||||
# reruns every RUN step after it. Docker keeps the layers above it
|
|
||||||
# until the base image, their own lines or go.mod and go.sum change.
|
|
||||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
|
||||||
# also mounts golangci-lint's; both survive from build to build.
|
|
||||||
|
|
||||||
# Lint phase
|
# Lint phase
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
|
ARG CACHEBUST
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
|
||||||
golangci-lint run --config .golangci.yml ./...
|
golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||||
@@ -229,10 +214,10 @@ style conventions are in separate documents:
|
|||||||
# cache also holds test results, which go test would otherwise replay.
|
# cache also holds test results, which go test would otherwise replay.
|
||||||
# golang:1.x, YYYY-MM-DD
|
# golang:1.x, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS test
|
FROM golang@sha256:... AS test
|
||||||
|
ARG CACHEBUST
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
ARG CACHEBUST
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||||
@@ -240,19 +225,19 @@ style conventions are in separate documents:
|
|||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
# Build stage. Nothing is wanted from either phase above; the copies
|
||||||
# are what make BuildKit build them first, so nothing after them can
|
# are what make BuildKit build them first, so this stage cannot run
|
||||||
# run unless lint and test passed.
|
# unless lint and test passed.
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
RUN apk add --no-cache git
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
ARG CACHEBUST
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
COPY --from=test /src/go.sum /dev/null
|
||||||
|
RUN apk add --no-cache git
|
||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||||
RUN git config --system --add safe.directory /src
|
RUN git config --system --add safe.directory /src
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# The VERSION build arg when one is given, otherwise
|
# The VERSION build arg when one is given, otherwise
|
||||||
@@ -289,26 +274,16 @@ style conventions are in separate documents:
|
|||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
same two copies. A plain `docker build .` builds the last stage's chain
|
||||||
and nothing else.
|
and nothing else.
|
||||||
- Check builds keep these, and none of them can hand back a stale result:
|
- Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
|
||||||
- The layers above `ARG CACHEBUST`: the system packages and Go's
|
it for root), and only on the steps that compile: `golangci-lint run`,
|
||||||
modules. Docker installs the packages again when the base image or the
|
both `go test` runs and `go build`. It keeps compiled packages between
|
||||||
install line changes, and downloads the modules again when `go.mod` or
|
check builds, and Go reuses one only when all its inputs are unchanged, so
|
||||||
`go.sum` changes. Go checks every module against `go.sum` as it
|
it saves the compile without skipping a check. Nothing else is kept:
|
||||||
downloads it, and no later build can change a kept layer. The modules
|
golangci-lint's own cache would replay stored findings in place of a lint
|
||||||
are kept in a layer rather than a cache mount because a cache mount is
|
run, and Go builds from whatever files a kept module cache holds without
|
||||||
shared by every build on the host, and Go builds from whatever files
|
checking them again (only `go mod verify` does), so a file changed there
|
||||||
it holds without checking them again (only `go mod verify` does), so a
|
would go unnoticed. Go's build cache also holds test results, which is why
|
||||||
file changed there would go unnoticed.
|
both `go test` runs carry `-count=1`.
|
||||||
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
|
|
||||||
root), mounted on the steps that compile: `golangci-lint run`, both
|
|
||||||
`go test` runs and `go build`. Go reuses a compiled package only when
|
|
||||||
all its inputs are unchanged. The cache also holds test results, which
|
|
||||||
is why both `go test` runs carry `-count=1`.
|
|
||||||
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
|
|
||||||
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
|
|
||||||
a package's findings only when the package's files and those of every
|
|
||||||
package it imports, `go.mod`, the linters and their settings in
|
|
||||||
`.golangci.yml`, and the golangci-lint version are all unchanged.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
@@ -317,20 +292,12 @@ style conventions are in separate documents:
|
|||||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||||
has no `apk`, so install with `apt-get` under the Debian package name
|
has no `apk`, so install with `apt-get` under the Debian package name
|
||||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||||
lists in the same `RUN`, so the layer does not keep them. The test phase,
|
lists in the same `RUN`, so the layer does not keep them:
|
||||||
on the Debian Go image, installs the libraries its tests need the same
|
|
||||||
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
|
|
||||||
Docker keeps it until the base image or the install line changes:
|
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
ARG CACHEBUST
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||||
|
|||||||
@@ -29,80 +29,20 @@
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- a file or directory removed while the scan runs is not an error, in the walk
|
- `scan` never records anything that is not on a persistent filesystem (`tmpfs`,
|
||||||
or in the hash phase. The scan prints no line for it and does not count it as
|
`proc`, `sysfs` and the other kernel or memory-backed types), matched by the
|
||||||
skipped. When the first of several hard links to a file is gone, the hash
|
device ID of each such mount point, listed as the scan starts from
|
||||||
phase reads the next one instead. On a terminal every progress display starts
|
`/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
|
||||||
with the spinner, the hash phase's included. A phase with an exact total ends
|
however the walk reaches it; no flag turns that off. Below an operand, its
|
||||||
its line with its percent and a 10-column bar at the terminal's right edge,
|
walk does not enter the default excludes either, so `sfdupes scan /` skips the
|
||||||
which no longer moves as the line's length changes. A line too wide for the
|
kernel's view of the running system and the temporary directories: on Linux
|
||||||
terminal now loses its rate before its bar. The time remaining is labelled
|
`/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`, on macOS `/dev`,
|
||||||
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
|
`/private/tmp`, `/private/var/tmp`, `/private/var/vm`, `/System/Volumes/VM`
|
||||||
|
and `/System/Volumes/Data`, and a Windows list, not used yet. An operand at or
|
||||||
- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
|
inside a default exclude is still walked. The scan prints a line for each path
|
||||||
bytes of a SHA-256, half the size of the hex text they held. The hashes stay
|
it leaves out, without counting it as skipped, and the records there count as
|
||||||
raw from the hash phase through the database to `report` and `trees`. Reading
|
outside the scanned operands, as for `-x`. `--no-default-excludes` turns off
|
||||||
a record whose hash is not 32 bytes is an error (2026-10-26,
|
the default excludes only, and `scan --help` lists them (2026-10-17,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/134)
|
|
||||||
|
|
||||||
- the hash phase gives every file the walk records three hashes: `head` and
|
|
||||||
`tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
|
|
||||||
of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
|
|
||||||
The 10 MiB and 50 MiB thresholds, the size census, the records stored without
|
|
||||||
hashes and the content phase are gone, so a scan has four phases and a rescan
|
|
||||||
reads only new and changed files. `info` no longer counts the records with
|
|
||||||
hashes, since every record has them (2026-10-25,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
|
||||||
|
|
||||||
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
|
|
||||||
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
|
|
||||||
stages of the `Dockerfile` the module download, and in `builder` the
|
|
||||||
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
|
|
||||||
until the base image, the install line, `go.mod` or `go.sum` changes, and
|
|
||||||
every step that lints, tests or compiles still comes after it;
|
|
||||||
`golangci-lint run` also keeps golangci-lint's cache. README says the same
|
|
||||||
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
|
|
||||||
|
|
||||||
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
|
|
||||||
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
|
|
||||||
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
|
|
||||||
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
|
|
||||||
early build without a database. When the linker set no version,
|
|
||||||
`sfdupes --version` prints the main module's version that Go recorded in the
|
|
||||||
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
|
|
||||||
when neither is known (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/116)
|
|
||||||
|
|
||||||
- `scan` creates the database's directory where the database path leads: the
|
|
||||||
path with its last element cut off as text, never cleaned, so a `..` after a
|
|
||||||
symlinked directory is followed through the symlink, as when the file is
|
|
||||||
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
|
|
||||||
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
|
|
||||||
used to create `/srv/new` and then fail (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
|
||||||
|
|
||||||
- `scan` never records its own database: once the database and its `.lock` file
|
|
||||||
are open, it notes the device and inode of the database file, its `-wal` and
|
|
||||||
`-shm` files and its `.lock` file, and the walk passes over a file with one of
|
|
||||||
them wherever the operands reach it, through a symlinked directory included,
|
|
||||||
printing no line and counting nothing. A record an earlier scan left for one
|
|
||||||
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
|
||||||
|
|
||||||
- `scan` never records or reads anything that is not on a persistent filesystem
|
|
||||||
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
|
|
||||||
matched by the device ID of each such filesystem, listed as the scan starts
|
|
||||||
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
|
|
||||||
however the walk or the content phase reaches it; no flag turns that off.
|
|
||||||
Below an operand, its walk does not enter the default excludes either, so
|
|
||||||
`sfdupes scan /` skips the kernel's view of the running system and the
|
|
||||||
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
|
|
||||||
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
|
|
||||||
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
|
|
||||||
Windows list, not used yet. An operand at or inside a default exclude is still
|
|
||||||
walked. The scan prints a line for each path it leaves out, without counting
|
|
||||||
it as skipped, and the records there count as outside the scanned operands, as
|
|
||||||
for `-x`. `--no-default-excludes` turns off the default excludes only, and
|
|
||||||
`scan --help` lists them (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
||||||
|
|
||||||
- the rate a progress display shows covers the items completed over the last 5
|
- the rate a progress display shows covers the items completed over the last 5
|
||||||
@@ -188,11 +128,7 @@
|
|||||||
and the build stage installs `git` and `make` itself; a new push cancels the
|
and the build stage installs `git` and `make` itself; a new push cancels the
|
||||||
workflow's older run on the same branch, and a run stops after 20 minutes;
|
workflow's older run on the same branch, and a run stops after 20 minutes;
|
||||||
`.claude/settings.json` is deleted (2026-10-15,
|
`.claude/settings.json` is deleted (2026-10-15,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
|
https://git.eeqj.de/sneak/sfdupes/issues/95)
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
|
|
||||||
`test` and the build stage, and the install of `git` and `make` in the build
|
|
||||||
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
|
|
||||||
every step that lints, tests or compiles still runs on every build
|
|
||||||
|
|
||||||
- with `-x`, an operand under another operand is walked on its own instead of
|
- with `-x`, an operand under another operand is walked on its own instead of
|
||||||
being dropped when a directory from it (from its directory, for a regular
|
being dropped when a directory from it (from its directory, for a regular
|
||||||
@@ -393,9 +329,9 @@
|
|||||||
a `CHECK_EPOCH` build argument; since
|
a `CHECK_EPOCH` build argument; since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
||||||
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
||||||
declares before every step that lints, tests or compiles. Run as root, the
|
declares first. Run as root, the tests fail
|
||||||
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
|
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
|
||||||
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
|
the test relies on, so the `test` phase runs them as `nobody`
|
||||||
- check the installed golangci-lint version in `script/bootstrap` instead of
|
- check the installed golangci-lint version in `script/bootstrap` instead of
|
||||||
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
|
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
|
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
|
||||||
|
|||||||
+14
-19
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -169,8 +168,8 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) {
|
|||||||
|
|
||||||
// TestSyncScanCancelledMidWalkKeepsRecords is the regression net under
|
// TestSyncScanCancelledMidWalkKeepsRecords is the regression net under
|
||||||
// the post-walk guard. The scan is cancelled part-way through the
|
// the post-walk guard. The scan is cancelled part-way through the
|
||||||
// walk, so it reaches the guard having walked a genuinely partial tree
|
// walk, so it reaches the guard holding a genuinely partial size
|
||||||
// and holding a still-populated index of records the walk never got to.
|
// census and a still-populated index of records the walk never got to.
|
||||||
// Every one of those records would look vanished to the update phase.
|
// Every one of those records would look vanished to the update phase.
|
||||||
// The guard is what stops the scan there, and this test is what
|
// The guard is what stops the scan there, and this test is what
|
||||||
// notices if it stops doing so: deleting the guard, or making it
|
// notices if it stops doing so: deleting the guard, or making it
|
||||||
@@ -209,10 +208,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// assertWalkGuardAborted checks that the scan stopped at the post-walk
|
// assertWalkGuardAborted checks that the scan stopped at the post-walk
|
||||||
// guard: with a walk that reached some files (it really ran) but not
|
// guard: with a census that is neither empty (the walk really ran)
|
||||||
// all of them (it really was cut short), and with no record counted for
|
// nor complete (it really was cut short), and with no record counted
|
||||||
// removal. A removal count means the partial walk was carried past
|
// for removal. A removal count means the partial census was carried
|
||||||
// the guard into the update phase, which is the failure this test
|
// past the guard into the update phase, which is the failure this test
|
||||||
// exists to catch.
|
// exists to catch.
|
||||||
func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
|
func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -223,30 +222,30 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if st.unchanged == 0 {
|
if st.unchanged == 0 {
|
||||||
t.Fatalf("stats = %+v: the walk reached no file, so it never "+
|
t.Fatalf("stats = %+v: the census is empty, so the walk never "+
|
||||||
"ran and the guard was reached for the wrong reason", st)
|
"ran and the guard was reached for the wrong reason", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
if st.unchanged >= walkCancelFiles {
|
if st.unchanged >= walkCancelFiles {
|
||||||
t.Fatalf("stats = %+v: the walk reached the whole tree, so it "+
|
t.Fatalf("stats = %+v: the census covers the whole tree, so the "+
|
||||||
"was not cut short", st)
|
"walk was not cut short", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The workers drop every directory still queued once the scan is
|
// The workers drop every directory still queued once the scan is
|
||||||
// cancelled, so only the files in the directories already in flight
|
// cancelled, so only the files in the directories already in flight
|
||||||
// can still be reached after the fact. A walk beyond that bound
|
// can add to the census after the fact. A census beyond that bound
|
||||||
// would mean the cancellation was not observed where it should have
|
// would mean the cancellation was not observed where it should have
|
||||||
// been.
|
// been.
|
||||||
limit := walkCancelAtDone + walkCancelInFlightFiles
|
limit := walkCancelAtDone + walkCancelInFlightFiles
|
||||||
if st.unchanged > limit {
|
if st.unchanged > limit {
|
||||||
t.Errorf("the walk reached %d files, want at most %d: it kept "+
|
t.Errorf("census covers %d files, want at most %d: the walk kept "+
|
||||||
"taking directories off the queue after cancellation",
|
"taking directories off the queue after cancellation",
|
||||||
st.unchanged, limit)
|
st.unchanged, limit)
|
||||||
}
|
}
|
||||||
|
|
||||||
if st.removed != 0 {
|
if st.removed != 0 {
|
||||||
t.Errorf("stats = %+v: the scan counted records for removal from "+
|
t.Errorf("stats = %+v: the scan counted records for removal from "+
|
||||||
"a partial walk", st)
|
"a partial census", st)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -763,9 +762,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
|
|||||||
|
|
||||||
var hashed atomic.Bool
|
var hashed atomic.Bool
|
||||||
|
|
||||||
hash := func(path string, size int64) (
|
hash := func(path string, size int64) (string, string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
hashed.Store(true)
|
hashed.Store(true)
|
||||||
|
|
||||||
return hashSignature(path, size)
|
return hashSignature(path, size)
|
||||||
@@ -805,9 +802,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
|
|||||||
|
|
||||||
// The scan is cancelled while the worker hashes, so the worker has
|
// The scan is cancelled while the worker hashes, so the worker has
|
||||||
// already passed the check that drops queued runs.
|
// already passed the check that drops queued runs.
|
||||||
hash := func(path string, size int64) (
|
hash := func(path string, size int64) (string, string, string, error) {
|
||||||
[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
|
|
||||||
) {
|
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
return hashSignature(path, size)
|
return hashSignature(path, size)
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -49,17 +48,16 @@ const lockFilePerm = 0o600
|
|||||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||||
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||||
// second. head, tail and content each hold the raw 32 bytes of a
|
// second.
|
||||||
// SHA-256.
|
|
||||||
const createTableSQL = `
|
const createTableSQL = `
|
||||||
CREATE TABLE files (
|
CREATE TABLE files (
|
||||||
path BLOB PRIMARY KEY,
|
path BLOB PRIMARY KEY,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
mtime INTEGER NOT NULL,
|
mtime INTEGER NOT NULL,
|
||||||
mtime_nsec INTEGER NOT NULL,
|
mtime_nsec INTEGER NOT NULL,
|
||||||
head BLOB NOT NULL,
|
head TEXT NOT NULL,
|
||||||
tail BLOB NOT NULL,
|
tail TEXT NOT NULL,
|
||||||
content BLOB NOT NULL
|
content TEXT NOT NULL
|
||||||
) WITHOUT ROWID
|
) WITHOUT ROWID
|
||||||
`
|
`
|
||||||
|
|
||||||
@@ -94,10 +92,6 @@ var errSchemaVersion = errors.New("unsupported database schema version")
|
|||||||
// database.
|
// database.
|
||||||
var errScanRunning = errors.New("another scan is running")
|
var errScanRunning = errors.New("another scan is running")
|
||||||
|
|
||||||
// errHashSize reports a record whose head, tail or content is not the
|
|
||||||
// 32 bytes of a SHA-256.
|
|
||||||
var errHashSize = errors.New("hash is not 32 bytes")
|
|
||||||
|
|
||||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||||
// path, with the working directory, its symlinks resolved, put in front
|
// path, with the working directory, its symlinks resolved, put in front
|
||||||
@@ -115,8 +109,10 @@ func databasePath() (string, string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The working directory as the kernel reports it, with no symlink
|
// The working directory as the kernel reports it, with no symlink
|
||||||
// in it, as README "Database" says. os.Getwd may return PWD
|
// in it. os.Getwd may return PWD instead, which can name it through
|
||||||
// instead, which can name it through a symlink.
|
// a symlink; a ".." after that symlink then leads somewhere else
|
||||||
|
// once the path is cleaned, as scan cleans it to create the
|
||||||
|
// database's directory.
|
||||||
wd, err := unix.Getwd()
|
wd, err := unix.Getwd()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||||
@@ -129,24 +125,6 @@ func databasePath() (string, string, error) {
|
|||||||
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createDatabaseDirectory creates the directory the database at path
|
|
||||||
// goes in, and any directory above it, if missing. path is absolute, as
|
|
||||||
// databasePath returns it. The directory is path with its last element
|
|
||||||
// cut off as text, by filepath.Split. filepath.Dir would clean it,
|
|
||||||
// dropping each ".." together with the directory name before it; when
|
|
||||||
// that name is a symlink, the cleaned path names another directory than
|
|
||||||
// the one the system reaches when it opens path.
|
|
||||||
func createDatabaseDirectory(path string) error {
|
|
||||||
dir, _ := filepath.Split(path)
|
|
||||||
|
|
||||||
err := os.MkdirAll(dir, dbDirPerm)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create database directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanParams are the connection parameters for scan: read-write, with
|
// scanParams are the connection parameters for scan: read-write, with
|
||||||
// WAL journaling and a busy timeout, so a report can run while a cron
|
// WAL journaling and a busy timeout, so a report can run while a cron
|
||||||
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
||||||
@@ -201,9 +179,9 @@ func openDB(path, params string) (*sql.DB, error) {
|
|||||||
// that deleted it would let the next scan lock a new file while
|
// that deleted it would let the next scan lock a new file while
|
||||||
// another still holds the old one.
|
// another still holds the old one.
|
||||||
func lockScanDatabase(path string) (*os.File, error) {
|
func lockScanDatabase(path string) (*os.File, error) {
|
||||||
err := createDatabaseDirectory(path)
|
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Like SQLite, this follows a symlink at path and creates its
|
// Like SQLite, this follows a symlink at path and creates its
|
||||||
@@ -247,9 +225,9 @@ func lockScanDatabase(path string) (*os.File, error) {
|
|||||||
// openScanDatabase opens the database for the scan subcommand, creating
|
// openScanDatabase opens the database for the scan subcommand, creating
|
||||||
// the file, its parent directory, and the schema as needed.
|
// the file, its parent directory, and the schema as needed.
|
||||||
func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
|
func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
|
||||||
err := createDatabaseDirectory(path)
|
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
db, err := openDB(path, scanParams)
|
db, err := openDB(path, scanParams)
|
||||||
@@ -267,42 +245,6 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
|
|||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseFiles returns, by device and inode, the files of the database
|
|
||||||
// at path that the scan keeps open: the database file, its -wal and
|
|
||||||
// -shm files and its .lock file, each named from the database's path
|
|
||||||
// with symlinks resolved, as SQLite and lockScanDatabase name them. The
|
|
||||||
// scan calls it once the database and the lock file are open, so that
|
|
||||||
// its walk passes over them however it reaches them, through a
|
|
||||||
// symlinked directory included. A file that does not exist then is left
|
|
||||||
// out, since no walk can reach it. There is no -journal file while the
|
|
||||||
// database is in WAL mode.
|
|
||||||
func databaseFiles(path string) ([]fileRec, error) {
|
|
||||||
resolved, err := filepath.EvalSymlinks(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("resolve database path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var files []fileRec
|
|
||||||
|
|
||||||
for _, p := range []string{
|
|
||||||
resolved, resolved + "-wal", resolved + "-shm", resolved + ".lock",
|
|
||||||
} {
|
|
||||||
fi, err := os.Lstat(p)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("database: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ino := inodeOfInfo(fi)
|
|
||||||
files = append(files, fileRec{path: p, dev: dev, ino: ino})
|
|
||||||
}
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// closeScanDatabase switches the database at path from WAL back to
|
// closeScanDatabase switches the database at path from WAL back to
|
||||||
// rollback-journal mode and closes it. Out of WAL mode the database
|
// rollback-journal mode and closes it. Out of WAL mode the database
|
||||||
// file alone holds the whole database, so a reader needs no -wal or
|
// file alone holds the whole database, so a reader needs no -wal or
|
||||||
@@ -498,24 +440,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path, head, tail, content []byte
|
path []byte
|
||||||
sec, nsec int64
|
sec, nsec int64
|
||||||
r scanRec
|
r scanRec
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
|
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
|
||||||
|
&r.content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
r.path = string(path)
|
r.path = string(path)
|
||||||
r.mtime = time.Unix(sec, nsec)
|
r.mtime = time.Unix(sec, nsec)
|
||||||
|
|
||||||
err = r.setHashes(head, tail, content)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn(r)
|
fn(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -572,22 +509,17 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path, head, tail, content []byte
|
path []byte
|
||||||
r scanRec
|
r scanRec
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&r.size, &head, &tail, &content, &path)
|
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
r.path = string(path)
|
r.path = string(path)
|
||||||
|
|
||||||
err = r.setHashes(head, tail, content)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = fn(r)
|
err = fn(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -602,22 +534,6 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// setHashes sets r's head, tail and content to the values read from
|
|
||||||
// those columns. Each must be the raw 32 bytes of a SHA-256.
|
|
||||||
func (r *scanRec) setHashes(head, tail, content []byte) error {
|
|
||||||
for _, b := range [][]byte{head, tail, content} {
|
|
||||||
if len(b) != sha256.Size {
|
|
||||||
return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
r.head = [sha256.Size]byte(head)
|
|
||||||
r.tail = [sha256.Size]byte(tail)
|
|
||||||
r.content = [sha256.Size]byte(content)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// countRecords returns the number of records in the database.
|
// countRecords returns the number of records in the database.
|
||||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||||
var n int64
|
var n int64
|
||||||
@@ -630,11 +546,15 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseCountsSQL counts the records and adds up their sizes, in one
|
// databaseCountsSQL counts the records, those with head and tail
|
||||||
|
// hashes and those with a content hash, and adds up their sizes, in one
|
||||||
// pass over the table. A SUM over no rows is NULL, so on an empty table
|
// pass over the table. A SUM over no rows is NULL, so on an empty table
|
||||||
// the SUM falls back to 0.
|
// each SUM falls back to 0.
|
||||||
const databaseCountsSQL = `
|
const databaseCountsSQL = `
|
||||||
SELECT COUNT(*), COALESCE(SUM(size), 0)
|
SELECT COUNT(*),
|
||||||
|
COALESCE(SUM(head <> '' AND tail <> ''), 0),
|
||||||
|
COALESCE(SUM(content <> ''), 0),
|
||||||
|
COALESCE(SUM(size), 0)
|
||||||
FROM files
|
FROM files
|
||||||
`
|
`
|
||||||
|
|
||||||
@@ -643,6 +563,8 @@ FROM files
|
|||||||
type databaseCounts struct {
|
type databaseCounts struct {
|
||||||
schemaVersion int
|
schemaVersion int
|
||||||
records int64 // every record
|
records int64 // every record
|
||||||
|
headTail int64 // records with head and tail hashes
|
||||||
|
content int64 // records with a content hash
|
||||||
size int64 // the sizes of all records added up
|
size int64 // the sizes of all records added up
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -670,7 +592,7 @@ func readDatabaseCounts(ctx context.Context,
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = tx.QueryRowContext(ctx, databaseCountsSQL).
|
err = tx.QueryRowContext(ctx, databaseCountsSQL).
|
||||||
Scan(&c.records, &c.size)
|
Scan(&c.records, &c.headTail, &c.content, &c.size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return c, fmt.Errorf("count records: %w", err)
|
return c, fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -678,15 +600,15 @@ func readDatabaseCounts(ctx context.Context,
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFileMeta streams every record's path, size and mtime to fn. Scan
|
// loadFileMeta streams every record's path, size, mtime, and whether
|
||||||
// change detection needs no hash values, and skipping the hash columns
|
// it carries hashes to fn. Scan change detection needs no hash
|
||||||
// keeps the scan's in-memory index small on multi-million-file
|
// values, and skipping the hash columns keeps the scan's in-memory
|
||||||
// databases.
|
// index small on multi-million-file databases.
|
||||||
func loadFileMeta(ctx context.Context, db *sql.DB,
|
func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||||
fn func(path string, size int64, mtime time.Time),
|
fn func(path string, size int64, mtime time.Time, hashed bool),
|
||||||
) error {
|
) error {
|
||||||
rows, err := db.QueryContext(ctx,
|
rows, err := db.QueryContext(ctx,
|
||||||
"SELECT path, size, mtime, mtime_nsec FROM files")
|
"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -697,14 +619,94 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
var (
|
var (
|
||||||
path []byte
|
path []byte
|
||||||
size, sec, nsec int64
|
size, sec, nsec int64
|
||||||
|
hashed int64
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &size, &sec, &nsec)
|
err = rows.Scan(&path, &size, &sec, &nsec, &hashed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn(string(path), size, time.Unix(sec, nsec))
|
fn(string(path), size, time.Unix(sec, nsec), hashed != 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = rows.Err()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read records: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// contentGroupsSQL selects each group (the records sharing a size,
|
||||||
|
// head, and tail) of records of at least headTailMin bytes that has more
|
||||||
|
// than one record and at least one record without a content hash, with
|
||||||
|
// the number of its records.
|
||||||
|
const contentGroupsSQL = `
|
||||||
|
SELECT size, head, tail, COUNT(*) AS records
|
||||||
|
FROM files
|
||||||
|
WHERE size >= ? AND head <> ''
|
||||||
|
GROUP BY size, head, tail
|
||||||
|
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||||
|
`
|
||||||
|
|
||||||
|
// contentCandidatesSQL selects every record of the groups
|
||||||
|
// contentGroupsSQL selects, with whether each record has a content
|
||||||
|
// hash. SQLite does the grouping, so no other record's hashes are
|
||||||
|
// loaded into memory; the rows come ordered by size, head, and tail, so
|
||||||
|
// each group's rows arrive together.
|
||||||
|
const contentCandidatesSQL = `
|
||||||
|
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
|
||||||
|
FROM files AS f
|
||||||
|
JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail)
|
||||||
|
ORDER BY size, head, tail
|
||||||
|
`
|
||||||
|
|
||||||
|
// countContentCandidates returns the number of rows of
|
||||||
|
// contentCandidatesSQL, adding up the records of the groups
|
||||||
|
// contentGroupsSQL selects, so it neither joins nor sorts.
|
||||||
|
func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
|
||||||
|
var n int64
|
||||||
|
|
||||||
|
err := db.QueryRowContext(ctx,
|
||||||
|
"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
|
||||||
|
headTailMin).Scan(&n)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("count content candidates: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
|
||||||
|
// each record, without its content hash, and whether it has one.
|
||||||
|
func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||||
|
fn func(r scanRec, hashed bool),
|
||||||
|
) error {
|
||||||
|
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read records: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var (
|
||||||
|
path []byte
|
||||||
|
sec, nsec int64
|
||||||
|
r scanRec
|
||||||
|
hashed int64
|
||||||
|
)
|
||||||
|
|
||||||
|
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
|
||||||
|
&hashed)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read record: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.path = string(path)
|
||||||
|
r.mtime = time.Unix(sec, nsec)
|
||||||
|
fn(r, hashed != 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
@@ -788,8 +790,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
|
|
||||||
for _, r := range upserts {
|
for _, r := range upserts {
|
||||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
|
||||||
r.head[:], r.tail[:], r.content[:])
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("upsert %s: %w", r.path, err)
|
return fmt.Errorf("upsert %s: %w", r.path, err)
|
||||||
}
|
}
|
||||||
|
|||||||
+67
-182
@@ -2,11 +2,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -120,116 +118,6 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
|
|
||||||
// through a symlink to a directory, then "..", to a missing directory.
|
|
||||||
// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
|
|
||||||
// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
|
|
||||||
// places scan creates the database's directory, must each create the
|
|
||||||
// database there, and nothing at srv/new, where the path leads once
|
|
||||||
// cleaned as text.
|
|
||||||
func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
creates := map[string]func(ctx context.Context, path string) error{
|
|
||||||
"lockScanDatabase": func(_ context.Context, path string) error {
|
|
||||||
lock, err := lockScanDatabase(path)
|
|
||||||
if err == nil {
|
|
||||||
_ = lock.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
"openScanDatabase": func(ctx context.Context, path string) error {
|
|
||||||
db, err := openScanDatabase(ctx, path)
|
|
||||||
if err == nil {
|
|
||||||
_ = db.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, create := range creates {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.Symlink(filepath.Join(dir, "data", "x"),
|
|
||||||
filepath.Join(dir, "srv", "link"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: %v", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("database the path leads to: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
|
|
||||||
if !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
t.Errorf("srv/new: stat error %v, want it missing", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDatabaseFiles opens a database through a symlink to its file:
|
|
||||||
// databaseFiles finds the database file and its -wal and -shm files
|
|
||||||
// beside the symlink's target, where SQLite keeps them, and leaves out
|
|
||||||
// the .lock file, which only a scan creates.
|
|
||||||
func TestDatabaseFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := testDBPath(t)
|
|
||||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
|
||||||
|
|
||||||
err := os.Symlink(path, link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := openScanDatabase(t.Context(), link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
files, err := databaseFiles(link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := make([]string, 0, len(files))
|
|
||||||
for _, f := range files {
|
|
||||||
got = append(got, f.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{path, path + "-wal", path + "-shm"}
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenDatabaseUnversionedForeign(t *testing.T) {
|
func TestOpenDatabaseUnversionedForeign(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -474,67 +362,6 @@ func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
|
|
||||||
// content is not the 32 bytes of a SHA-256: either one byte, or the 64
|
|
||||||
// bytes of a SHA-256 written as hex text. loadFileRows and
|
|
||||||
// loadSignatureRows must each fail on it with an error naming its path.
|
|
||||||
func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hexText := strings.Repeat("ab", sha256.Size)
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
update string
|
|
||||||
value any
|
|
||||||
}{
|
|
||||||
{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
|
|
||||||
{"hex-text head", "UPDATE files SET head = ?", hexText},
|
|
||||||
{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
|
|
||||||
{"hex-text tail", "UPDATE files SET tail = ?", hexText},
|
|
||||||
{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
|
|
||||||
{"hex-text content", "UPDATE files SET content = ?", hexText},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
err := applyChanges(t.Context(), db,
|
|
||||||
[]scanRec{{path: "/a/f"}}, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ExecContext(t.Context(), c.update, c.value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
loads := map[string]func() error{
|
|
||||||
"loadFileRows": func() error {
|
|
||||||
return loadFileRows(t.Context(), db, func(int64) {},
|
|
||||||
func(scanRec) {})
|
|
||||||
},
|
|
||||||
"loadSignatureRows": func() error {
|
|
||||||
return loadSignatureRows(t.Context(), db, func(int64) {},
|
|
||||||
func(scanRec) error { return nil })
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, load := range loads {
|
|
||||||
err := load()
|
|
||||||
if !errors.Is(err, errHashSize) ||
|
|
||||||
!strings.Contains(err.Error(), "/a/f") {
|
|
||||||
t.Errorf("%s: err = %v, want %v naming /a/f",
|
|
||||||
name, err, errHashSize)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
||||||
// from the files_signature index alone, with nothing to sort: the
|
// from the files_signature index alone, with nothing to sort: the
|
||||||
// first record comes at once, so report's read phase counts each
|
// first record comes at once, so report's read phase counts each
|
||||||
@@ -589,13 +416,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
|
|||||||
// written.
|
// written.
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{
|
{
|
||||||
size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
|
size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
|
||||||
tail: testHash("t2"), content: testHash("c2"),
|
content: "c2", path: "/a/tab\tnew\nline",
|
||||||
path: "/a/tab\tnew\nline",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
|
size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
|
||||||
tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
|
content: "c1", path: "/a/x",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -616,8 +442,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
|
|||||||
// An upsert for an existing path updates in place; a delete
|
// An upsert for an existing path updates in place; a delete
|
||||||
// removes exactly its path.
|
// removes exactly its path.
|
||||||
upd := scanRec{
|
upd := scanRec{
|
||||||
size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
|
size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
|
||||||
tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
|
path: "/a/x",
|
||||||
}
|
}
|
||||||
|
|
||||||
err = applyChanges(t.Context(), db, []scanRec{upd},
|
err = applyChanges(t.Context(), db, []scanRec{upd},
|
||||||
@@ -644,8 +470,8 @@ func TestApplyChangesBatching(t *testing.T) {
|
|||||||
recs := make([]scanRec, 0, n)
|
recs := make([]scanRec, 0, n)
|
||||||
for i := range n {
|
for i := range n {
|
||||||
recs = append(recs, scanRec{
|
recs = append(recs, scanRec{
|
||||||
size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
|
size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
|
||||||
tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
|
path: fmt.Sprintf("/batch/%07d", i),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -675,3 +501,62 @@ func TestApplyChangesBatching(t *testing.T) {
|
|||||||
t.Fatalf("records = %d, want 0", len(got))
|
t.Fatalf("records = %d, want 0", len(got))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCountContentCandidates checks that countContentCandidates counts
|
||||||
|
// the records the content phase's search returns: those of each group
|
||||||
|
// of two or more records of headTailMin or more with a head, where at
|
||||||
|
// least one has no content hash.
|
||||||
|
func TestCountContentCandidates(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := openTestDB(t)
|
||||||
|
|
||||||
|
// Each group's records: size, head, and content, one per record.
|
||||||
|
groups := []struct {
|
||||||
|
size int64
|
||||||
|
head string
|
||||||
|
contents []string
|
||||||
|
}{
|
||||||
|
{headTailMin, "h", []string{"", "c", "c"}}, // returned
|
||||||
|
{headTailMin + 1, "h", []string{"", ""}}, // returned
|
||||||
|
{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
|
||||||
|
{headTailMin + 3, "h", []string{""}}, // one record
|
||||||
|
{headTailMin - 1, "h", []string{"", ""}}, // too small
|
||||||
|
{headTailMin + 4, "", []string{"", ""}}, // no head
|
||||||
|
}
|
||||||
|
|
||||||
|
var recs []scanRec
|
||||||
|
|
||||||
|
for _, g := range groups {
|
||||||
|
for _, content := range g.contents {
|
||||||
|
recs = append(recs, scanRec{
|
||||||
|
size: g.size, mtime: time.Unix(1, 0), head: g.head,
|
||||||
|
tail: "t", content: content,
|
||||||
|
path: fmt.Sprintf("/r/%d", len(recs)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err := applyChanges(t.Context(), db, recs, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("applyChanges: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := countContentCandidates(t.Context(), db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("countContentCandidates: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var returned int64
|
||||||
|
|
||||||
|
err = loadContentCandidates(t.Context(), db,
|
||||||
|
func(scanRec, bool) { returned++ })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadContentCandidates: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got != 5 || returned != 5 {
|
||||||
|
t.Errorf("count = %d, records returned = %d, want 5 and 5",
|
||||||
|
got, returned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+47
-33
@@ -3,8 +3,8 @@ package main
|
|||||||
// This file holds two rules for the walk (README "Rules for the walk"):
|
// This file holds two rules for the walk (README "Rules for the walk"):
|
||||||
// the default excludes, the paths a scan's walk does not enter below an
|
// the default excludes, the paths a scan's walk does not enter below an
|
||||||
// operand unless scan is given --no-default-excludes, and the
|
// operand unless scan is given --no-default-excludes, and the
|
||||||
// filesystems that are not persistent, on which the scan never records
|
// filesystems that are not persistent, on which the walk never records
|
||||||
// or reads anything.
|
// anything.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
@@ -15,8 +15,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The systems that have default excludes, as runtime.GOOS names them.
|
// The systems that have default excludes, as runtime.GOOS names them.
|
||||||
@@ -27,11 +25,11 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// The fields of a line of mountInfoPath that readMountInfo reads,
|
// The fields of a line of mountInfoPath that readMountInfo reads,
|
||||||
// counted from 0 (see proc(5)): the device ID, as major:minor, is the
|
// counted from 0 (see proc(5)): the mount point is the fifth field, and
|
||||||
// third field, and after the sixth come optional fields, then a field
|
// after the sixth come optional fields, then a field holding "-", then
|
||||||
// holding "-", then the filesystem type.
|
// the filesystem type.
|
||||||
const (
|
const (
|
||||||
deviceField = 2
|
mountPointField = 4
|
||||||
optionalFieldsStart = 6
|
optionalFieldsStart = 6
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -43,17 +41,14 @@ const typesPerHelpLine = 6
|
|||||||
// readMountInfo reads.
|
// readMountInfo reads.
|
||||||
var errMountInfoLine = errors.New("unexpected line")
|
var errMountInfoLine = errors.New("unexpected line")
|
||||||
|
|
||||||
// walkRules holds the rules for the walk that a scan sets up as it
|
// walkRules holds the two rules for the walk that a scan sets up as it
|
||||||
// starts. excludes are the default excludes, or none with
|
// starts. excludes are the default excludes, or none with
|
||||||
// --no-default-excludes. nonPersistent maps the device ID of every
|
// --no-default-excludes. nonPersistent maps the device ID of every
|
||||||
// mounted filesystem that is not persistent to its type; the scan
|
// mounted filesystem that is not persistent to its type; the walk
|
||||||
// records and reads nothing on one of those, whatever the flags.
|
// records nothing on one of those, whatever the flags.
|
||||||
// databaseFiles are the scan's own database files, which the walk
|
|
||||||
// passes over (see databaseFiles in db.go).
|
|
||||||
type walkRules struct {
|
type walkRules struct {
|
||||||
excludes []string
|
excludes []string
|
||||||
nonPersistent map[uint64]string
|
nonPersistent map[uint64]string
|
||||||
databaseFiles []fileRec
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultExcludes returns the default excludes of the system goos
|
// defaultExcludes returns the default excludes of the system goos
|
||||||
@@ -110,6 +105,32 @@ func nonPersistentFilesystemTypes() []string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nonPersistentFilesystems returns the device ID of every mounted
|
||||||
|
// filesystem that is not persistent, with its type, read with lstat on
|
||||||
|
// its mount point. mountedFilesystems lists the mounts. A mount point
|
||||||
|
// that lstat cannot reach is left out.
|
||||||
|
func nonPersistentFilesystems() (map[uint64]string, error) {
|
||||||
|
mounts, err := mountedFilesystems()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
types := nonPersistentFilesystemTypes()
|
||||||
|
devices := make(map[uint64]string)
|
||||||
|
|
||||||
|
for mountPoint, fsType := range mounts {
|
||||||
|
if !slices.Contains(types, fsType) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if dev, ok := deviceOfPath(mountPoint); ok {
|
||||||
|
devices[dev] = fsType
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return devices, nil
|
||||||
|
}
|
||||||
|
|
||||||
// nonPersistentLine returns the line the scan prints for path when fi,
|
// nonPersistentLine returns the line the scan prints for path when fi,
|
||||||
// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
|
// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
|
||||||
// when it is on any other.
|
// when it is on any other.
|
||||||
@@ -129,13 +150,16 @@ func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
|
// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
|
||||||
// returns the device ID of every mounted filesystem that is not
|
// returns every mount point with the type of its filesystem. Where one
|
||||||
// persistent, with its type. The device ID is the one each line gives,
|
// filesystem is mounted over another, a later line names the one on
|
||||||
// not that of the file now at its mount point, so a mount that a later
|
// top, so the last line for a mount point decides.
|
||||||
// mount hides never adds the device of another filesystem.
|
func readMountInfo(mountinfo io.Reader) (map[string]string, error) {
|
||||||
func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
|
// The kernel writes a space, tab, newline or backslash in a mount
|
||||||
types := nonPersistentFilesystemTypes()
|
// point as an octal escape.
|
||||||
devices := make(map[uint64]string)
|
unescape := strings.NewReplacer(
|
||||||
|
`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`)
|
||||||
|
|
||||||
|
mounts := make(map[string]string)
|
||||||
lines := bufio.NewScanner(mountinfo)
|
lines := bufio.NewScanner(mountinfo)
|
||||||
|
|
||||||
for lines.Scan() {
|
for lines.Scan() {
|
||||||
@@ -146,20 +170,10 @@ func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
|
|||||||
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
|
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
|
||||||
}
|
}
|
||||||
|
|
||||||
var major, minor uint32
|
mounts[unescape.Replace(fields[mountPointField])] = fields[sep+1]
|
||||||
|
|
||||||
_, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
|
|
||||||
}
|
|
||||||
|
|
||||||
fsType := fields[sep+1]
|
|
||||||
if slices.Contains(types, fsType) {
|
|
||||||
devices[unix.Mkdev(major, minor)] = fsType
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return devices, lines.Err()
|
return mounts, lines.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultExcludesHelp describes, for the scan command's help, the
|
// defaultExcludesHelp describes, for the scan command's help, the
|
||||||
|
|||||||
+6
-26
@@ -2,19 +2,12 @@
|
|||||||
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import "golang.org/x/sys/unix"
|
||||||
"slices"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
// mountedFilesystems returns every mount point with the type of its
|
||||||
)
|
// filesystem, from the system's list of mounted filesystems. Where one
|
||||||
|
// filesystem is mounted over another, the later one in the list counts.
|
||||||
// nonPersistentFilesystems returns the device ID of every mounted
|
func mountedFilesystems() (map[string]string, error) {
|
||||||
// filesystem that is not persistent, with its type, from the system's
|
|
||||||
// list of mounted filesystems. The device ID is read with lstat on the
|
|
||||||
// mount point. Where one filesystem is mounted over another on the same
|
|
||||||
// mount point, the later one in the list counts. A mount point that
|
|
||||||
// lstat cannot reach is left out.
|
|
||||||
func nonPersistentFilesystems() (map[uint64]string, error) {
|
|
||||||
// The first call counts the filesystems, the second lists them.
|
// The first call counts the filesystems, the second lists them.
|
||||||
n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
|
n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -35,18 +28,5 @@ func nonPersistentFilesystems() (map[uint64]string, error) {
|
|||||||
unix.ByteSliceToString(st.Fstypename[:])
|
unix.ByteSliceToString(st.Fstypename[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
types := nonPersistentFilesystemTypes()
|
return mounts, nil
|
||||||
devices := make(map[uint64]string)
|
|
||||||
|
|
||||||
for mountPoint, fsType := range mounts {
|
|
||||||
if !slices.Contains(types, fsType) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if dev, ok := deviceOfPath(mountPoint); ok {
|
|
||||||
devices[dev] = fsType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return devices, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-7
@@ -12,12 +12,12 @@ import (
|
|||||||
// process sees.
|
// process sees.
|
||||||
const mountInfoPath = "/proc/self/mountinfo"
|
const mountInfoPath = "/proc/self/mountinfo"
|
||||||
|
|
||||||
// nonPersistentFilesystems returns the device ID of every mounted
|
// mountedFilesystems returns every mount point with the type of its
|
||||||
// filesystem that is not persistent, with its type: on Linux read from
|
// filesystem: on Linux read from mountInfoPath (see readMountInfo), and
|
||||||
// mountInfoPath (see readMountInfo), and on any other system none.
|
// on any other system none.
|
||||||
func nonPersistentFilesystems() (map[uint64]string, error) {
|
func mountedFilesystems() (map[string]string, error) {
|
||||||
if runtime.GOOS != goosLinux {
|
if runtime.GOOS != goosLinux {
|
||||||
return map[uint64]string{}, nil
|
return map[string]string{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := os.Open(mountInfoPath)
|
f, err := os.Open(mountInfoPath)
|
||||||
@@ -27,10 +27,10 @@ func nonPersistentFilesystems() (map[uint64]string, error) {
|
|||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
devices, err := readMountInfo(f)
|
mounts, err := readMountInfo(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
|
return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return devices, nil
|
return mounts, nil
|
||||||
}
|
}
|
||||||
|
|||||||
+43
-26
@@ -10,8 +10,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDefaultExcludes(t *testing.T) {
|
func TestDefaultExcludes(t *testing.T) {
|
||||||
@@ -63,17 +61,16 @@ func TestIsDefaultExclude(t *testing.T) {
|
|||||||
func TestReadMountInfo(t *testing.T) {
|
func TestReadMountInfo(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The second-to-last line has no optional fields. The tmpfs on
|
// The second-to-last line has no optional fields. /mnt/over has an
|
||||||
// /data/a/b is hidden by the root filesystem's /srv, mounted over
|
// ext4 filesystem mounted over its tmpfs.
|
||||||
// /data/a after it: the tmpfs still gives its own device, and the
|
|
||||||
// root filesystem's device is not listed.
|
|
||||||
mountinfo := strings.Join([]string{
|
mountinfo := strings.Join([]string{
|
||||||
"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
|
"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
|
||||||
"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
|
"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
|
||||||
"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
|
"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
|
||||||
"25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw",
|
`25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`,
|
||||||
"26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw",
|
"26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw",
|
||||||
"27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
|
"27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw",
|
||||||
|
"28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
|
||||||
}, "\n")
|
}, "\n")
|
||||||
|
|
||||||
got, err := readMountInfo(strings.NewReader(mountinfo))
|
got, err := readMountInfo(strings.NewReader(mountinfo))
|
||||||
@@ -81,23 +78,18 @@ func TestReadMountInfo(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
want := map[uint64]string{
|
want := map[string]string{
|
||||||
unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs",
|
"/": "ext4", "/proc": "proc", "/sys": "sysfs",
|
||||||
unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs",
|
"/mnt/ram disk": "tmpfs", "/mnt/over": "ext4", "/srv/auto": "autofs",
|
||||||
}
|
}
|
||||||
if !maps.Equal(got, want) {
|
if !maps.Equal(got, want) {
|
||||||
t.Errorf("readMountInfo = %v, want %v", got, want)
|
t.Errorf("readMountInfo = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, line := range []string{
|
_, err = readMountInfo(strings.NewReader("22 1 8:1 / / rw,relatime\n"))
|
||||||
"22 1 8:1 / / rw,relatime\n",
|
if !errors.Is(err, errMountInfoLine) {
|
||||||
"22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n",
|
t.Errorf("readMountInfo of a line without a type = %v, want %v",
|
||||||
} {
|
err, errMountInfoLine)
|
||||||
_, err = readMountInfo(strings.NewReader(line))
|
|
||||||
if !errors.Is(err, errMountInfoLine) {
|
|
||||||
t.Errorf("readMountInfo(%q) = %v, want %v", line, err,
|
|
||||||
errMountInfoLine)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,8 +156,9 @@ func TestDefaultExcludesHelp(t *testing.T) {
|
|||||||
// TestScanDoesNotEnterDefaultExclude checks that the walk does not
|
// TestScanDoesNotEnterDefaultExclude checks that the walk does not
|
||||||
// enter a default exclude below an operand. It says so on a line of its
|
// enter a default exclude below an operand. It says so on a line of its
|
||||||
// own and does not count it as skipped. The records below it count as
|
// own and does not count it as skipped. The records below it count as
|
||||||
// outside the operands: the record of a file gone from it is kept,
|
// outside the operands: the record of a file gone from it is kept, and
|
||||||
// while a file added to it is not recorded.
|
// its size joins the size census, while a file added to it is not
|
||||||
|
// recorded.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
||||||
@@ -185,6 +178,10 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
|||||||
|
|
||||||
unseen := writeFile(t, excluded, "new", pattern(2, 10))
|
unseen := writeFile(t, excluded, "new", pattern(2, 10))
|
||||||
|
|
||||||
|
// The same size as gone, so it is hashed only if the census holds the
|
||||||
|
// size of gone's record.
|
||||||
|
added := writeFile(t, dir, "added", pattern(3, 100))
|
||||||
|
|
||||||
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
|
st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
|
||||||
walkRules{excludes: []string{excluded}})
|
walkRules{excludes: []string{excluded}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -198,6 +195,11 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
|||||||
recs := dbRecords(t, db)
|
recs := dbRecords(t, db)
|
||||||
recordByPath(t, recs, gone)
|
recordByPath(t, recs, gone)
|
||||||
|
|
||||||
|
if recordByPath(t, recs, added).head == "" {
|
||||||
|
t.Errorf("%s not hashed: the census lacks the size of %s",
|
||||||
|
added, gone)
|
||||||
|
}
|
||||||
|
|
||||||
if slices.Contains(recordPaths(recs), unseen) {
|
if slices.Contains(recordPaths(recs), unseen) {
|
||||||
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
|
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
|
||||||
}
|
}
|
||||||
@@ -381,8 +383,9 @@ func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
|
|||||||
// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
|
// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
|
||||||
// that is not persistent beside one on the tests' temporary directory.
|
// that is not persistent beside one on the tests' temporary directory.
|
||||||
// The records an earlier scan left in it count as outside the operands:
|
// The records an earlier scan left in it count as outside the operands:
|
||||||
// the record of a file gone from it is kept, while a file added to it
|
// the record of a file gone from it is kept, and its size joins the size
|
||||||
// is not recorded.
|
// census, while a file added to it is not recorded, and its size does
|
||||||
|
// not join the census.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
|
func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
|
||||||
@@ -402,6 +405,11 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
|
|||||||
|
|
||||||
unseen := writeFile(t, shm, "new", pattern(2, 10))
|
unseen := writeFile(t, shm, "new", pattern(2, 10))
|
||||||
|
|
||||||
|
// added has the size of gone, and lonely that of unseen: each is
|
||||||
|
// hashed only if the census holds that size.
|
||||||
|
added := writeFile(t, dir, "added", pattern(3, 100))
|
||||||
|
lonely := writeFile(t, dir, "lonely", pattern(4, 10))
|
||||||
|
|
||||||
st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
|
st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -414,6 +422,15 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
|
|||||||
recs := dbRecords(t, db)
|
recs := dbRecords(t, db)
|
||||||
recordByPath(t, recs, gone)
|
recordByPath(t, recs, gone)
|
||||||
|
|
||||||
|
if recordByPath(t, recs, added).head == "" {
|
||||||
|
t.Errorf("%s not hashed: the census lacks the size of %s",
|
||||||
|
added, gone)
|
||||||
|
}
|
||||||
|
|
||||||
|
if recordByPath(t, recs, lonely).head != "" {
|
||||||
|
t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen)
|
||||||
|
}
|
||||||
|
|
||||||
if slices.Contains(recordPaths(recs), unseen) {
|
if slices.Contains(recordPaths(recs), unseen) {
|
||||||
t.Errorf("%s recorded", unseen)
|
t.Errorf("%s recorded", unseen)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,5 +24,3 @@ require (
|
|||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.11.0 // indirect
|
modernc.org/memory v1.11.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
retract v0.0.1 // early build without a database; does not work as the README describes
|
|
||||||
|
|||||||
@@ -113,6 +113,9 @@ func writeInfo(stdout io.Writer, dbPath, source string,
|
|||||||
dbFile.ModTime().UTC().Format(time.RFC3339))
|
dbFile.ModTime().UTC().Format(time.RFC3339))
|
||||||
fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
|
fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
|
||||||
fmt.Fprintf(&b, "records\t%d\n", counts.records)
|
fmt.Fprintf(&b, "records\t%d\n", counts.records)
|
||||||
|
fmt.Fprintf(&b, "records with head and tail hashes\t%d\n",
|
||||||
|
counts.headTail)
|
||||||
|
fmt.Fprintf(&b, "records with content hash\t%d\n", counts.content)
|
||||||
fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
|
fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
|
||||||
counts.size, humanBytes(counts.size))
|
counts.size, humanBytes(counts.size))
|
||||||
|
|
||||||
|
|||||||
+13
-11
@@ -12,25 +12,23 @@ import (
|
|||||||
// infoOpenLine is the plain line of info's open phase when it opens the
|
// infoOpenLine is the plain line of info's open phase when it opens the
|
||||||
// database at once; it prints it as the phase starts and as it ends.
|
// database at once; it prints it as the phase starts and as it ends.
|
||||||
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// infoCountLine is the plain line of info's count phase when the count
|
// infoCountLine is the plain line of info's count phase when the count
|
||||||
// takes under a second; it prints it as the phase starts and as it
|
// takes under a second; it prints it as the phase starts and as it
|
||||||
// ends.
|
// ends.
|
||||||
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
|
const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// infoRecs are four records: a duplicate pair, a file of 20 MiB and
|
// infoRecs are four records: a duplicate pair with every hash, a file
|
||||||
// another small file.
|
// of 20 MiB with head and tail hashes but no content hash yet, and a
|
||||||
|
// file never hashed.
|
||||||
func infoRecs() []scanRec {
|
func infoRecs() []scanRec {
|
||||||
h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
|
|
||||||
bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/a"},
|
||||||
{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/b"},
|
||||||
{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
|
{size: 20 << 20, head: "bh", tail: "bt", path: "/d/big"},
|
||||||
{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
|
{size: 7, path: "/d/unique"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,12 +45,16 @@ func TestRunInfo(t *testing.T) {
|
|||||||
"empty": {
|
"empty": {
|
||||||
want: "schema version\t1\n" +
|
want: "schema version\t1\n" +
|
||||||
"records\t0\n" +
|
"records\t0\n" +
|
||||||
|
"records with head and tail hashes\t0\n" +
|
||||||
|
"records with content hash\t0\n" +
|
||||||
"size of recorded files\t0 (0 B)\n",
|
"size of recorded files\t0 (0 B)\n",
|
||||||
},
|
},
|
||||||
"filled": {
|
"filled": {
|
||||||
recs: infoRecs(),
|
recs: infoRecs(),
|
||||||
want: "schema version\t1\n" +
|
want: "schema version\t1\n" +
|
||||||
"records\t4\n" +
|
"records\t4\n" +
|
||||||
|
"records with head and tail hashes\t3\n" +
|
||||||
|
"records with content hash\t2\n" +
|
||||||
"size of recorded files\t20971537 (20.0 MiB)\n",
|
"size of recorded files\t20971537 (20.0 MiB)\n",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
// Command sfdupes quickly identifies candidate duplicate files across
|
// Command sfdupes quickly identifies candidate duplicate files across
|
||||||
// very large filesystems without reading every byte of every file.
|
// very large filesystems without reading every byte of every file.
|
||||||
// Files are considered duplicates when their sizes are equal and they
|
// Files are considered duplicates when their sizes are equal and they
|
||||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
|
||||||
// KiB, and of the content. The content hash covers the whole file under
|
// hashed in full. A larger file is compared on the hashes of its first
|
||||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
// and last 64 KiB, and only when those match another file's is its
|
||||||
// file is never read in full. scan maintains a persistent SQLite
|
// content hash computed and compared: of the whole file when it is
|
||||||
// database of file signatures (SFDUPES_DATABASE, default
|
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
|
||||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
// or larger. scan maintains a persistent SQLite database of file
|
||||||
|
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
|
||||||
|
// that the other subcommands read.
|
||||||
//
|
//
|
||||||
// Usage:
|
// Usage:
|
||||||
//
|
//
|
||||||
@@ -26,7 +28,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/debug"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
@@ -57,32 +58,10 @@ var errNoSubcommand = errors.New("no subcommand")
|
|||||||
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
var errWorkersBelowOne = errors.New("--workers must be at least 1")
|
||||||
|
|
||||||
// Version is the build version, injected at link time via -ldflags
|
// Version is the build version, injected at link time via -ldflags
|
||||||
// (see the Makefile); empty when the linker set none, as for a binary
|
// (see the Makefile); "dev" for a plain go build.
|
||||||
// built by go install. buildVersion turns it into what --version prints.
|
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // written only by the linker
|
//nolint:gochecknoglobals // written only by the linker
|
||||||
var Version string
|
var Version = "dev"
|
||||||
|
|
||||||
// buildVersion returns the version that sfdupes --version prints
|
|
||||||
// (README "Install"): the version the linker set, when it set one;
|
|
||||||
// otherwise the main module's version that Go recorded in the binary,
|
|
||||||
// a tag such as v0.0.2 or the pseudo-version of an @main build; and
|
|
||||||
// "dev" when neither is known. info is what debug.ReadBuildInfo
|
|
||||||
// returns, nil when the binary carries none. Go records "(devel)" for
|
|
||||||
// a build that has no version, such as one inside a copy of the source
|
|
||||||
// without git.
|
|
||||||
func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
|
|
||||||
if linkerVersion != "" {
|
|
||||||
return linkerVersion
|
|
||||||
}
|
|
||||||
|
|
||||||
if info == nil || info.Main.Version == "" ||
|
|
||||||
info.Main.Version == "(devel)" {
|
|
||||||
return "dev"
|
|
||||||
}
|
|
||||||
|
|
||||||
return info.Main.Version
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
||||||
@@ -137,10 +116,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
var showVersion bool
|
var showVersion bool
|
||||||
|
|
||||||
printVersion := runE(func(context.Context, []string) error {
|
printVersion := runE(func(context.Context, []string) error {
|
||||||
info, _ := debug.ReadBuildInfo()
|
_, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version)
|
||||||
|
|
||||||
_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
|
|
||||||
buildVersion(Version, info))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("write stdout: %w", err)
|
return fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -236,7 +212,7 @@ func newScanCommand() *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
||||||
"concurrent workers for the walk and hash phases")
|
"concurrent workers for the walk, hash, and content phases")
|
||||||
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
||||||
"do not cross filesystem boundaries")
|
"do not cross filesystem boundaries")
|
||||||
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
|
scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
|
||||||
|
|||||||
+1
-38
@@ -9,7 +9,6 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime/debug"
|
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -715,8 +714,6 @@ func TestRunVersion(t *testing.T) {
|
|||||||
|
|
||||||
// README §Subcommands: the version is one line on stdout, with
|
// README §Subcommands: the version is one line on stdout, with
|
||||||
// nothing on stderr, and exits 0.
|
// nothing on stderr, and exits 0.
|
||||||
info, _ := debug.ReadBuildInfo()
|
|
||||||
|
|
||||||
for _, arg := range []string{"--version", "-v"} {
|
for _, arg := range []string{"--version", "-v"} {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -725,7 +722,7 @@ func TestRunVersion(t *testing.T) {
|
|||||||
t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
|
t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
want := "sfdupes " + buildVersion(Version, info) + "\n"
|
want := "sfdupes " + Version + "\n"
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
|
t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
|
||||||
}
|
}
|
||||||
@@ -736,40 +733,6 @@ func TestRunVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildVersion(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// README "Install": the linker's version wins; otherwise the main
|
|
||||||
// module's version that Go recorded; "dev" when neither is known.
|
|
||||||
tagged := &debug.BuildInfo{Main: debug.Module{Version: "v0.0.2"}}
|
|
||||||
pseudo := &debug.BuildInfo{Main: debug.Module{
|
|
||||||
Version: "v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1",
|
|
||||||
}}
|
|
||||||
devel := &debug.BuildInfo{Main: debug.Module{Version: "(devel)"}}
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
linker string
|
|
||||||
info *debug.BuildInfo
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"linker set", "v0.0.2-3-g4e4a94d", tagged, "v0.0.2-3-g4e4a94d"},
|
|
||||||
{"linker set, no build info", "4e4a94d", nil, "4e4a94d"},
|
|
||||||
{"build info with a tag", "", tagged, "v0.0.2"},
|
|
||||||
{"build info with a pseudo-version", "", pseudo,
|
|
||||||
"v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1"},
|
|
||||||
{"(devel)", "", devel, "dev"},
|
|
||||||
{"no build info", "", nil, "dev"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range cases {
|
|
||||||
if got := buildVersion(c.linker, c.info); got != c.want {
|
|
||||||
t.Errorf("%s: buildVersion(%q, ...) = %q, want %q",
|
|
||||||
c.name, c.linker, got, c.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+91
-103
@@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
@@ -35,17 +34,10 @@ const stallTime = 10 * time.Second
|
|||||||
// spinnerType selects the progressbar library's spinner style.
|
// spinnerType selects the progressbar library's spinner style.
|
||||||
const spinnerType = 14
|
const spinnerType = 14
|
||||||
|
|
||||||
// barWidth is the width of a phase's bar between its two ends, as in
|
// barWidth is the width of the progressbar library's bar between its
|
||||||
// the example line in README "Progress".
|
// two ends, as in the example line in README "Progress".
|
||||||
const barWidth = 10
|
const barWidth = 10
|
||||||
|
|
||||||
// barCell is one column of the filled part of a bar.
|
|
||||||
const barCell = "█"
|
|
||||||
|
|
||||||
// barColumns is the width of what percentBar returns: the percent in
|
|
||||||
// four columns and its sign, a space, and the bar with its two ends.
|
|
||||||
const barColumns = barWidth + 8
|
|
||||||
|
|
||||||
// percentScale converts a fraction to a percentage.
|
// percentScale converts a fraction to a percentage.
|
||||||
const percentScale = 100
|
const percentScale = 100
|
||||||
|
|
||||||
@@ -53,12 +45,17 @@ const percentScale = 100
|
|||||||
// read, as the progressbar library takes it.
|
// read, as the progressbar library takes it.
|
||||||
const fallbackWidth = 80
|
const fallbackWidth = 80
|
||||||
|
|
||||||
// spinnerMargin is the columns of a line besides its text: the spinner
|
// spinnerMargin is the columns of a spinner's line besides its text:
|
||||||
// and the three spaces the library draws around the text, and the
|
// the spinner and the three spaces the library draws around the text,
|
||||||
// terminal's last column, which every line stays clear of so that no
|
// and the terminal's last column, which every line stays clear of so
|
||||||
// terminal wraps it.
|
// that no terminal wraps it.
|
||||||
const spinnerMargin = 5
|
const spinnerMargin = 5
|
||||||
|
|
||||||
|
// barMargin is the columns of a bar's line besides its text: the
|
||||||
|
// percent and the bar the library draws after the text, the spaces
|
||||||
|
// around them, and the terminal's last column.
|
||||||
|
const barMargin = barWidth + 11
|
||||||
|
|
||||||
// countingRecords is what the display of a phase that reads the records
|
// countingRecords is what the display of a phase that reads the records
|
||||||
// says while it counts them, before it has its total.
|
// says while it counts them, before it has its total.
|
||||||
const countingRecords = "counting the records"
|
const countingRecords = "counting the records"
|
||||||
@@ -88,7 +85,7 @@ func terminalWidth() int {
|
|||||||
return width
|
return width
|
||||||
}
|
}
|
||||||
|
|
||||||
// phaseLabel names phase with its place among phases, as "[2/4] walk".
|
// phaseLabel names phase with its place among phases, as "[2/5] walk".
|
||||||
func phaseLabel(phases []string, phase string) string {
|
func phaseLabel(phases []string, phase string) string {
|
||||||
return fmt.Sprintf("[%d/%d] %s",
|
return fmt.Sprintf("[%d/%d] %s",
|
||||||
slices.Index(phases, phase)+1, len(phases), phase)
|
slices.Index(phases, phase)+1, len(phases), phase)
|
||||||
@@ -112,13 +109,13 @@ type drawnCount struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// progress renders one phase's progress on stderr. On a TTY the
|
// progress renders one phase's progress on stderr. On a TTY the
|
||||||
// progressbar library draws its spinner and then line, fitted to the
|
// progressbar library draws line, fitted to the terminal's width: after
|
||||||
// terminal's width. A phase with an exact total ends the line with its
|
// the library's spinner, or, for a phase with an exact total, before the
|
||||||
// percent and bar, at the terminal's right edge, when the terminal has
|
// library's bar when the terminal has room for the bar beside it. It is
|
||||||
// room for them. It is drawn again every redrawInterval, so its elapsed
|
// drawn again every redrawInterval, so its elapsed time keeps moving
|
||||||
// time keeps moving while the phase waits for its next item. When
|
// while the phase waits for its next item. When stderr is not a TTY it
|
||||||
// stderr is not a TTY it emits no ANSI redraws: it prints line as the
|
// emits no ANSI redraws: it prints line as the phase starts, every
|
||||||
// phase starts, every plainInterval while it runs, and as it ends.
|
// plainInterval while it runs, and as it ends.
|
||||||
//
|
//
|
||||||
// The draws come from a goroutine of their own, so mu guards the fields
|
// The draws come from a goroutine of their own, so mu guards the fields
|
||||||
// below it. Every method must be called from the main goroutine. A nil
|
// below it. Every method must be called from the main goroutine. A nil
|
||||||
@@ -134,6 +131,7 @@ type progress struct {
|
|||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
bar *progressbar.ProgressBar // what the library draws; nil until then
|
bar *progressbar.ProgressBar // what the library draws; nil until then
|
||||||
|
withBar bool // bar is the library's bar, not its spinner
|
||||||
widest int // the widest line bar has drawn, in columns
|
widest int // the widest line bar has drawn, in columns
|
||||||
label string
|
label string
|
||||||
unit string
|
unit string
|
||||||
@@ -143,7 +141,7 @@ type progress struct {
|
|||||||
stalledOn string
|
stalledOn string
|
||||||
count int64
|
count int64
|
||||||
start time.Time // when the phase started, for the elapsed time
|
start time.Time // when the phase started, for the elapsed time
|
||||||
rateStart time.Time // when the count started, for the time remaining
|
rateStart time.Time // when the count started, for the ETA
|
||||||
lastItem time.Time // when the last item was completed, or the count started
|
lastItem time.Time // when the last item was completed, or the count started
|
||||||
// drawn holds the count at each draw over the last rateWindow, oldest
|
// drawn holds the count at each draw over the last rateWindow, oldest
|
||||||
// first; the rate counts from the first. It keeps the draw before the
|
// first; the rate counts from the first. It keeps the draw before the
|
||||||
@@ -217,34 +215,55 @@ func (p *progress) draw() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if p.width == nil {
|
if p.width == nil {
|
||||||
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
|
fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, true))
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
width := p.width()
|
width := p.width()
|
||||||
text := p.line(now, width-spinnerMargin, true)
|
|
||||||
|
|
||||||
// Before each draw the library erases its line with as many spaces as
|
// A phase with an exact total gets the library's bar when its whole
|
||||||
// the widest line it has drawn. A display gets a new one when the
|
// line, without the percent the bar shows, fits beside the bar: the
|
||||||
// terminal has been made too narrow for that erase, which would wrap
|
// bar gives way before any part of the line. A total of 0 leaves the
|
||||||
// and leave a line behind at every draw.
|
// bar nothing to fill.
|
||||||
if p.bar == nil || p.widest >= width {
|
text, withBar := "", false
|
||||||
p.newBar(width)
|
if p.total > 0 && !p.estimate {
|
||||||
|
text = p.line(now, math.MaxInt, false)
|
||||||
|
withBar = len(text) <= width-barMargin
|
||||||
|
}
|
||||||
|
|
||||||
|
margin := barMargin
|
||||||
|
if !withBar {
|
||||||
|
margin = spinnerMargin
|
||||||
|
text = p.line(now, width-margin, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The library makes a bar or a spinner, and before each draw erases
|
||||||
|
// its line with as many spaces as the widest line it has drawn. A
|
||||||
|
// display gets a new one when it turns from one to the other, as its
|
||||||
|
// line grows or the terminal is resized, and when the terminal has
|
||||||
|
// been made too narrow for that erase, which would wrap and leave a
|
||||||
|
// line behind at every draw.
|
||||||
|
if p.bar == nil || p.withBar != withBar || p.widest >= width {
|
||||||
|
p.newBar(withBar, width)
|
||||||
|
}
|
||||||
|
|
||||||
|
if withBar {
|
||||||
|
_ = p.bar.Set64(p.count)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.bar.Describe(text)
|
p.bar.Describe(text)
|
||||||
|
|
||||||
// The library drew the spinner, the text and the spaces around it,
|
// The library drew the text and its margin, less the terminal's last
|
||||||
// less the terminal's last column, which spinnerMargin keeps clear. A
|
// column, which the margin keeps clear.
|
||||||
// bar's cells take one column each but more than one byte.
|
p.widest = max(p.widest, len(text)+margin-1)
|
||||||
p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBar has the library make what draw draws: its spinner, followed by
|
// newBar has the library make what draw draws: a bar for the phase's
|
||||||
// the text draw gives it and nothing of the library's own. p.mu must be
|
// total when withBar is set, and a spinner when not. Either shows the
|
||||||
// held.
|
// text draw gives it, and nothing of its own besides the spinner, or
|
||||||
func (p *progress) newBar(width int) {
|
// the bar and its percent. p.mu must be held.
|
||||||
|
func (p *progress) newBar(withBar bool, width int) {
|
||||||
if p.bar != nil {
|
if p.bar != nil {
|
||||||
// Erase what was drawn before, short of the last column of a
|
// Erase what was drawn before, short of the last column of a
|
||||||
// terminal that may have been made narrower since: the library's
|
// terminal that may have been made narrower since: the library's
|
||||||
@@ -253,36 +272,29 @@ func (p *progress) newBar(width int) {
|
|||||||
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
|
fmt.Fprint(os.Stderr, "\r"+blank+"\r")
|
||||||
}
|
}
|
||||||
|
|
||||||
// A total of -1 gives the library's spinner.
|
total := int64(-1) // the library's spinner
|
||||||
p.bar = progressbar.NewOptions64(-1,
|
if withBar {
|
||||||
|
total = p.total
|
||||||
|
}
|
||||||
|
|
||||||
|
p.bar = progressbar.NewOptions64(total,
|
||||||
progressbar.OptionSetWriter(os.Stderr),
|
progressbar.OptionSetWriter(os.Stderr),
|
||||||
|
progressbar.OptionSetWidth(barWidth),
|
||||||
progressbar.OptionSpinnerType(spinnerType),
|
progressbar.OptionSpinnerType(spinnerType),
|
||||||
// The spinner turns at each draw, not on a timer of the
|
// The spinner turns at each draw, not on a timer of the
|
||||||
// library's own.
|
// library's own.
|
||||||
progressbar.OptionSetSpinnerChangeInterval(0),
|
progressbar.OptionSetSpinnerChangeInterval(0),
|
||||||
// The text carries the elapsed time and the time remaining.
|
// The text carries the elapsed time and the ETA.
|
||||||
progressbar.OptionSetElapsedTime(false),
|
progressbar.OptionSetElapsedTime(false),
|
||||||
progressbar.OptionSetPredictTime(false),
|
progressbar.OptionSetPredictTime(false),
|
||||||
)
|
)
|
||||||
p.widest = 0
|
p.withBar, p.widest = withBar, 0
|
||||||
}
|
|
||||||
|
|
||||||
// percentBar returns the percent and the bar that end the text of a
|
|
||||||
// phase with an exact total, as " 25% |██ |": the bar's
|
|
||||||
// barWidth columns fill in proportion to the items done. A count past
|
|
||||||
// the total, as when a scan adds records while trees reads them, shows
|
|
||||||
// as 100%. p.mu must be held.
|
|
||||||
func (p *progress) percentBar() string {
|
|
||||||
done := min(float64(p.count)/float64(p.total), 1)
|
|
||||||
filled := strings.Repeat(barCell, int(done*barWidth))
|
|
||||||
|
|
||||||
return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// setTotal gives a phase that started without a total its total, once
|
// setTotal gives a phase that started without a total its total, once
|
||||||
// it is known. The rate, the time remaining and the time with no item
|
// it is known. The rate, the ETA and the time with no item completed
|
||||||
// completed count from then on: the time spent waiting for the total
|
// count from then on: the time spent waiting for the total says nothing
|
||||||
// says nothing about how fast items go.
|
// about how fast items go.
|
||||||
func (p *progress) setTotal(total int64) {
|
func (p *progress) setTotal(total int64) {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return
|
return
|
||||||
@@ -426,26 +438,22 @@ func (p *progress) finish() {
|
|||||||
// line formats the phase's progress as of now in at most room columns,
|
// line formats the phase's progress as of now in at most room columns,
|
||||||
// as
|
// as
|
||||||
//
|
//
|
||||||
// [3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
|
// [3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
|
||||||
//
|
//
|
||||||
// where "files" is the phase's unit. A phase with no total shows the
|
// where "files" is the phase's unit, without the percent when percent
|
||||||
// items done instead, or what it waits for. The rate is that since the
|
// is false, as before the library's bar, which shows its own. A phase
|
||||||
// first draw in drawn, and the time remaining comes from the rate since
|
// with no total shows the items done instead, or what it waits for. The
|
||||||
// the count started. Once no item has been completed for stallTime, the
|
// rate is that since the first draw in drawn, and the ETA comes from
|
||||||
// rate, which would read 0, gives way to the word "stalled" and where
|
// the rate since the count started. Once no item has been completed for
|
||||||
// the phase waits, as "stalled disk/db".
|
// stallTime, the rate, which would read 0, gives way to the word
|
||||||
|
// "stalled" and where the phase waits, as "stalled disk/db".
|
||||||
//
|
//
|
||||||
// When bar is true, as on a terminal, a phase with an exact total ends
|
// The phase and its place, the progress, the elapsed time and the ETA
|
||||||
// the line with its percent and bar, in place of the percent after its
|
// are kept whole. A line wider than room leaves out the rate, the words
|
||||||
// count. A total of 0 leaves the bar nothing to fill.
|
// of a stalled phase, or what the phase waits for; only if it is still
|
||||||
//
|
// wider is it cut at room. Every part is ASCII, so its length is its
|
||||||
// The phase and its place, the progress, the elapsed time and the time
|
// width in columns. p.mu must be held.
|
||||||
// remaining are kept whole. A line wider than room first leaves out the
|
func (p *progress) line(now time.Time, room int, percent bool) string {
|
||||||
// rate, the words of a stalled phase, or what the phase waits for. Next
|
|
||||||
// it leaves out the bar. Only if it is still wider is it cut at room.
|
|
||||||
// Every part but the bar is ASCII, so its length is its width in
|
|
||||||
// columns. p.mu must be held.
|
|
||||||
func (p *progress) line(now time.Time, room int, bar bool) string {
|
|
||||||
elapsed := now.Sub(p.start).Round(time.Second)
|
elapsed := now.Sub(p.start).Round(time.Second)
|
||||||
|
|
||||||
// head and tail hold the parts kept whole; extra, between them, is
|
// head and tail hold the parts kept whole; extra, between them, is
|
||||||
@@ -454,41 +462,21 @@ func (p *progress) line(now time.Time, room int, bar bool) string {
|
|||||||
|
|
||||||
if p.total < 0 && p.waitingFor != "" {
|
if p.total < 0 && p.waitingFor != "" {
|
||||||
head, extra = p.label+": ", p.waitingFor+", "
|
head, extra = p.label+": ", p.waitingFor+", "
|
||||||
tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
|
tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
|
||||||
} else {
|
} else {
|
||||||
since := p.drawn[0]
|
since := p.drawn[0]
|
||||||
rate := perSecond(p.count-since.count, now.Sub(since.at))
|
rate := perSecond(p.count-since.count, now.Sub(since.at))
|
||||||
average := perSecond(p.count, now.Sub(p.rateStart))
|
average := perSecond(p.count, now.Sub(p.rateStart))
|
||||||
|
|
||||||
head = p.label + ": " + p.itemsDone(true) + ", "
|
head = p.label + ": " + p.itemsDone(percent) + ", "
|
||||||
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
|
extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
|
||||||
tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
|
tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
|
||||||
|
|
||||||
if now.Sub(p.lastItem) >= stallTime {
|
if now.Sub(p.lastItem) >= stallTime {
|
||||||
extra = "stalled " + p.stalledOn + ", "
|
extra = "stalled " + p.stalledOn + ", "
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if bar && p.total > 0 && !p.estimate {
|
|
||||||
// Beside the bar, which shows the percent, the count shows none.
|
|
||||||
// The line is padded to the room the bar leaves it, so the bar
|
|
||||||
// stays at room's right edge however long the line is.
|
|
||||||
beside := room - barColumns
|
|
||||||
barHead := p.label + ": " + p.itemsDone(false) + ", "
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case len(barHead+extra+tail) <= beside:
|
|
||||||
return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
|
|
||||||
p.percentBar())
|
|
||||||
case len(barHead+tail) <= beside:
|
|
||||||
return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bar gives way. The rate, or the words of a stalled phase,
|
|
||||||
// gave way before it and stay out.
|
|
||||||
extra = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
full, kept := head+extra+tail, head+tail
|
full, kept := head+extra+tail, head+tail
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
@@ -538,10 +526,10 @@ func (p *progress) itemsDone(percent bool) string {
|
|||||||
return fmt.Sprintf("%s %.0f%%", done, pct)
|
return fmt.Sprintf("%s %.0f%%", done, pct)
|
||||||
}
|
}
|
||||||
|
|
||||||
// remain returns the time the phase has left at rate items a second, or
|
// eta returns the time the phase has left at rate items a second, or
|
||||||
// "unknown": with no total, before the first item, and once the count
|
// "unknown": with no total, before the first item, and once the count
|
||||||
// passes an estimate. p.mu must be held.
|
// passes an estimate. p.mu must be held.
|
||||||
func (p *progress) remain(rate float64) string {
|
func (p *progress) eta(rate float64) string {
|
||||||
switch {
|
switch {
|
||||||
case p.total < 0, p.estimate && p.count > p.total:
|
case p.total < 0, p.estimate && p.count > p.total:
|
||||||
return "unknown"
|
return "unknown"
|
||||||
@@ -551,8 +539,8 @@ func (p *progress) remain(rate float64) string {
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
left := time.Duration(
|
remaining := time.Duration(
|
||||||
float64(p.total-p.count) / rate * float64(time.Second))
|
float64(p.total-p.count) / rate * float64(time.Second))
|
||||||
|
|
||||||
return left.Round(time.Second).String()
|
return remaining.Round(time.Second).String()
|
||||||
}
|
}
|
||||||
|
|||||||
+120
-271
@@ -23,7 +23,7 @@ const testRedraw = time.Millisecond
|
|||||||
const testDeadline = 5 * time.Second
|
const testDeadline = 5 * time.Second
|
||||||
|
|
||||||
// hashStalledOn is where the hash phase says it waits once it has
|
// hashStalledOn is where the hash phase says it waits once it has
|
||||||
// stalled, as hashPhase gives it.
|
// stalled, as readRuns gives it.
|
||||||
const hashStalledOn = "disk/db"
|
const hashStalledOn = "disk/db"
|
||||||
|
|
||||||
// testClock is a clock a test moves by hand, for a display that reads
|
// testClock is a clock a test moves by hand, for a display that reads
|
||||||
@@ -117,38 +117,38 @@ func TestProgressLine(t *testing.T) {
|
|||||||
label: scanPhase("load"), total: -1,
|
label: scanPhase("load"), total: -1,
|
||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
},
|
},
|
||||||
want: "[1/4] load: counting the records, elapsed 10s, remain unknown",
|
want: "[1/5] load: counting the records, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
|
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
|
||||||
},
|
},
|
||||||
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
|
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
|
||||||
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, remain unknown",
|
want: "[3/5] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
|
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
|
||||||
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, remain 0s",
|
want: "[4/5] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
|
||||||
}, {
|
}, {
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
||||||
estimate: true, count: 20,
|
estimate: true, count: 20,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[2/5] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
// Past the estimate the walk cannot tell how long it has left.
|
// Past the estimate the walk cannot tell how long it has left.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
label: scanPhase("walk"), unit: scanUnit, total: 100,
|
||||||
estimate: true, count: 120,
|
estimate: true, count: 120,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
|
want: "[2/5] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
|
||||||
"remain unknown",
|
"eta unknown",
|
||||||
}, {
|
}, {
|
||||||
// A first scan's walk has no estimate.
|
// A first scan's walk has no estimate.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
|
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, remain unknown",
|
want: "[2/5] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
// The rate is in what the phase counts.
|
// The rate is in what the phase counts.
|
||||||
p: &progress{
|
p: &progress{
|
||||||
@@ -156,14 +156,14 @@ func TestProgressLine(t *testing.T) {
|
|||||||
count: 20,
|
count: 20,
|
||||||
},
|
},
|
||||||
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
|
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
|
||||||
"remain 40s",
|
"eta 40s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
tc.p.start, tc.p.rateStart, tc.p.lastItem = ago, ago, now
|
tc.p.start, tc.p.rateStart, tc.p.lastItem = ago, ago, now
|
||||||
tc.p.drawn = []drawnCount{{at: ago}}
|
tc.p.drawn = []drawnCount{{at: ago}}
|
||||||
|
|
||||||
if got := tc.p.line(now, math.MaxInt, false); got != tc.want {
|
if got := tc.p.line(now, math.MaxInt, true); got != tc.want {
|
||||||
t.Errorf("line = %q, want %q", got, tc.want)
|
t.Errorf("line = %q, want %q", got, tc.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -184,9 +184,9 @@ func TestLineFitsRoom(t *testing.T) {
|
|||||||
start: ago, rateStart: ago, lastItem: now,
|
start: ago, rateStart: ago, lastItem: now,
|
||||||
drawn: []drawnCount{{at: ago}},
|
drawn: []drawnCount{{at: ago}},
|
||||||
}
|
}
|
||||||
load := &progress{
|
content := &progress{
|
||||||
label: scanPhase("load"), total: -1,
|
label: scanPhase("content"), total: -1,
|
||||||
waitingFor: countingRecords, start: ago, rateStart: ago,
|
waitingFor: "searching the database", start: ago, rateStart: ago,
|
||||||
}
|
}
|
||||||
|
|
||||||
started := now.Add(-50 * time.Second)
|
started := now.Add(-50 * time.Second)
|
||||||
@@ -198,49 +198,48 @@ func TestLineFitsRoom(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
p *progress
|
p *progress
|
||||||
room int
|
room int
|
||||||
bar bool
|
percent bool
|
||||||
want string
|
want string
|
||||||
}{{
|
}{{
|
||||||
p: hash, room: 60,
|
p: hash, room: 57, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
|
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: hash, room: 59,
|
p: hash, room: 56, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40s",
|
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40s",
|
||||||
}, {
|
}, {
|
||||||
p: hash, room: 48,
|
p: hash, room: 45, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40",
|
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40",
|
||||||
}, {
|
}, {
|
||||||
// The bar shows the percent.
|
// Before the library's bar, which shows the percent.
|
||||||
p: hash, room: 74, bar: true,
|
p: hash, room: 57, percent: false,
|
||||||
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, remain 40s" +
|
want: "[3/5] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
|
||||||
" 20% |██ |",
|
|
||||||
}, {
|
}, {
|
||||||
p: load, room: 43,
|
p: content, room: 40, percent: true,
|
||||||
want: "[1/4] load: elapsed 10s, remain unknown",
|
want: "[5/5] content: elapsed 10s, eta unknown",
|
||||||
}, {
|
}, {
|
||||||
p: stalled, room: 68,
|
p: stalled, room: 65, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
|
want: "[3/5] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
|
||||||
"remain 3m20s",
|
"eta 3m20s",
|
||||||
}, {
|
}, {
|
||||||
p: stalled, room: 67,
|
p: stalled, room: 64, percent: true,
|
||||||
want: "[3/4] hash: [20/100] 20%, elapsed 50s, remain 3m20s",
|
want: "[3/5] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
if got := tc.p.line(now, tc.room, tc.bar); got != tc.want {
|
if got := tc.p.line(now, tc.room, tc.percent); got != tc.want {
|
||||||
t.Errorf("line(%d, %t) = %q, want %q", tc.room, tc.bar, got,
|
t.Errorf("line(%d, %t) = %q, want %q",
|
||||||
tc.want)
|
tc.room, tc.percent, got, tc.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
|
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
|
||||||
// its time remaining and its time with no record read from when the
|
// its ETA and its time with no record read from when the total becomes
|
||||||
// total becomes known, not from when the phase started waiting for it:
|
// known, not from when the phase started waiting for it: after 20
|
||||||
// after 20 seconds of waiting for the total, the phase has not stalled 5
|
// seconds of waiting for the total, the phase has not stalled 5 seconds
|
||||||
// seconds after it, and has 10 seconds after it with no record read.
|
// after it, and has 10 seconds after it with no record read.
|
||||||
func TestSetTotalStartsRate(t *testing.T) {
|
func TestSetTotalStartsRate(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -256,15 +255,15 @@ func TestSetTotalStartsRate(t *testing.T) {
|
|||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
|
|
||||||
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, remain unknown"
|
want := "[1/5] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
|
|
||||||
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, remain unknown"
|
want = "[1/5] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -273,8 +272,8 @@ func TestSetTotalStartsRate(t *testing.T) {
|
|||||||
p.increment()
|
p.increment()
|
||||||
}
|
}
|
||||||
|
|
||||||
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, remain 40s"
|
want = "[1/5] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
|
||||||
if got := p.line(clock.Now(), math.MaxInt, false); got != want {
|
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
|
||||||
t.Errorf("line = %q, want %q", got, want)
|
t.Errorf("line = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -317,9 +316,9 @@ func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
|
|||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
||||||
|
|
||||||
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown"
|
want := "[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
|
||||||
if len(lines) != 2 || lines[0] != want ||
|
if len(lines) != 2 || lines[0] != want ||
|
||||||
!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
|
!strings.HasPrefix(lines[1], "[3/5] hash: [1000/1000] 100%, ") {
|
||||||
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
|
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
|
||||||
lines, want)
|
lines, want)
|
||||||
}
|
}
|
||||||
@@ -337,7 +336,7 @@ func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
|
|||||||
}, nil, testRedraw, time.Now)
|
}, nil, testRedraw, time.Now)
|
||||||
|
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[1/4] load: counting the records, elapsed 0s, remain unknown\n", 3)
|
"[1/5] load: counting the records, elapsed 0s, eta unknown\n", 3)
|
||||||
p.finish()
|
p.finish()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -367,23 +366,23 @@ func TestPlainProgressStalledProducer(t *testing.T) {
|
|||||||
|
|
||||||
clock.Add(5 * time.Second)
|
clock.Add(5 * time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, remain 20s\n", 3)
|
"[3/5] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
|
||||||
|
|
||||||
clock.Add(time.Second)
|
clock.Add(time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, remain 24s\n", 3)
|
"[3/5] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
|
||||||
|
|
||||||
clock.Add(4 * time.Second)
|
clock.Add(4 * time.Second)
|
||||||
awaitOutput(t, stderr,
|
awaitOutput(t, stderr,
|
||||||
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, remain 40s\n",
|
"[3/5] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
|
||||||
3)
|
3)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
|
// TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
|
||||||
// little over plainInterval after the line before, as a late tick
|
// little over plainInterval after the line before, as a late tick
|
||||||
// does, through a stall and a burst of files after it: the rate covers
|
// does, through a stall and a burst of files after it: the rate covers
|
||||||
// the files read since the line before, while the time remaining comes
|
// the files read since the line before, while the ETA comes from the
|
||||||
// from the phase's average.
|
// phase's average.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestPlainProgressRateAfterStall(t *testing.T) {
|
func TestPlainProgressRateAfterStall(t *testing.T) {
|
||||||
@@ -411,10 +410,10 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
|
|||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
want := []string{
|
want := []string{
|
||||||
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown",
|
"[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
|
||||||
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, remain unknown",
|
"[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
|
||||||
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, remain unknown",
|
"[3/5] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
|
||||||
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, remain 2m15s",
|
"[3/5] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
|
||||||
@@ -427,8 +426,8 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
|
|||||||
// redrawInterval, as a terminal does, through a stall of stallTime and
|
// redrawInterval, as a terminal does, through a stall of stallTime and
|
||||||
// a burst of files after it: the rate covers the files read over the
|
// a burst of files after it: the rate covers the files read over the
|
||||||
// last rateWindow, and falls to 0 once the burst is older than that,
|
// last rateWindow, and falls to 0 once the burst is older than that,
|
||||||
// while the time remaining comes from the phase's average. Plain lines
|
// while the ETA comes from the phase's average. Plain lines show each
|
||||||
// show each draw.
|
// draw.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
||||||
@@ -458,8 +457,8 @@ func TestProgressRateAfterStallRedrawn(t *testing.T) {
|
|||||||
redraw(rateWindow + redrawInterval)
|
redraw(rateWindow + redrawInterval)
|
||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, remain 4m45s\n" +
|
want := "[3/5] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
|
||||||
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, remain 4m47s\n"
|
"[3/5] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
|
||||||
if !strings.Contains(stderr(), want) {
|
if !strings.Contains(stderr(), want) {
|
||||||
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
|
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
|
||||||
}
|
}
|
||||||
@@ -481,10 +480,10 @@ func lastFrame(out string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
|
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
|
||||||
// phase, with a bar or without, keeps being drawn while the phase waits
|
// phase, spinner or bar, keeps being drawn while the phase waits for its
|
||||||
// for its next item, with the elapsed time as of now, and shows every
|
// next item, with the elapsed time as of now, and shows every item
|
||||||
// item completed before the wait. The terminal is 120 columns wide, so
|
// completed before the wait. stderr is not a terminal here, so the
|
||||||
// the hash phase's display has room for its bar.
|
// displays take it to be fallbackWidth wide.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
||||||
@@ -507,8 +506,7 @@ func TestTerminalRedrawsWhileWaiting(t *testing.T) {
|
|||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
before := len(stderr())
|
before := len(stderr())
|
||||||
since := func() string { return stderr()[before:] }
|
since := func() string { return stderr()[before:] }
|
||||||
p := startProgress(tc.params, func() int { return 120 }, testRedraw,
|
p := startProgress(tc.params, terminalWidth, testRedraw, time.Now)
|
||||||
time.Now)
|
|
||||||
|
|
||||||
for range 50 {
|
for range 50 {
|
||||||
p.increment()
|
p.increment()
|
||||||
@@ -554,13 +552,13 @@ func drawLate(stderr func() string, params progressParams, width int,
|
|||||||
return stderr()[before:]
|
return stderr()[before:]
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalLinesKeepTimeRemaining checks the line each display shows
|
// TestTerminalLinesKeepETA checks the line each display shows on an
|
||||||
// on an 80-column terminal at a large scan's counts and times, and the
|
// 80-column terminal at a large scan's counts and times, and the line
|
||||||
// line of trees' read phase at the same counts: the rate is left out,
|
// of trees' read phase at the same counts: the rate is left out, so
|
||||||
// so that the elapsed time and the time remaining stay whole.
|
// that the elapsed time and the ETA stay whole.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
func TestTerminalLinesKeepETA(t *testing.T) {
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -572,20 +570,26 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
|||||||
label: scanPhase("walk"), unit: scanUnit, total: 1200000,
|
label: scanPhase("walk"), unit: scanUnit, total: 1200000,
|
||||||
estimate: true,
|
estimate: true,
|
||||||
},
|
},
|
||||||
want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
|
want: "[2/5] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
|
||||||
"remain unknown",
|
"eta unknown",
|
||||||
}, {
|
}, {
|
||||||
params: progressParams{
|
params: progressParams{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
||||||
},
|
},
|
||||||
want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
want: "[3/5] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
||||||
"remain 17h30m0s",
|
"eta 17h30m0s",
|
||||||
|
}, {
|
||||||
|
params: progressParams{
|
||||||
|
label: scanPhase("content"), unit: scanUnit, total: 9876543,
|
||||||
|
},
|
||||||
|
want: "[5/5] content: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
||||||
|
"eta 17h30m0s",
|
||||||
}, {
|
}, {
|
||||||
params: progressParams{
|
params: progressParams{
|
||||||
label: treesPhase("read"), unit: "records", total: 9876543,
|
label: treesPhase("read"), unit: "records", total: 9876543,
|
||||||
},
|
},
|
||||||
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
|
||||||
"remain 17h30m0s",
|
"eta 17h30m0s",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -596,32 +600,30 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalStalledLines draws every phase's line once the phase has
|
// TestTerminalStalledLines draws every phase's line on an 80-column
|
||||||
// stalled, its last item done 42 seconds before. At an ordinary scan's
|
// terminal once the phase has stalled, its last item done 42 seconds
|
||||||
// counts and times, 12345 of 45678 items done four and a half minutes
|
// before. At an ordinary scan's counts and times, 12345 of 45678 items
|
||||||
// in, on a terminal 100 columns wide, the line shows where the phase
|
// done four and a half minutes in, the line shows where the phase waits
|
||||||
// waits whole, beside the phase, progress, elapsed time and time
|
// whole, beside the phase, progress, elapsed time and ETA. At a large
|
||||||
// remaining. At a large scan's, 1234567 of 9876543 two and a half hours
|
// scan's, 1234567 of 9876543 two and a half hours in, those four still
|
||||||
// in, on an 80-column terminal, those four still show whole, as the
|
// show whole, as the words give way first where they do not fit.
|
||||||
// words give way first where they do not fit.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalStalledLines(t *testing.T) {
|
func TestTerminalStalledLines(t *testing.T) {
|
||||||
stderr := captureStderr(t)
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
// drawStalled draws the display params describes on a terminal width
|
// drawStalled draws the display params describes as it starts and
|
||||||
// columns wide as it starts and once more as it ends, with count of
|
// once more as it ends, with count of total items done elapsed into
|
||||||
// total items done elapsed into the phase, and returns what the
|
// the phase, and returns what the terminal shows.
|
||||||
// terminal shows.
|
drawStalled := func(params progressParams, total, count int64,
|
||||||
drawStalled := func(params progressParams, width int, total,
|
elapsed time.Duration,
|
||||||
count int64, elapsed time.Duration,
|
|
||||||
) string {
|
) string {
|
||||||
before := len(stderr())
|
before := len(stderr())
|
||||||
clock := &testClock{now: time.Now()}
|
clock := &testClock{now: time.Now()}
|
||||||
params.total = total
|
params.total = total
|
||||||
|
|
||||||
// No timed draws come between the two.
|
// No timed draws come between the two.
|
||||||
p := startProgress(params, func() int { return width }, time.Hour,
|
p := startProgress(params, func() int { return 80 }, time.Hour,
|
||||||
clock.Now)
|
clock.Now)
|
||||||
|
|
||||||
clock.Add(elapsed - 42*time.Second)
|
clock.Add(elapsed - 42*time.Second)
|
||||||
@@ -641,6 +643,9 @@ func TestTerminalStalledLines(t *testing.T) {
|
|||||||
{label: scanPhase("walk"), estimate: true, stalledOn: "disk"},
|
{label: scanPhase("walk"), estimate: true, stalledOn: "disk"},
|
||||||
{label: scanPhase("hash"), stalledOn: hashStalledOn},
|
{label: scanPhase("hash"), stalledOn: hashStalledOn},
|
||||||
{label: scanPhase("update"), stalledOn: "db"},
|
{label: scanPhase("update"), stalledOn: "db"},
|
||||||
|
// The content phase's checks and reads, then its last write.
|
||||||
|
{label: scanPhase("content"), stalledOn: "disk/db"},
|
||||||
|
{label: scanPhase("content"), stalledOn: "db"},
|
||||||
{label: treesPhase("read"), stalledOn: "db"},
|
{label: treesPhase("read"), stalledOn: "db"},
|
||||||
{label: treesPhase("group"), stalledOn: "cpu"},
|
{label: treesPhase("group"), stalledOn: "cpu"},
|
||||||
{label: treesPhase("write"), stalledOn: "out"},
|
{label: treesPhase("write"), stalledOn: "out"},
|
||||||
@@ -653,22 +658,19 @@ func TestTerminalStalledLines(t *testing.T) {
|
|||||||
approx = "~"
|
approx = "~"
|
||||||
}
|
}
|
||||||
|
|
||||||
// The percent is in the line, or beside the bar.
|
shown := drawStalled(params, 45678, 12345,
|
||||||
shown := drawStalled(params, 100, 45678, 12345,
|
|
||||||
4*time.Minute+30*time.Second)
|
4*time.Minute+30*time.Second)
|
||||||
for _, want := range []string{
|
want := params.label + ": [12345/" + approx + "45678] 27%, stalled " +
|
||||||
params.label + ": [12345/" + approx + "45678]",
|
params.stalledOn + ", elapsed 4m30s, eta 12m9s"
|
||||||
", stalled " + params.stalledOn + ", elapsed 4m30s, remain 12m9s",
|
|
||||||
} {
|
if !strings.Contains(shown, want) {
|
||||||
if !strings.Contains(shown, want) {
|
t.Errorf("terminal shows %q, want %q", shown, want)
|
||||||
t.Errorf("terminal shows %q, want %q", shown, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
shown = drawStalled(params, 80, 9876543, 1234567, 150*time.Minute)
|
shown = drawStalled(params, 9876543, 1234567, 150*time.Minute)
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
params.label + ": [1234567/" + approx + "9876543] 12%, ",
|
params.label + ": [1234567/" + approx + "9876543] 12%, ",
|
||||||
", elapsed 2h30m0s, remain 17h30m0s",
|
", elapsed 2h30m0s, eta 17h30m0s",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(shown, want) {
|
if !strings.Contains(shown, want) {
|
||||||
t.Errorf("terminal shows %q, want %q", shown, want)
|
t.Errorf("terminal shows %q, want %q", shown, want)
|
||||||
@@ -691,7 +693,7 @@ func TestTerminalLinesFitWidth(t *testing.T) {
|
|||||||
for _, width := range []int{120, 100, 80, 72, 60, 50, 40} {
|
for _, width := range []int{120, 100, 80, 72, 60, 50, 40} {
|
||||||
for _, total := range []int64{9876543, 20000} {
|
for _, total := range []int64{9876543, 20000} {
|
||||||
params := progressParams{
|
params := progressParams{
|
||||||
label: scanPhase("update"), unit: scanUnit, total: total,
|
label: scanPhase("content"), unit: scanUnit, total: total,
|
||||||
}
|
}
|
||||||
out := drawLate(stderr, params, width, total/8)
|
out := drawLate(stderr, params, width, total/8)
|
||||||
|
|
||||||
@@ -713,164 +715,13 @@ func TestTerminalLinesFitWidth(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTerminalLinesStartWithSpinner draws a display with no total and
|
|
||||||
// one with a bar, two and a half hours into the phase, on a terminal 120
|
|
||||||
// columns wide. Each line must start with the library's spinner in the
|
|
||||||
// first column, then a space and the phase's label.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalLinesStartWithSpinner(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
|
|
||||||
for _, params := range []progressParams{
|
|
||||||
{label: scanPhase("walk"), unit: scanUnit, total: -1},
|
|
||||||
{label: scanPhase("hash"), unit: scanUnit, total: 9876543},
|
|
||||||
} {
|
|
||||||
shown := lastFrame(drawLate(stderr, params, 120, 1234567))
|
|
||||||
spinner, rest, _ := strings.Cut(shown, " ")
|
|
||||||
|
|
||||||
if utf8.RuneCountInString(spinner) != 1 ||
|
|
||||||
!strings.HasPrefix(rest, params.label+": ") {
|
|
||||||
t.Errorf("terminal shows %q, want the spinner, a space and %q",
|
|
||||||
shown, params.label)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.total > 0 && !strings.Contains(shown, "% |") {
|
|
||||||
t.Errorf("terminal shows %q, want a bar", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalBarStaysAtRightEdge draws the hash phase's display two
|
|
||||||
// and a half hours in, at two counts whose lines differ in length, on
|
|
||||||
// terminals 110 and 120 columns wide. The bar is partly filled at the
|
|
||||||
// first count and full at the second. It must start the same number of
|
|
||||||
// columns from the terminal's right edge each time, and end the same
|
|
||||||
// number of columns from it: a line that grows or shrinks, as its time
|
|
||||||
// remaining does, never moves the bar, and nor do the bar's filled
|
|
||||||
// cells.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalBarStaysAtRightEdge(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
widths := []int{110, 120}
|
|
||||||
counts := []int64{1234567, 9876543}
|
|
||||||
startFromEdge := make([]int, 0, len(widths)*len(counts))
|
|
||||||
endFromEdge := make([]int, 0, len(widths)*len(counts))
|
|
||||||
|
|
||||||
for _, width := range widths {
|
|
||||||
for _, count := range counts {
|
|
||||||
shown := lastFrame(drawLate(stderr, params, width, count))
|
|
||||||
|
|
||||||
beforeBar, _, found := strings.Cut(shown, "% |")
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("terminal %d wide shows %q, want a bar", width, shown)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bar ends with the last "|".
|
|
||||||
throughBar := shown[:strings.LastIndex(shown, "|")+1]
|
|
||||||
|
|
||||||
startFromEdge = append(startFromEdge,
|
|
||||||
width-utf8.RuneCountInString(beforeBar))
|
|
||||||
endFromEdge = append(endFromEdge,
|
|
||||||
width-utf8.RuneCountInString(throughBar))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Min(startFromEdge) != slices.Max(startFromEdge) {
|
|
||||||
t.Errorf("bar starts %v columns from the right edge, want the same "+
|
|
||||||
"every time", startFromEdge)
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Min(endFromEdge) != slices.Max(endFromEdge) {
|
|
||||||
t.Errorf("bar ends %v columns from the right edge, want the same "+
|
|
||||||
"every time", endFromEdge)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalRateGivesWayBeforeBar draws the hash phase's display two
|
|
||||||
// and a half hours in. On a terminal 90 columns wide the line fits
|
|
||||||
// beside the bar without its rate, but not with it: the bar is drawn
|
|
||||||
// and the rate is not. On a terminal 85 columns wide the line does not
|
|
||||||
// fit beside the bar even without its rate: neither is drawn, though
|
|
||||||
// the rate would fit where the bar was.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalRateGivesWayBeforeBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
shown := lastFrame(drawLate(stderr, params, 90, 1234567))
|
|
||||||
if !strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
|
|
||||||
t.Errorf("terminal 90 wide shows %q, want the bar and no rate", shown)
|
|
||||||
}
|
|
||||||
|
|
||||||
shown = lastFrame(drawLate(stderr, params, 85, 1234567))
|
|
||||||
if strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
|
|
||||||
t.Errorf("terminal 85 wide shows %q, want neither bar nor rate", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalSpinnerTurnsBesideBar draws the hash phase's display, its
|
|
||||||
// bar partly filled, several times on a terminal 120 columns wide. The
|
|
||||||
// spinner must turn at every draw. A filled cell of the bar takes one
|
|
||||||
// column but three bytes. A line measured in bytes would seem too wide
|
|
||||||
// for the terminal, and every draw would start the display over, its
|
|
||||||
// spinner included.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestTerminalSpinnerTurnsBesideBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
params := progressParams{
|
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
|
||||||
}
|
|
||||||
|
|
||||||
// No timed draws: the test draws.
|
|
||||||
p := startProgress(params, func() int { return 120 }, time.Hour,
|
|
||||||
time.Now)
|
|
||||||
|
|
||||||
p.mu.Lock()
|
|
||||||
p.count = 1234567
|
|
||||||
p.mu.Unlock()
|
|
||||||
|
|
||||||
const draws = 4
|
|
||||||
|
|
||||||
spinners := make([]string, 0, draws)
|
|
||||||
|
|
||||||
for range draws {
|
|
||||||
before := len(stderr())
|
|
||||||
|
|
||||||
p.draw()
|
|
||||||
|
|
||||||
spinner, _, _ := strings.Cut(lastFrame(stderr()[before:]), " ")
|
|
||||||
spinners = append(spinners, spinner)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.finish()
|
|
||||||
|
|
||||||
for i := 1; i < len(spinners); i++ {
|
|
||||||
if spinners[i] == spinners[i-1] {
|
|
||||||
t.Errorf("spinners at successive draws are %q, want each to "+
|
|
||||||
"differ from the one before", spinners)
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
|
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
|
||||||
// and a half hours into the phase, on a terminal 120 columns wide, then
|
// and a half hours into the phase, on a terminal 120 columns wide, then
|
||||||
// makes the terminal narrower than that line and draws it again a few
|
// makes the terminal narrower than that line and draws it again a few
|
||||||
// times. Nothing written after that, the erase before each redraw
|
// times. Nothing written after that, the erase before each redraw
|
||||||
// included, may reach the narrower terminal's last column: it would
|
// included, may reach the narrower terminal's last column: it would
|
||||||
// wrap, and every redraw would leave a line behind. At the narrower
|
// wrap, and every redraw would leave a line behind. At these widths the
|
||||||
// width the hash phase's bar gives way.
|
// spinner stays a spinner and the bar gives way to the spinner.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
||||||
@@ -888,7 +739,7 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
|||||||
progressParams{
|
progressParams{
|
||||||
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
|
||||||
},
|
},
|
||||||
80,
|
90,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -928,10 +779,10 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProgressWarningsOnOwnLines drives the terminal displays, with a
|
// TestProgressWarningsOnOwnLines drives the terminal displays, spinner
|
||||||
// bar and without, through a run of warnings with no items between
|
// and bar, through a run of warnings with no items between them, as when
|
||||||
// them, as when the walk meets many unreadable paths, for many redraws:
|
// the walk meets many unreadable paths, for many redraws: every warning
|
||||||
// every warning must land on a line of its own, never inside a redraw.
|
// must land on a line of its own, never inside a redraw.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressWarningsOnOwnLines(t *testing.T) {
|
func TestProgressWarningsOnOwnLines(t *testing.T) {
|
||||||
@@ -977,12 +828,11 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProgressWriterRowsOnOwnLines drives the terminal displays, with a
|
// TestProgressWriterRowsOnOwnLines drives the terminal displays,
|
||||||
// bar and without, through a run of rows written through the display's
|
// spinner and bar, through a run of rows written through the display's
|
||||||
// writer to the same terminal, as stdout is in a plain "sfdupes trees",
|
// writer to the same terminal, as stdout is in a plain "sfdupes trees",
|
||||||
// each written out at once, for many redraws: the terminal must show
|
// each written out at once, for many redraws: the terminal must show
|
||||||
// every row whole, in order, on a line of its own. The terminal is 120
|
// every row whole, in order, on a line of its own.
|
||||||
// columns wide, so the write phase's display has room for its bar.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
||||||
@@ -999,8 +849,7 @@ func TestProgressWriterRowsOnOwnLines(t *testing.T) {
|
|||||||
var want []string
|
var want []string
|
||||||
|
|
||||||
for _, params := range cases {
|
for _, params := range cases {
|
||||||
p := startProgress(params, func() int { return 120 }, testRedraw,
|
p := startProgress(params, terminalWidth, testRedraw, time.Now)
|
||||||
time.Now)
|
|
||||||
out := p.writer(os.Stderr)
|
out := p.writer(os.Stderr)
|
||||||
|
|
||||||
// No pause between rows: one is garbled only if a redraw lands
|
// No pause between rows: one is garbled only if a redraw lands
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -25,9 +24,9 @@ const minGroupSize = 2
|
|||||||
type scanRec struct {
|
type scanRec struct {
|
||||||
size int64
|
size int64
|
||||||
mtime time.Time
|
mtime time.Time
|
||||||
head [sha256.Size]byte
|
head string
|
||||||
tail [sha256.Size]byte
|
tail string
|
||||||
content [sha256.Size]byte
|
content string
|
||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,6 +169,12 @@ type reportWriter struct {
|
|||||||
func (w *reportWriter) add(r scanRec) error {
|
func (w *reportWriter) add(r scanRec) error {
|
||||||
w.records++
|
w.records++
|
||||||
|
|
||||||
|
// A record without a content hash has unknown content (README
|
||||||
|
// "Database"), so it is never a duplicate.
|
||||||
|
if r.content == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if r.size == w.sig.size && r.head == w.sig.head &&
|
if r.size == w.sig.size && r.head == w.sig.head &&
|
||||||
r.tail == w.sig.tail && r.content == w.sig.content {
|
r.tail == w.sig.tail && r.content == w.sig.content {
|
||||||
w.paths = append(w.paths, r.path)
|
w.paths = append(w.paths, r.path)
|
||||||
|
|||||||
+47
-66
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -19,27 +18,14 @@ import (
|
|||||||
// awkwardDir is a directory name holding every byte the reports escape.
|
// awkwardDir is a directory name holding every byte the reports escape.
|
||||||
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
|
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
|
||||||
|
|
||||||
// testHash returns a stand-in hash for a test record: the bytes of s,
|
|
||||||
// then zeros. Stand-ins sort as their strings do, so a test can choose
|
|
||||||
// the order in which report reads its records.
|
|
||||||
func testHash(s string) [sha256.Size]byte {
|
|
||||||
var h [sha256.Size]byte
|
|
||||||
|
|
||||||
copy(h[:], s)
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
|
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
|
||||||
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
|
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
|
||||||
// sorts after it, so awkwardDir coming first shows that sorting uses
|
// sorts after it, so awkwardDir coming first shows that sorting uses
|
||||||
// the raw path.
|
// the raw path.
|
||||||
func awkwardPairRecs() []scanRec {
|
func awkwardPairRecs() []scanRec {
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
|
{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
|
||||||
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,8 +143,8 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
recs := make([]scanRec, 2*ioBufSize/len(dir))
|
recs := make([]scanRec, 2*ioBufSize/len(dir))
|
||||||
for i := range recs {
|
for i := range recs {
|
||||||
recs[i] = scanRec{
|
recs[i] = scanRec{
|
||||||
size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
|
size: int64(i/2) + 1, head: "h", tail: "t", content: "c",
|
||||||
content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
|
path: fmt.Sprintf("%s/%d", dir, i),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,7 +173,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
// reportOpenLine is the plain line of report's open phase when it opens
|
// reportOpenLine is the plain line of report's open phase when it opens
|
||||||
// the database at once; it prints it as the phase starts and as it ends.
|
// the database at once; it prints it as the phase starts and as it ends.
|
||||||
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// smokeReport is report's stdout for the smoke-test records.
|
// smokeReport is report's stdout for the smoke-test records.
|
||||||
const smokeReport = "first\tdupe\tsize\n" +
|
const smokeReport = "first\tdupe\tsize\n" +
|
||||||
@@ -225,7 +211,7 @@ func TestRunReportProgressNamesEachPhase(t *testing.T) {
|
|||||||
"report: database " + path + " (from " + databaseEnv + ")",
|
"report: database " + path + " (from " + databaseEnv + ")",
|
||||||
reportOpenLine,
|
reportOpenLine,
|
||||||
reportOpenLine,
|
reportOpenLine,
|
||||||
"[2/2] read: counting the records, elapsed 0s, remain unknown",
|
"[2/2] read: counting the records, elapsed 0s, eta unknown",
|
||||||
"[2/2] read: [6/6] 100%, ",
|
"[2/2] read: [6/6] 100%, ",
|
||||||
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
|
"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
|
||||||
"6.1 KiB reclaimable",
|
"6.1 KiB reclaimable",
|
||||||
@@ -291,14 +277,13 @@ func TestReadReportOnTerminal(t *testing.T) {
|
|||||||
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
|
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
|
||||||
// README §Constraints: identical database contents give identical
|
// README §Constraints: identical database contents give identical
|
||||||
// output, whatever order the records were inserted in.
|
// output, whatever order the records were inserted in.
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
|
|
||||||
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
|
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
|
||||||
recs = append(recs,
|
recs = append(recs,
|
||||||
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
|
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
|
||||||
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
|
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
|
||||||
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
|
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
|
||||||
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
|
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
|
||||||
|
scanRec{size: 50, path: "/x/unhashed"},
|
||||||
)
|
)
|
||||||
|
|
||||||
reversed := slices.Clone(recs)
|
reversed := slices.Clone(recs)
|
||||||
@@ -401,19 +386,16 @@ func TestWarnfEscapes(t *testing.T) {
|
|||||||
func TestDupeGroups(t *testing.T) {
|
func TestDupeGroups(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
|
|
||||||
u := testHash("u")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
|
||||||
{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
|
{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
|
||||||
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
|
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
|
||||||
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
|
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
|
||||||
// Same size as the /z group but a different head hash.
|
// Same size as the /z group but a different head hash.
|
||||||
{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
|
{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
|
||||||
// A singleton signature must not form a group.
|
// A singleton signature must not form a group.
|
||||||
{size: 7, head: u, tail: u, content: u, path: "/lonely"},
|
{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -437,13 +419,16 @@ func TestDupeGroups(t *testing.T) {
|
|||||||
func TestDupeGroupsContentSeparates(t *testing.T) {
|
func TestDupeGroupsContentSeparates(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Same size, head, and tail: /b differs from the others only in its
|
// Same size, head, and tail, but different content hashes: the final
|
||||||
// content hash, so it joins no group, while /a and /c group.
|
// rung keeps them apart, so no group forms. Matching content groups.
|
||||||
h, tl := testHash("h"), testHash("t")
|
// Records without a content hash never group, not even with each
|
||||||
|
// other.
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
|
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
|
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
|
||||||
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
|
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
|
||||||
|
{size: 100, head: "h", tail: "t", path: "/d"},
|
||||||
|
{size: 100, head: "h", tail: "t", path: "/e"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -462,15 +447,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
|
|||||||
|
|
||||||
// mtime is informational only; records differing only in mtime
|
// mtime is informational only; records differing only in mtime
|
||||||
// still group together.
|
// still group together.
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{
|
{
|
||||||
size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
|
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
|
||||||
content: c, path: "/m/1",
|
content: "c", path: "/m/1",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
|
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
|
||||||
content: c, path: "/m/2",
|
content: "c", path: "/m/2",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -485,12 +469,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
|
|||||||
|
|
||||||
// The hashes sort opposite to the first paths, so ordering the
|
// The hashes sort opposite to the first paths, so ordering the
|
||||||
// groups by hash instead of by first path fails this test.
|
// groups by hash instead of by first path fails this test.
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -512,16 +495,15 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
|
|||||||
// /z, /k, which is neither the order of their first paths nor its
|
// /z, /k, which is neither the order of their first paths nor its
|
||||||
// reverse, and must write them ordered by first path. The group of
|
// reverse, and must write them ordered by first path. The group of
|
||||||
// size 7 comes after them all, though its first path sorts first.
|
// size 7 comes after them all, though its first path sorts first.
|
||||||
a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/k/2"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
|
||||||
{size: 50, head: a, tail: a, content: a, path: "/k/1"},
|
{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/z/1"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
|
||||||
{size: 50, head: b, tail: b, content: b, path: "/z/2"},
|
{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
|
||||||
{size: 50, head: c, tail: c, content: c, path: "/a/2"},
|
{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
|
||||||
{size: 50, head: c, tail: c, content: c, path: "/a/1"},
|
{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
|
||||||
{size: 7, head: d, tail: d, content: d, path: "/0/2"},
|
{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
|
||||||
{size: 7, head: d, tail: d, content: d, path: "/0/1"},
|
{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
groups := dupeGroupsOf(t, recs)
|
groups := dupeGroupsOf(t, recs)
|
||||||
@@ -540,12 +522,11 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
|
|||||||
func TestDupeGroupsDeterministic(t *testing.T) {
|
func TestDupeGroupsDeterministic(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
a, b := testHash("a"), testHash("b")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 1, head: a, tail: a, content: a, path: "/p/1"},
|
{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
|
||||||
{size: 1, head: a, tail: a, content: a, path: "/p/2"},
|
{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
|
||||||
{size: 2, head: b, tail: b, content: b, path: "/q/1"},
|
{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
|
||||||
{size: 2, head: b, tail: b, content: b, path: "/q/2"},
|
{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
|
||||||
}
|
}
|
||||||
|
|
||||||
forward := dupeGroupsOf(t, recs)
|
forward := dupeGroupsOf(t, recs)
|
||||||
|
|||||||
+720
-445
File diff suppressed because it is too large
Load Diff
+4
-5
@@ -2,12 +2,11 @@
|
|||||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||||
# the formatter on the host, which a pristine checkout cannot do.
|
# the formatter on the host, which a pristine checkout cannot do.
|
||||||
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
|
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
||||||
# line, since a cached gate phase is a check that did not run; unlike
|
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
||||||
# --no-cache, it keeps the layers above that line and the cache mounts.
|
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||||
# before every step that lints, tests or compiles; a stage without that
|
# first; a stage without that line is served from cache on an unchanged
|
||||||
# line is served from cache on an unchanged tree, with no warning.
|
# tree, with no warning.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+4
-5
@@ -1,12 +1,11 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/docker: build the Docker image tagged with the project name.
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
# Identical in all repos; the tag comes from script/projectname.
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
|
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
||||||
# line, since a cached gate phase is a check that did not run; unlike
|
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
||||||
# --no-cache, it keeps the layers above that line and the cache mounts.
|
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||||
# before every step that lints, tests or compiles; a stage without that
|
# first; a stage without that line is served from cache on an unchanged
|
||||||
# line is served from cache on an unchanged tree, with no warning.
|
# tree, with no warning.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+5
-7
@@ -5,13 +5,11 @@
|
|||||||
# make its answer untrustworthy.
|
# make its answer untrustworthy.
|
||||||
#
|
#
|
||||||
# The phase is not the last stage in the file, so it is built only when
|
# The phase is not the last stage in the file, so it is built only when
|
||||||
# --target names it. A new CACHEBUST value reruns every RUN step after
|
# --target names it. A new CACHEBUST value reruns every RUN step, since
|
||||||
# the ARG CACHEBUST line, since a cached lint layer is a lint that did
|
# a cached lint layer is a lint that did not run; unlike --no-cache, it
|
||||||
# not run; unlike --no-cache, it keeps the layers above that line and
|
# keeps Go's build cache. This relies on every stage of the Dockerfile
|
||||||
# the cache mounts. This relies on every stage of the Dockerfile
|
# declaring ARG CACHEBUST first; a stage without that line is served
|
||||||
# declaring ARG CACHEBUST before every step that lints, tests or
|
# from cache on an unchanged tree, with no warning.
|
||||||
# compiles; a stage without that line is served from cache on an
|
|
||||||
# unchanged tree, with no warning.
|
|
||||||
# --output type=cacheonly writes no image, since nothing uses one.
|
# --output type=cacheonly writes no image, since nothing uses one.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
|||||||
+5
-6
@@ -3,12 +3,11 @@
|
|||||||
# and this builds that phase alone, on the same terms as script/lint:
|
# and this builds that phase alone, on the same terms as script/lint:
|
||||||
# --target because a phase that is not the last stage is built only when
|
# --target because a phase that is not the last stage is built only when
|
||||||
# named, and a new CACHEBUST value because a cached test layer is a test
|
# named, and a new CACHEBUST value because a cached test layer is a test
|
||||||
# that did not run; unlike --no-cache, it keeps the layers above the ARG
|
# that did not run; unlike --no-cache, it keeps Go's build cache. This
|
||||||
# CACHEBUST line and the cache mounts. This relies on every stage of the
|
# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
|
||||||
# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
|
# a stage without that line is served from cache on an unchanged tree,
|
||||||
# or compiles; a stage without that line is served from cache on an
|
# with no warning. --output type=cacheonly writes no image, since
|
||||||
# unchanged tree, with no warning. --output type=cacheonly writes no
|
# nothing uses one.
|
||||||
# image, since nothing uses one.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
@@ -286,9 +286,18 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
|
|||||||
// name and its signature (size, head, tail, content); mtime is
|
// name and its signature (size, head, tail, content); mtime is
|
||||||
// excluded.
|
// excluded.
|
||||||
func fileEntry(name string, r scanRec) string {
|
func fileEntry(name string, r scanRec) string {
|
||||||
|
content := r.content
|
||||||
|
|
||||||
|
// A record without a content hash has unknown content (README
|
||||||
|
// "Database"): give it a signature no other file can share, so
|
||||||
|
// trees containing it never compare equal. Real hashes are hex, so
|
||||||
|
// the NUL-prefixed form cannot collide.
|
||||||
|
if content == "" {
|
||||||
|
content = "unhashed\x00" + r.path
|
||||||
|
}
|
||||||
|
|
||||||
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
|
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
|
||||||
"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
|
"\x00" + r.head + "\x00" + r.tail + "\x00" + content
|
||||||
"\x00" + string(r.content[:])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// collectTreeGroups groups directories by digest, counting each one on
|
// collectTreeGroups groups directories by digest, counting each one on
|
||||||
@@ -338,8 +347,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
|||||||
// digest is the SHA-256 of its child entries — files serialized with
|
// digest is the SHA-256 of its child entries — files serialized with
|
||||||
// name and signature, subdirectories with name and recursive digest —
|
// name and signature, subdirectories with name and recursive digest —
|
||||||
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
|
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
|
||||||
// and every hash and digest is 32 bytes, so NUL delimiters are
|
// so NUL delimiters are unambiguous.
|
||||||
// unambiguous.
|
|
||||||
func (n *treeNode) computeDigest() {
|
func (n *treeNode) computeDigest() {
|
||||||
slices.Sort(n.entries)
|
slices.Sort(n.entries)
|
||||||
|
|
||||||
|
|||||||
+26
-21
@@ -10,17 +10,24 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Signature hashes shared by the smoke-test records.
|
||||||
|
const (
|
||||||
|
f1Head = "f1h"
|
||||||
|
f1Tail = "f1t"
|
||||||
|
f1Content = "f1c"
|
||||||
|
f2Head = "f2h"
|
||||||
|
f2Tail = "f2t"
|
||||||
|
f2Content = "f2c"
|
||||||
|
)
|
||||||
|
|
||||||
// treesOpenLine is the plain line of trees' open phase when it opens
|
// treesOpenLine is the plain line of trees' open phase when it opens
|
||||||
// the database at once; it prints it as the phase starts and as it ends.
|
// the database at once; it prints it as the phase starts and as it ends.
|
||||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"eta unknown"
|
||||||
|
|
||||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
||||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
||||||
@@ -146,14 +153,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
|
|||||||
|
|
||||||
for _, top := range []string{"/a", "/c"} {
|
for _, top := range []string{"/a", "/c"} {
|
||||||
for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
|
for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
|
||||||
content := testHash("c")
|
content := "c"
|
||||||
if p == "/b-x/f" {
|
if p == "/b-x/f" {
|
||||||
content = testHash("other")
|
content = "other"
|
||||||
}
|
}
|
||||||
|
|
||||||
recs = append(recs, scanRec{
|
recs = append(recs, scanRec{
|
||||||
size: 1, head: testHash("h"), tail: testHash("t"),
|
size: 1, head: "h", tail: "t", content: content, path: top + p,
|
||||||
content: content, path: top + p,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -236,11 +242,11 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
|||||||
"trees: database " + path + " (from " + databaseEnv + ")",
|
"trees: database " + path + " (from " + databaseEnv + ")",
|
||||||
treesOpenLine,
|
treesOpenLine,
|
||||||
treesOpenLine,
|
treesOpenLine,
|
||||||
"[2/4] read: counting the records, elapsed 0s, remain unknown",
|
"[2/4] read: counting the records, elapsed 0s, eta unknown",
|
||||||
"[2/4] read: [6/6] 100%, ",
|
"[2/4] read: [6/6] 100%, ",
|
||||||
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
|
"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, eta unknown",
|
||||||
"[3/4] group: [8/8] 100%, ",
|
"[3/4] group: [8/8] 100%, ",
|
||||||
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
|
"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, eta unknown",
|
||||||
"[4/4] write: [1/1] 100%, ",
|
"[4/4] write: [1/1] 100%, ",
|
||||||
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
|
"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
|
||||||
}
|
}
|
||||||
@@ -326,10 +332,11 @@ func TestTreeDigests(t *testing.T) {
|
|||||||
func TestTreeDigestContentSensitivity(t *testing.T) {
|
func TestTreeDigestContentSensitivity(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
same, c := testHash("same"), testHash("c")
|
const sharedTail = "same"
|
||||||
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
|
{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
|
||||||
{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
|
{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, dirs := treeOf(t, recs)
|
_, dirs := treeOf(t, recs)
|
||||||
@@ -390,10 +397,9 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
|
|||||||
|
|
||||||
// Identical sibling dirs share a parent, so their group cannot be
|
// Identical sibling dirs share a parent, so their group cannot be
|
||||||
// implied by a parent group and must be reported.
|
// implied by a parent group and must be reported.
|
||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
super, dirs := treeOf(t, recs)
|
||||||
@@ -412,11 +418,10 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
|||||||
// /p/a and /q/b contain identical x subtrees, but /p/a has an
|
// /p/a and /q/b contain identical x subtrees, but /p/a has an
|
||||||
// extra file, so the parents' digests differ and the x group must
|
// extra file, so the parents' digests differ and the x group must
|
||||||
// be reported.
|
// be reported.
|
||||||
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
|
|
||||||
recs := []scanRec{
|
recs := []scanRec{
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
|
||||||
{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
|
{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
|
||||||
{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
|
{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
|
||||||
}
|
}
|
||||||
|
|
||||||
super, dirs := treeOf(t, recs)
|
super, dirs := treeOf(t, recs)
|
||||||
|
|||||||
Reference in New Issue
Block a user