Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
a00b97e299 |
No files matched your search
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
|
||||
|
||||
BINARY := sfdupes
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
||||
LDFLAGS := -X main.Version=$(VERSION)
|
||||
|
||||
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
||||
|
||||
@@ -14,7 +14,7 @@ LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
||||
# Default target: build the binary. Phony so go build (which has its
|
||||
# own build cache) always decides what to recompile.
|
||||
sfdupes:
|
||||
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes
|
||||
go build -ldflags "$(LDFLAGS)" -o $(BINARY)
|
||||
|
||||
build: sfdupes
|
||||
|
||||
|
||||
@@ -17,13 +17,12 @@ gigabyte. A large file is never read in full. A match between two large files is
|
||||
a strong candidate, not proof.
|
||||
|
||||
`sfdupes` is meant for multi-terabyte servers, where reading every byte of every
|
||||
file takes too long. It has five subcommands. `scan` walks the filesystem and
|
||||
file takes too long. It has four subcommands. `scan` walks the filesystem and
|
||||
records the files it finds in an SQLite database. SQLite keeps a whole database
|
||||
in one file. The database is kept between runs. A rescan reads only new and
|
||||
changed files. That makes `scan` cheap enough to run every day from `cron`, the
|
||||
system's job scheduler. `report`, `trees` and `info` read the database alone and
|
||||
can run at any time. `dwim` scans `/` and then prints the `trees` report, for a
|
||||
first look at a new machine with one command.
|
||||
can run at any time.
|
||||
|
||||
## Getting Started
|
||||
|
||||
@@ -57,25 +56,25 @@ shell shows both on the terminal, and `>` redirects stdout alone.
|
||||
With Go installed, this builds and installs the newest tagged release:
|
||||
|
||||
```sh
|
||||
go install sneak.berlin/go/sfdupes/cmd/sfdupes@latest
|
||||
go install sneak.berlin/go/sfdupes@latest
|
||||
```
|
||||
|
||||
`@main` installs the tip of the `main` branch instead:
|
||||
|
||||
```sh
|
||||
go install sneak.berlin/go/sfdupes/cmd/sfdupes@main
|
||||
go install sneak.berlin/go/sfdupes@main
|
||||
```
|
||||
|
||||
The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set.
|
||||
|
||||
`sfdupes --version` prints `sfdupes VERSION` and exits. For a binary from
|
||||
`go install`, `VERSION` is the version Go recorded in it. For `@latest` that is
|
||||
a release tag. For `@main` it is the commit's tag when it has one. Otherwise it
|
||||
is a pseudo-version, Go's version string for an untagged commit. That string
|
||||
ends in the commit's hash. A binary built from a clone carries the output of
|
||||
`git describe --tags --always --dirty` in that clone, such as `v0.0.3` on a
|
||||
tagged commit. `VERSION` is `dev` when no version is known. To see everything Go
|
||||
recorded in an installed binary:
|
||||
a release tag such as `v0.0.3`. For `@main` it is the commit's tag when it has
|
||||
one. Otherwise it is a pseudo-version, Go's version string for an untagged
|
||||
commit. That string ends in the commit's hash. A binary built from a clone
|
||||
carries the output of `git describe --tags --always --dirty` in that clone, such
|
||||
as `v0.0.3` on a tagged commit. `VERSION` is `dev` when no version is known. To
|
||||
see everything Go recorded in an installed binary:
|
||||
|
||||
```sh
|
||||
go version -m "$(command -v sfdupes)"
|
||||
@@ -248,15 +247,12 @@ In order:
|
||||
- `report` prints the file-level duplicate report from the database.
|
||||
- `trees` prints the duplicate-tree report from the database.
|
||||
- `info` prints the database's details.
|
||||
- `dwim` scans `/` with the default excludes, then prints the duplicate-tree
|
||||
report.
|
||||
|
||||
```
|
||||
sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||
sfdupes report > dupes.tsv
|
||||
sfdupes trees > dupetrees.tsv
|
||||
sfdupes info
|
||||
sfdupes dwim
|
||||
sfdupes --version
|
||||
sfdupes [command] --help
|
||||
```
|
||||
@@ -352,26 +348,17 @@ default. An operand may be a directory or a regular file.
|
||||
|
||||
- `scan` with no operand is a usage error. The usage text goes to stderr, and
|
||||
the exit code is 2.
|
||||
- An operand that does not exist is a fatal error, and so is a symlink whose
|
||||
target does not exist. `scan` then stops before it walks anything, and exits
|
||||
with code 1.
|
||||
- Each operand is resolved to its real path before the walk, as `realpath`
|
||||
prints it. A relative operand is joined to the working directory as `pwd -P`
|
||||
prints it. Every symlink in the path is then followed. A `..` after a symlink
|
||||
counts from the directory the symlink leads to. From then on the operand is
|
||||
that real path. Its files are recorded under it, and warnings name it.
|
||||
- An operand that is a symlink is scanned at its target's real path. After a
|
||||
`cd` through a symlink, `scan .` records its files under the real path of the
|
||||
working directory.
|
||||
- An operand that does not exist is a fatal error. `scan` then stops before it
|
||||
walks anything, and exits 1.
|
||||
- Each operand is made absolute before the walk. A relative operand is joined to
|
||||
the working directory as `pwd` prints it, with its symlinks kept. A `.` or
|
||||
`..` in it is resolved as text, and symlinks are not followed. Each record is
|
||||
stored under that absolute path.
|
||||
- All operands belong to one scan, and one walk covers them all at once (see
|
||||
"Concurrency").
|
||||
- An operand that repeats another is dropped before the walk. Two operands
|
||||
repeat each other when their real paths are the same.
|
||||
- An operand that repeats another is dropped before the walk.
|
||||
- An operand under another operand is dropped the same way, unless one of the
|
||||
cases below applies. No path is then walked twice. Whether one operand is
|
||||
under another is judged on their real paths. With `/srv/link` a symlink to a
|
||||
directory outside `/srv`, `scan /srv /srv/link/sub` walks `sub` on its own,
|
||||
because the real path of `sub` is not under `/srv`.
|
||||
cases below applies. No path is then walked twice.
|
||||
- An operand under another is kept, and walked on its own, when the inner
|
||||
operand or a directory between the two is one of these:
|
||||
- a default exclude (see "Rules for the walk");
|
||||
@@ -382,19 +369,25 @@ default. An operand may be a directory or a regular file.
|
||||
memory (see "Rules for the walk"). The outer operand's walk does not enter
|
||||
such a directory, so the inner operand is walked on its own.
|
||||
|
||||
- No other case keeps an operand under another.
|
||||
- An operand that is neither a regular file nor a directory is not scanned. Such
|
||||
an operand is, for example, a symlink to a device node. `scan` prints a
|
||||
warning naming the real path and what it is. It counts the operand as skipped
|
||||
and drops it. The scan keeps the records beneath a dropped operand, unless
|
||||
another operand covers them.
|
||||
- No other case keeps an operand under another. An operand reached through a
|
||||
symlinked directory below the outer operand is dropped too. The outer
|
||||
operand's walk does not follow the symlink, so no file under the inner operand
|
||||
is scanned. The records that an earlier scan left under it are deleted. The
|
||||
`-x` case above also covers an operand reached that way.
|
||||
- An operand that is neither a regular file nor a directory is not scanned.
|
||||
`scan` prints a warning naming the path and what it is. It counts the operand
|
||||
as skipped and drops it. An operand beneath a symlink operand is scanned all
|
||||
the same. The scan keeps the records beneath a dropped operand, unless another
|
||||
operand covers them. After a `cd` through a symlink, `scan .` names the
|
||||
symlink and is skipped as a symlink operand. `scan sub` in that directory
|
||||
records its files under the symlink's path. `report` can then list each as a
|
||||
duplicate of the same file recorded under its real path.
|
||||
- An operand at or inside a directory named `.zfs` is never scanned, whether
|
||||
given alone or under another operand. The check looks at the real path, so a
|
||||
symlink into a `.zfs` directory is not scanned either. `scan` prints a warning
|
||||
naming the real path, counts the operand as skipped and drops it. The records
|
||||
beneath it are kept, unless it lies under another operand. Then they are
|
||||
deleted, because that operand's walk never enters `.zfs` (see "Rules for the
|
||||
walk").
|
||||
given alone or under another operand. The check looks at the path as given and
|
||||
at the path with its symlinks resolved. `scan` prints a warning naming the
|
||||
path, counts the operand as skipped and drops it. The records beneath it are
|
||||
kept, unless it lies under another operand. Then they are deleted, because
|
||||
that operand's walk never enters `.zfs` (see "Rules for the walk").
|
||||
- An operand on a filesystem that is not persistent is dropped too. `scan`
|
||||
prints a line naming it and its filesystem type, and does not count it as
|
||||
skipped.
|
||||
@@ -491,12 +484,12 @@ walk practical on large or busy storage pools.
|
||||
The walk knows them by device ID and inode number. The device ID is the
|
||||
number by which the kernel tells filesystems apart. The inode number tells
|
||||
the files of one filesystem apart. So the walk passes over them whatever
|
||||
path leads to them. The walk follows no symlink it finds. It opens each
|
||||
operand at its real path, not the path given. An operand given through a
|
||||
symlink can lead to the database files, and the walk passes over them there
|
||||
as well. The walk prints no line for the database files and does not count
|
||||
them as skipped. A record that an earlier scan left for one of them is
|
||||
deleted.
|
||||
path leads to them. The walk follows no symlink it finds. It does open each
|
||||
operand at the path given, even when that path runs through a symlinked
|
||||
directory. Such an operand can lead to the database files, and the walk
|
||||
passes over them there as well. The walk prints no line for the database
|
||||
files and does not count them as skipped. A record that an earlier scan left
|
||||
for one of them is deleted.
|
||||
|
||||
- By default the walk enters directories on other filesystems than its
|
||||
operand's. With `-x`, it never enters a directory on a different filesystem
|
||||
@@ -665,7 +658,6 @@ Processing:
|
||||
- Records are grouped by signature, the four values
|
||||
`(size, head, tail, content)`.
|
||||
- Every group with two or more paths is a duplicate group.
|
||||
- Empty files form no group, because removing one frees nothing.
|
||||
- Within a group, paths are sorted in byte order. The first is `first`, and
|
||||
every other path is a `dupe`.
|
||||
- Groups are ordered by size, largest first, then by `first` path. The output is
|
||||
@@ -714,7 +706,7 @@ The summary line on stderr gives:
|
||||
- the reclaimable bytes, the sum of `size` over all rows.
|
||||
|
||||
```
|
||||
report: 14 records read, 4 duplicate groups, 7 dupe files, 10.0 KiB reclaimable
|
||||
report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable
|
||||
```
|
||||
|
||||
### `trees` mode
|
||||
@@ -742,9 +734,6 @@ Processing:
|
||||
- `trees` builds the directory tree and computes every directory's digest. It
|
||||
then groups the directories by digest. Every group with two or more
|
||||
directories is a duplicate-tree group.
|
||||
- `trees` leaves out a group whose trees hold 0 bytes, because removing one
|
||||
frees nothing. An empty file still counts toward the digest of a tree that
|
||||
holds other files.
|
||||
- `trees` leaves out a group that a group of its members' parents already
|
||||
implies. A group is left out when its members' parents are distinct
|
||||
directories that all have the same digest. The parents' group, or an
|
||||
@@ -808,33 +797,6 @@ The summary line on stderr gives:
|
||||
trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable
|
||||
```
|
||||
|
||||
### `dwim` mode
|
||||
|
||||
`dwim`, short for "do what I mean", scans a new machine and reports its
|
||||
duplicate trees in one command. It takes no positional arguments and no flags of
|
||||
its own. Like `scan`, it should run as root, so that it can read every file.
|
||||
|
||||
`dwim` runs what `sfdupes scan /` runs with no flags. The walk skips the default
|
||||
excludes and crosses into other filesystems. `--workers` keeps its default. The
|
||||
database, its creation and the scan lock are those of `scan`. Then `dwim` runs
|
||||
what `sfdupes trees` runs. The largest trees come first.
|
||||
|
||||
The report goes to stdout, exactly as `trees` prints it. On stderr comes what
|
||||
`scan` prints, then what `trees` prints, then one last line:
|
||||
|
||||
```
|
||||
dwim: run "sfdupes trees" to print this report again
|
||||
```
|
||||
|
||||
The database keeps the scan, so `sfdupes trees` prints the same report later,
|
||||
even after the terminal is gone.
|
||||
|
||||
`SIGINT` or `SIGTERM` stops the scan as it stops `scan` (see "Stopping a scan").
|
||||
`dwim` then prints no report and exits 1. A fatal scan error also ends it before
|
||||
the report, with the same message and exit code as `scan`. Warnings for skipped
|
||||
files do not stop it. `dwim` stops catching signals when the scan ends, so a
|
||||
signal during the report ends it at once, as it ends `trees`.
|
||||
|
||||
### `info` mode
|
||||
|
||||
`info` takes no positional arguments and prints the database's details on
|
||||
@@ -875,7 +837,7 @@ scan is writing. `info` writes its lines after the `count` display has ended.
|
||||
|
||||
### Database
|
||||
|
||||
All five subcommands work on one SQLite database file.
|
||||
All four subcommands work on one SQLite database file.
|
||||
|
||||
#### Location
|
||||
|
||||
@@ -891,8 +853,6 @@ scan: new database /var/lib/sfdupes/db.sqlite (default)
|
||||
trees: database /srv/tmp/sfdupes/db.db (from SFDUPES_DATABASE)
|
||||
```
|
||||
|
||||
`dwim` prints the `scan` line first, and the `trees` line after the scan.
|
||||
|
||||
A relative path is printed with the working directory in front of it. The
|
||||
working directory is printed as `pwd -P` prints it, with its symlinks resolved.
|
||||
The rest of the path is printed as set, so a `..` in it stays. The line names
|
||||
@@ -1176,9 +1136,9 @@ showing the count it stopped at.
|
||||
### Error handling and exit codes
|
||||
|
||||
- `0`: success, even when files were skipped with warnings.
|
||||
- `1`: a fatal error, a `scan` stopped by `SIGINT` or `SIGTERM`, or a `dwim`
|
||||
stopped by either during its scan. A fatal error is one of these:
|
||||
- an operand that does not exist, or whose real path `scan` cannot find;
|
||||
- `1`: a fatal error, or a `scan` stopped by `SIGINT` or `SIGTERM`. A fatal
|
||||
error is one of these:
|
||||
- an operand that does not exist, or whose details `scan` cannot read;
|
||||
- a list of mounted filesystems that `scan` cannot read;
|
||||
- another `scan` holding the lock (see "The scan lock");
|
||||
- a database, its directory or its lock file that cannot be created or
|
||||
@@ -1198,7 +1158,7 @@ showing the count it stopped at.
|
||||
`--workers abc`;
|
||||
- `scan` with no operand;
|
||||
- `--workers` below 1;
|
||||
- `report`, `trees`, `info` or `dwim` with a positional argument.
|
||||
- `report`, `trees` or `info` with a positional argument.
|
||||
|
||||
A fatal error is reported on stderr, starting with `sfdupes:`. A failed write to
|
||||
stdout, such as a full disk, is reported the same way. Two cases never reach
|
||||
@@ -1386,7 +1346,7 @@ All of the following, run in this directory, must pass:
|
||||
head -c 2000 /dev/urandom > "$d/a/unique.bin" # same size, different content
|
||||
printf 'x' > "$d/tiny1"; printf 'x' > "$d/tiny2" # 1-byte duplicates
|
||||
printf 'y' > "$d/tiny3" # 1-byte non-duplicate
|
||||
: > "$d/empty1"; : > "$d/empty2" # empty files
|
||||
: > "$d/empty1"; : > "$d/empty2" # empty duplicates
|
||||
# duplicate trees: t1 and t2 are identical; t3 differs by one filename
|
||||
mkdir -p "$d/t1/sub" "$d/t2/sub" "$d/t3/sub"
|
||||
head -c 3000 /dev/urandom > "$d/t1/f1"
|
||||
@@ -1411,8 +1371,8 @@ All of the following, run in this directory, must pass:
|
||||
is the path that sorts first.
|
||||
- `t1/f1`, `t2/f1` and `t3/f1` form one group.
|
||||
- `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group.
|
||||
- `tiny1` and `tiny2` form a pair.
|
||||
- `unique.bin`, `tiny3`, `empty1` and `empty2` appear nowhere.
|
||||
- `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`.
|
||||
- `unique.bin` and `tiny3` appear nowhere.
|
||||
- The groups come in order of size, largest first.
|
||||
|
||||
`trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`.
|
||||
@@ -1425,9 +1385,8 @@ All of the following, run in this directory, must pass:
|
||||
because its content changed. `copy.bin` and `copy2.bin` remain as one pair.
|
||||
`unique.bin` is gone from the database.
|
||||
|
||||
The test suite automates this scenario (see
|
||||
`internal/sfdupes/scan_test.go`). It also checks that `report` and `trees`
|
||||
never touch the scanned filesystem.
|
||||
The test suite automates this scenario (see `scan_test.go`). It also checks
|
||||
that `report` and `trees` never touch the scanned filesystem.
|
||||
|
||||
## TODO
|
||||
|
||||
|
||||
@@ -29,40 +29,18 @@
|
||||
Newest first. An entry ends with its date and its issue or branch, where those
|
||||
are known.
|
||||
|
||||
- `report` and `trees` leave out empty files. `report` writes no group of size
|
||||
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
|
||||
counts toward a tree's digest (2026-11-11,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/178)
|
||||
- The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
|
||||
command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
|
||||
became `internal/sfdupes/cli.go`. Every other file kept its name. The install
|
||||
path is now `sneak.berlin/go/sfdupes/cmd/sfdupes` (2026-11-10,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/157)
|
||||
- `README.md` and a comment in `scan.go` no longer say ZFS counts a new file's
|
||||
blocks only seconds after the write. The count can land in under a second.
|
||||
Both now say up to seconds after the write, as the comment on `diskSizeOf` in
|
||||
`scan_test.go` does (2026-11-03, https://git.eeqj.de/sneak/sfdupes/issues/152)
|
||||
|
||||
- New subcommand `dwim`. It runs what `sfdupes scan /` runs with no flags, then
|
||||
what `sfdupes trees` runs. Its last line on stderr says that `sfdupes trees`
|
||||
prints the report again (2026-11-06,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/154)
|
||||
|
||||
- `scan` resolves each operand to its real path, symlinks followed, before the
|
||||
walk. A symlink operand is scanned at its target instead of skipped. With
|
||||
`/srv/link` a symlink to a directory outside `/srv`, `scan /srv /srv/link/sub`
|
||||
walks `sub` on its own instead of dropping it and deleting its records
|
||||
(2026-11-06, https://git.eeqj.de/sneak/sfdupes/issues/144)
|
||||
|
||||
- `README.md` and a comment in `internal/sfdupes/scan.go` no longer say ZFS
|
||||
counts a new file's blocks only seconds after the write. The count can land in
|
||||
under a second. Both now say up to seconds after the write, as the comment on
|
||||
`diskSizeOf` in `internal/sfdupes/scan_test.go` does (2026-11-03,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/152)
|
||||
|
||||
- The three disk size tests in `internal/sfdupes/scan_test.go` no longer fail at
|
||||
random on ZFS. `unix.Sync` can return before ZFS has counted a file's blocks,
|
||||
and one ZFS commit can take a file through several sizes. The test that needs
|
||||
the size to change waits for it, for at most 30 seconds. Each test reads the
|
||||
size before and after its scan. The rewrite test rescans until the two reads
|
||||
match, then compares exactly. The other two fail only on a `disk_size` outside
|
||||
the two reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150)
|
||||
- The three disk size tests in `scan_test.go` no longer fail at random on ZFS.
|
||||
`unix.Sync` can return before ZFS has counted a file's blocks, and one ZFS
|
||||
commit can take a file through several sizes. The test that needs the size to
|
||||
change waits for it, for at most 30 seconds. Each test reads the size before
|
||||
and after its scan. The rewrite test rescans until the two reads match, then
|
||||
compares exactly. The other two fail only on a `disk_size` outside the two
|
||||
reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150)
|
||||
|
||||
- `TODO.md` rewritten from scratch by the same rules as `README.md`, sentence by
|
||||
sentence. Every completed step is kept, with its date and its issue or branch.
|
||||
@@ -73,13 +51,9 @@ are known.
|
||||
it. It leads with what the program is and how to use it. A reference section
|
||||
covers every setting and every subcommand. Development and testing come last.
|
||||
Every statement was checked against the code on `next`. Five comments in
|
||||
`internal/sfdupes/cli_test.go` cite "Settings", where their rules now live.
|
||||
One comment each in these files cites what the README now says:
|
||||
- `internal/sfdupes/scan_test.go`;
|
||||
- `internal/sfdupes/trees_test.go`;
|
||||
- `internal/sfdupes/exclude_test.go`.
|
||||
|
||||
(2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
||||
`main_test.go` cite "Settings", where their rules now live. One comment each
|
||||
in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
|
||||
now says (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
||||
|
||||
- Each record holds `disk_size`, the bytes the file takes on disk. That is its
|
||||
allocated blocks, `st_blocks` × 512, from the `lstat` call the walk already
|
||||
@@ -152,12 +126,12 @@ are known.
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/109)
|
||||
|
||||
- The release rule was set. Until 1.0, each tip of `main` that passes the build
|
||||
gets the next `v0.0.N` tag.
|
||||
`go install sneak.berlin/go/sfdupes/cmd/sfdupes@latest` then installs it.
|
||||
`go.mod` retracts `v0.0.1`, the early build without a database. When the
|
||||
linker set no version, `sfdupes --version` prints the version Go recorded in
|
||||
the binary. That is the tag, or the pseudo-version of a `go install` build. It
|
||||
prints `dev` only when neither is known. README "Install" documented:
|
||||
gets the next `v0.0.N` tag. `go install sneak.berlin/go/sfdupes@latest` then
|
||||
installs it. `go.mod` retracts `v0.0.1`, the early build without a database.
|
||||
When the linker set no version, `sfdupes --version` prints the version Go
|
||||
recorded in the binary. That is the tag, or the pseudo-version of a
|
||||
`go install` build. It prints `dev` only when neither is known. README
|
||||
"Install" documented:
|
||||
- the rule;
|
||||
- what `@main` and `v0.0.1` give;
|
||||
- how to check an installed binary.
|
||||
@@ -408,9 +382,8 @@ are known.
|
||||
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
||||
|
||||
- A test fails when either `hashWorker` cancellation check in
|
||||
`internal/sfdupes/scan.go` is removed (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/83)
|
||||
- A test fails when either `hashWorker` cancellation check in `scan.go` is
|
||||
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
|
||||
|
||||
- A database path holding `?`, `#` or `%` opens exactly the file it names
|
||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
||||
@@ -418,15 +391,15 @@ are known.
|
||||
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
||||
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
||||
|
||||
- A test fails when either walk cancellation check in `internal/sfdupes/scan.go`
|
||||
is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
||||
- A test fails when either walk cancellation check in `scan.go` is removed
|
||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
||||
|
||||
- A test checks that `scan` refuses a database with another schema version
|
||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64)
|
||||
|
||||
- Four inaccurate comments in `internal/sfdupes/cancel_test.go` were corrected,
|
||||
and `walkCancelInFlightDirs` was renamed to `walkCancelInFlightFiles`
|
||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/33)
|
||||
- Four inaccurate comments in `cancel_test.go` were corrected, and
|
||||
`walkCancelInFlightDirs` was renamed to `walkCancelInFlightFiles` (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
||||
|
||||
- The `-x` filesystem-boundary rules in `subdirJob` are tested (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
||||
@@ -633,15 +606,15 @@ are known.
|
||||
- `loadRecords`;
|
||||
- `resolveRoots`.
|
||||
|
||||
The single exit point is `run` in `internal/sfdupes/cli.go`. It maps a
|
||||
`fatalError`, anything a subcommand returned, to exit 1, and cobra's own
|
||||
argument and flag errors to exit 2. So a runtime failure is never reported
|
||||
as a usage error, nor printed with the usage text.
|
||||
`internal/sfdupes/cli_test.go` drives the CLI in-process and asserts the
|
||||
exit codes from README "Error handling" and the stdout/stderr split. It also
|
||||
checks that a fatal error after the database is open leaves no `-wal` or
|
||||
`-shm` file behind, for `scan`, `report` or `trees` (2026-08-09, branch
|
||||
`db-close-on-fatal`, https://git.eeqj.de/sneak/sfdupes/issues/4)
|
||||
The single exit point is `run` in `main.go`. It maps a `fatalError`,
|
||||
anything a subcommand returned, to exit 1, and cobra's own argument and flag
|
||||
errors to exit 2. So a runtime failure is never reported as a usage error,
|
||||
nor printed with the usage text. `main_test.go` drives the CLI in-process
|
||||
and asserts the exit codes from README "Error handling" and the
|
||||
stdout/stderr split. It also checks that a fatal error after the database is
|
||||
open leaves no `-wal` or `-shm` file behind, for `scan`, `report` or `trees`
|
||||
(2026-08-09, branch `db-close-on-fatal`,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/4)
|
||||
|
||||
- `golangci-lint` was updated to v2.12.2 with the canonical config, merged as
|
||||
`38a01bd`. The pinned linter in the `Dockerfile` lint stage and
|
||||
@@ -774,4 +747,6 @@ are known.
|
||||
|
||||
# Accepted Divergences
|
||||
|
||||
None.
|
||||
- Flat single-package layout, with the `.go` files in the repo root. That is
|
||||
fine for a small single-binary tool per the Go styleguide, and the tracker
|
||||
audit agrees.
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,29 +0,0 @@
|
||||
// Command sfdupes quickly identifies candidate duplicate files across
|
||||
// very large filesystems without reading every byte of every file.
|
||||
// Files are considered duplicates when their sizes are equal and they
|
||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
||||
// KiB, and of the content. The content hash covers the whole file under
|
||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||
// file is never read in full. scan maintains a persistent SQLite
|
||||
// database of file signatures (SFDUPES_DATABASE, default
|
||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||
// sfdupes report > dupes.tsv
|
||||
// sfdupes trees > dupetrees.tsv
|
||||
// sfdupes info
|
||||
// sfdupes dwim
|
||||
// sfdupes --version
|
||||
//
|
||||
// See README.md for the complete specification.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"sneak.berlin/go/sfdupes/internal/sfdupes"
|
||||
)
|
||||
|
||||
func main() { os.Exit(sfdupes.Main()) }
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -146,44 +146,33 @@ var errHashSize = errors.New("hash is not 32 bytes")
|
||||
|
||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||
// path, made absolute by absolutePath, and where the path came from,
|
||||
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
||||
// prints first on stderr.
|
||||
// path, with the working directory, its symlinks resolved, put in front
|
||||
// of it when it is relative, and where the path came from, "default" or
|
||||
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
|
||||
// stderr.
|
||||
func databasePath() (string, string, error) {
|
||||
path, source := defaultDatabasePath, "default"
|
||||
if p := os.Getenv(databaseEnv); p != "" {
|
||||
path, source = p, "from "+databaseEnv
|
||||
}
|
||||
|
||||
abs, err := absolutePath(path)
|
||||
if filepath.IsAbs(path) {
|
||||
return path, source, nil
|
||||
}
|
||||
|
||||
// The working directory as the kernel reports it, with no symlink
|
||||
// in it, as README "Database" says. os.Getwd may return PWD
|
||||
// instead, which can name it through a symlink.
|
||||
wd, err := unix.Getwd()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||
}
|
||||
|
||||
return abs, source, nil
|
||||
}
|
||||
|
||||
// absolutePath returns path unchanged when it is absolute, and
|
||||
// otherwise the working directory, a "/" and path. The working
|
||||
// directory is the one the kernel reports, with no symlink in it;
|
||||
// os.Getwd may return PWD instead, which can name it through a symlink.
|
||||
// The result is not cleaned, as filepath.Abs and filepath.Join would
|
||||
// clean it. Cleaning drops each ".." together with the directory name
|
||||
// before it. When that name is a symlink, the cleaned path names
|
||||
// another file than the one the system reaches by following the symlink
|
||||
// first.
|
||||
func absolutePath(path string) (string, error) {
|
||||
if filepath.IsAbs(path) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
wd, err := unix.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Not filepath.Abs or filepath.Join: they clean the path, which
|
||||
// drops each ".." together with the directory name before it. When
|
||||
// that directory is a symlink, the cleaned path names another file.
|
||||
// The working directory ends in "/" only when it is the root.
|
||||
return strings.TrimSuffix(wd, "/") + "/" + path, nil
|
||||
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
||||
}
|
||||
|
||||
// createDatabaseDirectory creates the directory the database at path
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // calls the unexported database openers
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
// This file holds two rules for the walk (README "Rules for the walk"):
|
||||
// the default excludes, the paths a scan's walk does not enter below an
|
||||
@@ -1,6 +1,6 @@
|
||||
//go:build darwin
|
||||
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
@@ -1,6 +1,6 @@
|
||||
//go:build !darwin
|
||||
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -286,7 +286,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
||||
t.Skip(dir + " is not inside a default exclude")
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,37 +0,0 @@
|
||||
package sfdupes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
||||
// root" runs with no flags, then what "sfdupes trees" runs, and ends
|
||||
// with a line on stderr saying how to print the report again. The dwim
|
||||
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
|
||||
// does, but only during the scan. An error from the scan, an interrupt
|
||||
// included, ends it before the report. During the report, a signal
|
||||
// ends it at once, as it ends trees.
|
||||
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
|
||||
scanCtx, stop := interruptContext(ctx)
|
||||
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false)
|
||||
|
||||
stop()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = runTrees(ctx, stdout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stderr,
|
||||
`dwim: run "sfdupes trees" to print this report again`)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package sfdupes //nolint:testpackage // calls the unexported runDwim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
||||
// dwim scans its root, then prints the trees report, the larger of
|
||||
// two duplicate-tree pairs first, and ends by saying how to print
|
||||
// the report again.
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
root := realTempDir(t)
|
||||
|
||||
small := []string{
|
||||
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
||||
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
||||
}
|
||||
large := []string{
|
||||
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
|
||||
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
|
||||
stderr := captureStderr(t)
|
||||
|
||||
err := runDwim(t.Context(), root, &stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("stdout = %q, want a header and two groups",
|
||||
stdout.String())
|
||||
}
|
||||
|
||||
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
|
||||
if !strings.HasPrefix(lines[1], wantFirst) {
|
||||
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
|
||||
}
|
||||
|
||||
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
|
||||
if !strings.HasPrefix(lines[2], wantSecond) {
|
||||
t.Errorf("second group = %q, want it to start %q",
|
||||
lines[2], wantSecond)
|
||||
}
|
||||
|
||||
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
|
||||
if got := stderr(); !strings.HasSuffix(got, last) {
|
||||
t.Errorf("stderr = %q, want it to end %q", got, last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
||||
// A fatal scan error ends dwim before the report: a root that does
|
||||
// not exist fails the scan as it fails scan itself.
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
missing := filepath.Join(realTempDir(t), "missing")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
|
||||
stderr := captureStderr(t)
|
||||
|
||||
err := runDwim(t.Context(), missing, &stdout)
|
||||
if err == nil {
|
||||
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
||||
}
|
||||
|
||||
if got := stdout.String(); got != "" {
|
||||
t.Errorf("stdout = %q, want no report", got)
|
||||
}
|
||||
|
||||
if got := stderr(); strings.Contains(got, "trees:") ||
|
||||
strings.Contains(got, "dwim:") {
|
||||
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,23 @@
|
||||
// Package sfdupes implements the sfdupes command and the database its
|
||||
// subcommands share.
|
||||
package sfdupes
|
||||
// Command sfdupes quickly identifies candidate duplicate files across
|
||||
// very large filesystems without reading every byte of every file.
|
||||
// Files are considered duplicates when their sizes are equal and they
|
||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
||||
// KiB, and of the content. The content hash covers the whole file under
|
||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||
// file is never read in full. scan maintains a persistent SQLite
|
||||
// database of file signatures (SFDUPES_DATABASE, default
|
||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||
// sfdupes report > dupes.tsv
|
||||
// sfdupes trees > dupetrees.tsv
|
||||
// sfdupes info
|
||||
// sfdupes --version
|
||||
//
|
||||
// See README.md for the complete specification.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -28,7 +45,6 @@ const (
|
||||
cmdReport = "report"
|
||||
cmdTrees = "trees"
|
||||
cmdInfo = "info"
|
||||
cmdDwim = "dwim"
|
||||
)
|
||||
|
||||
// errNoSubcommand is returned by the root command when it is invoked
|
||||
@@ -68,22 +84,20 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
|
||||
return info.Main.Version
|
||||
}
|
||||
|
||||
// Main runs the command line in os.Args and returns the process exit
|
||||
// code.
|
||||
func Main() int {
|
||||
func main() {
|
||||
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
||||
// head", the Go runtime ends the process with SIGPIPE on the next
|
||||
// write instead of returning an error (README "Error handling").
|
||||
// Registering for SIGPIPE with os/signal would change that.
|
||||
return run(os.Args[1:], os.Stdout, os.Stderr)
|
||||
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
// run executes args against the command tree and returns the process
|
||||
// exit code. It is the program's single exit point: the subcommands
|
||||
// return their errors instead of exiting, so every deferred cleanup —
|
||||
// above all closing the database, which checkpoints the SQLite WAL —
|
||||
// runs before the process ends. The report, trees, info and dwim
|
||||
// subcommands write their data to stdout.
|
||||
// runs before the process ends. The report, trees and info subcommands
|
||||
// write their data to stdout.
|
||||
func run(args []string, stdout, stderr io.Writer) int {
|
||||
// A nil slice makes cobra fall back to os.Args, which would let a
|
||||
// test binary's own flags reach the command tree.
|
||||
@@ -187,16 +201,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
||||
}),
|
||||
}
|
||||
|
||||
dwimCmd := &cobra.Command{
|
||||
Use: cmdDwim,
|
||||
Short: "Scan / with the default excludes, then print the duplicate-tree report",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
||||
return runDwim(ctx, "/", stdout)
|
||||
}),
|
||||
}
|
||||
|
||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
|
||||
|
||||
return root
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // drives the unexported run
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -14,8 +14,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// usageMarker is the first line of cobra's usage text, which a usage
|
||||
@@ -622,7 +620,6 @@ func TestRunUsageErrors(t *testing.T) {
|
||||
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
||||
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
||||
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
||||
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
|
||||
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
||||
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
||||
}
|
||||
@@ -800,7 +797,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||
func scanFixture(t *testing.T) []string {
|
||||
t.Helper()
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
dupes := []string{
|
||||
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
||||
@@ -870,7 +867,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||
|
||||
// Same size as a.bin, so the scan reads it, and the read fails.
|
||||
@@ -898,278 +895,58 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
||||
assertNoSidecars(t, path)
|
||||
}
|
||||
|
||||
// realTempDir returns a new temporary directory by its real path, so
|
||||
// tests hold where the temporary directory is reached through a
|
||||
// symlink.
|
||||
func realTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// symlink creates link pointing at target.
|
||||
func symlink(t *testing.T, target, link string) {
|
||||
t.Helper()
|
||||
|
||||
err := os.Symlink(target, link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
|
||||
// The operand is scanned at its real path, and its file is recorded
|
||||
// there.
|
||||
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
|
||||
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, filepath.Join(dir, "target"), link)
|
||||
|
||||
stderr := scanOK(t, link)
|
||||
|
||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 0 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
// Scanning a directory through the symlink stores a record beneath
|
||||
// the symlink's own path for a file beneath its target.
|
||||
scanOK(t, filepath.Join(link, "sub"))
|
||||
|
||||
assertOperandSkipped(t, path, link, "symlink",
|
||||
filepath.Join(link, "sub", "f"))
|
||||
}
|
||||
|
||||
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
|
||||
// with link a symlink to target. Both resolve under target, so
|
||||
// target/sub is walked once, as part of target.
|
||||
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, filepath.Join(dir, "target"), link)
|
||||
|
||||
stderr := scanOK(t, link, filepath.Join(link, "sub"))
|
||||
|
||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 0 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
|
||||
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
|
||||
// with dir/link a symlink to a directory outside dir. The walk of dir
|
||||
// does not follow link, so dir/link/sub is walked on its own at its
|
||||
// real path, and the record an earlier scan stored there survives.
|
||||
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
a := writeFile(t, dir, "a", pattern(1, 10))
|
||||
|
||||
outside := realTempDir(t)
|
||||
f := writeFile(t, outside, "sub/f", pattern(2, 10))
|
||||
|
||||
symlink(t, outside, filepath.Join(dir, "link"))
|
||||
|
||||
scanOK(t, filepath.Join(outside, "sub"))
|
||||
|
||||
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
|
||||
|
||||
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"1 unchanged), 0 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
|
||||
want := []string{a, f}
|
||||
slices.Sort(want)
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||
t.Errorf("records = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
|
||||
// link, outside dir, a symlink to dir/real. link resolves inside dir,
|
||||
// so it is dropped as covered by dir, and its file is recorded once.
|
||||
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "real/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(realTempDir(t), "link")
|
||||
symlink(t, filepath.Join(dir, "real"), link)
|
||||
|
||||
stderr := scanOK(t, dir, link)
|
||||
|
||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 0 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
|
||||
// a symlink whose target does not exist. The scan fails with exit 1 and
|
||||
// an error naming the symlink as given, before it walks anything, so no
|
||||
// database is created.
|
||||
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
writeFile(t, dir, "f", pattern(1, 10))
|
||||
|
||||
dangling := filepath.Join(realTempDir(t), "dangling")
|
||||
symlink(t, filepath.Join(dir, "missing"), dangling)
|
||||
|
||||
stdout := captureStdout(t)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
|
||||
if code != exitFatal {
|
||||
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
|
||||
exitFatal)
|
||||
}
|
||||
|
||||
assertFatalOutput(t, stderr.String(), stdout())
|
||||
|
||||
want := "sfdupes: resolve " + dangling + ": "
|
||||
if !strings.Contains(stderr.String(), want) {
|
||||
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
||||
}
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("database: stat error %v, want it missing", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
|
||||
// after a cd through home/proj, a symlink to data/proj. The file is
|
||||
// recorded under data/proj, its real path.
|
||||
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||
|
||||
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
|
||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||
filepath.Join(dir, "home", "proj"))
|
||||
// link is dropped as a symlink, but link/sub must still be scanned,
|
||||
// not dropped as lying under link.
|
||||
scanOK(t, link, filepath.Join(link, "sub"))
|
||||
|
||||
// t.Chdir also sets PWD to the path through the symlink, as a
|
||||
// shell does after cd.
|
||||
t.Chdir(filepath.Join(dir, "home", "proj"))
|
||||
|
||||
scanOK(t, ".")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
|
||||
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
|
||||
// a cd through home/proj. As with realpath, the ".." counts from
|
||||
// data/proj, so data is scanned and home is not.
|
||||
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
|
||||
for _, tc := range []struct{ wd, operand string }{
|
||||
{"", "home/proj/.."},
|
||||
{"home/proj", ".."},
|
||||
} {
|
||||
t.Run(tc.operand, func(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||
g := writeFile(t, dir, "data/g", pattern(2, 10))
|
||||
writeFile(t, dir, "home/h", pattern(3, 10))
|
||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||
filepath.Join(dir, "home", "proj"))
|
||||
|
||||
// t.Chdir also sets PWD to the path given, as a shell does
|
||||
// after cd.
|
||||
t.Chdir(filepath.Join(dir, tc.wd))
|
||||
|
||||
scanOK(t, tc.operand)
|
||||
|
||||
want := []string{f, g}
|
||||
slices.Sort(want)
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||
t.Errorf("records = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
|
||||
// The scan warns about the FIFO by its real path, counts it as skipped,
|
||||
// and exits 0.
|
||||
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
dir := realTempDir(t)
|
||||
fifo := filepath.Join(dir, "fifo")
|
||||
|
||||
err := unix.Mkfifo(fifo, 0o600)
|
||||
db, err := openDB(path, reportParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, fifo, link)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
stderr := scanOK(t, link)
|
||||
|
||||
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
||||
if !strings.Contains(stderr, warning) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||
}
|
||||
|
||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 1 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
|
||||
}
|
||||
|
||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||
zfs := filepath.Join(realTempDir(t), ".zfs")
|
||||
zfs := filepath.Join(t.TempDir(), ".zfs")
|
||||
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
||||
|
||||
// No scan records a file inside a .zfs directory, so the record
|
||||
@@ -1220,7 +997,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
||||
writeFile(t, hourly, "f", pattern(1, 10))
|
||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
||||
@@ -1264,19 +1041,23 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
||||
|
||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
||||
// by its real path, counted as skipped, and not walked.
|
||||
// by the path given, counted as skipped, and not walked.
|
||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
||||
|
||||
snaps := filepath.Join(dir, "snaps")
|
||||
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||
|
||||
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
||||
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
daily := filepath.Join(snaps, "daily")
|
||||
stderr := scanOK(t, daily)
|
||||
|
||||
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 1 skipped\n"
|
||||
@@ -1288,19 +1069,24 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
||||
// a symlink to pool/data. The operand is scanned as usual, and its file
|
||||
// is recorded by its real path.
|
||||
// a symlink to a directory outside any .zfs directory. The operand is
|
||||
// scanned as usual, and its file is recorded by the path given.
|
||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||
|
||||
data := filepath.Join(dir, "data")
|
||||
symlink(t, filepath.Join(dir, "pool", "data"), data)
|
||||
|
||||
stderr := scanOK(t, filepath.Join(data, "daily"))
|
||||
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
daily := filepath.Join(data, "daily")
|
||||
stderr := scanOK(t, daily)
|
||||
|
||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 0 skipped\n"
|
||||
@@ -1308,9 +1094,14 @@ func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
db, err := openDB(path, reportParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
|
||||
}
|
||||
|
||||
func TestRunReportSucceeds(t *testing.T) {
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // drives the unexported startProgress
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -195,11 +195,10 @@ func (w *reportWriter) add(r scanRec) error {
|
||||
}
|
||||
|
||||
// endGroup ends the paths read with w.sig: two or more are a duplicate
|
||||
// group, held with its paths sorted until writeGroups writes it. Empty
|
||||
// files form no group, since removing one frees nothing.
|
||||
// group, held with its paths sorted until writeGroups writes it.
|
||||
func (w *reportWriter) endGroup() {
|
||||
if len(w.paths) < minGroupSize || w.sig.size == 0 {
|
||||
// The paths are dropped, and their slice reused.
|
||||
if len(w.paths) < minGroupSize {
|
||||
// A path with no duplicate is dropped, and its slice reused.
|
||||
w.paths = w.paths[:0]
|
||||
|
||||
return
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -434,27 +434,6 @@ func TestDupeGroups(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
|
||||
// duplicate group, while 1-byte files still do.
|
||||
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e, x := sha256.Sum256(nil), testHash("x")
|
||||
recs := []scanRec{
|
||||
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
|
||||
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
|
||||
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
|
||||
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
|
||||
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
|
||||
}
|
||||
|
||||
groups := dupeGroupsOf(t, recs)
|
||||
if len(groups) != 1 || groups[0].size != 1 ||
|
||||
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
|
||||
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDupeGroupsContentSeparates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
@@ -209,30 +209,29 @@ func interrupted(walked int) error {
|
||||
return errInterrupted
|
||||
}
|
||||
|
||||
// resolveRoots converts each PATH operand to its real path: absolute,
|
||||
// with every symlink in it followed. Database records are keyed by
|
||||
// that path, so scan results depend neither on the working directory
|
||||
// nor on the symlinks an operand was given through. An operand that
|
||||
// cannot be resolved, because it or a symlink's target does not exist
|
||||
// or cannot be read, is a fatal error before any scanning.
|
||||
// resolveRoots converts each PATH operand to an absolute, lexically
|
||||
// cleaned path (symlinks are not resolved) and verifies that it
|
||||
// exists. Database records are keyed by absolute path, so scan results
|
||||
// must not depend on the working directory.
|
||||
func resolveRoots(roots []string) ([]string, error) {
|
||||
resolved := make([]string, 0, len(roots))
|
||||
abs := make([]string, 0, len(roots))
|
||||
|
||||
for _, root := range roots {
|
||||
a, err := absolutePath(root)
|
||||
a, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||
}
|
||||
|
||||
r, err := filepath.EvalSymlinks(a)
|
||||
// A nonexistent operand is a fatal error before any scanning.
|
||||
_, err = os.Lstat(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resolved = append(resolved, r)
|
||||
abs = append(abs, a)
|
||||
}
|
||||
|
||||
return resolved, nil
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
// pruneRoots drops operands already covered by another operand:
|
||||
@@ -418,16 +417,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
|
||||
// walkableRoots returns the operands the walk can start from: regular
|
||||
// files, and directories not named .zfs, none of them inside a
|
||||
// directory named .zfs. The operands are real paths from resolveRoots,
|
||||
// so these checks apply to what a symlink operand points at. Every
|
||||
// other operand is warned about, counted as skipped, and dropped. A
|
||||
// dropped operand is no longer a root, so the records stored beneath it
|
||||
// are treated as if it had not been given: those under no other root
|
||||
// count as outside the roots and are not deleted as unverified. An
|
||||
// operand on a filesystem in rules that is not persistent is dropped
|
||||
// the same way, with a line saying so that is not a warning, and is not
|
||||
// counted as skipped. An operand that fails lstat here is kept, and the
|
||||
// walk warns about it unless it no longer exists.
|
||||
// directory named .zfs. Every other operand is warned about, counted as
|
||||
// skipped, and dropped. A dropped operand is no longer a root, so the
|
||||
// records stored beneath it are treated as if it had not been given:
|
||||
// those under no other root count as outside the roots and are not
|
||||
// deleted as unverified. An operand on a filesystem in rules that is not
|
||||
// persistent is dropped the same way, with a line saying so that is not
|
||||
// a warning, and is not counted as skipped. An operand that fails lstat
|
||||
// here is kept, and the walk warns about it unless it no longer exists.
|
||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||
kept := make([]string, 0, len(roots))
|
||||
|
||||
@@ -897,13 +894,19 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
||||
// operandWarning returns the one-line warning for an operand the walk
|
||||
// does not start from, naming the path and what it is, or "" for one it
|
||||
// does: a regular file, or a directory not named .zfs, that is not
|
||||
// inside a directory named .zfs. resolveRoots has already followed the
|
||||
// operand's symlinks, so a symlink here is an operand replaced by one
|
||||
// after it was resolved, and is not followed.
|
||||
// inside a directory named .zfs, neither as given nor with its symlinks
|
||||
// resolved. Symlinks are never followed, including as operands.
|
||||
func operandWarning(root string, fi fs.FileInfo) string {
|
||||
// The walk never enters a .zfs directory, so it starts from nothing
|
||||
// inside one either.
|
||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
||||
// inside one either, not even through a symlink. An operand whose
|
||||
// symlinks cannot be resolved is checked as given.
|
||||
resolved, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
resolved = root
|
||||
}
|
||||
|
||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
|
||||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
|
||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
||||
root)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
//go:build darwin
|
||||
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import "syscall"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//go:build !darwin
|
||||
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import "syscall"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes //nolint:testpackage // drives the unexported syncTree
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -1094,7 +1094,7 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
one := pattern(10, 2000)
|
||||
f1 := pattern(30, 3000)
|
||||
f2 := pattern(40, 100)
|
||||
@@ -1185,12 +1185,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
groups := dupeGroups(t, db)
|
||||
if len(groups) != 4 {
|
||||
t.Fatalf("len(groups) = %d, want 4", len(groups))
|
||||
if len(groups) != 5 {
|
||||
t.Fatalf("len(groups) = %d, want 5", len(groups))
|
||||
}
|
||||
|
||||
// The empty files empty1 and empty2 form no group.
|
||||
wantSizes := []int64{3000, 2000, 100, 1}
|
||||
wantSizes := []int64{3000, 2000, 100, 1, 0}
|
||||
for i, g := range groups {
|
||||
if g.size != wantSizes[i] {
|
||||
t.Errorf("groups[%d].size = %d, want %d",
|
||||
@@ -2311,24 +2310,33 @@ func TestUnderRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
||||
// database, and in one case also a hard link to the database file. The
|
||||
// walk knows the database's files by device ID and inode number, so it
|
||||
// passes over the database file, its -wal and -shm files, its .lock
|
||||
// file and the hard link with no line and no count. The record an
|
||||
// earlier scan left for the .lock file is deleted. A rescan of the
|
||||
// unchanged tree updates nothing, and report lists none of the
|
||||
// database's files.
|
||||
// database, reached directly and through a symlinked directory. The
|
||||
// walk passes over the database file, its -wal and -shm files and its
|
||||
// .lock file with no line and no count, and the record an earlier scan
|
||||
// left for the .lock file is deleted. A rescan of the unchanged tree
|
||||
// updates nothing, and report lists none of the database's files.
|
||||
func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
for _, viaHardLink := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
|
||||
tree := realTempDir(t)
|
||||
hardLink := filepath.Join(tree, "hardlink")
|
||||
for _, viaSymlink := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
tree := filepath.Join(base, "real", "tree")
|
||||
operand := tree
|
||||
|
||||
writeFile(t, tree, "a", pattern(1, 100))
|
||||
writeFile(t, tree, "b", pattern(1, 100))
|
||||
writeFile(t, tree, "empty1", nil)
|
||||
writeFile(t, tree, "empty2", nil)
|
||||
|
||||
if viaSymlink {
|
||||
err := os.Symlink(filepath.Join(base, "real"),
|
||||
filepath.Join(base, "link"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
operand = filepath.Join(base, "link", "tree")
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tree, "db.sqlite")
|
||||
t.Setenv(databaseEnv, dbPath)
|
||||
|
||||
@@ -2338,7 +2346,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
err = applyChanges(t.Context(), db, []scanRec{
|
||||
{path: filepath.Join(tree, "db.sqlite.lock")},
|
||||
{path: filepath.Join(operand, "db.sqlite.lock")},
|
||||
}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2346,30 +2354,17 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
|
||||
closeScanDatabase(t.Context(), db, dbPath)
|
||||
|
||||
if viaHardLink {
|
||||
err = os.Link(dbPath, hardLink)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
stderr := assertScanSummary(t,
|
||||
assertScanSummary(t,
|
||||
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
||||
tree)
|
||||
stderr += assertScanSummary(t,
|
||||
operand)
|
||||
assertScanSummary(t,
|
||||
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
||||
tree)
|
||||
|
||||
if strings.Contains(stderr, hardLink) {
|
||||
t.Errorf("scan stderr = %q, want no line naming %s",
|
||||
stderr, hardLink)
|
||||
}
|
||||
operand)
|
||||
|
||||
report := runStdout(t, cmdReport)
|
||||
if strings.Contains(report, "db.sqlite") ||
|
||||
strings.Contains(report, hardLink) ||
|
||||
!strings.Contains(report, filepath.Join(tree, "b")) {
|
||||
t.Errorf("report = %q, want a and b and none of "+
|
||||
!strings.Contains(report, filepath.Join(operand, "empty2")) {
|
||||
t.Errorf("report = %q, want the empty files and none of "+
|
||||
"the database's files", report)
|
||||
}
|
||||
})
|
||||
@@ -2393,9 +2388,8 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
|
||||
|
||||
// assertScanSummary scans operands and fails the test unless the
|
||||
// summary line gives counts with nothing skipped, and the line naming
|
||||
// the database is the only one that names one of its files. It returns
|
||||
// everything the scan printed to stderr.
|
||||
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||
// the database is the only one that names one of its files.
|
||||
func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
||||
t.Helper()
|
||||
|
||||
stderr := scanOK(t, operands...)
|
||||
@@ -2409,6 +2403,4 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
||||
"database to name one of its files", operands, stderr)
|
||||
}
|
||||
|
||||
return stderr
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package sfdupes
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -301,9 +301,7 @@ func fileEntry(name string, r scanRec) string {
|
||||
// collectTreeGroups groups directories by digest, counting each one on
|
||||
// prog as it is grouped, and returns every maximal group with two or
|
||||
// more members, each group's members sorted by path, groups ordered by
|
||||
// tree size descending then by first path ascending. A group whose
|
||||
// trees hold 0 bytes is left out, since removing one frees nothing. The
|
||||
// groups it implies hold 0 bytes too, so leaving it out hides nothing.
|
||||
// tree size descending then by first path ascending.
|
||||
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
||||
prog *progress,
|
||||
) [][]*treeNode {
|
||||
@@ -317,9 +315,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
||||
var dupes [][]*treeNode
|
||||
|
||||
for _, g := range groups {
|
||||
// The members share a digest, so they share a total size.
|
||||
if len(g) < minGroupSize || g[0].totalSize == 0 ||
|
||||
suppressed(g, super) {
|
||||
if len(g) < minGroupSize || suppressed(g, super) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package sfdupes //nolint:testpackage // checks the unexported treeNode results
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"os"
|
||||
"slices"
|
||||
@@ -436,35 +435,6 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
|
||||
// holding only empty files form no group, and that an empty file still
|
||||
// counts toward the digest of a tree that holds other files.
|
||||
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
|
||||
sha256.Sum256(nil)
|
||||
recs := []scanRec{
|
||||
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
|
||||
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
|
||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
|
||||
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
|
||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
|
||||
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
|
||||
// Without the empty file, /p/x3 is no copy of /p/x1.
|
||||
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
|
||||
}
|
||||
|
||||
super, dirs := treeOf(t, recs)
|
||||
|
||||
got := groupPaths(collectTreeGroups(dirs, super, nil))
|
||||
|
||||
want := [][]string{{"/p/x1", "/p/x2"}}
|
||||
if !slices.EqualFunc(got, want, slices.Equal) {
|
||||
t.Fatalf("groups = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
Reference in new issue
Block a user