diff --git a/Makefile b/Makefile index 67c7260..4d4c99e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ export CGO_ENABLED = 0 BINARY := sfdupes VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -LDFLAGS := -X main.Version=$(VERSION) +LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION) .PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean @@ -14,7 +14,7 @@ LDFLAGS := -X main.Version=$(VERSION) # Default target: build the binary. Phony so go build (which has its # own build cache) always decides what to recompile. sfdupes: - go build -ldflags "$(LDFLAGS)" -o $(BINARY) + go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes build: sfdupes diff --git a/README.md b/README.md index 1fdd3e7..6003f8a 100644 --- a/README.md +++ b/README.md @@ -57,25 +57,25 @@ shell shows both on the terminal, and `>` redirects stdout alone. With Go installed, this builds and installs the newest tagged release: ```sh -go install sneak.berlin/go/sfdupes@latest +go install sneak.berlin/go/sfdupes/cmd/sfdupes@latest ``` `@main` installs the tip of the `main` branch instead: ```sh -go install sneak.berlin/go/sfdupes@main +go install sneak.berlin/go/sfdupes/cmd/sfdupes@main ``` The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. `sfdupes --version` prints `sfdupes VERSION` and exits. For a binary from `go install`, `VERSION` is the version Go recorded in it. For `@latest` that is -a release tag such as `v0.0.3`. For `@main` it is the commit's tag when it has -one. Otherwise it is a pseudo-version, Go's version string for an untagged -commit. That string ends in the commit's hash. A binary built from a clone -carries the output of `git describe --tags --always --dirty` in that clone, such -as `v0.0.3` on a tagged commit. `VERSION` is `dev` when no version is known. To -see everything Go recorded in an installed binary: +a release tag. For `@main` it is the commit's tag when it has one. Otherwise it +is a pseudo-version, Go's version string for an untagged commit. That string +ends in the commit's hash. A binary built from a clone carries the output of +`git describe --tags --always --dirty` in that clone, such as `v0.0.3` on a +tagged commit. `VERSION` is `dev` when no version is known. To see everything Go +recorded in an installed binary: ```sh go version -m "$(command -v sfdupes)" @@ -1421,8 +1421,9 @@ All of the following, run in this directory, must pass: because its content changed. `copy.bin` and `copy2.bin` remain as one pair. `unique.bin` is gone from the database. - The test suite automates this scenario (see `scan_test.go`). It also checks - that `report` and `trees` never touch the scanned filesystem. + The test suite automates this scenario (see + `internal/sfdupes/scan_test.go`). It also checks that `report` and `trees` + never touch the scanned filesystem. ## TODO diff --git a/TODO.md b/TODO.md index 36d6907..8424492 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,12 @@ Newest first. An entry ends with its date and its issue or branch, where those are known. +- The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the + command, and it calls `Main` in the package `internal/sfdupes`. `main.go` + became `internal/sfdupes/cli.go`. Every other file kept its name. The install + path is now `sneak.berlin/go/sfdupes/cmd/sfdupes` (2026-11-10, + https://git.eeqj.de/sneak/sfdupes/issues/157) + - New subcommand `dwim`. It runs what `sfdupes scan /` runs with no flags, then what `sfdupes trees` runs. Its last line on stderr says that `sfdupes trees` prints the report again (2026-11-06, @@ -40,18 +46,19 @@ are known. walks `sub` on its own instead of dropping it and deleting its records (2026-11-06, https://git.eeqj.de/sneak/sfdupes/issues/144) -- `README.md` and a comment in `scan.go` no longer say ZFS counts a new file's - blocks only seconds after the write. The count can land in under a second. - Both now say up to seconds after the write, as the comment on `diskSizeOf` in - `scan_test.go` does (2026-11-03, https://git.eeqj.de/sneak/sfdupes/issues/152) +- `README.md` and a comment in `internal/sfdupes/scan.go` no longer say ZFS + counts a new file's blocks only seconds after the write. The count can land in + under a second. Both now say up to seconds after the write, as the comment on + `diskSizeOf` in `internal/sfdupes/scan_test.go` does (2026-11-03, + https://git.eeqj.de/sneak/sfdupes/issues/152) -- The three disk size tests in `scan_test.go` no longer fail at random on ZFS. - `unix.Sync` can return before ZFS has counted a file's blocks, and one ZFS - commit can take a file through several sizes. The test that needs the size to - change waits for it, for at most 30 seconds. Each test reads the size before - and after its scan. The rewrite test rescans until the two reads match, then - compares exactly. The other two fail only on a `disk_size` outside the two - reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150) +- The three disk size tests in `internal/sfdupes/scan_test.go` no longer fail at + random on ZFS. `unix.Sync` can return before ZFS has counted a file's blocks, + and one ZFS commit can take a file through several sizes. The test that needs + the size to change waits for it, for at most 30 seconds. Each test reads the + size before and after its scan. The rewrite test rescans until the two reads + match, then compares exactly. The other two fail only on a `disk_size` outside + the two reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150) - `TODO.md` rewritten from scratch by the same rules as `README.md`, sentence by sentence. Every completed step is kept, with its date and its issue or branch. @@ -62,9 +69,13 @@ are known. it. It leads with what the program is and how to use it. A reference section covers every setting and every subcommand. Development and testing come last. Every statement was checked against the code on `next`. Five comments in - `main_test.go` cite "Settings", where their rules now live. One comment each - in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README - now says (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135) + `internal/sfdupes/cli_test.go` cite "Settings", where their rules now live. + One comment each in these files cites what the README now says: + - `internal/sfdupes/scan_test.go`; + - `internal/sfdupes/trees_test.go`; + - `internal/sfdupes/exclude_test.go`. + + (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135) - Each record holds `disk_size`, the bytes the file takes on disk. That is its allocated blocks, `st_blocks` × 512, from the `lstat` call the walk already @@ -137,12 +148,12 @@ are known. https://git.eeqj.de/sneak/sfdupes/issues/109) - The release rule was set. Until 1.0, each tip of `main` that passes the build - gets the next `v0.0.N` tag. `go install sneak.berlin/go/sfdupes@latest` then - installs it. `go.mod` retracts `v0.0.1`, the early build without a database. - When the linker set no version, `sfdupes --version` prints the version Go - recorded in the binary. That is the tag, or the pseudo-version of a - `go install` build. It prints `dev` only when neither is known. README - "Install" documented: + gets the next `v0.0.N` tag. + `go install sneak.berlin/go/sfdupes/cmd/sfdupes@latest` then installs it. + `go.mod` retracts `v0.0.1`, the early build without a database. When the + linker set no version, `sfdupes --version` prints the version Go recorded in + the binary. That is the tag, or the pseudo-version of a `go install` build. It + prints `dev` only when neither is known. README "Install" documented: - the rule; - what `@main` and `v0.0.1` give; - how to check an installed binary. @@ -393,8 +404,9 @@ are known. `gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/26) -- A test fails when either `hashWorker` cancellation check in `scan.go` is - removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) +- A test fails when either `hashWorker` cancellation check in + `internal/sfdupes/scan.go` is removed (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/83) - A database path holding `?`, `#` or `%` opens exactly the file it names (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) @@ -402,15 +414,15 @@ are known. - `scan` rejects `--workers` below 1 as a usage error instead of running single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10) -- A test fails when either walk cancellation check in `scan.go` is removed - (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) +- A test fails when either walk cancellation check in `internal/sfdupes/scan.go` + is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) - A test checks that `scan` refuses a database with another schema version (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64) -- Four inaccurate comments in `cancel_test.go` were corrected, and - `walkCancelInFlightDirs` was renamed to `walkCancelInFlightFiles` (2026-10-04, - https://git.eeqj.de/sneak/sfdupes/issues/33) +- Four inaccurate comments in `internal/sfdupes/cancel_test.go` were corrected, + and `walkCancelInFlightDirs` was renamed to `walkCancelInFlightFiles` + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/33) - The `-x` filesystem-boundary rules in `subdirJob` are tested (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/17) @@ -617,15 +629,15 @@ are known. - `loadRecords`; - `resolveRoots`. - The single exit point is `run` in `main.go`. It maps a `fatalError`, - anything a subcommand returned, to exit 1, and cobra's own argument and flag - errors to exit 2. So a runtime failure is never reported as a usage error, - nor printed with the usage text. `main_test.go` drives the CLI in-process - and asserts the exit codes from README "Error handling" and the - stdout/stderr split. It also checks that a fatal error after the database is - open leaves no `-wal` or `-shm` file behind, for `scan`, `report` or `trees` - (2026-08-09, branch `db-close-on-fatal`, - https://git.eeqj.de/sneak/sfdupes/issues/4) + The single exit point is `run` in `internal/sfdupes/cli.go`. It maps a + `fatalError`, anything a subcommand returned, to exit 1, and cobra's own + argument and flag errors to exit 2. So a runtime failure is never reported + as a usage error, nor printed with the usage text. + `internal/sfdupes/cli_test.go` drives the CLI in-process and asserts the + exit codes from README "Error handling" and the stdout/stderr split. It also + checks that a fatal error after the database is open leaves no `-wal` or + `-shm` file behind, for `scan`, `report` or `trees` (2026-08-09, branch + `db-close-on-fatal`, https://git.eeqj.de/sneak/sfdupes/issues/4) - `golangci-lint` was updated to v2.12.2 with the canonical config, merged as `38a01bd`. The pinned linter in the `Dockerfile` lint stage and @@ -758,6 +770,4 @@ are known. # Accepted Divergences -- Flat single-package layout, with the `.go` files in the repo root. That is - fine for a small single-binary tool per the Go styleguide, and the tracker - audit agrees. +None. diff --git a/cmd/sfdupes/main.go b/cmd/sfdupes/main.go new file mode 100644 index 0000000..39a82dc --- /dev/null +++ b/cmd/sfdupes/main.go @@ -0,0 +1,29 @@ +// Command sfdupes quickly identifies candidate duplicate files across +// very large filesystems without reading every byte of every file. +// Files are considered duplicates when their sizes are equal and they +// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64 +// KiB, and of the content. The content hash covers the whole file under +// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large +// file is never read in full. scan maintains a persistent SQLite +// database of file signatures (SFDUPES_DATABASE, default +// /var/lib/sfdupes/db.sqlite) that the other subcommands read. +// +// Usage: +// +// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... +// sfdupes report > dupes.tsv +// sfdupes trees > dupetrees.tsv +// sfdupes info +// sfdupes dwim +// sfdupes --version +// +// See README.md for the complete specification. +package main + +import ( + "os" + + "sneak.berlin/go/sfdupes/internal/sfdupes" +) + +func main() { os.Exit(sfdupes.Main()) } diff --git a/cancel_test.go b/internal/sfdupes/cancel_test.go similarity index 99% rename from cancel_test.go rename to internal/sfdupes/cancel_test.go index 577c54d..a16b4e6 100644 --- a/cancel_test.go +++ b/internal/sfdupes/cancel_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker import ( "context" diff --git a/main.go b/internal/sfdupes/cli.go similarity index 90% rename from main.go rename to internal/sfdupes/cli.go index 1d4fed2..bc0539f 100644 --- a/main.go +++ b/internal/sfdupes/cli.go @@ -1,24 +1,6 @@ -// Command sfdupes quickly identifies candidate duplicate files across -// very large filesystems without reading every byte of every file. -// Files are considered duplicates when their sizes are equal and they -// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64 -// KiB, and of the content. The content hash covers the whole file under -// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large -// file is never read in full. scan maintains a persistent SQLite -// database of file signatures (SFDUPES_DATABASE, default -// /var/lib/sfdupes/db.sqlite) that the other subcommands read. -// -// Usage: -// -// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... -// sfdupes report > dupes.tsv -// sfdupes trees > dupetrees.tsv -// sfdupes info -// sfdupes dwim -// sfdupes --version -// -// See README.md for the complete specification. -package main +// Package sfdupes implements the sfdupes command and the database its +// subcommands share. +package sfdupes import ( "context" @@ -86,12 +68,14 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string { return info.Main.Version } -func main() { +// Main runs the command line in os.Args and returns the process exit +// code. +func Main() int { // Once the reader of a stdout pipe has gone, as in "sfdupes report | // head", the Go runtime ends the process with SIGPIPE on the next // write instead of returning an error (README "Error handling"). // Registering for SIGPIPE with os/signal would change that. - os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) + return run(os.Args[1:], os.Stdout, os.Stderr) } // run executes args against the command tree and returns the process diff --git a/main_test.go b/internal/sfdupes/cli_test.go similarity index 99% rename from main_test.go rename to internal/sfdupes/cli_test.go index e0abba1..6e65e7f 100644 --- a/main_test.go +++ b/internal/sfdupes/cli_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // drives the unexported run import ( "bytes" diff --git a/db.go b/internal/sfdupes/db.go similarity index 99% rename from db.go rename to internal/sfdupes/db.go index a27d163..a6fa2a0 100644 --- a/db.go +++ b/internal/sfdupes/db.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "context" diff --git a/db_test.go b/internal/sfdupes/db_test.go similarity index 99% rename from db_test.go rename to internal/sfdupes/db_test.go index 6d9da76..5d801ea 100644 --- a/db_test.go +++ b/internal/sfdupes/db_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // calls the unexported database openers import ( "context" diff --git a/dwim.go b/internal/sfdupes/dwim.go similarity index 98% rename from dwim.go rename to internal/sfdupes/dwim.go index d195f16..5158a46 100644 --- a/dwim.go +++ b/internal/sfdupes/dwim.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "context" diff --git a/dwim_test.go b/internal/sfdupes/dwim_test.go similarity index 97% rename from dwim_test.go rename to internal/sfdupes/dwim_test.go index 5622ad9..8345cd1 100644 --- a/dwim_test.go +++ b/internal/sfdupes/dwim_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // calls the unexported runDwim import ( "bytes" diff --git a/exclude.go b/internal/sfdupes/exclude.go similarity index 99% rename from exclude.go rename to internal/sfdupes/exclude.go index b023d16..743d4a8 100644 --- a/exclude.go +++ b/internal/sfdupes/exclude.go @@ -1,4 +1,4 @@ -package main +package sfdupes // This file holds two rules for the walk (README "Rules for the walk"): // the default excludes, the paths a scan's walk does not enter below an diff --git a/exclude_darwin.go b/internal/sfdupes/exclude_darwin.go similarity index 98% rename from exclude_darwin.go rename to internal/sfdupes/exclude_darwin.go index a669ab5..86b3bef 100644 --- a/exclude_darwin.go +++ b/internal/sfdupes/exclude_darwin.go @@ -1,6 +1,6 @@ //go:build darwin -package main +package sfdupes import ( "slices" diff --git a/exclude_other.go b/internal/sfdupes/exclude_other.go similarity index 97% rename from exclude_other.go rename to internal/sfdupes/exclude_other.go index 80b26a6..fa21041 100644 --- a/exclude_other.go +++ b/internal/sfdupes/exclude_other.go @@ -1,6 +1,6 @@ //go:build !darwin -package main +package sfdupes import ( "fmt" diff --git a/exclude_test.go b/internal/sfdupes/exclude_test.go similarity index 99% rename from exclude_test.go rename to internal/sfdupes/exclude_test.go index 25b593e..e40a4f1 100644 --- a/exclude_test.go +++ b/internal/sfdupes/exclude_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // checks the unexported defaultExcludes import ( "bytes" diff --git a/info.go b/internal/sfdupes/info.go similarity index 99% rename from info.go rename to internal/sfdupes/info.go index c30cac8..920aa0b 100644 --- a/info.go +++ b/internal/sfdupes/info.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "context" diff --git a/info_test.go b/internal/sfdupes/info_test.go similarity index 98% rename from info_test.go rename to internal/sfdupes/info_test.go index 5d6a58e..dc1fecc 100644 --- a/info_test.go +++ b/internal/sfdupes/info_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // seeds the database with unexported scanRec import ( "fmt" diff --git a/progress.go b/internal/sfdupes/progress.go similarity index 99% rename from progress.go rename to internal/sfdupes/progress.go index 3b112c3..8d24bf4 100644 --- a/progress.go +++ b/internal/sfdupes/progress.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "fmt" diff --git a/progress_test.go b/internal/sfdupes/progress_test.go similarity index 99% rename from progress_test.go rename to internal/sfdupes/progress_test.go index 4093aa6..91fb275 100644 --- a/progress_test.go +++ b/internal/sfdupes/progress_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // drives the unexported startProgress import ( "fmt" diff --git a/report.go b/internal/sfdupes/report.go similarity index 99% rename from report.go rename to internal/sfdupes/report.go index 0504cd8..9e8989b 100644 --- a/report.go +++ b/internal/sfdupes/report.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "context" diff --git a/report_test.go b/internal/sfdupes/report_test.go similarity index 99% rename from report_test.go rename to internal/sfdupes/report_test.go index 2023cb3..0257f34 100644 --- a/report_test.go +++ b/internal/sfdupes/report_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // checks the unexported dupeGroup results import ( "bytes" diff --git a/scan.go b/internal/sfdupes/scan.go similarity index 99% rename from scan.go rename to internal/sfdupes/scan.go index 3c2ec57..ba45468 100644 --- a/scan.go +++ b/internal/sfdupes/scan.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "cmp" diff --git a/scan_dev_darwin.go b/internal/sfdupes/scan_dev_darwin.go similarity index 95% rename from scan_dev_darwin.go rename to internal/sfdupes/scan_dev_darwin.go index a483441..5b1e9ab 100644 --- a/scan_dev_darwin.go +++ b/internal/sfdupes/scan_dev_darwin.go @@ -1,6 +1,6 @@ //go:build darwin -package main +package sfdupes import "syscall" diff --git a/scan_dev_other.go b/internal/sfdupes/scan_dev_other.go similarity index 90% rename from scan_dev_other.go rename to internal/sfdupes/scan_dev_other.go index bf16da3..44842c9 100644 --- a/scan_dev_other.go +++ b/internal/sfdupes/scan_dev_other.go @@ -1,6 +1,6 @@ //go:build !darwin -package main +package sfdupes import "syscall" diff --git a/scan_test.go b/internal/sfdupes/scan_test.go similarity index 99% rename from scan_test.go rename to internal/sfdupes/scan_test.go index 298b248..dafe42b 100644 --- a/scan_test.go +++ b/internal/sfdupes/scan_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // drives the unexported syncTree import ( "bytes" diff --git a/trees.go b/internal/sfdupes/trees.go similarity index 99% rename from trees.go rename to internal/sfdupes/trees.go index 554666f..adf1cd6 100644 --- a/trees.go +++ b/internal/sfdupes/trees.go @@ -1,4 +1,4 @@ -package main +package sfdupes import ( "context" diff --git a/trees_test.go b/internal/sfdupes/trees_test.go similarity index 99% rename from trees_test.go rename to internal/sfdupes/trees_test.go index 403762e..7425217 100644 --- a/trees_test.go +++ b/internal/sfdupes/trees_test.go @@ -1,4 +1,4 @@ -package main +package sfdupes //nolint:testpackage // checks the unexported treeNode results import ( "bytes"