From c9bf22d483e4603f89d300291197dc480e580357 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 2 Oct 2026 08:49:03 +0200 Subject: [PATCH 01/33] Stamp the git tag or short commit in a plain docker build (closes #67) .dockerignore now sends .git, without .git/config, which can hold a credential. The build stage takes the VERSION build argument when one is given, otherwise git describe --tags --always of that .git, and fails if the context carries .git and still yields no version. A plain docker build . used to stamp dev. The CI checkout fetches full history so CI sees the tag and stamps the same value as make build. Model: opus-5-5 --- .dockerignore | 7 ++++++- .gitea/workflows/check.yml | 2 ++ Dockerfile | 14 +++++++++++++- TODO.md | 11 +++++++++++ script/cibuild | 8 ++++---- 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index fd9fbce..e1d1d88 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,9 @@ -.git +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +.git/config + .claude .DS_Store sfdupes diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index ee73864..1be9221 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,4 +6,6 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 - run: script/cibuild diff --git a/Dockerfile b/Dockerfile index a6009f2..2885086 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,19 @@ ARG CHECK_EPOCH RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check -RUN make build +# The version stamped into the binary: the VERSION build argument when +# one is given, otherwise `git describe --tags --always` of the .git in +# the build context (git is installed by script/bootstrap above). A +# context that carries .git and still yields no version fails the build; +# with neither, as from a source tarball, it is "dev". +ARG VERSION +RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \ + if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ + [ "$version" = unknown ]; }; then \ + echo "no version could be derived although the build context carries .git" >&2; \ + exit 1; \ + fi; \ + make build VERSION="$version" # Runtime stage # alpine:3.22, 2026-07-23 diff --git a/TODO.md b/TODO.md index 7fd3083..b540d3a 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,17 @@ # Completed Steps +- stamp the git tag or short commit in a plain `docker build .` + instead of `dev` (2026-10-02, branch `next`, closes + https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now + sends `.git`, without `.git/config`, and the `Dockerfile` build + stage takes the `VERSION` build argument when one is given, + otherwise `git describe --tags --always` of that `.git`. The build + fails if the context carries `.git` and the version still comes out + empty, `dev` or `unknown`. The CI checkout step fetches the full + history (`fetch-depth: 0`) so CI sees the tag and stamps the same + value as `make build`. + - replace the 1 KiB end-window sampling with the head/tail plus content-hash ladder (2026-09-22, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is diff --git a/script/cibuild b/script/cibuild index 91fd8b2..55a90ba 100755 --- a/script/cibuild +++ b/script/cibuild @@ -16,10 +16,10 @@ # implies the repo is green. # # That implication holds only because of CHECK_EPOCH. A COPY layer is -# invalidated by changed content, and a merge commit's tree is -# byte-identical to the branch head it merges, so without a fresh value -# here Docker serves the gate layers from cache and the build reports a -# green it never earned. Passing the current epoch invalidates the gate +# invalidated only by changed content, and a rebuild of an unchanged +# checkout sends the same content, so without a fresh value here Docker +# serves the gate layers from cache and the build reports a green it +# never earned. Passing the current epoch invalidates the gate # layers on every run while leaving the pinned base images and # go mod download cached; see the Dockerfile for the placement. set -eu From c887f80f57663fd686bc5500466e6da3ecaa7cc1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sat, 3 Oct 2026 15:30:37 +0200 Subject: [PATCH 02/33] Escape tab, newline, CR and backslash in report paths (closes #7) A path holding a tab or newline split a row of the report or trees output. The path columns of both now write a backslash, tab, newline and carriage return as \\, \t, \n and \r; every other byte is written unchanged. Grouping and sorting still use the stored path. Warnings on stderr are escaped the same way in warnf, so each stays one line. In trees, the root directory's node now has the path "/" instead of an empty string, and its children's paths start with a single slash. README states the rule under "Report output format". Model: opus-5-5 --- README.md | 12 +++++ TODO.md | 4 ++ progress.go | 4 +- report.go | 17 ++++++- report_test.go | 118 +++++++++++++++++++++++++++++++++++++++++++++++++ trees.go | 21 ++++++--- trees_test.go | 41 +++++++++++++++++ 7 files changed, 209 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6508cbe..d9c042a 100644 --- a/README.md +++ b/README.md @@ -428,6 +428,15 @@ first dupe size /srv/a/big.iso /srv/c/big-copy2.iso 4294967296 ``` +Paths are raw bytes and may hold any byte except NUL, so the path +columns (`first` and `dupe`) are escaped to keep every row one line of +tab-separated fields: a backslash is written as `\\`, a tab as `\t`, a +newline as `\n`, and a carriage return as `\r`. Every other byte is +written unchanged, including bytes that are not valid UTF-8. Undoing +those four escapes gives back the stored path. Grouping and ordering +use the stored path, not the escaped one. The warnings `scan` prints on +stderr are escaped the same way, so each warning is one line. + Summary to stderr: records read, number of duplicate groups, number of dupe files, and total reclaimable bytes (sum of `size` over all dupe rows) in human units. @@ -499,6 +508,9 @@ first dupe files size /srv/a/project /srv/backup/project 3417 104857600 ``` +The `first` and `dupe` paths are escaped as described under "Report +output format". The root directory's path is `/`. + Summary to stderr: records read, number of duplicate-tree groups, number of dupe trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human units. diff --git a/TODO.md b/TODO.md index b540d3a..7724ba8 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- escape tabs, newlines, carriage returns and backslashes in report, + trees and warning paths; the root directory's path is `/` + (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/7) + - stamp the git tag or short commit in a plain `docker build .` instead of `dev` (2026-10-02, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now diff --git a/progress.go b/progress.go index ecb0585..f351d47 100644 --- a/progress.go +++ b/progress.go @@ -106,6 +106,8 @@ func (p *progress) increment() { } // warnf prints a one-line warning to stderr without corrupting the bar. +// The whole message is escaped like a report's path columns, so a path +// holding a newline cannot split the warning. func (p *progress) warnf(format string, args ...any) { if p == nil { return @@ -115,7 +117,7 @@ func (p *progress) warnf(format string, args ...any) { _ = p.bar.Clear() } - fmt.Fprintf(os.Stderr, format+"\n", args...) + fmt.Fprintln(os.Stderr, escapePath(fmt.Sprintf(format, args...))) } // finish terminates the pass's display. diff --git a/report.go b/report.go index c5761b5..f02ce2b 100644 --- a/report.go +++ b/report.go @@ -87,7 +87,7 @@ func runReport(ctx context.Context) error { for _, g := range dupes { for _, p := range g.paths[1:] { _, err = fmt.Fprintf(out, "%s\t%s\t%d\n", - g.paths[0], p, g.size) + escapePath(g.paths[0]), escapePath(p), g.size) if err != nil { return fmt.Errorf("write stdout: %w", err) } @@ -156,6 +156,21 @@ func collectDupeGroups(recs []scanRec) []dupeGroup { return dupes } +// escapePath returns a path as it is written in a report column (README +// "Report output format"): a backslash, tab, newline or carriage return +// becomes \\, \t, \n or \r, and every other byte is kept as it is. +// Grouping and sorting use the raw path, never this form. +func escapePath(p string) string { + // Most paths need no escaping; skip building a replacer for them. + if !strings.ContainsAny(p, "\\\t\n\r") { + return p + } + + return strings.NewReplacer( + `\`, `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`, + ).Replace(p) +} + // humanBytes formats a byte count in human units (binary prefixes). func humanBytes(n int64) string { const unit = 1024 diff --git a/report_test.go b/report_test.go index 1aa7cc5..d1aa290 100644 --- a/report_test.go +++ b/report_test.go @@ -1,10 +1,128 @@ package main import ( + "bytes" + "io" + "os" + "path/filepath" "slices" "testing" ) +// awkwardDir is a directory name holding every byte the reports escape. +const awkwardDir = "/d/\tone\ntwo\rthree\\four" + +// awkwardPairRecs is a duplicate pair in sibling directories /d/A and +// awkwardDir. A raw tab sorts before "A" but its escaped form `\t` +// sorts after it, so awkwardDir coming first shows that sorting uses +// the raw path. +func awkwardPairRecs() []scanRec { + return []scanRec{ + {size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"}, + {size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"}, + } +} + +// seedDatabase writes recs into a fresh database and returns its path. +func seedDatabase(t *testing.T, recs []scanRec) string { + t.Helper() + + path := testDBPath(t) + + db, err := openScanDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + err = applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + err = db.Close() + if err != nil { + t.Fatal(err) + } + + return path +} + +func TestRunReportEscapesPaths(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) + + var stderr bytes.Buffer + + stdout := captureStdout(t) + + code := run([]string{cmdReport}, &stderr) + if code != exitOK { + t.Fatalf("run(report) = %d, want %d; stderr: %s", + code, exitOK, stderr.String()) + } + + want := "first\tdupe\tsize\n" + + `/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n" + if got := stdout(); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + +func TestEscapePath(t *testing.T) { + t.Parallel() + + cases := map[string]string{ + "/srv/plain": "/srv/plain", + "/a\tb": `/a\tb`, + "/a\nb": `/a\nb`, + "/a\rb": `/a\rb`, + `/a\b`: `/a\\b`, + `/a\tb`: `/a\\tb`, + "/not-utf8\xff": "/not-utf8\xff", + } + for in, want := range cases { + if got := escapePath(in); got != want { + t.Errorf("escapePath(%q) = %q, want %q", in, got, want) + } + } +} + +// TestWarnfEscapes checks that a warning naming a path that holds a +// newline is still one line. +// +//nolint:paralleltest // replaces the process-wide os.Stderr +func TestWarnfEscapes(t *testing.T) { + f, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + if err != nil { + t.Fatal(err) + } + + saved := os.Stderr + os.Stderr = f + + t.Cleanup(func() { + os.Stderr = saved + + _ = f.Close() + }) + + (&progress{}).warnf("stat %s: %s", "/d/a\nb", "gone") + + _, err = f.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + want := `stat /d/a\nb: gone` + "\n" + if string(got) != want { + t.Errorf("warning = %q, want %q", got, want) + } +} + func TestCollectDupeGroups(t *testing.T) { t.Parallel() diff --git a/trees.go b/trees.go index f5337fc..f43374c 100644 --- a/trees.go +++ b/trees.go @@ -62,7 +62,8 @@ func runTrees(ctx context.Context) error { first := g[0] for _, n := range g[1:] { _, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", - first.path, n.path, first.fileCount, first.totalSize) + escapePath(first.path), escapePath(n.path), + first.fileCount, first.totalSize) if err != nil { return fmt.Errorf("write stdout: %w", err) } @@ -87,8 +88,8 @@ func runTrees(ctx context.Context) error { // buildHierarchy reconstructs the directory hierarchy from the record // paths under a synthetic super-root. Paths are split on "/"; for -// absolute paths the first component is empty, which simply becomes a -// top-level node representing "/". It returns the super-root and every +// absolute paths the first component is empty, which becomes the +// top-level node with path "/". It returns the super-root and every // directory node created. func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) { super := &treeNode{} @@ -102,9 +103,17 @@ func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) { for _, c := range comps[:len(comps)-1] { child := node.dirs[c] if child == nil { - childPath := c - if node != super { - childPath = node.path + "/" + c + childPath := node.path + "/" + c + + // The root directory's path is "/", not empty, and its + // children's paths start with one slash, not two. + switch { + case node == super && c == "": + childPath = "/" + case node == super: + childPath = c + case node.path == "/": + childPath = "/" + c } child = &treeNode{path: childPath, parent: node} diff --git a/trees_test.go b/trees_test.go index d0995ae..abf61f1 100644 --- a/trees_test.go +++ b/trees_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "slices" "testing" ) @@ -84,6 +85,46 @@ func TestBuildHierarchyCounts(t *testing.T) { } } +func TestBuildHierarchyRootPath(t *testing.T) { + t.Parallel() + + // The root directory's path is "/", never empty, and its + // children's paths start with a single slash. + _, dirs := buildHierarchy([]scanRec{{path: "/f"}, {path: "/srv/g"}}) + + got := make([]string, 0, len(dirs)) + for _, d := range dirs { + got = append(got, d.path) + } + + slices.Sort(got) + + want := []string{"/", "/srv"} + if !slices.Equal(got, want) { + t.Fatalf("directory paths = %q, want %q", got, want) + } +} + +func TestRunTreesEscapesPaths(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) + + var stderr bytes.Buffer + + stdout := captureStdout(t) + + code := run([]string{cmdTrees}, &stderr) + if code != exitOK { + t.Fatalf("run(trees) = %d, want %d; stderr: %s", + code, exitOK, stderr.String()) + } + + want := "first\tdupe\tfiles\tsize\n" + + `/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n" + if got := stdout(); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + func TestTreeDigests(t *testing.T) { t.Parallel() From d63d3cc7fc21f845b9b5f1b6cc442864474fa2bf Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sat, 3 Oct 2026 16:30:19 +0200 Subject: [PATCH 03/33] Open the database read-only for report and trees (closes #8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report and trees now connect read-only (mode=ro, query_only, the same busy timeout) and no longer set the journal mode, which is a write. A read-only connection to a WAL database still needs its -wal and -shm files, or write access to the directory to create them, so scan now switches the database back to rollback-journal mode whenever it closes it: between scans the file alone holds the database. If a report has the database open at that moment the switch is refused; scan warns and the database stays in WAL mode, with its -wal and -shm files, until the next scan. README §Database states what readers need. Model: opus-5-5 --- README.md | 32 +++++++++++++------ TODO.md | 4 +++ db.go | 48 +++++++++++++++++++++------- db_test.go | 44 +++++++++++++++++++++++++ main_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++---- report.go | 6 ++-- scan.go | 7 ++-- 7 files changed, 197 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d9c042a..d48f695 100644 --- a/README.md +++ b/README.md @@ -152,16 +152,28 @@ All three subcommands operate on a single SQLite database file: use. `report` and `trees` require an existing database; a missing database file is a fatal error (exit 1) telling the user to run `scan` first. -- The database uses WAL journal mode and a busy timeout, so running a - report while a cron `scan` is in progress is safe. The filesystem - is authoritative; the database is an eventually-consistent - reflection of it. Hashed records are committed in batched - transactions while the scan is still running (keeping the WAL - small and letting concurrent reports observe progress), so a - report may see a scan's changes partially applied, and a scan - that dies partway leaves a valid database holding everything - hashed so far; the next scan skips those records and converges - toward the filesystem. +- While `scan` runs, the database is in WAL journal mode with a busy + timeout, so running a report while a cron `scan` is in progress is + safe. The filesystem is authoritative; the database is an + eventually-consistent reflection of it. Hashed records are + committed in batched transactions while the scan is still running + (keeping the WAL small and letting concurrent reports observe + progress), so a report may see a scan's changes partially applied, + and a scan that dies partway leaves a valid database holding + everything hashed so far; the next scan skips those records and + converges toward the filesystem. +- `scan` switches the database back to rollback-journal mode when it + closes it, so between scans the database file alone holds the whole + database. Each switch needs the database to itself: a `scan` that + starts while a report is still reading waits for it up to the + 10-second busy timeout, then fails; a `scan` that ends while a + report has the database open warns and leaves the database in WAL + mode until the next scan. +- `report` and `trees` open the database read-only and need only read + access to the database file, and no write access to its directory. + While the database is in WAL mode they also read the `-wal` and + `-shm` files beside it, which SQLite creates with the database + file's permissions. - Schema (`PRAGMA user_version` is the schema version, currently 1; a database with any other version is a fatal error): diff --git a/TODO.md b/TODO.md index 7724ba8..c6de6e9 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `report` and `trees` open the database read-only, and `scan` leaves it + out of WAL mode, so reading needs only read access (2026-10-03, closes + https://git.eeqj.de/sneak/sfdupes/issues/8) + - escape tabs, newlines, carriage returns and backslashes in report, trees and warning paths; the root directory's path is `/` (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/7) diff --git a/db.go b/db.go index f732608..2060dc4 100644 --- a/db.go +++ b/db.go @@ -74,16 +74,24 @@ func databasePath() string { return defaultDatabasePath } -// openDB opens the SQLite database at path with WAL journaling and a -// busy timeout, so a report can run while a cron scan is in progress. -// It does not create or verify the schema. -func openDB(path string) (*sql.DB, error) { - dsn := "file:" + path + - "?_pragma=busy_timeout(10000)" + - "&_pragma=journal_mode(WAL)" + - "&_pragma=synchronous(NORMAL)" +// scanParams are the connection parameters for scan: read-write, with +// WAL journaling and a busy timeout, so a report can run while a cron +// scan is in progress. closeScanDatabase leaves WAL mode again. +const scanParams = "_pragma=busy_timeout(10000)" + + "&_pragma=journal_mode(WAL)" + + "&_pragma=synchronous(NORMAL)" - db, err := sql.Open("sqlite", dsn) +// reportParams are the connection parameters for report and trees: +// read-only, with the same busy timeout. They set no journal mode, +// because setting one is a write. +const reportParams = "mode=ro" + + "&_pragma=busy_timeout(10000)" + + "&_pragma=query_only(1)" + +// openDB opens the SQLite database at path with the connection +// parameters params. It does not create or verify the schema. +func openDB(path, params string) (*sql.DB, error) { + db, err := sql.Open("sqlite", "file:"+path+"?"+params) if err != nil { return nil, fmt.Errorf("open database %s: %w", path, err) } @@ -104,7 +112,7 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { return nil, fmt.Errorf("create database directory: %w", err) } - db, err := openDB(path) + db, err := openDB(path, scanParams) if err != nil { return nil, err } @@ -119,6 +127,24 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { return db, nil } +// closeScanDatabase switches the database at path from WAL back to +// rollback-journal mode and closes it. Out of WAL mode the database +// file alone holds the whole database, so a reader needs no -wal or +// -shm file beside it, nor write access to create them. The switch +// fails while a report has the database open; the database then stays +// in WAL mode, still readable, until a later scan closes it. +func closeScanDatabase(ctx context.Context, db *sql.DB, path string) { + // Runs on the way out of a cancelled scan too. + _, err := db.ExecContext(context.WithoutCancel(ctx), + "PRAGMA journal_mode = DELETE") + if err != nil { + fmt.Fprintf(os.Stderr, "scan: database %s left in WAL mode: %v\n", + path, err) + } + + _ = db.Close() +} + // openReportDatabase opens an existing database for the report and // trees subcommands. A missing database file is an error directing the // user to run scan first; the schema version must match exactly. @@ -134,7 +160,7 @@ func openReportDatabase(ctx context.Context, return nil, fmt.Errorf("database: %w", err) } - db, err := openDB(path) + db, err := openDB(path, reportParams) if err != nil { return nil, err } diff --git a/db_test.go b/db_test.go index b68bddd..bc66c9c 100644 --- a/db_test.go +++ b/db_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "os" "path/filepath" "slices" "strings" @@ -130,6 +131,49 @@ func TestOpenReportDatabaseOK(t *testing.T) { _ = db.Close() } +func TestCloseScanDatabaseWhileReportOpen(t *testing.T) { + t.Parallel() + + // A report holding the database open stops scan from taking it out + // of WAL mode. The -wal and -shm files must then stay beside it, so + // that a later report still needs only read access. + path := testDBPath(t) + + scanDB, err := openScanDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + reportDB, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + closeScanDatabase(t.Context(), scanDB, path) + + _ = reportDB.Close() + + _, err = os.Stat(path + "-wal") + if err != nil { + t.Fatalf("no -wal left: the switch out of WAL mode was not "+ + "stopped: %v", err) + } + + makeReadOnly(t, path) + + reportDB, err = openReportDatabase(t.Context(), path) + if err != nil { + t.Fatalf("openReportDatabase: %v", err) + } + + defer func() { _ = reportDB.Close() }() + + _, err = loadFileRows(t.Context(), reportDB) + if err != nil { + t.Fatalf("loadFileRows: %v", err) + } +} + func TestApplyChangesRoundTrip(t *testing.T) { t.Parallel() diff --git a/main_test.go b/main_test.go index b307a82..432e9e4 100644 --- a/main_test.go +++ b/main_test.go @@ -44,6 +44,44 @@ func assertNoSidecars(t *testing.T, path string) { } } +// makeReadOnly takes write permission away from the database at path, +// from any WAL sidecar beside it, and from their directory, as for a +// user reading a database that a root cron scan keeps. Root ignores +// file permissions, so it skips the test when run as root. +func makeReadOnly(t *testing.T, path string) { + t.Helper() + + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + + err := os.Chmod(path, 0o400) + if err != nil { + t.Fatal(err) + } + + for _, suffix := range walSuffixes { + err = os.Chmod(path+suffix, 0o400) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + t.Fatal(err) + } + } + + dir := filepath.Dir(path) + + //nolint:gosec // reaching the database needs the search bit + err = os.Chmod(dir, 0o500) + if err != nil { + t.Fatal(err) + } + + // Runs before t.TempDir's own cleanup, which must delete the files. + t.Cleanup(func() { + //nolint:gosec // removing the directory needs its search bit back + _ = os.Chmod(dir, 0o700) + }) +} + // captureStdout redirects os.Stdout to a file for the rest of the test // and returns a function reading back everything written to it. Only // machine-readable data belongs on stdout (README design goal 4), so @@ -91,13 +129,13 @@ func captureStdout(t *testing.T) func() string { // brokenDatabase writes a database that opens cleanly and passes the // schema-version check but has no files table, so the first query // fails with the database already open: a fatal error on a path that -// owns an open database. +// owns an open database. It closes the database the way scan does. func brokenDatabase(t *testing.T) string { t.Helper() path := testDBPath(t) - db, err := openDB(path) + db, err := openDB(path, scanParams) if err != nil { t.Fatal(err) } @@ -108,10 +146,7 @@ func brokenDatabase(t *testing.T) string { t.Fatal(err) } - err = db.Close() - if err != nil { - t.Fatal(err) - } + closeScanDatabase(t.Context(), db, path) return path } @@ -144,7 +179,10 @@ func TestOpenDatabaseKeepsWALWhileOpen(t *testing.T) { func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { // Every subcommand that owns an open database must close it when - // it fails: no os.Exit between the open and the return. + // it fails: no os.Exit between the open and the return. The + // sidecar check is evidence of the close only for scan: report and + // trees only read a database that is out of WAL mode, which leaves + // nothing on disk whether they close it or not. cases := map[string][]string{ cmdScan: {cmdScan}, cmdReport: {cmdReport}, @@ -395,3 +433,41 @@ func TestRunTreesSucceeds(t *testing.T) { assertNoSidecars(t, path) } + +func TestRunReportsNeedOnlyReadAccess(t *testing.T) { + // README §Database: report and trees need only read access to the + // database file. With its directory read-only as well, SQLite + // cannot create any file beside it. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dupes := scanFixture(t) + assertNoSidecars(t, path) + makeReadOnly(t, path) + + cases := map[string]string{ + cmdReport: "first\tdupe\tsize\n" + + dupes[0] + "\t" + dupes[1] + "\t300\n", + cmdTrees: "first\tdupe\tfiles\tsize\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + + "\t1\t300\n", + } + + for name, want := range cases { + var stderr bytes.Buffer + + stdout := captureStdout(t) + + code := run([]string{name}, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + + continue + } + + if got := stdout(); got != want { + t.Errorf("%s stdout = %q, want %q", name, got, want) + } + } +} diff --git a/report.go b/report.go index f02ce2b..00b7b01 100644 --- a/report.go +++ b/report.go @@ -31,9 +31,9 @@ type scanRec struct { // loadRecords opens the database and reads every file record for the // report and trees subcommands. Any database problem — including a // missing database — is fatal. The error is returned rather than -// exiting, so that the deferred close — which checkpoints the SQLite -// WAL — always runs; the database is closed before the caller formats -// its output, so it stays closed even if that output fails. +// exiting, so that the deferred close always runs; the database is +// closed before the caller formats its output, so it stays closed even +// if that output fails. func loadRecords(ctx context.Context) ([]scanRec, error) { dbPath := databasePath() diff --git a/scan.go b/scan.go index 6d7aaf4..a215c5a 100644 --- a/scan.go +++ b/scan.go @@ -87,8 +87,9 @@ type fileMeta struct { // hash only when its size, head, and tail match another file's. Flag // parsing and the at-least-one-operand check are done by cobra. Errors // are returned rather than exiting, so that the deferred close — which -// checkpoints the SQLite WAL — always runs. Cancelling ctx unwinds the -// worker pools and aborts the scan with the context's error. +// takes the database out of WAL mode — always runs. Cancelling ctx +// unwinds the worker pools and aborts the scan with the context's +// error. func runScan(ctx context.Context, roots []string, workers int, oneFS bool, ) error { @@ -108,7 +109,7 @@ func runScan(ctx context.Context, roots []string, workers int, return err } - defer func() { _ = db.Close() }() + defer closeScanDatabase(ctx, db, dbPath) st, err := syncScan(ctx, db, roots, workers, oneFS) if err != nil { From 01ff3bb5f04410aafdeacf2873c1e2aa152172b4 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sat, 3 Oct 2026 18:01:27 +0200 Subject: [PATCH 04/33] Test report and trees stdout write failures (closes #30) report and trees already checked every stdout write and the final flush. run now takes the stdout it hands to them, so tests pass a closed file or a failing writer instead of swapping os.Stdout: a closed stdout exits 1 with a one-line diagnostic, and the writer's error reaches the caller. README "Error handling" now states the two cases that never reach sfdupes as a failed write: a pipe reader that exits early ends the process with SIGPIPE, as with cat; and stdout closed with >&- is replaced by /dev/null by the Go runtime before main runs, so the run succeeds. Model: opus-5-5 --- README.md | 12 ++++ TODO.md | 4 ++ main.go | 19 ++++-- main_test.go | 175 +++++++++++++++++++++++++------------------------ report.go | 5 +- report_test.go | 8 +-- scan_test.go | 3 +- trees.go | 5 +- trees_test.go | 8 +-- 9 files changed, 131 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index d48f695..527b3fd 100644 --- a/README.md +++ b/README.md @@ -573,6 +573,18 @@ Additional requirements: - `2`: usage error (including `scan` with no `PATH` operand and `report`/`trees` with any positional argument). +A stdout write failure, such as a full disk, is reported in one line on +stderr and exits 1. Two cases never reach sfdupes as a failed write: + +- When the reader of a stdout pipe exits early, as in + `sfdupes report | head`, the next write ends sfdupes with `SIGPIPE`, + quietly and without a summary, the way `cat` or `sort` end. The + shell reports the signal (status 141 in most shells), not exit 1. +- When stdout is closed outright (`sfdupes report >&-`), the Go + runtime opens `/dev/null` in its place before sfdupes starts, so + the output is discarded and the run succeeds, as with + `> /dev/null`. + ## Entrypoints This repository adheres to the diff --git a/TODO.md b/TODO.md index c6de6e9..5928fe5 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- test stdout write failures in `report` and `trees`; README states that + `| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null` + (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30) + - `report` and `trees` open the database read-only, and `scan` leaves it out of WAL mode, so reading needs only read access (2026-10-03, closes https://git.eeqj.de/sneak/sfdupes/issues/8) diff --git a/main.go b/main.go index d2160c1..3a8ce57 100644 --- a/main.go +++ b/main.go @@ -57,22 +57,27 @@ var errNoSubcommand = errors.New("no subcommand") var Version = "dev" func main() { - os.Exit(run(os.Args[1:], os.Stderr)) + // Once the reader of a stdout pipe has gone, as in "sfdupes report | + // head", the Go runtime ends the process with SIGPIPE on the next + // write instead of returning an error (README "Error handling"). + // Registering for SIGPIPE with os/signal would change that. + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } // run executes args against the command tree and returns the process // exit code. It is the program's single exit point: the subcommands // return their errors instead of exiting, so every deferred cleanup — // above all closing the database, which checkpoints the SQLite WAL — -// runs before the process ends. -func run(args []string, stderr io.Writer) int { +// runs before the process ends. The report and trees subcommands write +// their data to stdout. +func run(args []string, stdout, stderr io.Writer) int { // A nil slice makes cobra fall back to os.Args, which would let a // test binary's own flags reach the command tree. if args == nil { args = []string{} } - root := newRootCommand(stderr) + root := newRootCommand(stdout, stderr) root.SetArgs(args) err := root.Execute() @@ -98,7 +103,7 @@ func run(args []string, stderr io.Writer) int { // newRootCommand builds the command tree. Everything on stdout is // machine-readable data; all human-facing output (help, usage, errors) // goes to stderr. -func newRootCommand(stderr io.Writer) *cobra.Command { +func newRootCommand(stdout, stderr io.Writer) *cobra.Command { root := &cobra.Command{ Use: "sfdupes", Short: "Find candidate duplicate files by size and head/tail/content SHA-256", @@ -140,7 +145,7 @@ func newRootCommand(stderr io.Writer) *cobra.Command { Short: "Read the scan database and print the file-level duplicates report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { - return runReport(ctx) + return runReport(ctx, stdout) }), } @@ -149,7 +154,7 @@ func newRootCommand(stderr io.Writer) *cobra.Command { Short: "Read the scan database and print the duplicate-tree report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { - return runTrees(ctx) + return runTrees(ctx, stdout) }), } diff --git a/main_test.go b/main_test.go index 432e9e4..f04801b 100644 --- a/main_test.go +++ b/main_test.go @@ -82,50 +82,6 @@ func makeReadOnly(t *testing.T, path string) { }) } -// captureStdout redirects os.Stdout to a file for the rest of the test -// and returns a function reading back everything written to it. Only -// machine-readable data belongs on stdout (README design goal 4), so -// the tests assert on it directly. -func captureStdout(t *testing.T) func() string { - t.Helper() - - f, err := os.Create(filepath.Join(t.TempDir(), "stdout")) - if err != nil { - t.Fatal(err) - } - - saved := os.Stdout - os.Stdout = f - - t.Cleanup(func() { - os.Stdout = saved - - _ = f.Close() - }) - - return func() string { - // Read what has been written without disturbing the write - // offset, so the capture can be inspected more than once. - size, err := f.Seek(0, io.SeekCurrent) - if err != nil { - t.Fatal(err) - } - - if size == 0 { - return "" - } - - b := make([]byte, size) - - _, err = f.ReadAt(b, 0) - if err != nil { - t.Fatal(err) - } - - return string(b) - } -} - // brokenDatabase writes a database that opens cleanly and passes the // schema-version check but has no files table, so the first query // fails with the database already open: a fatal error on a path that @@ -198,17 +154,15 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { args = append(args, t.TempDir()) } - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run(args, &stderr) + code := run(args, &stdout, &stderr) if code != exitFatal { t.Errorf("run(%v) = %d, want %d", args, code, exitFatal) } assertNoSidecars(t, path) - assertFatalOutput(t, stderr.String(), stdout()) + assertFatalOutput(t, stderr.String(), stdout.String()) // Proof that the failure happened after the open: only a // query against the opened database can report this. @@ -226,18 +180,16 @@ func TestRunMissingOperandIsFatalNotUsage(t *testing.T) { // must not dump the usage text. t.Setenv(databaseEnv, testDBPath(t)) - var stderr bytes.Buffer - - stdout := captureStdout(t) + var stdout, stderr bytes.Buffer missing := filepath.Join(t.TempDir(), "nope") - code := run([]string{cmdScan, missing}, &stderr) + code := run([]string{cmdScan, missing}, &stdout, &stderr) if code != exitFatal { t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal) } - assertFatalOutput(t, stderr.String(), stdout()) + assertFatalOutput(t, stderr.String(), stdout.String()) } // assertFatalOutput checks that a fatal error was reported the way @@ -281,11 +233,9 @@ func TestRunUsageErrors(t *testing.T) { // path that does not exist. t.Setenv(databaseEnv, testDBPath(t)) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run(tc.args, &stderr) + code := run(tc.args, &stdout, &stderr) if code != exitUsage { t.Errorf("run(%v) = %d, want %d", tc.args, code, exitUsage) } @@ -294,7 +244,7 @@ func TestRunUsageErrors(t *testing.T) { t.Errorf("stderr = %q, want %q", stderr.String(), tc.want) } - if got := stdout(); got != "" { + if got := stdout.String(); got != "" { t.Errorf("stdout = %q, want nothing (data only)", got) } }) @@ -303,9 +253,9 @@ func TestRunUsageErrors(t *testing.T) { // TestRunHelpAndVersionSucceed checks that the two informational flags // exit 0 and keep their human-facing output on stderr. -// -//nolint:paralleltest // captureStdout replaces the process-wide os.Stdout func TestRunHelpAndVersionSucceed(t *testing.T) { + t.Parallel() + assertHumanOutput(t, "--help") assertHumanOutput(t, "--version") } @@ -316,11 +266,9 @@ func TestRunHelpAndVersionSucceed(t *testing.T) { func assertHumanOutput(t *testing.T, arg string) { t.Helper() - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{arg}, &stderr) + code := run([]string{arg}, &stdout, &stderr) if code != exitOK { t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) } @@ -329,7 +277,7 @@ func assertHumanOutput(t *testing.T, arg string) { t.Errorf("run(%s) wrote nothing to stderr", arg) } - if got := stdout(); got != "" { + if got := stdout.String(); got != "" { t.Errorf("stdout = %q, want nothing (data only)", got) } } @@ -358,17 +306,15 @@ func scanFixture(t *testing.T) []string { t.Fatal(err) } - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{cmdScan, dir}, &stderr) + code := run([]string{cmdScan, dir}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(scan) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) } - if got := stdout(); got != "" { + if got := stdout.String(); got != "" { t.Errorf("scan stdout = %q, want nothing (data only)", got) } @@ -389,18 +335,16 @@ func TestRunReportSucceeds(t *testing.T) { dupes := scanFixture(t) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{cmdReport}, &stderr) + code := run([]string{cmdReport}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) } want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } @@ -413,11 +357,9 @@ func TestRunTreesSucceeds(t *testing.T) { dupes := scanFixture(t) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{cmdTrees}, &stderr) + code := run([]string{cmdTrees}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(trees) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) @@ -427,7 +369,7 @@ func TestRunTreesSucceeds(t *testing.T) { // trees of each other. want := "first\tdupe\tfiles\tsize\n" + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } @@ -454,11 +396,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) { } for name, want := range cases { - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{name}, &stderr) + code := run([]string{name}, &stdout, &stderr) if code != exitOK { t.Errorf("run(%s) = %d, want %d; stderr: %s", name, code, exitOK, stderr.String()) @@ -466,8 +406,71 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) { continue } - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("%s stdout = %q, want %q", name, got, want) } } } + +func TestRunStdoutClosedIsFatal(t *testing.T) { + // README §Error handling: a stdout write failure exits 1, reported + // in one line on stderr. + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + t.Setenv(databaseEnv, testDBPath(t)) + + scanFixture(t) + + stdout, err := os.Create(filepath.Join(t.TempDir(), "stdout")) + if err != nil { + t.Fatal(err) + } + + err = stdout.Close() + if err != nil { + t.Fatal(err) + } + + var stderr bytes.Buffer + + code := run([]string{name}, stdout, &stderr) + if code != exitFatal { + t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) + } + + got := stderr.String() + if !strings.HasPrefix(got, "sfdupes: write stdout: ") || + !strings.Contains(got, os.ErrClosed.Error()) || + strings.Count(got, "\n") != 1 { + t.Errorf("stderr = %q, want one line reporting the "+ + "failed stdout write", got) + } + }) + } +} + +// errWriteFailed is the error failingWriter returns. +var errWriteFailed = errors.New("write failed") + +// failingWriter is a stdout that fails every write. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errWriteFailed } + +func TestStdoutWriteErrorPropagates(t *testing.T) { + t.Setenv(databaseEnv, testDBPath(t)) + + scanFixture(t) + + cases := map[string]func(context.Context, io.Writer) error{ + cmdReport: runReport, + cmdTrees: runTrees, + } + + for name, fn := range cases { + err := fn(t.Context(), failingWriter{}) + if !errors.Is(err, errWriteFailed) { + t.Errorf("%s: error = %v, want %v", name, err, errWriteFailed) + } + } +} diff --git a/report.go b/report.go index 00b7b01..68b4ecb 100644 --- a/report.go +++ b/report.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "io" "os" "slices" "strings" @@ -65,7 +66,7 @@ type dupeGroup struct { // from the database and prints the file-level duplicates report as TSV // on stdout. It never touches the scanned filesystem; its only I/O is // the database, stdout, and stderr. -func runReport(ctx context.Context) error { +func runReport(ctx context.Context, stdout io.Writer) error { recs, err := loadRecords(ctx) if err != nil { return err @@ -73,7 +74,7 @@ func runReport(ctx context.Context) error { dupes := collectDupeGroups(recs) - out := bufio.NewWriterSize(os.Stdout, ioBufSize) + out := bufio.NewWriterSize(stdout, ioBufSize) _, err = fmt.Fprintln(out, "first\tdupe\tsize") if err != nil { diff --git a/report_test.go b/report_test.go index d1aa290..9aa092d 100644 --- a/report_test.go +++ b/report_test.go @@ -50,11 +50,9 @@ func seedDatabase(t *testing.T, recs []scanRec) string { func TestRunReportEscapesPaths(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{cmdReport}, &stderr) + code := run([]string{cmdReport}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) @@ -62,7 +60,7 @@ func TestRunReportEscapesPaths(t *testing.T) { want := "first\tdupe\tsize\n" + `/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } } diff --git a/scan_test.go b/scan_test.go index f47005d..5e2a259 100644 --- a/scan_test.go +++ b/scan_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/hex" "fmt" + "io" "os" "path/filepath" "runtime" @@ -1548,7 +1549,7 @@ func TestScanHashWriteFailureUnwindsPool(t *testing.T) { code := run([]string{ cmdScan, "--workers", strconv.Itoa(hashLeakWorkers), dir, - }, &stderr) + }, io.Discard, &stderr) if code != exitFatal { t.Fatalf("run(scan) = %d, want %d; stderr: %s", code, exitFatal, stderr.String()) diff --git a/trees.go b/trees.go index f43374c..735c5b5 100644 --- a/trees.go +++ b/trees.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "fmt" + "io" "os" "slices" "strconv" @@ -36,7 +37,7 @@ type treeNode struct { // maximal duplicate-tree groups as TSV on stdout. It never touches the // scanned filesystem; its only I/O is the database, stdout, and // stderr. -func runTrees(ctx context.Context) error { +func runTrees(ctx context.Context, stdout io.Writer) error { recs, err := loadRecords(ctx) if err != nil { return err @@ -47,7 +48,7 @@ func runTrees(ctx context.Context) error { dupes := collectTreeGroups(allDirs, super) - out := bufio.NewWriterSize(os.Stdout, ioBufSize) + out := bufio.NewWriterSize(stdout, ioBufSize) _, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize") if err != nil { diff --git a/trees_test.go b/trees_test.go index abf61f1..5cc8817 100644 --- a/trees_test.go +++ b/trees_test.go @@ -108,11 +108,9 @@ func TestBuildHierarchyRootPath(t *testing.T) { func TestRunTreesEscapesPaths(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer - stdout := captureStdout(t) - - code := run([]string{cmdTrees}, &stderr) + code := run([]string{cmdTrees}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(trees) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) @@ -120,7 +118,7 @@ func TestRunTreesEscapesPaths(t *testing.T) { want := "first\tdupe\tfiles\tsize\n" + `/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n" - if got := stdout(); got != want { + if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } } From 705c8729cac4a46f751cc4353472328131f59059 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 02:30:21 +0200 Subject: [PATCH 05/33] Hold a lock so a second scan fails at once (closes #53) scan takes an exclusive flock(2) on a lock file beside the database (its path with .lock appended) before it walks anything or opens the database, and holds it until it returns. A second scan against the same database fails at once with a one-line error naming the lock file and exits 1. report and trees never take the lock. The lock ends with the process, so a fatal error or an interrupt releases it; the file is never deleted. golang.org/x/sys becomes a direct dependency. The README smoke test now keeps the database outside the scanned tree, where its empty lock file would have joined the empty-file group. Model: opus-5-5 --- README.md | 34 ++++++++++++++------ TODO.md | 4 +++ db.go | 47 +++++++++++++++++++++++++++ go.mod | 2 +- main_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ scan.go | 19 ++++++++--- 6 files changed, 180 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 527b3fd..36a99ce 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,9 @@ Goals, in order: `sfdupes`. - Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one progress-bar library** - (`github.com/schollz/progressbar/v3`), and **one SQLite driver** - (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled). + (`github.com/schollz/progressbar/v3`), **one SQLite driver** + (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and + `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). `github.com/spf13/viper` is permitted if configuration-file support is ever needed, but is not currently used. No other third-party deps. @@ -152,6 +153,20 @@ All three subcommands operate on a single SQLite database file: use. `report` and `trees` require an existing database; a missing database file is a fatal error (exit 1) telling the user to run `scan` first. +- Only one `scan` runs against a database at a time. For its whole + run, `scan` holds an exclusive `flock(2)` lock on a lock file + beside the database, named by appending `.lock` to the database + path (`/var/lib/sfdupes/db.sqlite.lock` by default), taken before + it walks the filesystem or opens the database. A second `scan` + against the same database does not wait: it fails at once with a + one-line error naming the lock file and exits 1, without walking + anything or opening the database, and the running scan carries on. + The lock file is created on first use, open to its owner only, and + left in place: a leftover file blocks nothing, because the lock + ends with the process holding it however it ends, a fatal error or + an interrupt included, and deleting the file while a scan runs + would let a second scan start. `report` and `trees` never take the + lock, so they run during a scan. - While `scan` runs, the database is in WAL journal mode with a busy timeout, so running a report while a cron `scan` is in progress is safe. The filesystem is authoritative; the database is an @@ -567,9 +582,10 @@ Additional requirements: ### Error handling and exit codes - `0`: success, even if individual files were skipped with warnings. -- `1`: fatal error (e.g., a `PATH` operand does not exist, the - database cannot be created/opened/read/written, a missing database - for `report`/`trees`, stdout write failure). +- `1`: fatal error (e.g., a `PATH` operand does not exist, another + `scan` is already running against the same database, the database + cannot be created/opened/read/written, a missing database for + `report`/`trees`, stdout write failure). - `2`: usage error (including `scan` with no `PATH` operand and `report`/`trees` with any positional argument). @@ -722,7 +738,7 @@ All of the following, run in this directory, must pass: ```sh d=$(mktemp -d) - export SFDUPES_DATABASE="$d/db.sqlite" + export SFDUPES_DATABASE="$(mktemp -d)/db.sqlite" mkdir -p "$d/a" "$d/b" head -c 2000 /dev/urandom > "$d/a/one.bin" cp "$d/a/one.bin" "$d/b/copy.bin" @@ -750,9 +766,9 @@ All of the following, run in this directory, must pass: ./sfdupes report ``` - (The scan database lives inside `$d` here purely for test hygiene; - scanning `$d` therefore also records the SQLite file itself, which - is harmless.) + (The database lives in a temp directory of its own: inside `$d`, + the scan would record it, and its empty lock file would join the + `empty1`/`empty2` group.) Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one group (two dupe rows, `first` is the lexicographically diff --git a/TODO.md b/TODO.md index 5928fe5..57b026f 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `scan` holds a lock on a lock file beside the database for its whole run, + so a second `scan` fails at once with exit 1 (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/53) + - test stdout write failures in `report` and `trees`; README states that `| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null` (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30) diff --git a/db.go b/db.go index 2060dc4..1c1ffac 100644 --- a/db.go +++ b/db.go @@ -11,6 +11,7 @@ import ( "slices" "strconv" + "golang.org/x/sys/unix" // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo // disabled. _ "modernc.org/sqlite" @@ -32,6 +33,11 @@ const schemaVersion = 1 // scan. const dbDirPerm = 0o755 +// lockFilePerm is the mode for the scan lock file. Anyone who can open +// the file can hold the lock and keep every scan from running, so it +// is open to its owner only. +const lockFilePerm = 0o600 + // createTableSQL is the schema applied to a fresh database. Paths are // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. const createTableSQL = ` @@ -64,6 +70,10 @@ var errNoDatabase = errors.New( // does not understand. var errSchemaVersion = errors.New("unsupported database schema version") +// errScanRunning reports that another scan holds the lock on the +// database. +var errScanRunning = errors.New("another scan is running") + // databasePath resolves the database location: SFDUPES_DATABASE when // set and non-empty, the compiled-in default otherwise. func databasePath() string { @@ -104,6 +114,43 @@ func openDB(path, params string) (*sql.DB, error) { return db, nil } +// lockScanDatabase takes the lock that keeps a second scan off the +// database at path: an exclusive flock(2) on the file beside it named +// path with ".lock" appended, created along with the database's parent +// directory if missing. A lock held by another scan fails at once +// instead of waiting. The lock lasts until the returned file is closed +// or the process ends. The file is never deleted: a scan that deleted +// it would let the next scan lock a new file while another still holds +// the old one. +func lockScanDatabase(path string) (*os.File, error) { + err := os.MkdirAll(filepath.Dir(path), dbDirPerm) + if err != nil { + return nil, fmt.Errorf("create database directory: %w", err) + } + + lockPath := path + ".lock" + + //nolint:gosec // the operator chooses the database path + f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm) + if err != nil { + return nil, err + } + + err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err != nil { + _ = f.Close() + + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, fmt.Errorf("%w (lock held on %s)", + errScanRunning, lockPath) + } + + return nil, fmt.Errorf("lock %s: %w", lockPath, err) + } + + return f, nil +} + // openScanDatabase opens the database for the scan subcommand, creating // the file, its parent directory, and the schema as needed. func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) { diff --git a/go.mod b/go.mod index 9b0faaa..5a13062 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.7 require ( github.com/schollz/progressbar/v3 v3.19.1 github.com/spf13/cobra v1.10.2 + golang.org/x/sys v0.46.0 modernc.org/sqlite v1.54.0 ) @@ -18,7 +19,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/main_test.go b/main_test.go index f04801b..ac53a0e 100644 --- a/main_test.go +++ b/main_test.go @@ -412,6 +412,95 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) { } } +// holdScanLock takes the lock on the database at path, as a running +// scan does, and holds it until the test ends. It fails the test when +// the lock is already held. +func holdScanLock(t *testing.T, path string) { + t.Helper() + + lock, err := lockScanDatabase(path) + if err != nil { + t.Fatalf("lock %s: %v", path, err) + } + + t.Cleanup(func() { _ = lock.Close() }) +} + +func TestRunSecondScanFails(t *testing.T) { + // README §Database: while one scan holds the lock, a second scan + // fails at once, naming the lock file, without creating the + // database. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + holdScanLock(t, path) + + var stdout, stderr bytes.Buffer + + code := run([]string{cmdScan, t.TempDir()}, &stdout, &stderr) + if code != exitFatal { + t.Errorf("run(scan) = %d, want %d", code, exitFatal) + } + + want := "sfdupes: another scan is running (lock held on " + + path + ".lock)\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout.String(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + _, err := os.Stat(path) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("stat %s = %v, want the database not created", path, err) + } +} + +func TestRunScanReleasesLock(t *testing.T) { + // README §Database: a scan releases the lock however it ends. + t.Run("success", func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + holdScanLock(t, path) + }) + + t.Run("fatal error", func(t *testing.T) { + path := brokenDatabase(t) + t.Setenv(databaseEnv, path) + + code := run([]string{cmdScan, t.TempDir()}, io.Discard, io.Discard) + if code != exitFatal { + t.Fatalf("run(scan) = %d, want %d", code, exitFatal) + } + + holdScanLock(t, path) + }) +} + +func TestRunReportsDuringScan(t *testing.T) { + // README §Database: report and trees never take the lock, so they + // run while a scan holds it. + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + holdScanLock(t, path) + + for _, name := range []string{cmdReport, cmdTrees} { + var stderr bytes.Buffer + + code := run([]string{name}, io.Discard, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + } + } +} + func TestRunStdoutClosedIsFatal(t *testing.T) { // README §Error handling: a stdout write failure exits 1, reported // in one line on stderr. diff --git a/scan.go b/scan.go index a215c5a..c46c42a 100644 --- a/scan.go +++ b/scan.go @@ -85,11 +85,13 @@ type fileMeta struct { // least one other file shares are ever hashed: a size-unique file // cannot be a duplicate. A file of headTailMin or more gets its content // hash only when its size, head, and tail match another file's. Flag -// parsing and the at-least-one-operand check are done by cobra. Errors -// are returned rather than exiting, so that the deferred close — which -// takes the database out of WAL mode — always runs. Cancelling ctx -// unwinds the worker pools and aborts the scan with the context's -// error. +// parsing and the at-least-one-operand check are done by cobra. The +// scan holds the lock on the database for its whole run, so a second +// scan fails before it walks the filesystem or opens the database. +// Errors are returned rather than exiting, so that the deferred close — +// which takes the database out of WAL mode — always runs, and the lock +// is released after it. Cancelling ctx unwinds the worker pools and +// aborts the scan with the context's error. func runScan(ctx context.Context, roots []string, workers int, oneFS bool, ) error { @@ -104,6 +106,13 @@ func runScan(ctx context.Context, roots []string, workers int, dbPath := databasePath() + lock, err := lockScanDatabase(dbPath) + if err != nil { + return err + } + + defer func() { _ = lock.Close() }() + db, err := openScanDatabase(ctx, dbPath) if err != nil { return err From 2dd1194f33db55d1f1289408b4e6362c69ed5c82 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 04:01:43 +0200 Subject: [PATCH 06/33] Warn about and skip non-regular and .zfs operands, keeping their records (closes #9) A symlink, socket, FIFO or device-node operand, or a directory operand named .zfs, was silently ignored yet stayed in the scanned operands, so the update phase deleted every record stored beneath it. Such an operand now gets a one-line warning, counts as skipped, and is dropped before overlapping operands are pruned and the database index is loaded: another operand beneath it is still scanned, and the records beneath it count as outside the scanned operands and are not deleted, unless it lies under another operand. The exit status stays 0. An operand that turns into one of these after that check is warned about and skipped by the walk instead. README "scan mode" and "Rules for the walk" say so. Model: opus-5-5 --- README.md | 27 +++++++-- TODO.md | 4 ++ main_test.go | 163 +++++++++++++++++++++++++++++++++++++++++++++++++-- scan.go | 117 ++++++++++++++++++++++++++++-------- scan_test.go | 6 +- 5 files changed, 279 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 36a99ce..f4e7667 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,18 @@ duplicates another or lies under another is dropped before walking, so every file is reached exactly once and produces one database record. +An operand that is a symlink (never followed, not even as an operand), +socket, FIFO, or device node, or a directory named `.zfs`, is not +scanned. `scan` prints a one-line warning naming the path and what it +is, counts it as skipped, and drops it from the scanned operands before +reading the database. Another operand beneath it is still scanned. The +records stored beneath it are not deleted: they are treated like any +other record outside the scanned operands, including the content-phase +exception below. If it lies under another operand, they are under that +operand instead, and are deleted like any other record there that this +scan did not verify. This is not an error: a scan whose every operand +is dropped walks nothing and exits 0. + `scan` synchronizes the database with the filesystem state under the scanned operands: @@ -383,9 +395,12 @@ during the hash phase: Rules for the walk: - Only regular files. Skip directories, symlinks (do not follow, - including symlink operands), sockets, FIFOs, and device nodes. + including symlink operands), sockets, FIFOs, and device nodes. An + operand that is a symlink, socket, FIFO, or device node is dropped + as described in "`scan` mode" above. - Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; - walking them would list every file once per snapshot). + walking them would list every file once per snapshot), not even + when it is an operand; such an operand is dropped the same way. - Filesystem boundaries are crossed by default. With `-x` (long form `--one-file-system`, following the GNU `du`/`rsync` convention), never descend into a directory on a different @@ -396,9 +411,11 @@ Rules for the walk: path, and continue. Per-file errors never abort the run; the final summary reports how many were skipped. As specified above, a skipped path that has a database record from an earlier scan loses - that record, unless it failed only in the content phase; an - unreadable directory subtree likewise loses its records (accepted: - the database mirrors what the latest scan could actually verify). + that record, unless it failed only in the content phase, or is an + operand dropped before the database was read that lies under no + other operand; an unreadable directory subtree likewise loses its + records (accepted: the database mirrors what the latest scan could + actually verify). Concurrency: the walk phase (which also stats files), the hash phase, and the content phase each use a worker pool of `--workers` workers diff --git a/TODO.md b/TODO.md index 57b026f..b07d9a1 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- warn about and skip symlink, socket, FIFO, device and `.zfs` + operands, keeping the records beneath them (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/9) + - `scan` holds a lock on a lock file beside the database for its whole run, so a second `scan` fails at once with exit 1 (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/53) diff --git a/main_test.go b/main_test.go index ac53a0e..d6716de 100644 --- a/main_test.go +++ b/main_test.go @@ -82,6 +82,50 @@ func makeReadOnly(t *testing.T, path string) { }) } +// captureStderr redirects os.Stderr to a file for the rest of the test +// and returns a function reading back everything written to it. scan +// writes its warnings and summary straight to os.Stderr, not to the +// stderr writer run is given. +func captureStderr(t *testing.T) func() string { + t.Helper() + + f, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + if err != nil { + t.Fatal(err) + } + + saved := os.Stderr + os.Stderr = f + + t.Cleanup(func() { + os.Stderr = saved + + _ = f.Close() + }) + + return func() string { + // Read what has been written without disturbing the write + // offset, so the capture can be inspected more than once. + size, err := f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + + if size == 0 { + return "" + } + + b := make([]byte, size) + + _, err = f.ReadAt(b, 0) + if err != nil { + t.Fatal(err) + } + + return string(b) + } +} + // brokenDatabase writes a database that opens cleanly and passes the // schema-version check but has no files table, so the first query // fails with the database already open: a fatal error on a path that @@ -306,19 +350,31 @@ func scanFixture(t *testing.T) []string { t.Fatal(err) } - var stdout, stderr bytes.Buffer + scanOK(t, dir) - code := run([]string{cmdScan, dir}, &stdout, &stderr) + return dupes +} + +// scanOK runs scan over operands, fails the test unless it exits 0 with +// nothing on stdout, and returns everything it printed to stderr. +func scanOK(t *testing.T, operands ...string) string { + t.Helper() + + var stdout bytes.Buffer + + stderr := captureStderr(t) + + code := run(append([]string{cmdScan}, operands...), &stdout, os.Stderr) if code != exitOK { - t.Fatalf("run(scan) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + t.Fatalf("run(scan %q) = %d, want %d; stderr: %s", + operands, code, exitOK, stderr()) } if got := stdout.String(); got != "" { t.Errorf("scan stdout = %q, want nothing (data only)", got) } - return dupes + return stderr() } func TestRunScanSucceedsDespiteWarnings(t *testing.T) { @@ -329,6 +385,103 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) { assertNoSidecars(t, path) } +func TestRunScanSkipsSymlinkOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + writeFile(t, dir, "target/sub/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(filepath.Join(dir, "target"), link) + if err != nil { + t.Fatal(err) + } + + // Scanning a directory through the symlink stores a record beneath + // the symlink's own path for a file beneath its target. + scanOK(t, filepath.Join(link, "sub")) + + assertOperandSkipped(t, path, link, "symlink", + filepath.Join(link, "sub", "f")) +} + +func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + writeFile(t, dir, "target/sub/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(filepath.Join(dir, "target"), link) + if err != nil { + t.Fatal(err) + } + + // link is dropped as a symlink, but link/sub must still be scanned, + // not dropped as lying under link. + scanOK(t, link, filepath.Join(link, "sub")) + + db, err := openDB(path, reportParams) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f")) +} + +func TestRunScanSkipsZFSOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + zfs := filepath.Join(t.TempDir(), ".zfs") + snapshot := filepath.Join(zfs, "snapshot", "hourly") + f := writeFile(t, snapshot, "f", pattern(1, 10)) + + // An operand beneath a .zfs directory is walked, because it is not + // itself named .zfs. + scanOK(t, snapshot) + + assertOperandSkipped(t, path, zfs, ".zfs directory", f) +} + +// assertOperandSkipped scans operand alone and checks that it is skipped +// as kind: a warning naming it, one skip in the summary, exit 0, and the +// record for kept, which an earlier scan stored beneath operand, still +// in the database at dbPath. +func assertOperandSkipped(t *testing.T, dbPath, operand, kind, + kept string, +) { + t.Helper() + + stderr := scanOK(t, operand) + + warning := "walk " + operand + ": skipping " + kind + " operand\n" + if !strings.Contains(stderr, warning) { + t.Errorf("stderr = %q, want %q", stderr, warning) + } + + summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " + + "0 unchanged), 1 skipped\n" + if !strings.Contains(stderr, summary) { + t.Errorf("stderr = %q, want %q", stderr, summary) + } + + db, err := openDB(dbPath, reportParams) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = db.Close() }) + + recordByPath(t, dbRecords(t, db), kept) +} + func TestRunReportSucceeds(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) diff --git a/scan.go b/scan.go index c46c42a..c7406b2 100644 --- a/scan.go +++ b/scan.go @@ -220,14 +220,18 @@ type scanState struct { // in the content hash of every record of headTailMin or more whose // size, head, and tail match another record's). Records outside the // roots are never touched, except that the content phase fills in -// their content hash. +// their content hash. Operands the walk cannot start from are dropped +// first, so the records beneath them count as outside the roots unless +// they lie under another root. func syncScan(ctx context.Context, db *sql.DB, roots []string, workers int, oneFS bool, ) (scanStats, error) { - roots = pruneRoots(roots) - s := &scanState{db: db} + // Types are checked before pruning so that an operand under a + // dropped one is still scanned, not dropped as lying under it. + roots = pruneRoots(s.walkableRoots(roots)) + err := s.loadIndex(ctx, roots) if err != nil { return s.st, err @@ -263,6 +267,35 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, return s.st, s.contentPhase(ctx, workers) } +// walkableRoots returns the operands the walk can start from: regular +// files, and directories not named .zfs. Every other operand is warned +// about, counted as skipped, and dropped. A dropped operand is no +// longer a root, so the records stored beneath it count as outside the +// roots and are not deleted as unverified, unless it lies under another +// root. An operand that fails lstat here is kept, and the walk warns +// about it. +func (s *scanState) walkableRoots(roots []string) []string { + kept := make([]string, 0, len(roots)) + + for _, root := range roots { + fi, err := os.Lstat(root) + if err == nil { + warn := operandWarning(root, fi) + if warn != "" { + s.st.skipped++ + + fmt.Fprintln(os.Stderr, escapePath(warn)) + + continue + } + } + + kept = append(kept, root) + } + + return kept +} + // loadIndex indexes the database records under the scan roots for // change detection and collects the sizes of every record outside // them: out-of-scope records join the size census so a scanned file @@ -799,11 +832,43 @@ func sendEvent(ctx context.Context, events chan<- walkEvent, } } +// operandWarning returns the one-line warning for an operand the walk +// does not start from, naming the path and what it is, or "" for one it +// does: a regular file, or a directory not named .zfs. Symlinks are +// never followed, including as operands. +func operandWarning(root string, fi fs.FileInfo) string { + var kind string + + switch mode := fi.Mode(); { + case mode.IsRegular(): + return "" + case mode.IsDir(): + if filepath.Base(root) != ".zfs" { + return "" + } + + kind = ".zfs directory" + case mode&fs.ModeSymlink != 0: + kind = "symlink" + case mode&fs.ModeSocket != 0: + kind = "socket" + case mode&fs.ModeNamedPipe != 0: + kind = "FIFO" + case mode&fs.ModeDevice != 0: + kind = "device node" + default: + kind = "non-regular file" + } + + return fmt.Sprintf("walk %s: skipping %s operand", root, kind) +} + // seedRoot turns one PATH operand into the walk's starting state: a -// regular-file operand is statted and emitted directly, a directory -// operand becomes an initial job, and a symlink or other non-regular -// operand yields nothing (symlinks are never followed, including as -// operands). +// regular-file operand is statted and emitted directly, and a directory +// operand becomes an initial job. walkableRoots has already dropped +// every other operand. One that has changed into something else since +// is warned about and skipped here; it is still a root, so the records +// stored beneath it are deleted as unverified. func seedRoot(ctx context.Context, root string, events chan<- walkEvent, ) []dirJob { @@ -817,30 +882,30 @@ func seedRoot(ctx context.Context, root string, return nil } - switch { - case fi.IsDir(): - if filepath.Base(root) == ".zfs" { - return nil - } + warn := operandWarning(root, fi) + if warn != "" { + sendEvent(ctx, events, walkEvent{warn: warn, fail: true}) + return nil + } + + if fi.IsDir() { dev, ok := deviceOfInfo(fi) return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}} - case fi.Mode().IsRegular(): - dev, ino := inodeOfInfo(fi) - - sendEvent(ctx, events, walkEvent{rec: fileRec{ - path: root, - size: fi.Size(), - mtime: fi.ModTime().Unix(), - dev: dev, - ino: ino, - }}) - - return nil - default: - return nil } + + dev, ino := inodeOfInfo(fi) + + sendEvent(ctx, events, walkEvent{rec: fileRec{ + path: root, + size: fi.Size(), + mtime: fi.ModTime().Unix(), + dev: dev, + ino: ino, + }}) + + return nil } // startWalkWorkers starts the walk worker pool. Each worker processes diff --git a/scan_test.go b/scan_test.go index 5e2a259..249989a 100644 --- a/scan_test.go +++ b/scan_test.go @@ -857,9 +857,11 @@ func TestWalkFileAndSymlinkOperands(t *testing.T) { t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs) } - // A symlink operand is not followed and yields nothing. + // A symlink operand that reaches the walk (it became one after + // walkableRoots checked it) is not followed: it yields a warning and + // no records. recs, errs = collectWalk(t, []string{link}, false, 2) - if errs != 0 || len(recs) != 0 { + if errs != 1 || len(recs) != 0 { t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs) } } From 9abf81535a516478943d054827bd277e9b4668bf Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 04:47:35 +0200 Subject: [PATCH 07/33] Print progress at once off a terminal, keep warnings out of redraws (closes #13) When stderr is not a terminal, each phase prints its zero-state line as it starts instead of after its first item. stderrIsTTY uses term.IsTerminal from golang.org/x/term, now a direct dependency, so /dev/null is no longer taken for a terminal. A spinner keeps the library's background redraw, so its count and elapsed time stay current while a phase waits for its next item. A warning printed during a spinner phase goes through the bar (progressbar.Bprintln), which prints it before its next redraw instead of racing it. Bars with a total have no background redraw and still print warnings directly. Model: opus-5-5 --- README.md | 15 +++-- TODO.md | 4 ++ go.mod | 2 +- progress.go | 53 +++++++++++----- progress_test.go | 161 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 progress_test.go diff --git a/README.md b/README.md index f4e7667..ae992d3 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,8 @@ Goals, in order: `sfdupes`. - Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one progress-bar library** - (`github.com/schollz/progressbar/v3`), **one SQLite driver** + (`github.com/schollz/progressbar/v3`), `golang.org/x/term` to tell + whether stderr is a terminal, **one SQLite driver** (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). `github.com/spf13/viper` is permitted if configuration-file support @@ -589,10 +590,16 @@ hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54 Additional requirements: -- When stderr is not a TTY, do not emit ANSI redraws: print a plain - one-line progress update no more often than every 5 seconds instead. +- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not + emit ANSI redraws: print a plain one-line progress update the moment + each phase starts, then no more often than every 5 seconds. - Progress updates are driven from the main goroutine and must be - non-blocking with respect to the worker pool. + non-blocking with respect to the worker pool. On a terminal the + spinner-style displays also redraw on their own several times a + second, so their count and elapsed time stay current while a phase + waits for its next item. +- A warning printed during a phase always lands on a line of its own, + never inside the progress display. - `report` and `trees` modes need no progress display, only their stderr summaries. diff --git a/TODO.md b/TODO.md index b07d9a1..95fefb7 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- progress prints at once on a non-terminal, uses a real terminal test, + and prints warnings through a spinner instead of racing its redraw + (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/13) + - warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping the records beneath them (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/9) diff --git a/go.mod b/go.mod index 5a13062..cf8c37c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/schollz/progressbar/v3 v3.19.1 github.com/spf13/cobra v1.10.2 golang.org/x/sys v0.46.0 + golang.org/x/term v0.44.0 modernc.org/sqlite v1.54.0 ) @@ -19,7 +20,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/term v0.44.0 // indirect modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/progress.go b/progress.go index f351d47..e6ca868 100644 --- a/progress.go +++ b/progress.go @@ -6,6 +6,7 @@ import ( "time" "github.com/schollz/progressbar/v3" + "golang.org/x/term" ) // plainInterval is the minimum time between progress lines when stderr @@ -24,24 +25,23 @@ const percentScale = 100 // stderrIsTTY reports whether stderr is attached to a terminal. func stderrIsTTY() bool { - fi, err := os.Stderr.Stat() - if err != nil { - return false - } - - return fi.Mode()&os.ModeCharDevice != 0 + return term.IsTerminal(int(os.Stderr.Fd())) } // progress renders one scan pass's progress on stderr. On a TTY it // delegates to the progressbar library (spinner style when the total is // unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When // stderr is not a TTY it emits no ANSI redraws: it prints a plain -// one-line update no more often than every plainInterval. +// one-line update as the pass starts, then no more often than every +// plainInterval. // -// All methods must be called from the main goroutine only. A nil -// *progress is a valid no-display receiver: every method is a no-op, -// so batched database flushes during the streaming pass can reuse the -// update-pass helpers without rendering anything. +// All methods must be called from the main goroutine only. On a TTY +// the library also redraws a spinner from its own goroutine, several +// times a second, so its count and elapsed time stay current while a +// pass waits for its next item. A nil *progress is a valid +// no-display receiver: every method is a no-op, so batched database +// flushes during the streaming pass can reuse the update-pass helpers +// without rendering anything. type progress struct { label string total int64 // -1 when unknown (walk pass) @@ -53,10 +53,22 @@ type progress struct { func newProgress(label string, total int64) *progress { p := &progress{label: label, total: total, start: time.Now()} - if !stderrIsTTY() { + if stderrIsTTY() { + p.bar = newBar(label, total) + return p } + // Print the zero state at once: the first item may take minutes, + // and a pass must never look hung. + p.last = p.start + fmt.Fprintln(os.Stderr, p.plainLine()) + + return p +} + +// newBar builds the TTY display for newProgress. +func newBar(label string, total int64) *progressbar.ProgressBar { opts := []progressbar.Option{ progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetDescription(label), @@ -81,9 +93,7 @@ func newProgress(label string, total int64) *progress { ) } - p.bar = progressbar.NewOptions64(total, opts...) - - return p + return progressbar.NewOptions64(total, opts...) } // increment records one completed item and refreshes the display. @@ -113,11 +123,22 @@ func (p *progress) warnf(format string, args ...any) { return } + msg := escapePath(fmt.Sprintf(format, args...)) + + if p.bar != nil && p.total < 0 { + // The library also redraws a spinner from its own goroutine, so + // a direct write could land inside a redraw. The bar prints the + // warning itself, just before its next redraw. + _, _ = progressbar.Bprintln(p.bar, msg) + + return + } + if p.bar != nil { _ = p.bar.Clear() } - fmt.Fprintln(os.Stderr, escapePath(fmt.Sprintf(format, args...))) + fmt.Fprintln(os.Stderr, msg) } // finish terminates the pass's display. diff --git a/progress_test.go b/progress_test.go new file mode 100644 index 0000000..363759a --- /dev/null +++ b/progress_test.go @@ -0,0 +1,161 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// spinnerIdle comfortably outlasts the 100ms interval at which the +// progressbar library redraws a spinner from its own goroutine. +const spinnerIdle = 500 * time.Millisecond + +//nolint:paralleltest // replaces the process-wide os.Stderr +func TestStderrIsTTYFalseForNonTerminals(t *testing.T) { + r, pipe, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + regular, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + if err != nil { + t.Fatal(err) + } + + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + + saved := os.Stderr + + t.Cleanup(func() { + os.Stderr = saved + + for _, f := range []*os.File{r, pipe, regular, devNull} { + _ = f.Close() + } + }) + + cases := map[string]*os.File{ + "a pipe": pipe, + "a regular file": regular, + os.DevNull: devNull, + } + + for name, f := range cases { + os.Stderr = f + + if stderrIsTTY() { + t.Errorf("stderrIsTTY() = true with stderr on %s", name) + } + } +} + +// TestNewProgressPrintsBeforeFirstItem checks that each pass shows its +// zero state the moment it starts when stderr is not a terminal, and +// that the next line still waits for plainInterval. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestNewProgressPrintsBeforeFirstItem(t *testing.T) { + stderr := captureStderr(t) + + newProgress("walk", -1).increment() + newProgress("hash", 10).increment() + + want := "walk: 0 files, elapsed 0s\n" + + "hash: [0/10] 0% 0 files/s elapsed 0s eta ?\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } +} + +// newWalkSpinner returns the walk pass's terminal display, writing to +// os.Stderr whether or not it is a terminal, and stops the library's +// redraws when the test ends. +func newWalkSpinner(t *testing.T) *progress { + t.Helper() + + p := &progress{ + label: "walk", total: -1, start: time.Now(), + bar: newBar("walk", -1), + } + t.Cleanup(p.finish) + + return p +} + +// TestProgressWarningsOnOwnLines drives the terminal display of the walk +// pass through a run of warnings with no items between them, as when the +// walk meets many unreadable paths, for several of the spinner's +// redraws: every warning must land on a line of its own, never inside a +// redraw. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestProgressWarningsOnOwnLines(t *testing.T) { + stderr := captureStderr(t) + p := newWalkSpinner(t) + + // No pause between warnings: one written straight to stderr is + // garbled only if a redraw lands while it is being written. + issued := 0 + for start := time.Now(); time.Since(start) < spinnerIdle; issued++ { + p.warnf("warning") + } + + // The spinner prints the warnings at its next redraw. + time.Sleep(spinnerIdle) + + // A terminal shows each line as the text after its last carriage + // return. + shown := 0 + + for line := range strings.SplitSeq(stderr(), "\n") { + if !strings.Contains(line, "warning") { + continue + } + + shown++ + + if text := line[strings.LastIndex(line, "\r")+1:]; text != "warning" { + t.Errorf("terminal shows %q, want %q", text, "warning") + } + } + + if shown != issued { + t.Errorf("%d warning lines, want %d", shown, issued) + } +} + +// TestSpinnerShowsCountAfterBurst checks that once a burst of items +// faster than the redraw limit is over, the walk display shows every +// item completed while it waits for the next one. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestSpinnerShowsCountAfterBurst(t *testing.T) { + stderr := captureStderr(t) + p := newWalkSpinner(t) + + for range 50 { + p.increment() + } + + time.Sleep(spinnerIdle) + + // A terminal shows the last frame drawn. The library starts each + // frame with a carriage return and erases the previous one with + // spaces first. + var shown string + + for frame := range strings.SplitSeq(stderr(), "\r") { + if strings.TrimSpace(frame) != "" { + shown = frame + } + } + + if !strings.Contains(shown, "(50/-,") { + t.Errorf("terminal shows %q, want a count of 50", shown) + } +} From 33cf3dd29aee4721c154ff06c69833b20a7ec937 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 06:47:25 +0200 Subject: [PATCH 08/33] Stream report and trees instead of loading every record (closes #14) report now has SQLite group the records and put the rows in report order, helped by a new files_signature index on (size, head, tail, content), and writes each row as it reads it. trees reads the records in path order, where all the paths under a directory come together, so it computes each directory's digest as soon as the stream leaves it and keeps only its path, parent, digest and totals. Output is unchanged. The tests that called the removed in-memory grouping functions now group records stored in a database. New tests check that both commands give the same output whatever order the records were inserted in, and that a stdout failure partway through a long report is reported as one. Model: opus-5-5 --- README.md | 28 ++++-- TODO.md | 4 + db.go | 104 ++++++++++++++++++-- db_test.go | 35 ++----- report.go | 129 +++++++------------------ report_test.go | 159 ++++++++++++++++++++++++++++--- scan_test.go | 49 +++++----- trees.go | 254 +++++++++++++++++++++++++++++-------------------- trees_test.go | 111 +++++++++++++++++---- 9 files changed, 576 insertions(+), 297 deletions(-) diff --git a/README.md b/README.md index ae992d3..f9905fa 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,15 @@ Goals, in order: millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool under resilver). Holding one small record (path, size, mtime) per file in memory during a scan is acceptable; holding - every file's hashes is not (they stay in the database). + every file's hashes is not (they stay in the database). The + reporting commands do not hold every file's hashes either: `report` + lets SQLite group and order the records and writes each row as it + reads it, so its memory does not grow with the database, and + `trees` reads the records in path order and keeps each directory's + path, digest and totals, plus the hashes of only the files in the + directories holding the record being read, so its memory grows with + the number of directories and with the size of the largest + directory. 3. **Scan incrementally, analyze offline.** The expensive filesystem scan maintains a persistent database; an unchanged file is never read again on a rescan, except to compute its content hash once a @@ -184,7 +192,10 @@ All three subcommands operate on a single SQLite database file: starts while a report is still reading waits for it up to the 10-second busy timeout, then fails; a `scan` that ends while a report has the database open warns and leaves the database in WAL - mode until the next scan. + mode until the next scan. `report` writes each row as it reads it, + so it is still reading while its output is paused (a pager, a + stalled pipe), and a `scan` started then fails after the busy + timeout. - `report` and `trees` open the database read-only and need only read access to the database file, and no write access to its directory. While the database is in WAL mode they also read the `-wal` and @@ -202,6 +213,7 @@ All three subcommands operate on a single SQLite database file: tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples ) WITHOUT ROWID; + CREATE INDEX files_signature ON files (size, head, tail, content); ``` Paths are stored as BLOBs because Unix paths are raw bytes, not @@ -216,7 +228,8 @@ All three subcommands operate on a single SQLite database file: until the content phase of a scan (see "`scan` mode" below) has read the file. A record with an empty `content` is never part of a duplicate group, though it still defines the file for tree - reconstruction. + reconstruction. The `files_signature` index lets SQLite group the + records by signature for `report` without sorting the whole table. ### Duplicate detection @@ -444,9 +457,12 @@ arguments. **`report` must never touch the filesystem being analyzed.** It does not stat, open, or otherwise access any path that appears in the records; its -only I/O is reading the database and writing stdout/stderr. It must -produce identical output whether or not the scanned filesystem is still -mounted. +only I/O is reading the database, writing stdout/stderr, and the +temporary file SQLite sorts in when the duplicate rows do not fit in +memory. SQLite puts that file in `$SQLITE_TMPDIR` or `$TMPDIR` when set, +otherwise in `/var/tmp` (or `/tmp`), and deletes it as soon as it has +opened it. `report` must produce identical output whether or not the +scanned filesystem is still mounted. Processing: diff --git a/TODO.md b/TODO.md index 95fefb7..18e9bc3 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `report` and `trees` stream the records instead of holding them all in + memory; the schema gains the `files_signature` index (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/14) + - progress prints at once on a non-terminal, uses a real terminal test, and prints warnings through a spinner instead of racing its redraw (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/13) diff --git a/db.go b/db.go index 1c1ffac..7633ac0 100644 --- a/db.go +++ b/db.go @@ -51,6 +51,12 @@ CREATE TABLE files ( ) WITHOUT ROWID ` +// createIndexSQL indexes the records by signature, so report can have +// SQLite group them without sorting the whole table. +const createIndexSQL = ` +CREATE INDEX files_signature ON files (size, head, tail, content) +` + // upsertSQL inserts one file record, replacing any existing record for // the same path. const upsertSQL = ` @@ -256,6 +262,11 @@ func createSchema(ctx context.Context, db *sql.DB) error { return fmt.Errorf("create schema: %w", err) } + _, err = db.ExecContext(ctx, createIndexSQL) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + _, err = db.ExecContext(ctx, "PRAGMA user_version = "+strconv.Itoa(schemaVersion)) if err != nil { @@ -277,18 +288,18 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) { return v, nil } -// loadFileRows reads every record from the files table. -func loadFileRows(ctx context.Context, db *sql.DB) ([]scanRec, error) { +// loadFileRows streams every record to fn in path order: byte order, +// which is the order of the primary key, so SQLite does not sort. +func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error { rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, head, tail, content FROM files") + "SELECT path, size, mtime, head, tail, content FROM files "+ + "ORDER BY path") if err != nil { - return nil, fmt.Errorf("read records: %w", err) + return fmt.Errorf("read records: %w", err) } defer func() { _ = rows.Close() }() - var recs []scanRec - for rows.Next() { var ( path []byte @@ -298,19 +309,92 @@ func loadFileRows(ctx context.Context, db *sql.DB) ([]scanRec, error) { err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &r.content) if err != nil { - return nil, fmt.Errorf("read record: %w", err) + return fmt.Errorf("read record: %w", err) } r.path = string(path) - recs = append(recs, r) + fn(r) } err = rows.Err() if err != nil { - return nil, fmt.Errorf("read records: %w", err) + return fmt.Errorf("read records: %w", err) } - return recs, nil + return nil +} + +// dupeRowsSQL selects every record in a duplicate group, with the +// group's first path. A group is the records with a content hash that +// share a size, head, tail, and content, when there are two or more of +// them. The rows come in report order: groups by size descending, then +// by first path, and each group's paths ascending. +const dupeRowsSQL = ` +SELECT g.first, f.path, f.size +FROM files AS f +JOIN ( + SELECT size, head, tail, content, MIN(path) AS first + FROM files + WHERE content <> '' + GROUP BY size, head, tail, content + HAVING COUNT(*) > 1 +) AS g USING (size, head, tail, content) +ORDER BY f.size DESC, g.first, f.path +` + +// loadDupeRows streams the rows of dupeRowsSQL to fn and returns the +// number of records in the database. The count and the rows are read +// in one transaction, so they agree while a scan is committing. An +// error from fn stops the reading and is returned as it is. +func loadDupeRows(ctx context.Context, db *sql.DB, + fn func(first, path string, size int64) error, +) (int, error) { + // Everything goes through tx: the report connection is the only + // one, so a query on db would wait for tx forever. + tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + var records int + + err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + rows, err := tx.QueryContext(ctx, dupeRowsSQL) + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ( + first, path []byte + size int64 + ) + + err = rows.Scan(&first, &path, &size) + if err != nil { + return 0, fmt.Errorf("read record: %w", err) + } + + err = fn(string(first), string(path), size) + if err != nil { + return 0, err + } + } + + err = rows.Err() + if err != nil { + return 0, fmt.Errorf("read records: %w", err) + } + + return records, nil } // loadFileMeta streams every record's path, size, mtime, and whether diff --git a/db_test.go b/db_test.go index bc66c9c..7bbb2b8 100644 --- a/db_test.go +++ b/db_test.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "slices" - "strings" "testing" ) @@ -73,9 +72,8 @@ func TestOpenScanDatabaseCreates(t *testing.T) { defer func() { _ = db.Close() }() - recs, err := loadFileRows(t.Context(), db) - if err != nil || len(recs) != 0 { - t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err) + if recs := dbRecords(t, db); len(recs) != 0 { + t.Fatalf("records = %v, want none", recs) } } @@ -168,7 +166,7 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) { defer func() { _ = reportDB.Close() }() - _, err = loadFileRows(t.Context(), reportDB) + err = loadFileRows(t.Context(), reportDB, func(scanRec) {}) if err != nil { t.Fatalf("loadFileRows: %v", err) } @@ -196,15 +194,8 @@ func TestApplyChangesRoundTrip(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err := loadFileRows(t.Context(), db) - if err != nil { - t.Fatal(err) - } - - slices.SortFunc(got, func(a, b scanRec) int { - return strings.Compare(a.path, b.path) - }) - + // The records come back in path order, which is the order of recs. + got := dbRecords(t, db) if !slices.Equal(got, recs) { t.Fatalf("rows = %+v, want %+v", got, recs) } @@ -221,11 +212,7 @@ func TestApplyChangesRoundTrip(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err = loadFileRows(t.Context(), db) - if err != nil { - t.Fatal(err) - } - + got = dbRecords(t, db) if len(got) != 1 || got[0] != upd { t.Fatalf("rows = %+v, want just %+v", got, upd) } @@ -254,9 +241,8 @@ func TestApplyChangesBatching(t *testing.T) { t.Fatalf("applyChanges: %v", err) } - got, err := loadFileRows(t.Context(), db) - if err != nil || len(got) != n { - t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n) + if got := dbRecords(t, db); len(got) != n { + t.Fatalf("records = %d, want %d", len(got), n) } deletes := make([]string, 0, n) @@ -270,8 +256,7 @@ func TestApplyChangesBatching(t *testing.T) { t.Fatalf("applyChanges deletes: %v", err) } - got, err = loadFileRows(t.Context(), db) - if err != nil || len(got) != 0 { - t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err) + if got := dbRecords(t, db); len(got) != 0 { + t.Fatalf("records = %d, want 0", len(got)) } } diff --git a/report.go b/report.go index 68b4ecb..d2a0c7b 100644 --- a/report.go +++ b/report.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "slices" "strings" ) @@ -29,50 +28,21 @@ type scanRec struct { path string } -// loadRecords opens the database and reads every file record for the -// report and trees subcommands. Any database problem — including a -// missing database — is fatal. The error is returned rather than -// exiting, so that the deferred close always runs; the database is -// closed before the caller formats its output, so it stays closed even -// if that output fails. -func loadRecords(ctx context.Context) ([]scanRec, error) { +// runReport implements the report subcommand: it prints the file-level +// duplicates report as TSV on stdout. SQLite groups and orders the +// records, and each row is written as it is read, so no group is held +// in memory. It never touches the scanned filesystem; its only I/O is +// the database (with SQLite's temporary sort file), stdout, and stderr. +// Any database problem, including a missing database, is fatal. +func runReport(ctx context.Context, stdout io.Writer) error { dbPath := databasePath() db, err := openReportDatabase(ctx, dbPath) - if err != nil { - return nil, err - } - - defer func() { _ = db.Close() }() - - recs, err := loadFileRows(ctx, db) - if err != nil { - return nil, fmt.Errorf("database %s: %w", dbPath, err) - } - - return recs, nil -} - -// dupeGroup is one set of candidate-duplicate files: identical size, -// head hash, tail hash, and content hash. paths is sorted -// lexicographically; the first entry is the group's "first", the rest -// are dupes. -type dupeGroup struct { - size int64 - paths []string -} - -// runReport implements the report subcommand: it reads every record -// from the database and prints the file-level duplicates report as TSV -// on stdout. It never touches the scanned filesystem; its only I/O is -// the database, stdout, and stderr. -func runReport(ctx context.Context, stdout io.Writer) error { - recs, err := loadRecords(ctx) if err != nil { return err } - dupes := collectDupeGroups(recs) + defer func() { _ = db.Close() }() out := bufio.NewWriterSize(stdout, ioBufSize) @@ -81,21 +51,36 @@ func runReport(ctx context.Context, stdout io.Writer) error { return fmt.Errorf("write stdout: %w", err) } - dupeFiles := 0 + var ( + groups, dupeFiles int + reclaimable int64 + writeErr error + ) - var reclaimable int64 + records, err := loadDupeRows(ctx, db, + func(first, path string, size int64) error { + // A group's first path is its first row; every other + // path is a dupe. + if path == first { + groups++ - for _, g := range dupes { - for _, p := range g.paths[1:] { - _, err = fmt.Fprintf(out, "%s\t%s\t%d\n", - escapePath(g.paths[0]), escapePath(p), g.size) - if err != nil { - return fmt.Errorf("write stdout: %w", err) + return nil } + _, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n", + escapePath(first), escapePath(path), size) dupeFiles++ - reclaimable += g.size - } + reclaimable += size + + return writeErr + }) + + if writeErr != nil { + return fmt.Errorf("write stdout: %w", writeErr) + } + + if err != nil { + return fmt.Errorf("database %s: %w", dbPath, err) } err = out.Flush() @@ -106,57 +91,11 @@ func runReport(ctx context.Context, stdout io.Writer) error { fmt.Fprintf(os.Stderr, "report: %d records read, %d duplicate groups, %d dupe files, "+ "%s reclaimable\n", - len(recs), len(dupes), dupeFiles, humanBytes(reclaimable)) + records, groups, dupeFiles, humanBytes(reclaimable)) return nil } -// collectDupeGroups groups records by signature and returns every group -// with two or more paths, each group's paths sorted lexicographically, -// groups ordered by size descending then by first path ascending. -func collectDupeGroups(recs []scanRec) []dupeGroup { - groups := make(map[fileSig][]string) - - for _, r := range recs { - // A record without a content hash has unknown content and is - // never reported as a duplicate (README "Database"). - if r.content == "" { - continue - } - - k := fileSig{ - size: r.size, head: r.head, tail: r.tail, content: r.content, - } - groups[k] = append(groups[k], r.path) - } - - var dupes []dupeGroup - - for k, paths := range groups { - if len(paths) < minGroupSize { - continue - } - - slices.Sort(paths) - dupes = append(dupes, dupeGroup{size: k.size, paths: paths}) - } - - // Biggest reclaimable space first; ties broken by first path. - slices.SortFunc(dupes, func(a, b dupeGroup) int { - if a.size != b.size { - if a.size > b.size { - return -1 - } - - return 1 - } - - return strings.Compare(a.paths[0], b.paths[0]) - }) - - return dupes -} - // escapePath returns a path as it is written in a report column (README // "Report output format"): a backslash, tab, newline or carriage return // becomes \\, \t, \n or \r, and every other byte is kept as it is. diff --git a/report_test.go b/report_test.go index 9aa092d..796dbbf 100644 --- a/report_test.go +++ b/report_test.go @@ -2,10 +2,14 @@ package main import ( "bytes" + "database/sql" + "errors" + "fmt" "io" "os" "path/filepath" "slices" + "strings" "testing" ) @@ -47,6 +51,53 @@ func seedDatabase(t *testing.T, recs []scanRec) string { return path } +// dupeGroup is one duplicate group as report reads it: the size, and +// the paths in report order, first path first. +type dupeGroup struct { + size int64 + paths []string +} + +// dupeGroups returns the duplicate groups report reads from db, in +// report order. +func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup { + t.Helper() + + var groups []dupeGroup + + _, err := loadDupeRows(t.Context(), db, + func(first, path string, size int64) error { + if path == first { + groups = append(groups, dupeGroup{size: size}) + } + + g := &groups[len(groups)-1] + g.paths = append(g.paths, path) + + return nil + }) + if err != nil { + t.Fatal(err) + } + + return groups +} + +// dupeGroupsOf writes recs into a fresh database and returns the +// duplicate groups report reads from it. +func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup { + t.Helper() + + db := openTestDB(t) + + err := applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + return dupeGroups(t, db) +} + func TestRunReportEscapesPaths(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) @@ -65,6 +116,82 @@ func TestRunReportEscapesPaths(t *testing.T) { } } +func TestReportStdoutFailsWhileReading(t *testing.T) { + // Each row holds two paths longer than dir, so the report is more + // than twice the stdout buffer and stdout fails while rows are + // still being read, not at the final flush. + dir := "/" + strings.Repeat("d", 4096) + + recs := make([]scanRec, ioBufSize/len(dir)) + for i := range recs { + recs[i] = scanRec{ + size: 1, head: "h", tail: "t", content: "c", + path: fmt.Sprintf("%s/%d", dir, i), + } + } + + t.Setenv(databaseEnv, seedDatabase(t, recs)) + + err := runReport(t.Context(), failingWriter{}) + if !errors.Is(err, errWriteFailed) || + !strings.HasPrefix(err.Error(), "write stdout: ") { + t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed) + } +} + +func TestRunReportsIgnoreInsertionOrder(t *testing.T) { + // README §Constraints: identical database contents give identical + // output, whatever order the records were inserted in. + recs := append(smokeTreeRecs(), awkwardPairRecs()...) + recs = append(recs, + scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"}, + scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"}, + scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"}, + scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"}, + scanRec{size: 50, path: "/x/unhashed"}, + ) + + reversed := slices.Clone(recs) + slices.Reverse(reversed) + + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + t.Setenv(databaseEnv, seedDatabase(t, recs)) + + forward := runStdout(t, name) + + t.Setenv(databaseEnv, seedDatabase(t, reversed)) + + backward := runStdout(t, name) + + if strings.Count(forward, "\n") < 3 { + t.Errorf("stdout = %q, want at least two rows", forward) + } + + if forward != backward { + t.Errorf("stdout depends on insertion order: %q vs %q", + forward, backward) + } + }) + } +} + +// runStdout runs the subcommand name and returns its stdout, failing +// the test unless it succeeds. +func runStdout(t *testing.T, name string) string { + t.Helper() + + var stdout, stderr bytes.Buffer + + code := run([]string{name}, &stdout, &stderr) + if code != exitOK { + t.Fatalf("run(%s) = %d, want %d; stderr: %s", + name, code, exitOK, stderr.String()) + } + + return stdout.String() +} + func TestEscapePath(t *testing.T) { t.Parallel() @@ -121,7 +248,7 @@ func TestWarnfEscapes(t *testing.T) { } } -func TestCollectDupeGroups(t *testing.T) { +func TestDupeGroups(t *testing.T) { t.Parallel() recs := []scanRec{ @@ -136,7 +263,7 @@ func TestCollectDupeGroups(t *testing.T) { {size: 7, head: "u", tail: "u", content: "u", path: "/lonely"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } @@ -154,7 +281,7 @@ func TestCollectDupeGroups(t *testing.T) { } } -func TestCollectDupeGroupsContentSeparates(t *testing.T) { +func TestDupeGroupsContentSeparates(t *testing.T) { t.Parallel() // Same size, head, and tail, but different content hashes: the final @@ -169,7 +296,7 @@ func TestCollectDupeGroupsContentSeparates(t *testing.T) { {size: 100, head: "h", tail: "t", path: "/e"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1 (only the matching content)", len(groups)) @@ -180,7 +307,7 @@ func TestCollectDupeGroupsContentSeparates(t *testing.T) { } } -func TestCollectDupeGroupsMtimeExcluded(t *testing.T) { +func TestDupeGroupsMtimeExcluded(t *testing.T) { t.Parallel() // mtime is informational only; records differing only in mtime @@ -190,23 +317,25 @@ func TestCollectDupeGroupsMtimeExcluded(t *testing.T) { {size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1", len(groups)) } } -func TestCollectDupeGroupsTieBreak(t *testing.T) { +func TestDupeGroupsTieBreak(t *testing.T) { t.Parallel() + // The hashes sort opposite to the first paths, so ordering the + // groups by hash instead of by first path fails this test. recs := []scanRec{ - {size: 50, head: "b", tail: "b", content: "b", path: "/beta/2"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/beta/1"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/alpha/2"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/alpha/1"}, + {size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"}, + {size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"}, + {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"}, + {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"}, } - groups := collectDupeGroups(recs) + groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } @@ -218,7 +347,7 @@ func TestCollectDupeGroupsTieBreak(t *testing.T) { } } -func TestCollectDupeGroupsDeterministic(t *testing.T) { +func TestDupeGroupsDeterministic(t *testing.T) { t.Parallel() recs := []scanRec{ @@ -228,12 +357,12 @@ func TestCollectDupeGroupsDeterministic(t *testing.T) { {size: 2, head: "b", tail: "b", content: "b", path: "/q/2"}, } - forward := collectDupeGroups(recs) + forward := dupeGroupsOf(t, recs) reversed := slices.Clone(recs) slices.Reverse(reversed) - backward := collectDupeGroups(reversed) + backward := dupeGroupsOf(t, reversed) if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool { return a.size == b.size && slices.Equal(a.paths, b.paths) }) { diff --git a/scan_test.go b/scan_test.go index 249989a..3b4e73b 100644 --- a/scan_test.go +++ b/scan_test.go @@ -400,7 +400,7 @@ func TestScanContentGate(t *testing.T) { } } - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, same) { t.Fatalf("groups = %+v, want only the identical pair %q", groups, same) @@ -435,7 +435,7 @@ func TestScanContentAcrossOperands(t *testing.T) { want := []string{a, b} slices.Sort(want) - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q", groups, want) } @@ -462,7 +462,7 @@ func TestScanContentWithinOperand(t *testing.T) { want := []string{stored, added} - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q", groups, want) } @@ -520,7 +520,7 @@ func TestScanContentStalePartners(t *testing.T) { } } - if groups := collectDupeGroups(recs); len(groups) != 0 { + if groups := dupeGroups(t, db); len(groups) != 0 { t.Errorf("groups = %+v, want none", groups) } } @@ -571,7 +571,7 @@ func TestScanContentHashedStalePartners(t *testing.T) { // The stored records lie outside the operand and are left as they // are, so they still group with each other, but not with the copy. - groups := collectDupeGroups(recs) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, stored) { t.Errorf("groups = %+v, want only the stored pair %q", groups, stored) } @@ -620,7 +620,7 @@ func TestScanContentReadFailure(t *testing.T) { want := []string{a, b} slices.Sort(want) - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 || !slices.Equal(groups[0].paths, want) { t.Fatalf("groups = %+v, want the pair %q after the retry", groups, want) @@ -956,11 +956,16 @@ func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { return st } -// dbRecords returns every record currently in the database. +// dbRecords returns every record currently in the database, in path +// order. func dbRecords(t *testing.T, db *sql.DB) []scanRec { t.Helper() - recs, err := loadFileRows(t.Context(), db) + var recs []scanRec + + err := loadFileRows(t.Context(), db, func(r scanRec) { + recs = append(recs, r) + }) if err != nil { t.Fatal(err) } @@ -997,10 +1002,10 @@ func recordPaths(recs []scanRec) []string { // assertSmokeDupeGroups checks the file-level duplicate groups for the // smoke tree rooted at dir. -func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { +func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) { t.Helper() - groups := collectDupeGroups(parsed) + groups := dupeGroups(t, db) if len(groups) != 5 { t.Fatalf("len(groups) = %d, want 5", len(groups)) } @@ -1025,11 +1030,10 @@ func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) { // assertSmokeTreeGroups checks the duplicate-tree groups for the smoke // tree rooted at dir. -func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) { +func assertSmokeTreeGroups(t *testing.T, dir string, db *sql.DB) { t.Helper() - super, dirs := buildHierarchy(parsed) - super.compute() + super, dirs := dbTree(t, db) tg := collectTreeGroups(dirs, super) if len(tg) != 1 { @@ -1063,8 +1067,8 @@ func TestScanPipeline(t *testing.T) { t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles) } - assertSmokeDupeGroups(t, dir, parsed) - assertSmokeTreeGroups(t, dir, parsed) + assertSmokeDupeGroups(t, dir, db) + assertSmokeTreeGroups(t, dir, db) } func TestSyncScanUnchangedReuse(t *testing.T) { @@ -1298,7 +1302,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { } } - if groups := collectDupeGroups(recs); len(groups) != 0 { + if groups := dupeGroups(t, db); len(groups) != 0 { t.Fatalf("groups = %+v, want none from unhashed records", groups) } @@ -1313,7 +1317,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { st) } - groups := collectDupeGroups(dbRecords(t, db)) + groups := dupeGroups(t, db) if len(groups) != 1 { t.Fatalf("groups = %+v, want the a/c pair", groups) } @@ -1349,8 +1353,7 @@ func TestTreesUnhashedNeverEqual(t *testing.T) { } for name, unknown := range cases { - super, dirs := buildHierarchy(append(slices.Clone(shared), unknown...)) - super.compute() + super, dirs := treeOf(t, append(slices.Clone(shared), unknown...)) if tg := collectTreeGroups(dirs, super); len(tg) != 0 { t.Errorf("%s: tree groups = %d, want 0 (the files may differ)", @@ -1387,7 +1390,7 @@ func TestScanHardlinksReadOnce(t *testing.T) { t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb) } - if groups := collectDupeGroups(recs); len(groups) != 1 { + if groups := dupeGroups(t, db); len(groups) != 1 { t.Fatalf("groups = %+v, want the hardlink pair", groups) } } @@ -1628,10 +1631,8 @@ func TestReportsNeverTouchFilesystem(t *testing.T) { t.Fatal(err) } - recs := dbRecords(t, db) - - assertSmokeDupeGroups(t, dir, recs) - assertSmokeTreeGroups(t, dir, recs) + assertSmokeDupeGroups(t, dir, db) + assertSmokeTreeGroups(t, dir, db) } func TestUnderRoot(t *testing.T) { diff --git a/trees.go b/trees.go index 735c5b5..5c1bbf7 100644 --- a/trees.go +++ b/trees.go @@ -12,39 +12,48 @@ import ( "strings" ) -// fileSig is a file's duplicate signature; mtime is excluded. -type fileSig struct { - size int64 - head string - tail string - content string -} - -// treeNode is one directory reconstructed from the scan stream. +// treeNode is one directory reconstructed from the record paths. type treeNode struct { - path string - parent *treeNode - dirs map[string]*treeNode - files map[string]fileSig + path string + parent *treeNode + // entries holds the serialized child entries until the digest is + // computed from them, and is then dropped. + entries []string digest [sha256.Size]byte fileCount int64 totalSize int64 } // runTrees implements the trees subcommand: it reads every record from -// the database, reconstructs the directory hierarchy from the record -// paths, computes a Merkle-style digest per directory, and prints -// maximal duplicate-tree groups as TSV on stdout. It never touches the -// scanned filesystem; its only I/O is the database, stdout, and -// stderr. +// the database in path order, reconstructs the directory hierarchy from +// the record paths, computes a Merkle-style digest per directory, and +// prints maximal duplicate-tree groups as TSV on stdout. It never +// touches the scanned filesystem; its only I/O is the database, stdout, +// and stderr. Any database problem, including a missing database, is +// fatal. func runTrees(ctx context.Context, stdout io.Writer) error { - recs, err := loadRecords(ctx) + dbPath := databasePath() + + db, err := openReportDatabase(ctx, dbPath) if err != nil { return err } - super, allDirs := buildHierarchy(recs) - super.compute() + defer func() { _ = db.Close() }() + + records := 0 + tree := newTreeBuilder() + + err = loadFileRows(ctx, db, func(r scanRec) { + records++ + + tree.add(r) + }) + if err != nil { + return fmt.Errorf("database %s: %w", dbPath, err) + } + + super, allDirs := tree.finish() dupes := collectTreeGroups(allDirs, super) @@ -82,73 +91,128 @@ func runTrees(ctx context.Context, stdout io.Writer) error { fmt.Fprintf(os.Stderr, "trees: %d records read, %d duplicate tree groups, %d dupe trees, "+ "%s reclaimable\n", - len(recs), len(dupes), dupeTrees, humanBytes(reclaimable)) + records, len(dupes), dupeTrees, humanBytes(reclaimable)) return nil } -// buildHierarchy reconstructs the directory hierarchy from the record -// paths under a synthetic super-root. Paths are split on "/"; for -// absolute paths the first component is empty, which becomes the -// top-level node with path "/". It returns the super-root and every -// directory node created. -func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) { +// treeBuilder reconstructs the directory hierarchy from records added +// in path order, under a synthetic super-root. Paths are split on "/"; +// for absolute paths the first component is empty, which becomes the +// top-level directory with path "/". In path order all the paths under +// one directory come together, so a directory is complete once a path +// outside it is added: its digest is computed then and its entries are +// dropped. Only the directories holding the latest path keep entries. +type treeBuilder struct { + super *treeNode + // open lists the directories holding the latest path, outermost + // first, starting with the super-root; names[i] is open[i]'s name. + open []*treeNode + names []string + // dirs lists every completed directory. + dirs []*treeNode +} + +func newTreeBuilder() *treeBuilder { super := &treeNode{} - var allDirs []*treeNode + return &treeBuilder{ + super: super, + open: []*treeNode{super}, + names: []string{""}, + } +} - for _, r := range recs { - comps := strings.Split(r.path, "/") +// add adds one record. Each record must come after the previous one in +// path order (byte order); otherwise a completed directory would be +// started again as a second directory with the same path. +func (b *treeBuilder) add(r scanRec) { + comps := strings.Split(r.path, "/") + dirNames, name := comps[:len(comps)-1], comps[len(comps)-1] - node := super - for _, c := range comps[:len(comps)-1] { - child := node.dirs[c] - if child == nil { - childPath := node.path + "/" + c - - // The root directory's path is "/", not empty, and its - // children's paths start with one slash, not two. - switch { - case node == super && c == "": - childPath = "/" - case node == super: - childPath = c - case node.path == "/": - childPath = "/" + c - } - - child = &treeNode{path: childPath, parent: node} - if node.dirs == nil { - node.dirs = make(map[string]*treeNode) - } - - node.dirs[c] = child - allDirs = append(allDirs, child) - } - - node = child - } - - if node.files == nil { - node.files = make(map[string]fileSig) - } - - sig := fileSig{ - size: r.size, head: r.head, tail: r.tail, content: r.content, - } - - // A record without a content hash has unknown content (README - // "Database"): give it a signature no other file can share, so - // trees containing it never compare equal. Real hashes are - // hex, so the NUL-prefixed form cannot collide. - if sig.content == "" { - sig.content = "unhashed\x00" + r.path - } - - node.files[comps[len(comps)-1]] = sig + // Keep the open directories that hold this path; complete the rest. + depth := 1 + for depth < len(b.open) && depth <= len(dirNames) && + b.names[depth] == dirNames[depth-1] { + depth++ } - return super, allDirs + b.closeTo(depth) + + for _, c := range dirNames[depth-1:] { + b.openDir(c) + } + + dir := b.open[len(b.open)-1] + dir.entries = append(dir.entries, fileEntry(name, r)) + dir.fileCount++ + dir.totalSize += r.size +} + +// openDir starts the directory called name inside the innermost open +// one. +func (b *treeBuilder) openDir(name string) { + parent := b.open[len(b.open)-1] + path := parent.path + "/" + name + + // The root directory's path is "/", not empty, and its children's + // paths start with one slash, not two. + switch { + case parent == b.super && name == "": + path = "/" + case parent == b.super: + path = name + case parent.path == "/": + path = "/" + name + } + + b.open = append(b.open, &treeNode{path: path, parent: parent}) + b.names = append(b.names, name) +} + +// closeTo completes the open directories after the first n, innermost +// first: each one's digest is computed and entered in its parent along +// with its totals. +func (b *treeBuilder) closeTo(n int) { + for len(b.open) > n { + last := len(b.open) - 1 + dir, name := b.open[last], b.names[last] + b.open, b.names = b.open[:last], b.names[:last] + + dir.computeDigest() + + dir.parent.entries = append(dir.parent.entries, + "d\x00"+name+"\x00"+string(dir.digest[:])) + dir.parent.fileCount += dir.fileCount + dir.parent.totalSize += dir.totalSize + b.dirs = append(b.dirs, dir) + } +} + +// finish completes every open directory and returns the super-root and +// every directory. +func (b *treeBuilder) finish() (*treeNode, []*treeNode) { + b.closeTo(1) + + return b.super, b.dirs +} + +// fileEntry serializes a file child for its directory's digest: its +// name and its signature (size, head, tail, content); mtime is +// excluded. +func fileEntry(name string, r scanRec) string { + content := r.content + + // A record without a content hash has unknown content (README + // "Database"): give it a signature no other file can share, so + // trees containing it never compare equal. Real hashes are hex, so + // the NUL-prefixed form cannot collide. + if content == "" { + content = "unhashed\x00" + r.path + } + + return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) + + "\x00" + r.head + "\x00" + r.tail + "\x00" + content } // collectTreeGroups groups directories by digest and returns every @@ -190,38 +254,22 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode { return dupes } -// compute fills in digest, fileCount, and totalSize for n and all of -// its descendants. A directory's digest is the SHA-256 of its child -// entries — files serialized with name and signature, subdirectories -// with name and recursive digest — sorted byte-lexicographically. -// Filenames cannot contain NUL or "/", so NUL delimiters are -// unambiguous. -func (n *treeNode) compute() { - entries := make([]string, 0, len(n.dirs)+len(n.files)) - for name, sig := range n.files { - entries = append(entries, - "f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+ - "\x00"+sig.head+"\x00"+sig.tail+"\x00"+sig.content) - n.fileCount++ - n.totalSize += sig.size - } - - for name, child := range n.dirs { - child.compute() - entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:])) - n.fileCount += child.fileCount - n.totalSize += child.totalSize - } - - slices.Sort(entries) +// computeDigest sets n's digest and drops its entries. A directory's +// digest is the SHA-256 of its child entries — files serialized with +// name and signature, subdirectories with name and recursive digest — +// sorted byte-lexicographically. Filenames cannot contain NUL or "/", +// so NUL delimiters are unambiguous. +func (n *treeNode) computeDigest() { + slices.Sort(n.entries) h := sha256.New() - for _, e := range entries { + for _, e := range n.entries { h.Write([]byte(e)) h.Write([]byte{0}) } copy(n.digest[:], h.Sum(nil)) + n.entries = nil } // suppressed reports whether a duplicate-tree group is non-maximal: its diff --git a/trees_test.go b/trees_test.go index 5cc8817..969c438 100644 --- a/trees_test.go +++ b/trees_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "database/sql" "slices" "testing" ) @@ -30,6 +31,36 @@ func smokeTreeRecs() []scanRec { } } +// dbTree builds the directory hierarchy from the records in db the way +// trees does, and returns the super-root and every directory. +func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) { + t.Helper() + + tree := newTreeBuilder() + + err := loadFileRows(t.Context(), db, tree.add) + if err != nil { + t.Fatal(err) + } + + return tree.finish() +} + +// treeOf writes recs into a fresh database and builds the directory +// hierarchy from it the way trees does. +func treeOf(t *testing.T, recs []scanRec) (*treeNode, []*treeNode) { + t.Helper() + + db := openTestDB(t) + + err := applyChanges(t.Context(), db, recs, nil, nil) + if err != nil { + t.Fatal(err) + } + + return dbTree(t, db) +} + // nodeByPath finds the directory node with the given path. func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode { t.Helper() @@ -60,11 +91,10 @@ func groupPaths(groups [][]*treeNode) [][]string { return out } -func TestBuildHierarchyCounts(t *testing.T) { +func TestTreeCounts(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + _, dirs := treeOf(t, smokeTreeRecs()) d := nodeByPath(t, dirs, "/d") if d.fileCount != 6 || d.totalSize != 9300 { @@ -85,12 +115,12 @@ func TestBuildHierarchyCounts(t *testing.T) { } } -func TestBuildHierarchyRootPath(t *testing.T) { +func TestTreeRootPath(t *testing.T) { t.Parallel() // The root directory's path is "/", never empty, and its // children's paths start with a single slash. - _, dirs := buildHierarchy([]scanRec{{path: "/f"}, {path: "/srv/g"}}) + _, dirs := treeOf(t, []scanRec{{path: "/f"}, {path: "/srv/g"}}) got := make([]string, 0, len(dirs)) for _, d := range dirs { @@ -105,6 +135,56 @@ func TestBuildHierarchyRootPath(t *testing.T) { } } +func TestTreeNamesSortingBeforeSlash(t *testing.T) { + t.Parallel() + + // In path order "/a/b-x/f" and "/a/b.txt" come between the file + // "/a/b" and "/a/b/f", because "-" and "." sort before "/". Each + // directory must still be built once, whole, so /a matches /c. + recs := make([]scanRec, 0, 8) + + for _, top := range []string{"/a", "/c"} { + for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} { + content := "c" + if p == "/b-x/f" { + content = "other" + } + + recs = append(recs, scanRec{ + size: 1, head: "h", tail: "t", content: content, path: top + p, + }) + } + } + + super, dirs := treeOf(t, recs) + + got := make([]string, 0, len(dirs)) + for _, d := range dirs { + got = append(got, d.path) + } + + slices.Sort(got) + + want := []string{"/", "/a", "/a/b", "/a/b-x", "/c", "/c/b", "/c/b-x"} + if !slices.Equal(got, want) { + t.Fatalf("directory paths = %q, want %q", got, want) + } + + groups := collectTreeGroups(dirs, super) + + gotGroups := groupPaths(groups) + wantGroups := [][]string{{"/a", "/c"}} + + if !slices.EqualFunc(gotGroups, wantGroups, slices.Equal) { + t.Fatalf("groups = %v, want %v", gotGroups, wantGroups) + } + + if groups[0][0].fileCount != 4 || groups[0][0].totalSize != 4 { + t.Errorf("group totals: %d files %d bytes, want 4 4", + groups[0][0].fileCount, groups[0][0].totalSize) + } +} + func TestRunTreesEscapesPaths(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) @@ -126,8 +206,7 @@ func TestRunTreesEscapesPaths(t *testing.T) { func TestTreeDigests(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + _, dirs := treeOf(t, smokeTreeRecs()) t1 := nodeByPath(t, dirs, "/d/t1") t2 := nodeByPath(t, dirs, "/d/t2") @@ -160,8 +239,7 @@ func TestTreeDigestContentSensitivity(t *testing.T) { {size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + _, dirs := treeOf(t, recs) a := nodeByPath(t, dirs, "/r/a") b := nodeByPath(t, dirs, "/r/b") @@ -174,8 +252,7 @@ func TestTreeDigestContentSensitivity(t *testing.T) { func TestCollectTreeGroupsMaximal(t *testing.T) { t.Parallel() - super, dirs := buildHierarchy(smokeTreeRecs()) - super.compute() + super, dirs := treeOf(t, smokeTreeRecs()) groups := collectTreeGroups(dirs, super) @@ -199,16 +276,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) { recs := smokeTreeRecs() - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) forward := groupPaths(collectTreeGroups(dirs, super)) reversed := slices.Clone(recs) slices.Reverse(reversed) - superR, dirsR := buildHierarchy(reversed) - superR.compute() + superR, dirsR := treeOf(t, reversed) backward := groupPaths(collectTreeGroups(dirsR, superR)) if !slices.EqualFunc(forward, backward, slices.Equal) { @@ -227,8 +302,7 @@ func TestCollectTreeGroupsSiblings(t *testing.T) { {size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) got := groupPaths(collectTreeGroups(dirs, super)) @@ -250,8 +324,7 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) { {size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"}, } - super, dirs := buildHierarchy(recs) - super.compute() + super, dirs := treeOf(t, recs) got := groupPaths(collectTreeGroups(dirs, super)) From 4847882e462c48e536af856d34d3c545c6a423ae Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 09:30:38 +0200 Subject: [PATCH 09/33] Stop scan cleanly on SIGINT or SIGTERM (closes #5) A first SIGINT or SIGTERM cancels the scan. It commits the hashed records still in its batch, with a context that is not cancelled for that one write, and starts no other write or deletion; deletions need a complete walk, so records under paths an interrupted walk never reached are kept. A batch whose commit failed is now kept for that final commit instead of dropped. The progress display is finished (a bar stopped short is no longer filled up), `scan: interrupted after N files` goes to stderr, and the exit code is 1. A second signal ends the process at once. A SIGINT inherited as ignored stays ignored. Model: opus-5-5 --- README.md | 28 ++++- TODO.md | 4 + cancel_test.go | 260 ++++++++++++++++++++++++++++++++++++++++++++--- main.go | 6 ++ progress.go | 10 +- progress_test.go | 40 ++++++-- scan.go | 119 ++++++++++++++++++---- scan_test.go | 41 +++++--- 8 files changed, 446 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index f9905fa..52668c9 100644 --- a/README.md +++ b/README.md @@ -184,8 +184,9 @@ All three subcommands operate on a single SQLite database file: (keeping the WAL small and letting concurrent reports observe progress), so a report may see a scan's changes partially applied, and a scan that dies partway leaves a valid database holding - everything hashed so far; the next scan skips those records and - converges toward the filesystem. + every batch committed so far (an interrupted scan also commits the + batch in progress, see "Error handling and exit codes"); the next + scan skips those records and converges toward the filesystem. - `scan` switches the database back to rollback-journal mode when it closes it, so between scans the database file alone holds the whole database. Each switch needs the database to itself: a `scan` that @@ -616,6 +617,8 @@ Additional requirements: waits for its next item. - A warning printed during a phase always lands on a line of its own, never inside the progress display. +- A bar whose phase stops short of its total, as an interrupted one + does, is left as last drawn rather than filled up. - `report` and `trees` modes need no progress display, only their stderr summaries. @@ -625,7 +628,8 @@ Additional requirements: - `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is already running against the same database, the database cannot be created/opened/read/written, a missing database for - `report`/`trees`, stdout write failure). + `report`/`trees`, stdout write failure), or a `scan` stopped by + `SIGINT` or `SIGTERM` (see below). - `2`: usage error (including `scan` with no `PATH` operand and `report`/`trees` with any positional argument). @@ -641,6 +645,24 @@ stderr and exits 1. Two cases never reach sfdupes as a failed write: the output is discarded and the run succeeds, as with `> /dev/null`. +`scan` stops cleanly on `SIGINT` (Ctrl-C) or `SIGTERM`. Its workers +stop taking work, each finishing at most the directory listing or file +it is reading; the progress display is finished; and the records it has +hashed but not yet committed are committed, so the next scan does not +hash them again. Apart from that commit it starts no further writes or +deletions: records are deleted only after a complete walk, so those +under paths an interrupted walk never reached are kept. The database is +closed and the lock released as on any other exit, the line +`scan: interrupted after N files` goes to stderr, N being the number of +files the walk reached, and the exit code is 1. The next scan skips the +records already written and converges as usual. + +After the first signal `scan` stops catching them, so a second one ends +it at once, as an uncaught signal does: the records not yet committed +are lost, and the database is left valid, as when any scan dies (see +"Database"). A `SIGINT` that `scan` inherits as ignored, as a script's +background job does, stays ignored. + ## Entrypoints This repository adheres to the diff --git a/TODO.md b/TODO.md index 18e9bc3..2dd0179 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has + hashed, deletes nothing more, exits 1 (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/5) + - `report` and `trees` stream the records instead of holding them all in memory; the schema gains the `files_signature` index (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/14) diff --git a/cancel_test.go b/cancel_test.go index f8c18d6..e4ecd29 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -4,11 +4,16 @@ import ( "context" "database/sql" "errors" + "fmt" "os" + "os/signal" "path/filepath" + "slices" "strconv" + "strings" "sync" "sync/atomic" + "syscall" "testing" "time" ) @@ -150,9 +155,8 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) { // Every one of those records would look vanished to the update phase. // The guard is what stops the scan there, and this test is what // notices if it stops doing so: deleting the guard, or making it -// unreachable, makes the scan carry its truncated view into a later -// phase and fail there instead, with a wrapped error rather than the -// bare cancellation. +// unreachable, makes the scan carry its truncated view into the update +// phase, which counts every record the walk never reached for removal. // //nolint:paralleltest // counts goroutines: must not run beside others func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { @@ -182,10 +186,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { // assertWalkGuardAborted checks that the scan stopped at the post-walk // guard: with a census that is neither empty (the walk really ran) -// nor complete (it really was cut short), and with the guard's own -// bare cancellation as the error. A wrapped error means the partial -// census was carried past the guard into the hash or update phase, -// which is the failure this test exists to catch. +// nor complete (it really was cut short), and with no record counted +// for removal. A removal count means the partial census was carried +// past the guard into the update phase, which is the failure this test +// exists to catch. func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { t.Helper() @@ -194,12 +198,6 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { err, context.Canceled) } - if errors.Unwrap(err) != nil { - t.Errorf("syncScan reported %q, want the guard's bare "+ - "cancellation: a wrapped error means the truncated census "+ - "reached a later phase", err) - } - if st.unchanged == 0 { t.Fatalf("stats = %+v: the census is empty, so the walk never "+ "ran and the guard was reached for the wrong reason", st) @@ -260,6 +258,242 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) { assertRecordsIntact(t, db, before) } +// hashCancelAtDone is the consultation on which the mid-hash test's +// context cancels itself. The walk of buildWalkCancelTree spends about +// one per file and three per directory, and the hash phase then one per +// file hashed, so this lands about half way through the hash phase. +const hashCancelAtDone = walkCancelFiles + 3*walkCancelDirs + + walkCancelFiles/2 + +// TestSyncScanCancelledMidHashKeepsHashedRecords cancels a first scan +// part-way through its hash phase. The fixture holds fewer files than a +// batch, so every file hashed is still waiting to be committed: the scan +// must commit them all before it returns, and the next scan must hash +// only the rest. +func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) { + t.Parallel() + + dir := buildWalkCancelTree(t) + db := openTestDB(t) + + st, err := syncScan(newWalkClock(hashCancelAtDone), db, + []string{dir}, walkCancelWorkers, false) + if !errors.Is(err, context.Canceled) { + t.Fatalf("syncScan cancelled mid-hash = %v, want %v", + err, context.Canceled) + } + + if st.walked != walkCancelFiles || st.added == 0 || + st.added >= walkCancelFiles { + t.Fatalf("stats = %+v: want the walk complete and the hash phase "+ + "cut short", st) + } + + if got := len(dbRecords(t, db)); got != st.added { + t.Errorf("%d records after the cancelled scan, want the %d it hashed", + got, st.added) + } + + hashed := st.added + + st = syncTree(t, db, dir) + if st.added != walkCancelFiles-hashed || st.unchanged != hashed { + t.Errorf("next scan stats = %+v, want %d added %d unchanged", + st, walkCancelFiles-hashed, hashed) + } +} + +// storedPaths opens the database at path as report does, which fails +// unless it is a valid database, and returns its records' paths. +func storedPaths(t *testing.T, path string) []string { + t.Helper() + + db, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + return recordPaths(dbRecords(t, db)) +} + +// TestRunScanInterrupted calls the scan entrypoint with a context that +// is already cancelled, as when a signal arrives at once. It must return +// errInterrupted promptly with its one line on stderr, leave the +// database valid and as it was, and leave nothing in the way of the +// next scan, which must bring the database up to date. +func TestRunScanInterrupted(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + stderr := captureStderr(t) + dir := buildSmokeTree(t) + + err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + if err != nil { + t.Fatal(err) + } + + before := storedPaths(t, path) + + // A vanished file and a new one: the interrupted scan records + // neither. + gone := filepath.Join(dir, "a", "unique.bin") + + err = os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "a/new.bin", pattern(50, 10)) + shown := len(stderr()) + done := make(chan struct{}) + + go func() { + defer close(done) + + err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, + false) + }() + + awaitReturn(t, done, "runScan") + + if !errors.Is(err, errInterrupted) { + t.Fatalf("runScan on a cancelled context = %v, want %v", + err, errInterrupted) + } + + want := "scan: interrupted after 0 files\n" + if got := stderr()[shown:]; got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + assertNoSidecars(t, path) + + if got := storedPaths(t, path); !slices.Equal(got, before) { + t.Errorf("records = %q after the interrupted scan, want %q", + got, before) + } + + err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + if err != nil { + t.Fatal(err) + } + + got := storedPaths(t, path) + if slices.Contains(got, gone) || !slices.Contains(got, added) { + t.Errorf("records = %q after the next scan, want %q gone and %q "+ + "added", got, gone, added) + } +} + +// TestRunScanInterruptedMidHash interrupts the scan entrypoint part-way +// through its hash phase, after the database is open. It must return +// errInterrupted, release the lock, end stderr with its line counting +// every file the walk reached, close the database out of WAL mode, and +// keep the records it hashed. +func TestRunScanInterruptedMidHash(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + stderr := captureStderr(t) + dir := buildWalkCancelTree(t) + + err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, + walkCancelWorkers, false) + if !errors.Is(err, errInterrupted) { + t.Fatalf("runScan interrupted mid-hash = %v, want %v", + err, errInterrupted) + } + + holdScanLock(t, path) + + want := fmt.Sprintf("scan: interrupted after %d files\n", walkCancelFiles) + if got := stderr(); !strings.HasSuffix(got, want) { + t.Errorf("stderr = %q, want it to end with %q", got, want) + } + + assertNoSidecars(t, path) + + db, err := openReportDatabase(t.Context(), path) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + // A plain close also removes the sidecars, but leaves WAL mode on. + var mode string + + err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&mode) + if err != nil { + t.Fatal(err) + } + + if mode != "delete" { + t.Errorf("journal mode = %q after the interrupted scan, want %q", + mode, "delete") + } + + kept := len(dbRecords(t, db)) + if kept == 0 || kept >= walkCancelFiles { + t.Errorf("%d records after the interrupted scan, want those it "+ + "hashed: some but not all of the %d files", kept, walkCancelFiles) + } +} + +// TestInterruptContextCatchesSIGTERM sends SIGTERM to the test process +// while the scan's handler is installed, and checks that it cancels the +// scan's context. +// +//nolint:paralleltest // signals the whole process: must not run beside a scan +func TestInterruptContextCatchesSIGTERM(t *testing.T) { + // Caught here as well, so that a handler that misses SIGTERM fails + // this test instead of ending the test process. + caught := make(chan os.Signal, 1) + signal.Notify(caught, syscall.SIGTERM) + + defer signal.Stop(caught) + + ctx, stop := interruptContext(t.Context()) + defer stop() + + err := syscall.Kill(os.Getpid(), syscall.SIGTERM) + if err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + case <-time.After(poolUnwind): + t.Fatal("SIGTERM did not cancel the scan's context") + } +} + +// TestCommitFullBatchKeepsFailedBatch checks that a full batch whose +// commit fails, as it does once the scan is interrupted, stays in the +// batch, so that syncScan's final commit saves it. +func TestCommitFullBatchKeepsFailedBatch(t *testing.T) { + t.Parallel() + + s := &scanState{db: openTestDB(t)} + for i := range updateBatchSize { + s.batch = append(s.batch, scanRec{path: "/f" + strconv.Itoa(i)}) + } + + err := s.commitFullBatch(cancelledContext(t)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("commitFullBatch on a cancelled context = %v, want %v", + err, context.Canceled) + } + + if len(s.batch) != updateBatchSize { + t.Errorf("batch holds %d records after the failed commit, want %d", + len(s.batch), updateBatchSize) + } +} + // drainClosed counts the values received from ch until it closes, // failing the test if it does not close within poolUnwind. A pool that // ignored its cancellation leaves its channel open with its goroutines diff --git a/main.go b/main.go index 3a8ce57..f1c54df 100644 --- a/main.go +++ b/main.go @@ -87,6 +87,9 @@ func run(args []string, stdout, stderr io.Writer) int { switch { case err == nil: return exitOK + case errors.Is(err, errInterrupted): + // The interrupted scan has printed its own line. + return exitFatal case errors.As(err, &fatal): // The command ran and failed: a runtime error, reported // without the usage text that a usage error gets. @@ -132,6 +135,9 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { Short: "Walk trees and synchronize the scan database", Args: cobra.MinimumNArgs(1), RunE: runE(func(ctx context.Context, args []string) error { + ctx, stop := interruptContext(ctx) + defer stop() + return runScan(ctx, args, scanWorkers, scanOneFS) }), } diff --git a/progress.go b/progress.go index e6ca868..bc042e8 100644 --- a/progress.go +++ b/progress.go @@ -141,14 +141,20 @@ func (p *progress) warnf(format string, args ...any) { fmt.Fprintln(os.Stderr, msg) } -// finish terminates the pass's display. +// finish terminates the pass's display. A bar whose pass stopped short +// of its total, as an interrupted one does, is left as last drawn; the +// library's Finish would fill it up. func (p *progress) finish() { if p == nil { return } if p.bar != nil { - _ = p.bar.Finish() + if p.total >= 0 && p.count < p.total { + _ = p.bar.Exit() + } else { + _ = p.bar.Finish() + } fmt.Fprintln(os.Stderr) diff --git a/progress_test.go b/progress_test.go index 363759a..2d36e5f 100644 --- a/progress_test.go +++ b/progress_test.go @@ -144,18 +144,46 @@ func TestSpinnerShowsCountAfterBurst(t *testing.T) { time.Sleep(spinnerIdle) - // A terminal shows the last frame drawn. The library starts each - // frame with a carriage return and erases the previous one with - // spaces first. + if shown := lastFrame(stderr()); !strings.Contains(shown, "(50/-,") { + t.Errorf("terminal shows %q, want a count of 50", shown) + } +} + +// lastFrame returns what a terminal shows of the frames a bar drew: the +// last one. The library starts each frame with a carriage return and +// erases the previous one with spaces first. +func lastFrame(out string) string { var shown string - for frame := range strings.SplitSeq(stderr(), "\r") { + for frame := range strings.SplitSeq(out, "\r") { if strings.TrimSpace(frame) != "" { shown = frame } } - if !strings.Contains(shown, "(50/-,") { - t.Errorf("terminal shows %q, want a count of 50", shown) + return shown +} + +// TestBarStoppedShortKeepsCount checks that the terminal display of a +// pass that stops before its total, as an interrupted one does, is left +// as last drawn instead of being filled up. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestBarStoppedShortKeepsCount(t *testing.T) { + stderr := captureStderr(t) + p := &progress{ + label: "hash", total: 10, start: time.Now(), + bar: newBar("hash", 10), + } + + p.increment() + + // Past the redraw limit, so the bar draws the next count. + time.Sleep(2 * barThrottle) + p.increment() + p.finish() + + if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/10,") { + t.Errorf("terminal shows %q, want a count of 2 of 10", shown) } } diff --git a/scan.go b/scan.go index c7406b2..c61a9d8 100644 --- a/scan.go +++ b/scan.go @@ -11,6 +11,7 @@ import ( "io" "io/fs" "os" + "os/signal" "path/filepath" "slices" "strings" @@ -57,6 +58,10 @@ const sampleWindow = 1024 * 1024 // and hash worker pools. const workQueueDepth = 1024 +// errInterrupted reports a scan stopped by SIGINT or SIGTERM. runScan +// has already printed its line, so run prints nothing more. +var errInterrupted = errors.New("scan interrupted") + // fileRec carries one statted file between the scan phases. dev and // ino identify the underlying inode so hard-linked paths can share // one read; both are zero when the platform exposes no inode. @@ -90,8 +95,10 @@ type fileMeta struct { // scan fails before it walks the filesystem or opens the database. // Errors are returned rather than exiting, so that the deferred close — // which takes the database out of WAL mode — always runs, and the lock -// is released after it. Cancelling ctx unwinds the worker pools and -// aborts the scan with the context's error. +// is released after it. When ctx is cancelled, as by the SIGINT or +// SIGTERM that interruptContext catches, the scan keeps what it has +// hashed (see syncScan), prints how many files its walk reached, and +// returns errInterrupted. func runScan(ctx context.Context, roots []string, workers int, oneFS bool, ) error { @@ -114,6 +121,12 @@ func runScan(ctx context.Context, roots []string, workers int, defer func() { _ = lock.Close() }() db, err := openScanDatabase(ctx, dbPath) + if err != nil && ctx.Err() != nil { + // Interrupted while opening; SQLite may report that with an + // error of its own rather than the context's. + return interrupted(0) + } + if err != nil { return err } @@ -121,6 +134,10 @@ func runScan(ctx context.Context, roots []string, workers int, defer closeScanDatabase(ctx, db, dbPath) st, err := syncScan(ctx, db, roots, workers, oneFS) + if errors.Is(err, context.Canceled) { + return interrupted(st.walked) + } + if err != nil { return fmt.Errorf("update database %s: %w", dbPath, err) } @@ -134,6 +151,34 @@ func runScan(ctx context.Context, roots []string, workers int, return nil } +// interruptContext returns a copy of ctx that the first SIGINT or +// SIGTERM cancels; the scan command runs the scan under it. stop +// releases the signals. +func interruptContext(ctx context.Context) (context.Context, func()) { + // A SIGINT ignored from the start, as by a script's background job, + // stays ignored. + signals := []os.Signal{syscall.SIGTERM} + if !signal.Ignored(syscall.SIGINT) { + signals = append(signals, syscall.SIGINT) + } + + ctx, stop := signal.NotifyContext(ctx, signals...) + + // Stopping restores the default handling, so a second signal ends + // the process at once. + context.AfterFunc(ctx, stop) + + return ctx, stop +} + +// interrupted prints the line for a scan stopped by a signal after its +// walk reached walked files, and returns errInterrupted. +func interrupted(walked int) error { + fmt.Fprintf(os.Stderr, "scan: interrupted after %d files\n", walked) + + return errInterrupted +} + // resolveRoots converts each PATH operand to an absolute, lexically // cleaned path (symlinks are not resolved) and verifies that it // exists. Database records are keyed by absolute path, so scan results @@ -187,8 +232,9 @@ func pruneRoots(roots []string) []string { } // scanStats summarizes one scan's database synchronization for the -// final stderr summary. +// final stderr summary, or for the line an interrupted scan prints. type scanStats struct { + walked int // files the walk reached added int updated int removed int @@ -210,7 +256,30 @@ type scanState struct { st scanStats } -// syncScan synchronizes the database with the filesystem under roots +// syncScan synchronizes the database with the filesystem under roots; +// see runPhases. When ctx is cancelled, as by an interrupt, it commits +// the hashed records still waiting in the batch, starts no other write +// or deletion, and returns the cancellation. +func syncScan(ctx context.Context, db *sql.DB, roots []string, + workers int, oneFS bool, +) (scanStats, error) { + s := &scanState{db: db} + + err := s.runPhases(ctx, roots, workers, oneFS) + if err == nil || ctx.Err() == nil { + return s.st, err + } + + // The one write made after the cancellation, so it cannot use ctx. + err = applyChanges(context.WithoutCancel(ctx), db, s.batch, nil, nil) + if err != nil { + return s.st, err + } + + return s.st, ctx.Err() +} + +// runPhases synchronizes the database with the filesystem under roots // in four sequential phases: walk (enumerate and stat every file, // building a complete size census), hash (read only the new or // changed — or previously unhashed — files whose size at least one @@ -223,48 +292,41 @@ type scanState struct { // their content hash. Operands the walk cannot start from are dropped // first, so the records beneath them count as outside the roots unless // they lie under another root. -func syncScan(ctx context.Context, db *sql.DB, roots []string, +func (s *scanState) runPhases(ctx context.Context, roots []string, workers int, oneFS bool, -) (scanStats, error) { - s := &scanState{db: db} - +) error { // Types are checked before pruning so that an operand under a // dropped one is still scanned, not dropped as lying under it. roots = pruneRoots(s.walkableRoots(roots)) err := s.loadIndex(ctx, roots) if err != nil { - return s.st, err + return err } changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers)) // A cancelled walk stops early, so its size census covers only part - // of the roots, and every file it never reached looks vanished to - // the update phase. Defence in depth rather than the only barrier: - // that phase would today fail on its first BeginTx with the same - // cancelled context before deleting anything. But it is the barrier - // that survives a later decision to let an interrupted scan commit - // what it has, and it turns a confusing failure deep in the update - // phase into a clean abort at the phase boundary. + // of the roots, and every file it never reached would look vanished + // to the update phase. Stop before anything is written or deleted. err = ctx.Err() if err != nil { - return s.st, err + return err } s.partition(changed, unhashed) err = s.hashPhase(ctx, workers) if err != nil { - return s.st, err + return err } err = s.updatePhase(ctx) if err != nil { - return s.st, err + return err } - return s.st, s.contentPhase(ctx, workers) + return s.contentPhase(ctx, workers) } // walkableRoots returns the operands the walk can start from: regular @@ -349,6 +411,7 @@ func (s *scanState) walkPhase( } s.sizes = append(s.sizes, ev.rec.size) + s.st.walked++ prog.increment() @@ -552,16 +615,22 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error { } // commitFullBatch commits the running batch once it holds -// updateBatchSize records. +// updateBatchSize records. A batch that fails to commit is kept: the +// commit fails when the scan is interrupted, and syncScan then commits +// the batch itself. func (s *scanState) commitFullBatch(ctx context.Context) error { if len(s.batch) < updateBatchSize { return nil } err := applyBatch(ctx, s.db, s.batch, nil, nil) + if err != nil { + return err + } + s.batch = s.batch[:0] - return err + return nil } // updatePhase writes the scan's tail under one progress display: the @@ -1004,6 +1073,12 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, var subs []dirJob for _, e := range entries { + // A cancelled scan wants nothing more from this directory: stop + // rather than lstat the rest of a large one. + if ctx.Err() != nil { + return nil + } + p := filepath.Join(job.path, e.Name()) if e.IsDir() { diff --git a/scan_test.go b/scan_test.go index 3b4e73b..99d9b15 100644 --- a/scan_test.go +++ b/scan_test.go @@ -9,11 +9,13 @@ import ( "fmt" "io" "os" + "os/signal" "path/filepath" "runtime" "slices" "strconv" "strings" + "syscall" "testing" "time" ) @@ -456,7 +458,7 @@ func TestScanContentWithinOperand(t *testing.T) { added := sparseFile(t, dir, "d2", headTailMin) st := syncTree(t, db, dir) - if st != (scanStats{added: 1, unchanged: 2}) { + if st != (scanStats{walked: 3, added: 1, unchanged: 2}) { t.Fatalf("rescan stats = %+v, want 1 added 2 unchanged", st) } @@ -502,7 +504,7 @@ func TestScanContentStalePartners(t *testing.T) { sparseFile(t, dirB, "changed-copy", headTailMin+1) st := syncTree(t, db, dirB) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Errorf("stats = %+v, want 2 added and nothing skipped", st) } @@ -559,7 +561,7 @@ func TestScanContentHashedStalePartners(t *testing.T) { b := sparseFile(t, t.TempDir(), "copy", headTailMin) st := syncTree(t, db, filepath.Dir(b)) - if st != (scanStats{added: 1}) { + if st != (scanStats{walked: 1, added: 1}) { t.Errorf("stats = %+v, want 1 added and nothing skipped", st) } @@ -599,7 +601,7 @@ func TestScanContentReadFailure(t *testing.T) { b := sparseFile(t, dirB, "b", headTailMin) st := syncTree(t, db, dirB) - if st != (scanStats{added: 1, skipped: 1}) { + if st != (scanStats{walked: 1, added: 1, skipped: 1}) { t.Fatalf("stats = %+v, want 1 added 1 skipped", st) } @@ -613,7 +615,7 @@ func TestScanContentReadFailure(t *testing.T) { } st = syncTree(t, db, dirB) - if st != (scanStats{unchanged: 1}) { + if st != (scanStats{walked: 1, unchanged: 1}) { t.Fatalf("rescan stats = %+v, want 1 unchanged", st) } @@ -659,7 +661,7 @@ func TestScanContentCheckError(t *testing.T) { b := sparseFile(t, t.TempDir(), "b", headTailMin) st := syncTree(t, db, filepath.Dir(b)) - if st != (scanStats{added: 1, skipped: 1}) { + if st != (scanStats{walked: 1, added: 1, skipped: 1}) { t.Fatalf("stats = %+v, want 1 added 1 skipped", st) } @@ -687,7 +689,7 @@ func TestScanContentHardlinks(t *testing.T) { c := sparseFile(t, dir, "copy", headTailMin) st := syncTree(t, db, dir) - if st != (scanStats{added: 3}) { + if st != (scanStats{walked: 3, added: 3}) { t.Fatalf("stats = %+v, want 3 added", st) } @@ -1058,7 +1060,7 @@ func TestScanPipeline(t *testing.T) { db := openTestDB(t) st := syncTree(t, db, dir) - if st != (scanStats{added: smokeTreeFiles}) { + if st != (scanStats{walked: smokeTreeFiles, added: smokeTreeFiles}) { t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles) } @@ -1081,7 +1083,7 @@ func TestSyncScanUnchangedReuse(t *testing.T) { writeFile(t, dir, "b.bin", pattern(2, 600)) st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("first scan stats = %+v, want 2 added", st) } @@ -1095,7 +1097,7 @@ func TestSyncScanUnchangedReuse(t *testing.T) { } st = syncTree(t, db, dir) - if st != (scanStats{unchanged: 2}) { + if st != (scanStats{walked: 2, unchanged: 2}) { t.Fatalf("rescan stats = %+v, want 2 unchanged", st) } @@ -1124,7 +1126,7 @@ func TestSyncScanMtimeBump(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{updated: 1}) { + if st != (scanStats{walked: 1, updated: 1}) { t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) } @@ -1152,7 +1154,7 @@ func TestSyncScanAddRemove(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{added: 1, removed: 1, unchanged: 1}) { + if st != (scanStats{walked: 2, added: 1, removed: 1, unchanged: 1}) { t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged", st) } @@ -1269,7 +1271,7 @@ func TestSyncScanOverlappingRoots(t *testing.T) { // A file reachable via two overlapping operands is deduplicated // by path in the shared walk and processed once. st := syncTree(t, db, dir, filepath.Join(dir, "sub")) - if st != (scanStats{added: 1}) { + if st != (scanStats{walked: 1, added: 1}) { t.Fatalf("stats = %+v, want 1 added", st) } @@ -1290,7 +1292,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { // Neither size is shared, so neither file is read: both records // are written without hashes and no duplicates are reported. st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("stats = %+v, want 2 added", st) } @@ -1312,7 +1314,7 @@ func TestScanSkipsUniqueSizes(t *testing.T) { c := writeFile(t, dir, "c.bin", pattern(1, 500)) st = syncTree(t, db, dir) - if st != (scanStats{added: 1, updated: 1, unchanged: 1}) { + if st != (scanStats{walked: 3, added: 1, updated: 1, unchanged: 1}) { t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged", st) } @@ -1376,7 +1378,7 @@ func TestScanHardlinksReadOnce(t *testing.T) { } st := syncTree(t, db, dir) - if st != (scanStats{added: 2}) { + if st != (scanStats{walked: 2, added: 2}) { t.Fatalf("stats = %+v, want 2 added", st) } @@ -1498,6 +1500,13 @@ func injectWriteFailure(t *testing.T, path string) { func baselineGoroutines(t *testing.T) int { t.Helper() + // The first scan command in a process starts os/signal's goroutine, + // which never exits. Start it now, so the baseline counts it + // instead of the scan seeming to leave it behind. + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT) + signal.Stop(ch) + deadline := time.Now().Add(goroutineSettle) last := runtime.NumGoroutine() From cb5dda4f4506473b1b68e80d4dac5f859bbba231 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 09:47:28 +0200 Subject: [PATCH 10/33] Print --version to stdout (closes #15) Cobra's built-in version flag prints through the writer that carries help and usage, which is stderr here. The root command now defines its own -v/--version flag and prints one line, "sfdupes VERSION", to stdout; a failed write is a fatal error (exit 1). Help and usage stay on stderr. README documents --version and --help, their streams and exit codes. Tests cover both flags and the failed write. Model: opus-5-5 --- README.md | 10 +++++-- TODO.md | 3 ++ main.go | 48 ++++++++++++++++++++++---------- main_test.go | 78 ++++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 103 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 52668c9..c7d4101 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,8 @@ Goals, in order: again. `scan` is designed to be cronned; the reports run at any time against the last completed scan. 4. **Clean stream separation.** Everything on stdout is machine-readable - data. All progress, warnings, and summaries go to stderr. Never mix - them. + data. All progress, warnings, summaries, and help and usage text go + to stderr. Never mix them. ### Constraints @@ -149,8 +149,14 @@ Three subcommands, all implemented: sfdupes scan [--workers N] [-x] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv +sfdupes --version ``` +`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to +stdout and exits 0, writing nothing to stderr. `-h` or `--help`, alone +or after a subcommand, prints the help text to stderr and exits 0, +writing nothing to stdout. + ### Database All three subcommands operate on a single SQLite database file: diff --git a/TODO.md b/TODO.md index 2dd0179..7e77fb3 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- `--version` prints `sfdupes VERSION` to stdout; README documents it and + `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) + - `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed, deletes nothing more, exits 1 (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/5) diff --git a/main.go b/main.go index f1c54df..9379ede 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ // sfdupes scan [--workers N] [-x] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv +// sfdupes --version // // See README.md for the complete specification. package main @@ -104,15 +105,29 @@ func run(args []string, stdout, stderr io.Writer) int { } // newRootCommand builds the command tree. Everything on stdout is -// machine-readable data; all human-facing output (help, usage, errors) -// goes to stderr. +// machine-readable data, the version line included; all human-facing +// output (help, usage, errors) goes to stderr. func newRootCommand(stdout, stderr io.Writer) *cobra.Command { + var showVersion bool + + printVersion := runE(func(context.Context, []string) error { + _, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version) + if err != nil { + return fmt.Errorf("write stdout: %w", err) + } + + return nil + }) + root := &cobra.Command{ - Use: "sfdupes", - Short: "Find candidate duplicate files by size and head/tail/content SHA-256", - Version: Version, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { + Use: "sfdupes", + Short: "Find candidate duplicate files by size and head/tail/content SHA-256", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if showVersion { + return printVersion(cmd, args) + } + // A missing subcommand prints usage and exits 2: cobra // prints the usage text for the returned error, and run // maps everything that is not a fatal error to exit 2. @@ -125,6 +140,11 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { root.SetErr(stderr) root.CompletionOptions.DisableDefaultCmd = true + // Cobra's built-in version flag prints through the help writer, + // stderr; this one prints to stdout. + root.Flags().BoolVarP(&showVersion, "version", "v", false, + "print the version to stdout") + var ( scanWorkers int scanOneFS bool @@ -169,13 +189,13 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { return root } -// runE adapts a subcommand implementation to cobra's RunE. Cobra -// prints the error and the command's usage text for every error RunE -// returns, but a subcommand that ran and failed has no usage problem -// to report: both are silenced here, and the error is marked fatal so -// that run reports it on stderr and exits 1 rather than 2. The command's -// context is handed to the implementation: cancelling it unwinds the -// scan's worker pools. +// runE adapts a subcommand implementation, or the version print, to +// cobra's RunE. Cobra prints the error and the command's usage text for +// every error RunE returns, but a subcommand that ran and failed has no +// usage problem to report: both are silenced here, and the error is +// marked fatal so that run reports it on stderr and exits 1 rather than +// 2. The command's context is handed to the implementation: cancelling +// it unwinds the scan's worker pools. func runE( fn func(ctx context.Context, args []string) error, ) func(*cobra.Command, []string) error { diff --git a/main_test.go b/main_test.go index d6716de..bddb2a9 100644 --- a/main_test.go +++ b/main_test.go @@ -295,34 +295,72 @@ func TestRunUsageErrors(t *testing.T) { } } -// TestRunHelpAndVersionSucceed checks that the two informational flags -// exit 0 and keep their human-facing output on stderr. -func TestRunHelpAndVersionSucceed(t *testing.T) { +func TestRunHelp(t *testing.T) { t.Parallel() - assertHumanOutput(t, "--help") - assertHumanOutput(t, "--version") + // README §Subcommands: help goes to stderr, exits 0, and leaves + // stdout empty. + cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}} + + for _, args := range cases { + var stdout, stderr bytes.Buffer + + code := run(args, &stdout, &stderr) + if code != exitOK { + t.Errorf("run(%v) = %d, want %d", args, code, exitOK) + } + + if !strings.Contains(stderr.String(), usageMarker) { + t.Errorf("run(%v) stderr = %q, want the help text", + args, stderr.String()) + } + + if got := stdout.String(); got != "" { + t.Errorf("run(%v) stdout = %q, want nothing (data only)", + args, got) + } + } } -// assertHumanOutput runs sfdupes with one informational flag and checks -// that it succeeds with its output on stderr and stdout untouched -// (README design goal 4). -func assertHumanOutput(t *testing.T, arg string) { - t.Helper() +func TestRunVersion(t *testing.T) { + t.Parallel() - var stdout, stderr bytes.Buffer + // README §Subcommands: the version is one line on stdout, with + // nothing on stderr, and exits 0. + for _, arg := range []string{"--version", "-v"} { + var stdout, stderr bytes.Buffer - code := run([]string{arg}, &stdout, &stderr) - if code != exitOK { - t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) + code := run([]string{arg}, &stdout, &stderr) + if code != exitOK { + t.Errorf("run(%s) = %d, want %d", arg, code, exitOK) + } + + want := "sfdupes " + Version + "\n" + if got := stdout.String(); got != want { + t.Errorf("run(%s) stdout = %q, want %q", arg, got, want) + } + + if got := stderr.String(); got != "" { + t.Errorf("run(%s) stderr = %q, want nothing", arg, got) + } + } +} + +func TestRunVersionWriteFailureIsFatal(t *testing.T) { + t.Parallel() + + // README §Error handling: a stdout write failure exits 1, reported + // in one line on stderr. + var stderr bytes.Buffer + + code := run([]string{"--version"}, failingWriter{}, &stderr) + if code != exitFatal { + t.Errorf("run(--version) = %d, want %d", code, exitFatal) } - if stderr.Len() == 0 { - t.Errorf("run(%s) wrote nothing to stderr", arg) - } - - if got := stdout.String(); got != "" { - t.Errorf("stdout = %q, want nothing (data only)", got) + want := "sfdupes: write stdout: " + errWriteFailed.Error() + "\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) } } From 2eeba3df6fb96d744d265bdf47026a7c73bc5157 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 10:01:28 +0200 Subject: [PATCH 11/33] Keep the module cache out of the build stage's chown (closes #43) The build stage handed /src and the whole Go module cache to the unprivileged user with chown -R, in a layer that re-ran on every source change. On this host that step took from about 80 s to over ten minutes, depending on load. The module cache now sits at /go/pkg/mod and stays root's: bootstrap fills it as root. The sources are copied with --chown. A small chown, cached with bootstrap, hands builder the /src directory itself, the module cache's cache/download directory (where make build saves its lookup of this module's own version), and the telemetry files root's go commands left in its home. Tests and the build still run as builder. Model: opus-5-5 --- Dockerfile | 34 ++++++++++++++++++++++++++-------- TODO.md | 4 ++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2885086..98a2174 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,14 +51,21 @@ RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \ FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder # We never build or run as root. Create an unprivileged user and point -# HOME and the Go caches at its home so go build and go test can write -# their caches when we drop to it below. $GOPATH/bin is deliberately not -# on PATH: script/bootstrap no longer `go install`s anything (the linter -# runs from a pinned image, never from a host install), so nothing lands +# HOME and the build cache at its home so go build and go test can write +# it when we drop to it below. $GOPATH/bin is deliberately not on PATH: +# script/bootstrap no longer `go install`s anything (the linter runs +# from a pinned image, never from a host install), so nothing lands # there and adding it would only widen what this image resolves. +# +# The module cache is kept outside that home, at the base image's +# default /go/pkg/mod, and belongs to root: script/bootstrap fills it as +# root. Do not move it into the home and hand it over with `chown -R`: +# that walks every file in it, which took from about 80 s to over ten +# minutes on a shared host, depending on load. RUN adduser -D -u 1000 builder ENV HOME=/home/builder ENV GOPATH=/home/builder/go +ENV GOMODCACHE=/go/pkg/mod ENV GOCACHE=/home/builder/.cache/go-build WORKDIR /src @@ -83,11 +90,22 @@ COPY script/ script/ COPY go.mod go.sum ./ RUN script/bootstrap -COPY . . +# Hand builder only what it writes to, without walking the module cache. +# This layer stays cached with bootstrap. +# - /src itself: make build writes the binary into it, and git refuses +# a repository whose top directory belongs to another user. +# - the module cache's cache/download directory itself, not what is in +# it: Go only reads the downloaded modules, but make build saves its +# lookup of this module's own version from git there, in a new +# directory named after the module path. +# - builder's home: the go commands bootstrap ran as root left Go's +# telemetry files there, a few small files. +RUN chown builder:builder /src /go/pkg/mod/cache/download && \ + chown -R builder:builder /home/builder -# Hand the sources and caches to the unprivileged user, then drop root -# before running any checks or builds. -RUN chown -R builder:builder /src /home/builder +# The sources are handed to builder as they are copied, so no layer has +# to walk them. Then drop root before running any checks or builds. +COPY --chown=builder:builder . . USER builder # Fail the build unless the branch is green. Runs as non-root so the diff --git a/TODO.md b/TODO.md index 7e77fb3..03b7b68 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- the `Dockerfile` build stage keeps the Go module cache out of `builder`'s + home and copies the sources with `--chown`, so no `chown -R` walks them + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43) + - `--version` prints `sfdupes VERSION` to stdout; README documents it and `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) From 33f8607e17076e8fe70e529c1275c524ed75b9b5 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 11:47:20 +0200 Subject: [PATCH 12/33] Document install, a daily cron scan and reading the reports (closes #54) Getting Started gains three parts: installing with go install, from a clone or as the Docker image; a crontab line for a daily root scan, with where its stderr and failures go; and how to read the two reports, why a row is a candidate rather than proof, and how to check a pair with cmp before removing anything. The usage block names --help, and the text below it --workers and -x. The install line uses @main, not @latest: @latest resolves to the v0.0.1 tag, which predates the database. Model: opus-5-5 --- README.md | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- TODO.md | 4 ++ 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c7d4101..afbf225 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,122 @@ daily `sfdupes scan` cron job, with the reporting commands run interactively whenever needed; their results are as fresh as the last completed scan. +### Install + +With Go installed, this builds and installs the current `main` branch: + +```sh +go install sneak.berlin/go/sfdupes@main +``` + +The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is +set. A binary installed this way reports its version as `dev`; one +built from a clone or into the Docker image carries the git tag or +commit it was built from. + +From a clone, `make build` writes the binary to `./sfdupes`: + +```sh +git clone https://git.eeqj.de/sneak/sfdupes.git +cd sfdupes +make build +``` + +Copy the binary to `/usr/local/bin` for the cron job below. + +`make docker` builds the Docker image, tagged `sfdupes`, after running +the tests and the linter (see "Build"). The image runs `sfdupes` as +root with the database at its default path, so a bind mount of +`/var/lib/sfdupes` keeps the database between runs. Mount the scanned +tree at the same path inside the container as on the host; read-only +is enough. The database records paths as the container sees them, so +the reports then name the host's paths. + +```sh +make docker +docker run --rm -v /srv:/srv:ro -v /var/lib/sfdupes:/var/lib/sfdupes \ + sfdupes scan /srv +docker run --rm -v /var/lib/sfdupes:/var/lib/sfdupes sfdupes report > dupes.tsv +``` + +### Daily scan from cron + +Run `scan` as root, so that it can read every file: a path it cannot +read is skipped with a warning and loses its database record (see +"Rules for the walk"). As a file `/etc/cron.d/sfdupes`: + +``` +30 3 * * * root /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log +``` + +- The database is `/var/lib/sfdupes/db.sqlite`, created with its + directory by the first scan. To keep it elsewhere, set + `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same + line. +- `scan` writes nothing to stdout. Its stderr, appended here to + `/var/log/sfdupes.log`, holds a plain progress line as each phase + starts and then at most every 5 seconds, a warning for each path it + skips, and the summary line (see "Progress" and "`scan` mode"). The + log grows with every scan; rotate it like any other. +- Skipped paths do not fail a scan: it still exits 0, and cron sends + nothing. A scan that fails, or is stopped by `SIGINT` or `SIGTERM`, + exits 1 with the reason among the last lines of the log; `tail` + prints them, and cron mails them to root if the host can send mail. +- A scan still running when the next one starts carries on. The new + one fails at once, and the lines cron mails include + `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. +- `report` and `trees` need only read access to the database (see + "Database"). Under the usual umask of `022` the first scan creates + it readable by every user, so an unprivileged user can run them + against root's database. + +### Reading the reports + +Each row of `report` names two copies of one file, and each row of +`trees` two copies of one directory tree (see "Report output format" +and "Trees output format"). In a group of copies, the path that sorts +first byte by byte is `first` and every other path is a `dupe` of it. +`first` says nothing about which copy is the original or the oldest; +which copy to keep is your choice. + +A row is a candidate, not proof: + +- The reports read only the database, so they show the files as of + the last scan; a file may have changed or gone since. +- A file of 50 MiB or more is compared only on samples of its content + (see "Duplicate detection"). +- Paths that are hard links to one file are listed as duplicates, but + they share their data, so removing one frees nothing. + +Compare a pair byte for byte before removing either copy. For the row +`/srv/a/big.iso`, `/srv/b/big-copy.iso`, `4294967296`: + +```sh +cmp /srv/a/big.iso /srv/b/big-copy.iso && echo identical +[ /srv/a/big.iso -ef /srv/b/big-copy.iso ] && echo "hard links" +``` + +`cmp` prints nothing and exits 0 only when every byte matches, and +otherwise reports where the files differ. The second line prints +`hard links` when the two paths are the same file, so removing either +frees nothing. + +A path holding a backslash, tab, newline or carriage return is escaped +in the reports (see "Report output format"). Undo the escapes before +using it. `printf '%b'` does exactly that, because every backslash in +an escaped path starts one of the four escapes. Command substitution +drops trailing newlines, so print an `x` after the path and remove it +afterwards, or a path that ends in a newline names a different file: + +```sh +p="$(printf '%bx' '/srv/a/tab\tname.txt')"; p="${p%x}" +cmp "$p" /srv/b/tab-copy.txt +``` + +Check a `trees` row with `diff -r`, which compares the two trees file +by file and also names anything present in only one of them, such as +an empty directory or a symlink, which `trees` does not see. + ## Rationale Duplicate finders that hash entire files do not scale to the target @@ -150,12 +266,16 @@ sfdupes scan [--workers N] [-x] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv sfdupes --version +sfdupes [command] --help ``` -`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to -stdout and exits 0, writing nothing to stderr. `-h` or `--help`, alone -or after a subcommand, prints the help text to stderr and exits 0, -writing nothing to stdout. +`--workers N` sets the size of each `scan` worker pool (default: the +number of CPUs), and `-x` (`--one-file-system`) keeps the walk of each +operand on that operand's filesystem; both are described under +"`scan` mode". `sfdupes --version` (or `-v`) prints one line, +`sfdupes VERSION`, to stdout and exits 0, writing nothing to stderr. +`-h` or `--help`, alone or after a subcommand, prints the help text to +stderr and exits 0, writing nothing to stdout. ### Database diff --git a/TODO.md b/TODO.md index 03b7b68..c912a5c 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- README documents install, Docker, a daily cron scan and how to read + and check the reports (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/54) + - the `Dockerfile` build stage keeps the Go module cache out of `builder`'s home and copies the sources with `--chown`, so no `chown -R` walks them (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43) From bccc14ffef3170dd2a3f6b6792738d8f20d6b1ca Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 12:01:36 +0200 Subject: [PATCH 13/33] Refuse an unversioned database that already has a files table (closes #11) A database at user_version 0 that already has a files table was made by something else: scan used to run its CREATE TABLE on it and fail with a raw SQLite error, and report and trees gave only a bare version mismatch. All three now refuse such a database with the schema-version error telling the operator to remove the file and rescan. scan creates the table and index and sets the version in one transaction, so a first scan stopped partway leaves an empty database the next scan sets up, never a files table at version 0. A genuinely empty database is unchanged. Model: opus-4-8 (implementation); opus-5-5 (rebase) --- README.md | 7 +++++- TODO.md | 4 +++ db.go | 59 ++++++++++++++++++++++++++++++++++++++++++--- db_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index afbf225..5859aed 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,12 @@ All three subcommands operate on a single SQLite database file: `-shm` files beside it, which SQLite creates with the database file's permissions. - Schema (`PRAGMA user_version` is the schema version, currently 1; a - database with any other version is a fatal error): + database with any other version is a fatal error. `scan` creates + the schema and sets the version in one transaction, so a first scan + stopped while doing so leaves an empty database the next scan sets + up. A database at version 0 that already has a `files` table was + therefore not made by sfdupes; every subcommand refuses it with an + error telling the user to remove the file and rescan): ```sql CREATE TABLE files ( diff --git a/TODO.md b/TODO.md index c912a5c..ac052fc 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `scan` creates the schema in one transaction; a version-0 database with a + `files` table is refused with a clear schema-version error (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/11) + - README documents install, Docker, a daily cron scan and how to read and check the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) diff --git a/db.go b/db.go index 7633ac0..c2ec0a3 100644 --- a/db.go +++ b/db.go @@ -219,6 +219,12 @@ func openReportDatabase(ctx context.Context, } v, err := userVersion(ctx, db) + if err == nil && v == 0 { + // An empty database passes this check and fails the version + // check below. + err = checkUnversioned(ctx, db) + } + if err != nil { _ = db.Close() @@ -245,6 +251,11 @@ func initSchema(ctx context.Context, db *sql.DB) error { switch v { case 0: + err = checkUnversioned(ctx, db) + if err != nil { + return err + } + return createSchema(ctx, db) case schemaVersion: return nil @@ -254,25 +265,65 @@ func initSchema(ctx context.Context, db *sql.DB) error { } } +// checkUnversioned checks a database at user_version 0 before it is +// taken for an empty one. createSchema creates the files table and +// sets the version together, so a files table at version 0 was made by +// something else. Adopting it could corrupt unrelated data, so that is +// a schema-version error telling the operator to remove the file and +// rescan. +func checkUnversioned(ctx context.Context, db *sql.DB) error { + var name string + + err := db.QueryRowContext(ctx, + "SELECT name FROM sqlite_master "+ + "WHERE type = 'table' AND name = 'files'").Scan(&name) + + switch { + case err == nil: + return fmt.Errorf( + "has a files table but no schema version; "+ + "remove the file and rescan: %w", errSchemaVersion) + case errors.Is(err, sql.ErrNoRows): + return nil + default: + return fmt.Errorf("check for files table: %w", err) + } +} + // createSchema applies the schema to a fresh database and stamps the -// schema version. +// schema version in one transaction, so a creation stopped partway, by +// an interrupt or an error, leaves an empty database the next scan +// sets up, never a files table at version 0, which checkUnversioned +// refuses. func createSchema(ctx context.Context, db *sql.DB) error { - _, err := db.ExecContext(ctx, createTableSQL) + tx, err := db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("create schema: %w", err) } - _, err = db.ExecContext(ctx, createIndexSQL) + defer func() { _ = tx.Rollback() }() + + _, err = tx.ExecContext(ctx, createTableSQL) if err != nil { return fmt.Errorf("create schema: %w", err) } - _, err = db.ExecContext(ctx, + _, err = tx.ExecContext(ctx, createIndexSQL) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + + _, err = tx.ExecContext(ctx, "PRAGMA user_version = "+strconv.Itoa(schemaVersion)) if err != nil { return fmt.Errorf("set schema version: %w", err) } + err = tx.Commit() + if err != nil { + return fmt.Errorf("create schema: %w", err) + } + return nil } diff --git a/db_test.go b/db_test.go index 7bbb2b8..2fc6df5 100644 --- a/db_test.go +++ b/db_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" ) @@ -77,6 +78,76 @@ func TestOpenScanDatabaseCreates(t *testing.T) { } } +func TestOpenDatabaseUnversionedForeign(t *testing.T) { + t.Parallel() + + // A database that has a files table but user_version 0, written by + // some other tool. report, trees and scan must refuse it with the + // schema-version error, not adopt it and not emit a raw SQLite + // "table files already exists". + path := testDBPath(t) + + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + + _, err = db.ExecContext(t.Context(), "CREATE TABLE files (x INTEGER)") + if err != nil { + t.Fatal(err) + } + + _ = db.Close() + + _, err = openReportDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) || + !strings.Contains(err.Error(), "remove the file and rescan") { + t.Fatalf("report: err = %v, want errSchemaVersion telling the "+ + "operator to remove the file and rescan", err) + } + + _, err = openScanDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) || + !strings.Contains(err.Error(), "remove the file and rescan") { + t.Fatalf("scan: err = %v, want errSchemaVersion telling the "+ + "operator to remove the file and rescan", err) + } +} + +func TestSchemaCreationStoppedPartway(t *testing.T) { + t.Parallel() + + // A first scan stopped while creating the schema must leave a + // database the next scan accepts. max_page_count(2) leaves room for + // the files table but not its index, so schema creation fails right + // after CREATE TABLE, a point an interrupt could also stop it at. + path := testDBPath(t) + + db, err := openDB(path, scanParams+"&_pragma=max_page_count(2)") + if err != nil { + t.Fatal(err) + } + + err = initSchema(t.Context(), db) + _ = db.Close() + + if err == nil { + t.Fatal("initSchema with no room for the index succeeded") + } + + db, err = openScanDatabase(t.Context(), path) + if err != nil { + t.Fatalf("next scan: %v", err) + } + + defer func() { _ = db.Close() }() + + v, err := userVersion(t.Context(), db) + if err != nil || v != schemaVersion { + t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion) + } +} + func TestOpenReportDatabaseMissing(t *testing.T) { t.Parallel() From 7ff0dbb6e18461cc3b15321cb2f4ca95d804b3aa Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 12:13:23 +0200 Subject: [PATCH 14/33] Test the -x filesystem-boundary rejection branch (closes #17) No test ever ran the part of subdirJob that refuses to descend across a filesystem boundary under -x. The new tests call subdirJob directly with a made-up device for the operand, so no second real filesystem is needed. They cover refusal across a boundary, skipping the check when the operand's device is unknown, the warning when the subdirectory cannot be statted, and crossing by default without -x. When descent is accepted, the whole returned job is compared, so losing the operand's device on the way down fails a test. scan.go is unchanged. Model: opus-4-8 (implementation); opus-5-5 (rebase) --- TODO.md | 3 + scan_test.go | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/TODO.md b/TODO.md index ac052fc..721fc15 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/17) + - `scan` creates the schema in one transaction; a version-0 database with a `files` table is refused with a clear schema-version error (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/11) diff --git a/scan_test.go b/scan_test.go index 99d9b15..d2560b5 100644 --- a/scan_test.go +++ b/scan_test.go @@ -6,8 +6,10 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" "io" + "io/fs" "os" "os/signal" "path/filepath" @@ -912,6 +914,159 @@ func TestDeviceOfInfo(t *testing.T) { } } +// dirEntryFor returns the fs.DirEntry for name within dir, obtained via +// the same os.ReadDir the walk uses, so it carries a real Info(). +func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry { + t.Helper() + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + for _, e := range entries { + if e.Name() == name { + return e + } + } + + t.Fatalf("entry %q not found in %q", name, dir) + + return nil +} + +// callSubdirJob runs subdirJob against e under parent, collecting any +// warning events it emits (subdirJob emits at most one). +func callSubdirJob(t *testing.T, p string, e fs.DirEntry, + parent dirJob, oneFS bool, +) (dirJob, bool, []walkEvent) { + t.Helper() + + events := make(chan walkEvent, 1) + job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events) + close(events) + + var evs []walkEvent + for ev := range events { + evs = append(evs, ev) + } + + return job, ok, evs +} + +// TestSubdirJobOneFilesystem exercises the -x boundary check in +// subdirJob directly, so no second real filesystem is needed. The +// subdirectory's real device is compared against a fabricated operand +// device. +func TestSubdirJobOneFilesystem(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + sub := filepath.Join(dir, "sub") + + err := os.Mkdir(sub, 0o750) + if err != nil { + t.Fatal(err) + } + + info, err := os.Lstat(sub) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + // A device the subdirectory is not on, standing in for an operand + // rooted on a different filesystem. + otherDev := dev + 1 + e := dirEntryFor(t, dir, "sub") + + cases := []struct { + name string + oneFS bool + parent dirJob + wantOK bool + }{ + // -x on, subdirectory on a different device than its operand: + // descent is refused. + {"reject across boundary", true, + dirJob{rootDev: otherDev, rootDevOK: true}, false}, + // -x on but the operand's own device is unknown: the boundary + // check is bypassed and descent proceeds. + {"bypass when root device unknown", true, + dirJob{rootDev: otherDev, rootDevOK: false}, true}, + // Default (no -x): boundaries are crossed even onto a different + // device. + {"cross by default", false, + dirJob{rootDev: otherDev, rootDevOK: true}, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + job, ok, evs := callSubdirJob(t, sub, e, tc.parent, tc.oneFS) + if ok != tc.wantOK { + t.Fatalf("accepted = %v, want %v", ok, tc.wantOK) + } + + if len(evs) != 0 { + t.Fatalf("unexpected events: %+v", evs) + } + + // The accepted job must carry the operand's device down, or -x + // stops checking below the first level. + want := dirJob{ + path: sub, + rootDev: tc.parent.rootDev, + rootDevOK: tc.parent.rootDevOK, + } + if ok && job != want { + t.Fatalf("job = %+v, want %+v", job, want) + } + }) + } +} + +// errInfoUnavailable is returned by errDirEntry.Info(). +var errInfoUnavailable = errors.New("info unavailable") + +// errDirEntry is a directory entry whose Info() always fails, driving +// subdirJob's stat-error branch deterministically. +type errDirEntry struct{ name string } + +func (e errDirEntry) Name() string { return e.name } +func (errDirEntry) IsDir() bool { return true } +func (errDirEntry) Type() fs.FileMode { + return fs.ModeDir +} + +func (errDirEntry) Info() (fs.FileInfo, error) { + return nil, errInfoUnavailable +} + +// TestSubdirJobStatError asserts that when a subdirectory's Info() +// fails under -x, subdirJob warns and refuses descent. +func TestSubdirJobStatError(t *testing.T) { + t.Parallel() + + p := "/does/not/matter/sub" + + _, ok, evs := callSubdirJob(t, p, errDirEntry{name: "sub"}, + dirJob{rootDev: 1, rootDevOK: true}, true) + if ok { + t.Fatal("descent accepted after stat error, want refused") + } + + if len(evs) != 1 || !evs[0].fail || !strings.Contains(evs[0].warn, p) { + t.Fatalf("want one warning naming the path, got %+v", evs) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { From 948fb0363055f8ef60031ff6f725b4f79a71d177 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 13:01:31 +0200 Subject: [PATCH 15/33] Correct four inaccurate comments in cancel_test.go and rename a constant (closes #33) Fix the comments the re-review of https://git.eeqj.de/sneak/sfdupes/issues/6 found misdescribing their tests; no test's behaviour changes. State the property the walkClock tests rely on, that the index load's Done cost does not grow with the record count, instead of a wrong fixed figure. Describe poolUnwind so it is true of every use, and mark the tests that have no bound. Record that hashWorker's results-send exit is reachable through stop but has no test that fails without it. Rename walkCancelInFlightDirs to walkCancelInFlightFiles, since it counts files. Note at the top why the file departs from the one-test-file-per-source-file convention. Model: opus-4-8 (implementation); opus-5-5 (rework) --- TODO.md | 4 +++ cancel_test.go | 79 ++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index 721fc15..57c7daf 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- correct four inaccurate comments in `cancel_test.go` and rename + `walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/33) + - test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/17) diff --git a/cancel_test.go b/cancel_test.go index e4ecd29..59e4210 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -18,11 +18,21 @@ import ( "time" ) -// poolUnwind bounds how long a goroutine is given to leave a pool -// after its context is cancelled. Only a failing run ever waits this -// long: a pool that ignored its cancellation parks forever, and this -// is what turns that into a failed assertion instead of a suite that -// hangs until the test binary's own timeout. +// This file gathers the tests for scan cancellation and worker-pool +// unwinding. Everything it exercises lives in scan.go, so by the repo's +// convention of one test file per source file it would belong in +// scan_test.go. It is kept separate on purpose: cancellation behaviour +// cuts across both the walk pool and the hash pool as a single concern, +// and scan_test.go is already over 1,600 lines. That is the deliberate +// exception the convention otherwise expects to be stated. + +// poolUnwind bounds how long a test waits for a cancellation to take +// effect: for a goroutine to return or a channel to close once its +// context is cancelled, or for a signal to cancel the scan's context. +// Only a failing run waits this long, and the bound is what makes that +// failure an assertion instead of a hang. A call made without it, as +// most of this file's scans are, has no bound: a regression that parks +// it is caught only as the test binary's own timeout. const poolUnwind = 2 * time.Second // walkClock is a context whose cancellation is driven by the scan's @@ -33,9 +43,14 @@ const poolUnwind = 2 * time.Second // // The accounting behind the n chosen by each test: every blocking // channel operation in the walk selects on Done, so the walk spends -// one consultation per file event plus a couple per directory, while -// the index load that runs ahead of it spends a small fixed number -// (three) whatever the record count. +// one consultation per file event plus a couple per directory. The +// index load that runs ahead of it also consults Done, but a bounded +// number of times that does not grow with the record count. The tests +// depend on that property, not on the bound's exact value: each test +// sets n from the consultations of the walk, plus those of the hash +// phase when it cancels mid-hash, far from both ends of the phase it +// interrupts, so the cancellation lands inside that phase whatever the +// record count. type walkClock struct { n int64 seen atomic.Int64 @@ -91,11 +106,14 @@ func (c *walkClock) Value(_ any) any { // directory still queued and only the handful already in flight can // emit anything more. const ( - walkCancelDirs = 100 - walkCancelFilesPerDir = 20 - walkCancelFiles = walkCancelDirs * walkCancelFilesPerDir - walkCancelWorkers = 4 - walkCancelInFlightDirs = walkCancelWorkers * walkCancelFilesPerDir + walkCancelDirs = 100 + walkCancelFilesPerDir = 20 + walkCancelFiles = walkCancelDirs * walkCancelFilesPerDir + walkCancelWorkers = 4 + // The most files the walkCancelWorkers directories already in + // flight when the scan is cancelled can still emit, at + // walkCancelFilesPerDir each. A file count, not a directory count. + walkCancelInFlightFiles = walkCancelWorkers * walkCancelFilesPerDir ) // walkCancelAtDone is the consultation on which the fixture's context @@ -158,6 +176,11 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) { // unreachable, makes the scan carry its truncated view into the update // phase, which counts every record the walk never reached for removal. // +// The syncScan call here is not bounded by poolUnwind: a regression +// that left a worker pool parked would hang it, and that regression is +// caught only by the test binary's own timeout, not by a quick +// assertion. +// //nolint:paralleltest // counts goroutines: must not run beside others func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { dir := buildWalkCancelTree(t) @@ -209,10 +232,11 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) { } // The workers drop every directory still queued once the scan is - // cancelled, so only the directories already in flight can add to - // the census after the fact. A census beyond that bound would mean - // the cancellation was not observed where it should have been. - limit := walkCancelAtDone + walkCancelInFlightDirs + // cancelled, so only the files in the directories already in flight + // can add to the census after the fact. A census beyond that bound + // would mean the cancellation was not observed where it should have + // been. + limit := walkCancelAtDone + walkCancelInFlightFiles if st.unchanged > limit { t.Errorf("census covers %d files, want at most %d: the walk kept "+ "taking directories off the queue after cancellation", @@ -646,7 +670,11 @@ func TestDispatchDirsClosesJobsWhenCancelled(t *testing.T) { // TestFeedHashJobsClosesJobsWhenCancelled checks that the hash feeder // abandons the runs it has not queued yet and still closes the job -// channel, which is what lets the workers' range terminate. +// channel, which is what lets the workers' range terminate. The +// receive on jobs below is not bounded: a feeder that returned without +// closing jobs would leave that receive with no sender and no close, so +// this regression is caught by the test binary's timeout rather than by +// a bounded assertion. func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) { t.Parallel() @@ -674,6 +702,16 @@ func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) { // nobody wants the hashes of — while still letting the range run out // so the pool tears down. The queued run names a file that does not // exist, so a worker that hashed it anyway would produce a result. +// +// hashWorker's other cancellation exit, abandoning the send of a +// result, is reachable from the scan: stop cancels the pool before it +// drains results, so a worker waiting on that send can leave through +// it. The tests that stop a scan mid-hash, among them +// TestScanHashWriteFailureUnwindsPool, reach it in some runs only, +// depending on timing, and no test fails without it, since stop's +// drain frees a waiting worker anyway. This test, for its part, catches +// a removed drop check in some runs only: a worker that hashes the run +// anyway then picks at random between sending the result and leaving. func TestHashWorkerDropsQueuedRuns(t *testing.T) { t.Parallel() @@ -706,7 +744,10 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) { // TestHashPhaseCancelledReturnsContextError checks the result loop's // own exit: with the pool cancelled, no result will ever arrive, and // the loop must leave through the cancellation rather than wait for a -// receive that cannot happen. +// receive that cannot happen. This call is not bounded by poolUnwind: a +// loop that dropped its cancellation case would block on that receive, +// so the regression surfaces as the test binary's timeout rather than +// as a bounded assertion. func TestHashPhaseCancelledReturnsContextError(t *testing.T) { t.Parallel() From 8032ea682b0875433befa90400afb2a63d2f4c56 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 13:30:23 +0200 Subject: [PATCH 16/33] Test that scan refuses another schema version (closes #64) The version-mismatch test only opened its database through openReportDatabase, so nothing exercised the branch of initSchema that stops scan on a database stamped with an unknown schema version. The test now opens the same database through openScanDatabase too and requires errSchemaVersion, and is renamed to match the unversioned-file test beside it, which also covers both paths. Model: opus-5-5 --- TODO.md | 3 +++ db_test.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 57c7daf..d824ca3 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- test that `scan` refuses a database with another schema version + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64) + - correct four inaccurate comments in `cancel_test.go` and rename `walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/33) diff --git a/db_test.go b/db_test.go index 2fc6df5..ef7e833 100644 --- a/db_test.go +++ b/db_test.go @@ -157,9 +157,11 @@ func TestOpenReportDatabaseMissing(t *testing.T) { } } -func TestOpenReportDatabaseVersionMismatch(t *testing.T) { +func TestOpenDatabaseVersionMismatch(t *testing.T) { t.Parallel() + // A database stamped with a schema version other than 0 and + // schemaVersion. report, trees and scan must all refuse it. path := testDBPath(t) db, err := openScanDatabase(t.Context(), path) @@ -176,7 +178,12 @@ func TestOpenReportDatabaseVersionMismatch(t *testing.T) { _, err = openReportDatabase(t.Context(), path) if !errors.Is(err, errSchemaVersion) { - t.Fatalf("err = %v, want errSchemaVersion", err) + t.Fatalf("report: err = %v, want errSchemaVersion", err) + } + + _, err = openScanDatabase(t.Context(), path) + if !errors.Is(err, errSchemaVersion) { + t.Fatalf("scan: err = %v, want errSchemaVersion", err) } } From 7278c354f055bea8d27196a18671f92b25ecfa5d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 14:01:38 +0200 Subject: [PATCH 17/33] Test both walk cancellation checks on their own (closes #81) walkOneDir's check was hiding the worker's: a worker that walked a queued directory on a cancelled scan still emitted nothing, because walkOneDir stopped at its first entry. The worker test now queues a missing directory, whose read fails and sends a warning before walkOneDir's check is reached. A new test calls walkOneDir directly on a cancelled scan and checks it returns no subdirectory to descend into. Model: opus-5-5 --- TODO.md | 3 +++ cancel_test.go | 45 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/TODO.md b/TODO.md index d824ca3..53e2e68 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- a test fails when either walk cancellation check in `scan.go` is + removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) + - test that `scan` refuses a database with another schema version (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64) diff --git a/cancel_test.go b/cancel_test.go index 59e4210..8723bb4 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -586,20 +586,49 @@ func TestSendEventAbandonsBlockedSend(t *testing.T) { awaitReturn(t, done, "sendEvent") } -// TestWalkWorkersDropQueuedDirs checks that cancelled walk workers keep -// reading jobs and drop the directories rather than stopping their -// read: the range over jobs has to run out for the pool to tear down -// and close its event stream. -func TestWalkWorkersDropQueuedDirs(t *testing.T) { +// TestWalkOneDirStopsWhenCancelled checks that a cancelled scan stops +// reading a directory instead of going through the rest of its +// entries. A walk that kept going would return the subdirectory below +// to descend into. Unlike a file event, that return is not a send the +// cancellation can abandon, so the test catches the regression every +// time. +func TestWalkOneDirStopsWhenCancelled(t *testing.T) { t.Parallel() dir := t.TempDir() - writeEmptyFiles(t, dir, walkCancelFilesPerDir) + + err := os.Mkdir(filepath.Join(dir, "sub"), 0o750) + if err != nil { + t.Fatal(err) + } + + // Unbuffered and unread: on a cancelled scan every send gives up. + events := make(chan walkEvent) + + subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events) + if len(subs) != 0 { + t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ + "want none", subs) + } +} + +// TestWalkWorkersDropQueuedDirs checks that cancelled walk workers keep +// reading jobs and drop the directories rather than stopping their +// read: the range over jobs has to run out for the pool to tear down +// and close its event stream. The queued directory does not exist, so +// a worker that walked it anyway would send a warning before +// walkOneDir's own cancellation check could stop it. On a cancelled +// scan that send delivers or gives up at random, so with 64 jobs +// queued the regression has a one in 2^64 chance of passing. +func TestWalkWorkersDropQueuedDirs(t *testing.T) { + t.Parallel() + + missing := filepath.Join(t.TempDir(), "missing") jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false) - for range 4 { - jobs <- dirJob{path: dir} + for range 64 { + jobs <- dirJob{path: missing} } close(jobs) From c9c8b1d06c37cc0384c5588bec7b9068cd97fcfc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 14:30:21 +0200 Subject: [PATCH 18/33] Reject scan --workers below 1 as a usage error (closes #10) A --workers value of 0 or less used to be quietly raised to 1, so a typo ran the whole scan on one worker with nothing on stderr to say why. scan now refuses it before anything is scanned: one line on stderr and exit 2, like the other usage errors. The clamp in runScan is gone, and README states the rule and the default. Model: opus-5-5 --- README.md | 9 ++++++--- TODO.md | 4 ++++ main.go | 23 +++++++++++++++++++++-- main_test.go | 36 ++++++++++++++++++++++++++++++++++++ scan.go | 7 ++----- 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5859aed..4e4ac42 100644 --- a/README.md +++ b/README.md @@ -566,7 +566,9 @@ Rules for the walk: Concurrency: the walk phase (which also stats files), the hash phase, and the content phase each use a worker pool of `--workers` workers (default `runtime.NumCPU()`); the walk parallelizes across -directories, hashing across files. All three phases are seek-bound on +directories, hashing across files. `--workers` must be at least 1: a +smaller value is a usage error, reported in one line on stderr with +exit 2 before anything is scanned. All three phases are seek-bound on spinning disks, so raising `--workers` well past the core count can help on pools with many spindles. The main goroutine owns partitioning, database writes, and progress rendering; progress @@ -761,8 +763,9 @@ Additional requirements: cannot be created/opened/read/written, a missing database for `report`/`trees`, stdout write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below). -- `2`: usage error (including `scan` with no `PATH` operand and - `report`/`trees` with any positional argument). +- `2`: usage error (including `scan` with no `PATH` operand, `scan` + with `--workers` below 1, and `report`/`trees` with any positional + argument). A stdout write failure, such as a full disk, is reported in one line on stderr and exits 1. Two cases never reach sfdupes as a failed write: diff --git a/TODO.md b/TODO.md index 53e2e68..b465821 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `scan` rejects `--workers` below 1 as a usage error instead of + running single-threaded (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/10) + - a test fails when either walk cancellation check in `scan.go` is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) diff --git a/main.go b/main.go index 9379ede..6133b50 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,10 @@ const ( // cobra prints for it is the whole message. var errNoSubcommand = errors.New("no subcommand") +// errWorkersBelowOne is the usage error for a scan --workers value +// below 1. +var errWorkersBelowOne = errors.New("--workers must be at least 1") + // Version is the build version, injected at link time via -ldflags // (see the Makefile); "dev" for a plain go build. // @@ -98,8 +102,7 @@ func run(args []string, stdout, stderr io.Writer) int { return exitFatal default: - // A usage error: cobra has already printed the message and - // the usage text. + // A usage error, which cobra has already reported on stderr. return exitUsage } } @@ -154,6 +157,9 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { Use: cmdScan + " [--workers N] [-x] PATH...", Short: "Walk trees and synchronize the scan database", Args: cobra.MinimumNArgs(1), + PreRunE: func(cmd *cobra.Command, _ []string) error { + return checkScanWorkers(cmd, scanWorkers) + }, RunE: runE(func(ctx context.Context, args []string) error { ctx, stop := interruptContext(ctx) defer stop() @@ -189,6 +195,19 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { return root } +// checkScanWorkers rejects a scan --workers value below 1. That is a +// usage error reported in one line: cobra prints the returned message +// without the usage text, and run exits 2. +func checkScanWorkers(cmd *cobra.Command, workers int) error { + if workers >= 1 { + return nil + } + + cmd.SilenceUsage = true + + return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers) +} + // runE adapts a subcommand implementation, or the version print, to // cobra's RunE. Cobra prints the error and the command's usage text for // every error RunE returns, but a subcommand that ran and failed has no diff --git a/main_test.go b/main_test.go index bddb2a9..8a09878 100644 --- a/main_test.go +++ b/main_test.go @@ -295,6 +295,42 @@ func TestRunUsageErrors(t *testing.T) { } } +func TestRunScanRejectsWorkersBelowOne(t *testing.T) { + // README §scan mode: --workers below 1 is a usage error reported in + // one line on stderr, before the scan opens the database. + for _, workers := range []string{"0", "-1"} { + t.Run(workers, func(t *testing.T) { + dbPath := testDBPath(t) + t.Setenv(databaseEnv, dbPath) + + var stdout, stderr bytes.Buffer + + args := []string{cmdScan, "--workers", workers, t.TempDir()} + + code := run(args, &stdout, &stderr) + if code != exitUsage { + t.Errorf("run(%v) = %d, want %d", args, code, exitUsage) + } + + want := "Error: --workers must be at least 1, got " + workers + + "\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout.String(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + + _, err := os.Stat(dbPath) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("stat %s: %v, want the database never created", + dbPath, err) + } + }) + } +} + func TestRunHelp(t *testing.T) { t.Parallel() diff --git a/scan.go b/scan.go index c61a9d8..f181f0a 100644 --- a/scan.go +++ b/scan.go @@ -98,14 +98,11 @@ type fileMeta struct { // is released after it. When ctx is cancelled, as by the SIGINT or // SIGTERM that interruptContext catches, the scan keeps what it has // hashed (see syncScan), prints how many files its walk reached, and -// returns errInterrupted. +// returns errInterrupted. workers must be at least 1; the scan command +// rejects anything less. func runScan(ctx context.Context, roots []string, workers int, oneFS bool, ) error { - if workers < 1 { - workers = 1 - } - roots, err := resolveRoots(roots) if err != nil { return err From 722675f1533d586f8534d209b8c615da258d9a45 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 15:13:19 +0200 Subject: [PATCH 19/33] Escape the database path in the SQLite connection string (closes #55) openDB put the path into the connection string unescaped, so a ? or # in it ended the file name and a % started an escape: scan could silently fill a database under a shortened name. The path now goes through net/url as a file: URI. An absolute path gets an empty host and a relative path none, because SQLite reads what follows file:// up to the next slash as a host name. The path is not cleaned, so it stays exactly what the operator gave. A test runs scan, report and trees against such a file name given as an absolute path, as one starting with //, and as a relative path, and checks that only that file and its lock file exist afterwards. Model: opus-5-5 --- README.md | 4 +++- TODO.md | 3 +++ db.go | 15 +++++++++++- main_test.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e4ac42..98d7024 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,9 @@ All three subcommands operate on a single SQLite database file: - Location: the value of the `SFDUPES_DATABASE` environment variable when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. - There is no command-line flag. + There is no command-line flag. The path names the file exactly, + whatever characters it holds (`?`, `#` and `%` included); a + relative path is relative to the working directory. - `scan` creates the database (and its parent directory) on first use. `report` and `trees` require an existing database; a missing database file is a fatal error (exit 1) telling the user to run diff --git a/TODO.md b/TODO.md index b465821..d146d28 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- a database path holding `?`, `#` or `%` opens exactly the file it names + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) + - `scan` rejects `--workers` below 1 as a usage error instead of running single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10) diff --git a/db.go b/db.go index c2ec0a3..ca42037 100644 --- a/db.go +++ b/db.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "net/url" "os" "path/filepath" "slices" @@ -107,7 +108,19 @@ const reportParams = "mode=ro" + // openDB opens the SQLite database at path with the connection // parameters params. It does not create or verify the schema. func openDB(path, params string) (*sql.DB, error) { - db, err := sql.Open("sqlite", "file:"+path+"?"+params) + // The path is escaped into a file: URI, so ?, # and % in it stay + // part of the file name. SQLite reads what follows file:// up to + // the next / as a host name, so an absolute path goes after an + // empty host (file:///abs) and a relative path goes without one + // (file:rel). + uri := url.URL{ + Scheme: "file", + OmitHost: !filepath.IsAbs(path), + Path: path, + RawQuery: params, + } + + db, err := sql.Open("sqlite", uri.String()) if err != nil { return nil, fmt.Errorf("open database %s: %w", path, err) } diff --git a/main_test.go b/main_test.go index 8a09878..bf4150d 100644 --- a/main_test.go +++ b/main_test.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -639,6 +640,73 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) { } } +// assertRunsUseDatabase runs scan, then report and trees, against the +// database that SFDUPES_DATABASE names, the file name in dir. It fails +// unless the reports find the duplicate pair the scan recorded and dir +// then holds only that file and its lock file: nothing was created +// under a shortened name. +func assertRunsUseDatabase(t *testing.T, dir, name string) { + t.Helper() + + dupes := scanFixture(t) + + want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n" + if got := runStdout(t, cmdReport); got != want { + t.Errorf("report stdout = %q, want %q", got, want) + } + + want = "first\tdupe\tfiles\tsize\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" + if got := runStdout(t, cmdTrees); got != want { + t.Errorf("trees stdout = %q, want %q", got, want) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + got := make([]string, 0, len(entries)) + for _, e := range entries { + got = append(got, e.Name()) + } + + if wantFiles := []string{name, name + ".lock"}; !slices.Equal(got, wantFiles) { + t.Errorf("%s holds %q, want %q", dir, got, wantFiles) + } +} + +func TestRunDatabasePathUsedAsGiven(t *testing.T) { + // README §Database: the path names the database file exactly. In + // SQLite's connection string an unescaped ? or # would end the file + // name and % would start an escape, and a path starting with // + // could be read as a host name. %25 is a valid escape, so unescaped + // this name opens a file named a without any error. + const name = "a?b#c%25d e.sqlite" + + t.Run("absolute", func(t *testing.T) { + dir := t.TempDir() + t.Setenv(databaseEnv, filepath.Join(dir, name)) + + assertRunsUseDatabase(t, dir, name) + }) + + t.Run("leading double slash", func(t *testing.T) { + dir := t.TempDir() + t.Setenv(databaseEnv, "/"+filepath.Join(dir, name)) + + assertRunsUseDatabase(t, dir, name) + }) + + t.Run("relative", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + t.Setenv(databaseEnv, name) + + assertRunsUseDatabase(t, dir, name) + }) +} + // holdScanLock takes the lock on the database at path, as a running // scan does, and holds it until the test ends. It fails the test when // the lock is already held. From 4a16a41bd7e8ec11a30cef895b2562fde508092d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 16:01:36 +0200 Subject: [PATCH 20/33] Test both hashWorker cancellation checks on their own (closes #83) TestHashWorkerDropsQueuedRuns now passes hashWorker a hash function that records being called, so a worker that hashes a run after the scan is cancelled fails the test every time instead of only when it then chose to send its result. TestHashWorkerAbandonsBlockedSend cancels the scan from inside the hash function and leaves the result channel unread, so the worker can only return through the cancellation case beside its send. The scan tests could not show this, because stop drains results and frees a parked worker anyway. Model: opus-5-5 --- TODO.md | 3 ++ cancel_test.go | 74 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/TODO.md b/TODO.md index d146d28..d37d1e8 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- a test fails when either `hashWorker` cancellation check in `scan.go` + is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) + - a database path holding `?`, `#` or `%` opens exactly the file it names (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) diff --git a/cancel_test.go b/cancel_test.go index 8723bb4..6058973 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -729,47 +729,77 @@ func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) { // TestHashWorkerDropsQueuedRuns checks that a cancelled hash worker // keeps reading jobs and drops the runs rather than reading files // nobody wants the hashes of — while still letting the range run out -// so the pool tears down. The queued run names a file that does not -// exist, so a worker that hashed it anyway would produce a result. -// -// hashWorker's other cancellation exit, abandoning the send of a -// result, is reachable from the scan: stop cancels the pool before it -// drains results, so a worker waiting on that send can leave through -// it. The tests that stop a scan mid-hash, among them -// TestScanHashWriteFailureUnwindsPool, reach it in some runs only, -// depending on timing, and no test fails without it, since stop's -// drain frees a waiting worker anyway. This test, for its part, catches -// a removed drop check in some runs only: a worker that hashes the run -// anyway then picks at random between sending the result and leaving. +// so the pool tears down. The hash function records that it was +// called, so a worker that hashed the queued run anyway is caught +// every time. func TestHashWorkerDropsQueuedRuns(t *testing.T) { t.Parallel() done := make(chan struct{}) jobs := make(chan []fileRec, 1) - results := make(chan hashResult, 1) + results := make(chan hashResult) - run := []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} - - jobs <- run + jobs <- []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} close(jobs) + var hashed atomic.Bool + + hash := func(path string, size int64) (string, string, string, error) { + hashed.Store(true) + + return hashSignature(path, size) + } + go func() { defer close(done) - hashWorker(cancelledContext(t), jobs, results, hashSignature) + hashWorker(cancelledContext(t), jobs, results, hash) }() awaitReturn(t, done, "hashWorker") - select { - case r := <-results: - t.Errorf("cancelled hash worker produced %+v, want the run dropped", - r) - default: + if hashed.Load() { + t.Error("cancelled hash worker hashed the queued run, want it dropped") } } +// TestHashWorkerAbandonsBlockedSend checks that a hash worker with a +// result to deliver and nobody to deliver it to leaves once the scan +// is cancelled, instead of holding the pool open. The scan tests do +// not catch this: stop drains results, which frees a parked worker +// anyway. +func TestHashWorkerAbandonsBlockedSend(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + done := make(chan struct{}) + jobs := make(chan []fileRec, 1) + // Unbuffered and unread, with jobs left open: the worker's only way + // out is the cancellation case beside its send. + results := make(chan hashResult) + + jobs <- []fileRec{{path: filepath.Join(t.TempDir(), "missing"), size: 1}} + + // The scan is cancelled while the worker hashes, so the worker has + // already passed the check that drops queued runs. + hash := func(path string, size int64) (string, string, string, error) { + cancel() + + return hashSignature(path, size) + } + + go func() { + defer close(done) + + hashWorker(ctx, jobs, results, hash) + }() + + awaitReturn(t, done, "hashWorker") +} + // TestHashPhaseCancelledReturnsContextError checks the result loop's // own exit: with the pool cancelled, no result will ever arrive, and // the loop must leave through the cancellation rather than wait for a From e2227ac07ec9b64024823205dd7ef71d890bdf9f Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 16:13:20 +0200 Subject: [PATCH 21/33] Copy the current canonical .golangci.yml (closes #26) The shared lint config in the prompts repo moved from the deprecated gomodguard linter to gomodguard_v2, with a module block list, and now enables depguard to keep test-support packages out of non-test files. This replaces the repo's copy with that file unchanged, so lint no longer prints the gomodguard deprecation warning. Model: opus-5-5 --- .golangci.yml | 68 +++++++++++++++++++++++++++++++++++++++++++++++++-- TODO.md | 4 +++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 26b1610..a7a74c2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,14 +10,20 @@ run: linters: default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields - - depguard # Dependency allow/block lists - godot # Requires comments to end with periods - - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 settings: lll: line-length: 88 @@ -28,6 +34,64 @@ linters: max-complexity: 15 dupl: threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 diff --git a/TODO.md b/TODO.md index d37d1e8..1305b90 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- `.golangci.yml` replaced with the current canonical copy, which uses + `gomodguard_v2`, so lint no longer prints a deprecation warning + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/26) + - a test fails when either `hashWorker` cancellation check in `scan.go` is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) From 0b7078301d5cecdaf002a1a0ed2289664ea0b7ce Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 17:47:26 +0200 Subject: [PATCH 22/33] Test the remaining CLI cases (closes #16) Most of the command-line contract was already tested through run. This adds what was missing: report and trees with no database exit 1 with the message telling the user to run scan; a scan that skips an unreadable file prints its warning and counts the skip in its summary; the report and trees summary lines are checked exactly. The scan tests now capture the process's own stdout, which scan would write to directly, so a stray stdout write in scan fails them. The fatal-path test takes its subcommands from the command tree, so a new subcommand wired without runE fails it. The nonexistent-operand test gets an accurate name. Model: opus-5-5 --- TODO.md | 4 ++ cancel_test.go | 20 ++++-- main_test.go | 167 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 153 insertions(+), 38 deletions(-) diff --git a/TODO.md b/TODO.md index 1305b90..2e1fa6b 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- tests cover a missing database, `scan` keeping stdout empty, its skip + warning, the `report` and `trees` summary lines, and every subcommand + going through `runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16) + - `.golangci.yml` replaced with the current canonical copy, which uses `gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/26) diff --git a/cancel_test.go b/cancel_test.go index 6058973..556e150 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -344,13 +344,14 @@ func storedPaths(t *testing.T, path string) []string { // TestRunScanInterrupted calls the scan entrypoint with a context that // is already cancelled, as when a signal arrives at once. It must return -// errInterrupted promptly with its one line on stderr, leave the -// database valid and as it was, and leave nothing in the way of the -// next scan, which must bring the database up to date. +// errInterrupted promptly with its one line on stderr and nothing on +// stdout, leave the database valid and as it was, and leave nothing in +// the way of the next scan, which must bring the database up to date. func TestRunScanInterrupted(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) + stdout := captureStdout(t) stderr := captureStderr(t) dir := buildSmokeTree(t) @@ -393,6 +394,10 @@ func TestRunScanInterrupted(t *testing.T) { t.Errorf("stderr = %q, want %q", got, want) } + if got := stdout(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + assertNoSidecars(t, path) if got := storedPaths(t, path); !slices.Equal(got, before) { @@ -415,12 +420,13 @@ func TestRunScanInterrupted(t *testing.T) { // TestRunScanInterruptedMidHash interrupts the scan entrypoint part-way // through its hash phase, after the database is open. It must return // errInterrupted, release the lock, end stderr with its line counting -// every file the walk reached, close the database out of WAL mode, and -// keep the records it hashed. +// every file the walk reached, write nothing to stdout, close the +// database out of WAL mode, and keep the records it hashed. func TestRunScanInterruptedMidHash(t *testing.T) { path := testDBPath(t) t.Setenv(databaseEnv, path) + stdout := captureStdout(t) stderr := captureStderr(t) dir := buildWalkCancelTree(t) @@ -438,6 +444,10 @@ func TestRunScanInterruptedMidHash(t *testing.T) { t.Errorf("stderr = %q, want it to end with %q", got, want) } + if got := stdout(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + assertNoSidecars(t, path) db, err := openReportDatabase(t.Context(), path) diff --git a/main_test.go b/main_test.go index bf4150d..7742365 100644 --- a/main_test.go +++ b/main_test.go @@ -85,21 +85,41 @@ func makeReadOnly(t *testing.T, path string) { // captureStderr redirects os.Stderr to a file for the rest of the test // and returns a function reading back everything written to it. scan -// writes its warnings and summary straight to os.Stderr, not to the -// stderr writer run is given. +// writes its warnings and summary, and report and trees their +// summaries, straight to os.Stderr, not to the stderr writer run is +// given. func captureStderr(t *testing.T) func() string { t.Helper() - f, err := os.Create(filepath.Join(t.TempDir(), "stderr")) + return capture(t, &os.Stderr) +} + +// captureStdout does for os.Stdout what captureStderr does for +// os.Stderr. scan is never given run's stdout writer, so anything it +// printed would go straight to os.Stdout. The scan tests pass os.Stdout +// as run's stdout too, so the one capture sees both. +func captureStdout(t *testing.T) func() string { + t.Helper() + + return capture(t, &os.Stdout) +} + +// capture redirects *std, which is os.Stdout or os.Stderr, to a file +// for the rest of the test and returns a function reading back +// everything written to it. +func capture(t *testing.T, std **os.File) func() string { + t.Helper() + + f, err := os.Create(filepath.Join(t.TempDir(), "output")) if err != nil { t.Fatal(err) } - saved := os.Stderr - os.Stderr = f + saved := *std + *std = f t.Cleanup(func() { - os.Stderr = saved + *std = saved _ = f.Close() }) @@ -184,30 +204,33 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { // sidecar check is evidence of the close only for scan: report and // trees only read a database that is out of WAL mode, which leaves // nothing on disk whether they close it or not. - cases := map[string][]string{ - cmdScan: {cmdScan}, - cmdReport: {cmdReport}, - cmdTrees: {cmdTrees}, - } + // + // The subcommands come from the command tree, so a new one is + // checked too: one wired with a bare RunE instead of runE reports + // its failure as a usage error, exit 2 with the usage text. + for _, cmd := range newRootCommand(io.Discard, io.Discard).Commands() { + name := cmd.Name() - for name, args := range cases { t.Run(name, func(t *testing.T) { path := brokenDatabase(t) t.Setenv(databaseEnv, path) + args := []string{name} if name == cmdScan { args = append(args, t.TempDir()) } - var stdout, stderr bytes.Buffer + stdout := captureStdout(t) - code := run(args, &stdout, &stderr) + var stderr bytes.Buffer + + code := run(args, os.Stdout, &stderr) if code != exitFatal { t.Errorf("run(%v) = %d, want %d", args, code, exitFatal) } assertNoSidecars(t, path) - assertFatalOutput(t, stderr.String(), stdout.String()) + assertFatalOutput(t, stderr.String(), stdout()) // Proof that the failure happened after the open: only a // query against the opened database can report this. @@ -219,22 +242,24 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { } } -func TestRunMissingOperandIsFatalNotUsage(t *testing.T) { +func TestRunNonexistentPathIsFatalNotUsage(t *testing.T) { // README §Error handling: a PATH operand that does not exist is a // fatal error (1), not a usage error (2) — and a runtime failure // must not dump the usage text. t.Setenv(databaseEnv, testDBPath(t)) - var stdout, stderr bytes.Buffer + stdout := captureStdout(t) + + var stderr bytes.Buffer missing := filepath.Join(t.TempDir(), "nope") - code := run([]string{cmdScan, missing}, &stdout, &stderr) + code := run([]string{cmdScan, missing}, os.Stdout, &stderr) if code != exitFatal { t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal) } - assertFatalOutput(t, stderr.String(), stdout.String()) + assertFatalOutput(t, stderr.String(), stdout()) } // assertFatalOutput checks that a fatal error was reported the way @@ -258,6 +283,34 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) { } } +func TestRunMissingDatabaseIsFatal(t *testing.T) { + // README §Database: report and trees need an existing database; a + // missing one exits 1 with a message telling the user to run scan. + for _, name := range []string{cmdReport, cmdTrees} { + t.Run(name, func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + var stdout, stderr bytes.Buffer + + code := run([]string{name}, &stdout, &stderr) + if code != exitFatal { + t.Errorf("run(%s) = %d, want %d", name, code, exitFatal) + } + + want := "sfdupes: " + path + ": no database (run \"sfdupes " + + "scan\" first, or set " + databaseEnv + ")\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + + if got := stdout.String(); got != "" { + t.Errorf("stdout = %q, want nothing (data only)", got) + } + }) + } +} + func TestRunUsageErrors(t *testing.T) { // Usage errors keep exiting 2 with cobra's own report on stderr. cases := map[string]struct { @@ -435,17 +488,16 @@ func scanFixture(t *testing.T) []string { func scanOK(t *testing.T, operands ...string) string { t.Helper() - var stdout bytes.Buffer - + stdout := captureStdout(t) stderr := captureStderr(t) - code := run(append([]string{cmdScan}, operands...), &stdout, os.Stderr) + code := run(append([]string{cmdScan}, operands...), os.Stdout, os.Stderr) if code != exitOK { t.Fatalf("run(scan %q) = %d, want %d; stderr: %s", operands, code, exitOK, stderr()) } - if got := stdout.String(); got != "" { + if got := stdout(); got != "" { t.Errorf("scan stdout = %q, want nothing (data only)", got) } @@ -453,10 +505,41 @@ func scanOK(t *testing.T, operands ...string) string { } func TestRunScanSucceedsDespiteWarnings(t *testing.T) { + // README §Error handling: a scan that skips a file it cannot read + // warns, counts the skip in its summary, and still exits 0, which + // scanOK checks along with the empty stdout. + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + path := testDBPath(t) t.Setenv(databaseEnv, path) - scanFixture(t) + dir := t.TempDir() + writeFile(t, dir, "a.bin", pattern(1, 300)) + + // Same size as a.bin, so the scan reads it, and the read fails. + unreadable := writeFile(t, dir, "unreadable.bin", pattern(2, 300)) + + err := os.Chmod(unreadable, 0) + if err != nil { + t.Fatal(err) + } + + stderr := scanOK(t, dir) + + warning := "hash " + unreadable + ": open " + unreadable + + ": permission denied\n" + if !strings.Contains(stderr, warning) { + t.Errorf("stderr = %q, want %q", stderr, warning) + } + + summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " + + "0 unchanged), 1 skipped\n" + if !strings.Contains(stderr, summary) { + t.Errorf("stderr = %q, want %q", stderr, summary) + } + assertNoSidecars(t, path) } @@ -563,12 +646,14 @@ func TestRunReportSucceeds(t *testing.T) { dupes := scanFixture(t) - var stdout, stderr bytes.Buffer + var stdout bytes.Buffer - code := run([]string{cmdReport}, &stdout, &stderr) + stderr := captureStderr(t) + + code := run([]string{cmdReport}, &stdout, os.Stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + code, exitOK, stderr()) } want := "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n" @@ -576,6 +661,12 @@ func TestRunReportSucceeds(t *testing.T) { t.Errorf("stdout = %q, want %q", got, want) } + want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " + + "300 B reclaimable\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + assertNoSidecars(t, path) } @@ -585,12 +676,14 @@ func TestRunTreesSucceeds(t *testing.T) { dupes := scanFixture(t) - var stdout, stderr bytes.Buffer + var stdout bytes.Buffer - code := run([]string{cmdTrees}, &stdout, &stderr) + stderr := captureStderr(t) + + code := run([]string{cmdTrees}, &stdout, os.Stderr) if code != exitOK { t.Fatalf("run(trees) = %d, want %d; stderr: %s", - code, exitOK, stderr.String()) + code, exitOK, stderr()) } // The two directories holding the duplicate pair are duplicate @@ -601,6 +694,12 @@ func TestRunTreesSucceeds(t *testing.T) { t.Errorf("stdout = %q, want %q", got, want) } + want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " + + "300 B reclaimable\n" + if got := stderr(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } + assertNoSidecars(t, path) } @@ -730,9 +829,11 @@ func TestRunSecondScanFails(t *testing.T) { holdScanLock(t, path) - var stdout, stderr bytes.Buffer + stdout := captureStdout(t) - code := run([]string{cmdScan, t.TempDir()}, &stdout, &stderr) + var stderr bytes.Buffer + + code := run([]string{cmdScan, t.TempDir()}, os.Stdout, &stderr) if code != exitFatal { t.Errorf("run(scan) = %d, want %d", code, exitFatal) } @@ -743,7 +844,7 @@ func TestRunSecondScanFails(t *testing.T) { t.Errorf("stderr = %q, want %q", got, want) } - if got := stdout.String(); got != "" { + if got := stdout(); got != "" { t.Errorf("stdout = %q, want nothing (data only)", got) } From 1317d66589c2ecdecf46b00db19f6b0404e24acc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 18:01:29 +0200 Subject: [PATCH 23/33] Stop script/lint writing an image it never uses (closes #48) script/lint builds Dockerfile.lint only for the exit status, but every run exported the result as an image: seconds spent exporting, and one untagged image left behind each time. It now builds with --output=type=cacheonly, so nothing is exported. CHECK_EPOCH still changes on every run, so the gate steps still run each time; the build cache is kept as before. Model: opus-5-5 --- README.md | 4 +++- TODO.md | 3 +++ script/lint | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 98d7024..824bef5 100644 --- a/README.md +++ b/README.md @@ -831,7 +831,9 @@ and may be invoked directly. The provided entrypoints are: copies the repository into the digest-pinned `golangci/golangci-lint` image and runs `golangci-lint config verify` and `golangci-lint run` as build - steps, so a successful build is a clean lint. The linter is never + steps, so a successful build is a clean lint. That exit status is + all it produces, so it runs with `--output=type=cacheonly` and + writes no image; a run leaves only build cache. The linter is never run on the host, which makes a working `docker` the one prerequisite for linting — and therefore for `make check` and the pre-commit hook. Offline machines: the gate steps themselves make diff --git a/TODO.md b/TODO.md index 2e1fa6b..06d931a 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,9 @@ # Completed Steps +- `script/lint` writes no image, so a run no longer leaves an untagged + one behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48) + - tests cover a missing database, `scan` keeping stdout empty, its skip warning, the `report` and `trees` summary lines, and every subcommand going through `runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16) diff --git a/script/lint b/script/lint index d402d8a..20b021a 100755 --- a/script/lint +++ b/script/lint @@ -14,6 +14,10 @@ # 0 in well under a second having run no linter. The PID is in the value # as well as the epoch because two lint runs land inside the same second # easily, and `date +%s` alone would cache the second one. +# +# The result is the build's exit status and the image is never used, so +# --output=type=cacheonly writes none. Without it every run spends +# seconds exporting an image and leaves it behind untagged. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -22,6 +26,7 @@ main() { cd "$ROOT" docker build \ --build-arg CHECK_EPOCH="$(date +%s)-$$" \ + --output=type=cacheonly \ -f Dockerfile.lint \ . } From 313aa0fc128a3a9a028f9d90a10c44ead1042a96 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 18:30:25 +0200 Subject: [PATCH 24/33] Format Markdown with prettier in make fmt and make fmt-check (closes #19) script/fmt and script/fmt-check run prettier over every Markdown file again, next to gofmt. prettier is pinned by hash through package.json and yarn.lock, copied from the prompts repo with .prettierrc and .prettierignore, and is never installed on a host: a new prettier stage of the Dockerfile installs it into a digest-pinned node image, and both scripts build that stage and run it with the repository mounted. CI checks the Markdown in a markdown stage that the build stage waits on. Because make fmt-check now runs docker, the Dockerfile runs gofmt directly in its lint stage instead. All Markdown is reformatted. Model: opus-5-5 --- .prettierignore | 2 + .prettierrc | 4 + Dockerfile | 75 ++- README.md | 1399 ++++++++++++++++++++++------------------------ TODO.md | 843 +++++++++++++--------------- package.json | 5 + script/bootstrap | 24 +- script/cibuild | 20 +- script/docker | 10 +- script/fmt | 14 +- script/fmt-check | 29 +- yarn.lock | 8 + 12 files changed, 1195 insertions(+), 1238 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 package.json create mode 100644 yarn.lock diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..23d67fc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +node_modules/ +yarn.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/Dockerfile b/Dockerfile index 98a2174..b911abe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,9 @@ COPY . . # build would exit 0 having run nothing. script/cibuild and # script/docker pass a fresh CHECK_EPOCH on every invocation. # -# Two properties this depends on. ARG is per-stage, so the build stage -# below declares it again; one declaration here would leave that -# stage's gate cacheable. And each gate RUN must reference the value, +# Two properties this depends on. ARG is per-stage, so the markdown and +# build stages below declare it again; one declaration here would leave +# their gates cacheable. And each gate RUN must reference the value, # because BuildKit hashes the expanded command: a declared but # unreferenced ARG invalidates nothing. # @@ -27,9 +27,16 @@ ARG CHECK_EPOCH # target now runs `docker build -f Dockerfile.lint`, and a docker build # cannot run a docker build: routing the gate through make would mean # nesting docker inside this image. Same reason `make check` is gone -# from the build stage below. `make fmt-check` stays as it is — it is a -# gate, not the aggregate, and it shells out to nothing. -RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check +# from the build stage below, and `make fmt-check` from both stages: it +# runs prettier through docker too. Its gofmt half is the step below, +# its Markdown half the markdown stage further down. gofmt's output is +# assigned to a variable first so that its own exit status, as when it +# cannot parse a file, still fails the step. +RUN echo "gate gofmt, epoch ${CHECK_EPOCH}" && \ + files="$(gofmt -s -l .)" && \ + if [ -n "$files" ]; then \ + echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \ + fi # The FROM above and the one in Dockerfile.lint pin the same linter # twice, and nothing else keeps them in sync; this fails the build when @@ -46,6 +53,30 @@ RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \ RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \ golangci-lint run --config .golangci.yml ./... +# Prettier stage: the prettier that formats this repository's Markdown, +# never installed on a host. script/fmt and script/fmt-check build this +# stage alone and run it with the repository mounted on /src. prettier +# is installed in /tools so that the repository, mounted or copied onto +# /src, cannot hide it. +# node:22-alpine, 2026-02-22 +FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS prettier +WORKDIR /tools +# yarn.lock pins prettier by hash, and --frozen-lockfile fails rather +# than install anything yarn.lock does not name. +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile +ENV PATH=/tools/node_modules/.bin:$PATH +WORKDIR /src + +# Markdown stage: the Markdown half of `make fmt-check`, as a gate. +FROM prettier AS markdown +COPY . . +# Second per-stage declaration of the gate cache-buster; see the lint +# stage above. +ARG CHECK_EPOCH +RUN echo "gate prettier, epoch ${CHECK_EPOCH}" && \ + prettier --check '**/*.md' --tab-width 4 --prose-wrap always + # Build stage # golang:1.25-alpine, 2026-07-23 FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder @@ -70,15 +101,17 @@ ENV GOCACHE=/home/builder/.cache/go-build WORKDIR /src -# No-op file copy whose only purpose is the build-graph edge: it is what -# makes this stage depend on the lint stage, and so what forces BuildKit -# to finish fmt-check, the pin guard and lint before compilation and -# tests start. Remove it and the fail-fast design dies silently — the -# build stops gating on lint and still exits 0. It replaces a copy of -# the linter binary itself, which is no longer wanted here: nothing in -# this stage runs the linter, because `make lint` is now a docker build -# and a docker build cannot run inside one. +# No-op file copies whose only purpose is the build-graph edge: they are +# what make this stage depend on the lint and markdown stages, and so +# what forces BuildKit to finish gofmt, the pin guard, lint and prettier +# before compilation and tests start. Remove one and the fail-fast +# design dies silently — the build stops gating on that stage and still +# exits 0. The first replaces a copy of the linter binary itself, which +# is no longer wanted here: nothing in this stage runs the linter, +# because `make lint` is now a docker build and a docker build cannot +# run inside one. COPY --from=lint /src/go.sum /dev/null +COPY --from=markdown /src/go.sum /dev/null # Install development prerequisites the same way a developer does, # rather than duplicating the installs inline. Only script/ and the @@ -112,19 +145,19 @@ USER builder # permission-denied test paths are exercised legitimately (root would # bypass the chmod(0) the tests rely on). # -# The gates are the individual targets, not `make check`: that aggregate -# runs `script/lint`, which is now a docker build, and nothing inside an -# image build may shell out to docker. Lint is not skipped by this — it -# ran in the lint stage above, which this stage's COPY --from makes a -# prerequisite. `make`, not the scripts directly, because the Makefile's +# The gate is `make test`, not `make check`: that aggregate runs +# `script/lint` and `script/fmt-check`, which both run docker, and +# nothing inside an image build may shell out to docker. Lint and the +# format checks are not skipped by this — they ran in the lint and +# markdown stages above, which this stage's COPY --from lines make +# prerequisites. `make`, not the script directly, because the Makefile's # `export CGO_ENABLED = 0` applies only to what it invokes. # -# Second per-stage declaration of the gate cache-buster; see the lint +# Third per-stage declaration of the gate cache-buster; see the lint # stage above for why one is not enough. It is placed after USER so the # drop to the unprivileged user still happens before the checks run. ARG CHECK_EPOCH RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test -RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check # The version stamped into the binary: the VERSION build argument when # one is given, otherwise `git describe --tags --always` of the .git in diff --git a/README.md b/README.md index 824bef5..8e6c030 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,23 @@ ## Description -`sfdupes` is an MIT-licensed Go CLI tool by -[@sneak](https://sneak.berlin) that quickly identifies *candidate* -duplicate files — and, ultimately, entire duplicate directory trees — -across very large filesystems without reading every byte of every file. -Files are considered duplicates when their sizes are equal and they -agree on a short ladder of hashes. A file under 10 MiB is hashed in full -and compared directly. A larger file is gated first on the SHA-256 of -its first 64 KiB and of its last 64 KiB, and only when its size and -both of those match another file's is it read for a content hash to -compare — the SHA-256 of the whole file when it is under 50 MiB, or of -gigabyte-spaced 1 MiB samples when it is 50 MiB or larger. Below 50 MiB -the content hash is proof of identical content; at or above 50 MiB it is -a strong candidate signal rather than proof, because the gaps between -samples are never read. The intended use is -finding duplicate downloads and duplicated directory trees on -multi-terabyte ZFS servers where reading every byte of every file is -prohibitively expensive. `scan` maintains a persistent SQLite database of file -signatures that survives between runs, so it can be run from cron and -the reports can be generated at any time from the most recent scan. +`sfdupes` is an MIT-licensed Go CLI tool by [@sneak](https://sneak.berlin) that +quickly identifies _candidate_ duplicate files — and, ultimately, entire +duplicate directory trees — across very large filesystems without reading every +byte of every file. Files are considered duplicates when their sizes are equal +and they agree on a short ladder of hashes. A file under 10 MiB is hashed in +full and compared directly. A larger file is gated first on the SHA-256 of its +first 64 KiB and of its last 64 KiB, and only when its size and both of those +match another file's is it read for a content hash to compare — the SHA-256 of +the whole file when it is under 50 MiB, or of gigabyte-spaced 1 MiB samples when +it is 50 MiB or larger. Below 50 MiB the content hash is proof of identical +content; at or above 50 MiB it is a strong candidate signal rather than proof, +because the gaps between samples are never read. The intended use is finding +duplicate downloads and duplicated directory trees on multi-terabyte ZFS servers +where reading every byte of every file is prohibitively expensive. `scan` +maintains a persistent SQLite database of file signatures that survives between +runs, so it can be run from cron and the reports can be generated at any time +from the most recent scan. This README is the complete and authoritative specification. @@ -34,22 +32,19 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite" ./sfdupes trees > dupetrees.tsv ``` -`scan` walks one or more filesystem trees and maintains one database -record per regular file (path, size, mtime, head hash, tail hash, -content hash). The -database persists between runs; a rescan only hashes files that are new -or changed, or that may have gained a duplicate since the last scan, -and removes records for files that no longer exist. -`report` reads the database and prints the file-level duplicates -report. `trees` reads the same database and prints the duplicate-tree -report. A missing/invalid subcommand — or a `scan` invocation with no -`PATH` operand — prints a usage message and exits 2. +`scan` walks one or more filesystem trees and maintains one database record per +regular file (path, size, mtime, head hash, tail hash, content hash). The +database persists between runs; a rescan only hashes files that are new or +changed, or that may have gained a duplicate since the last scan, and removes +records for files that no longer exist. `report` reads the database and prints +the file-level duplicates report. `trees` reads the same database and prints the +duplicate-tree report. A missing/invalid subcommand — or a `scan` invocation +with no `PATH` operand — prints a usage message and exits 2. -The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed -anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a -daily `sfdupes scan` cron job, with the reporting commands run -interactively whenever needed; their results are as fresh as the last -completed scan. +The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere +by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` +cron job, with the reporting commands run interactively whenever needed; their +results are as fresh as the last completed scan. ### Install @@ -59,10 +54,9 @@ With Go installed, this builds and installs the current `main` branch: go install sneak.berlin/go/sfdupes@main ``` -The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is -set. A binary installed this way reports its version as `dev`; one -built from a clone or into the Docker image carries the git tag or -commit it was built from. +The binary goes to `$(go env GOPATH)/bin`, or to `$GOBIN` when that is set. A +binary installed this way reports its version as `dev`; one built from a clone +or into the Docker image carries the git tag or commit it was built from. From a clone, `make build` writes the binary to `./sfdupes`: @@ -74,13 +68,12 @@ make build Copy the binary to `/usr/local/bin` for the cron job below. -`make docker` builds the Docker image, tagged `sfdupes`, after running -the tests and the linter (see "Build"). The image runs `sfdupes` as -root with the database at its default path, so a bind mount of -`/var/lib/sfdupes` keeps the database between runs. Mount the scanned -tree at the same path inside the container as on the host; read-only -is enough. The database records paths as the container sees them, so -the reports then name the host's paths. +`make docker` builds the Docker image, tagged `sfdupes`, after running the tests +and the linter (see "Build"). The image runs `sfdupes` as root with the database +at its default path, so a bind mount of `/var/lib/sfdupes` keeps the database +between runs. Mount the scanned tree at the same path inside the container as on +the host; read-only is enough. The database records paths as the container sees +them, so the reports then name the host's paths. ```sh make docker @@ -91,52 +84,49 @@ docker run --rm -v /var/lib/sfdupes:/var/lib/sfdupes sfdupes report > dupes.tsv ### Daily scan from cron -Run `scan` as root, so that it can read every file: a path it cannot -read is skipped with a warning and loses its database record (see -"Rules for the walk"). As a file `/etc/cron.d/sfdupes`: +Run `scan` as root, so that it can read every file: a path it cannot read is +skipped with a warning and loses its database record (see "Rules for the walk"). +As a file `/etc/cron.d/sfdupes`: ``` 30 3 * * * root /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log ``` -- The database is `/var/lib/sfdupes/db.sqlite`, created with its - directory by the first scan. To keep it elsewhere, set - `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same - line. -- `scan` writes nothing to stdout. Its stderr, appended here to - `/var/log/sfdupes.log`, holds a plain progress line as each phase - starts and then at most every 5 seconds, a warning for each path it - skips, and the summary line (see "Progress" and "`scan` mode"). The - log grows with every scan; rotate it like any other. -- Skipped paths do not fail a scan: it still exits 0, and cron sends - nothing. A scan that fails, or is stopped by `SIGINT` or `SIGTERM`, - exits 1 with the reason among the last lines of the log; `tail` - prints them, and cron mails them to root if the host can send mail. -- A scan still running when the next one starts carries on. The new - one fails at once, and the lines cron mails include - `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. -- `report` and `trees` need only read access to the database (see - "Database"). Under the usual umask of `022` the first scan creates - it readable by every user, so an unprivileged user can run them - against root's database. +- The database is `/var/lib/sfdupes/db.sqlite`, created with its directory by + the first scan. To keep it elsewhere, set + `SFDUPES_DATABASE=/path/to/db.sqlite` before the command on the same line. +- `scan` writes nothing to stdout. Its stderr, appended here to + `/var/log/sfdupes.log`, holds a plain progress line as each phase starts and + then at most every 5 seconds, a warning for each path it skips, and the + summary line (see "Progress" and "`scan` mode"). The log grows with every + scan; rotate it like any other. +- Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A + scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the + reason among the last lines of the log; `tail` prints them, and cron mails + them to root if the host can send mail. +- A scan still running when the next one starts carries on. The new one fails at + once, and the lines cron mails include + `sfdupes: another scan is running (lock held on /var/lib/sfdupes/db.sqlite.lock)`. +- `report` and `trees` need only read access to the database (see "Database"). + Under the usual umask of `022` the first scan creates it readable by every + user, so an unprivileged user can run them against root's database. ### Reading the reports -Each row of `report` names two copies of one file, and each row of -`trees` two copies of one directory tree (see "Report output format" -and "Trees output format"). In a group of copies, the path that sorts -first byte by byte is `first` and every other path is a `dupe` of it. -`first` says nothing about which copy is the original or the oldest; -which copy to keep is your choice. +Each row of `report` names two copies of one file, and each row of `trees` two +copies of one directory tree (see "Report output format" and "Trees output +format"). In a group of copies, the path that sorts first byte by byte is +`first` and every other path is a `dupe` of it. `first` says nothing about which +copy is the original or the oldest; which copy to keep is your choice. A row is a candidate, not proof: -- The reports read only the database, so they show the files as of - the last scan; a file may have changed or gone since. -- A file of 50 MiB or more is compared only on samples of its content - (see "Duplicate detection"). -- Paths that are hard links to one file are listed as duplicates, but - they share their data, so removing one frees nothing. +- The reports read only the database, so they show the files as of the last + scan; a file may have changed or gone since. +- A file of 50 MiB or more is compared only on samples of its content (see + "Duplicate detection"). +- Paths that are hard links to one file are listed as duplicates, but they share + their data, so removing one frees nothing. Compare a pair byte for byte before removing either copy. For the row `/srv/a/big.iso`, `/srv/b/big-copy.iso`, `4294967296`: @@ -146,120 +136,110 @@ cmp /srv/a/big.iso /srv/b/big-copy.iso && echo identical [ /srv/a/big.iso -ef /srv/b/big-copy.iso ] && echo "hard links" ``` -`cmp` prints nothing and exits 0 only when every byte matches, and -otherwise reports where the files differ. The second line prints -`hard links` when the two paths are the same file, so removing either -frees nothing. +`cmp` prints nothing and exits 0 only when every byte matches, and otherwise +reports where the files differ. The second line prints `hard links` when the two +paths are the same file, so removing either frees nothing. -A path holding a backslash, tab, newline or carriage return is escaped -in the reports (see "Report output format"). Undo the escapes before -using it. `printf '%b'` does exactly that, because every backslash in -an escaped path starts one of the four escapes. Command substitution -drops trailing newlines, so print an `x` after the path and remove it -afterwards, or a path that ends in a newline names a different file: +A path holding a backslash, tab, newline or carriage return is escaped in the +reports (see "Report output format"). Undo the escapes before using it. +`printf '%b'` does exactly that, because every backslash in an escaped path +starts one of the four escapes. Command substitution drops trailing newlines, so +print an `x` after the path and remove it afterwards, or a path that ends in a +newline names a different file: ```sh p="$(printf '%bx' '/srv/a/tab\tname.txt')"; p="${p%x}" cmp "$p" /srv/b/tab-copy.txt ``` -Check a `trees` row with `diff -r`, which compares the two trees file -by file and also names anything present in only one of them, such as -an empty directory or a symlink, which `trees` does not see. +Check a `trees` row with `diff -r`, which compares the two trees file by file +and also names anything present in only one of them, such as an empty directory +or a symlink, which `trees` does not see. ## Rationale -Duplicate finders that hash entire files do not scale to the target -environment: ~10 million files and ~150 TB on possibly slow or busy -disks (a ZFS pool under resilver). sfdupes spends disk I/O only on files -whose size at least one other file shares, since a size-unique file -cannot be a duplicate. Of those, a file under 10 MiB is read in full; a -larger one has its cheap end windows read first, and is read for a -content hash only when its size and both end windows match another -file's — the whole file below 50 MiB, but only gigabyte-spaced samples -at or above 50 MiB, so the largest files are never read in full. This -keeps a full-filesystem sweep tractable, and the signatures are kept in -a persistent database, so the expensive filesystem pass is incremental: -a rescan re-hashes only files whose recorded mtime or size changed, plus -— for its content hash — a file of 10 MiB or more whose size and end -windows have come to match another file's. All analysis happens offline -from the database alone. The end goal is -not individual files but whole duplicated trees — duplicate -extractions, duplicate downloads, copied project trees — which an -operator can consider removing as a unit. +Duplicate finders that hash entire files do not scale to the target environment: +~10 million files and ~150 TB on possibly slow or busy disks (a ZFS pool under +resilver). sfdupes spends disk I/O only on files whose size at least one other +file shares, since a size-unique file cannot be a duplicate. Of those, a file +under 10 MiB is read in full; a larger one has its cheap end windows read first, +and is read for a content hash only when its size and both end windows match +another file's — the whole file below 50 MiB, but only gigabyte-spaced samples +at or above 50 MiB, so the largest files are never read in full. This keeps a +full-filesystem sweep tractable, and the signatures are kept in a persistent +database, so the expensive filesystem pass is incremental: a rescan re-hashes +only files whose recorded mtime or size changed, plus — for its content hash — a +file of 10 MiB or more whose size and end windows have come to match another +file's. All analysis happens offline from the database alone. The end goal is +not individual files but whole duplicated trees — duplicate extractions, +duplicate downloads, copied project trees — which an operator can consider +removing as a unit. ## Design Goals, in order: -1. **Find whole duplicate trees, not just files.** The end goal is to - identify places where the exact same set of files and directories - exists at two or more paths (duplicate extractions, duplicate - downloads, copied project trees), so the operator can consider - removing an entire subtree at once. File-level duplicate detection is - the foundation; tree-level detection is built on top of it. -2. **Spend I/O in proportion to duplicate likelihood.** Only files - whose size at least one other file shares are read at all — a - size-unique file cannot be a duplicate. Those are compared by the - ladder in "Duplicate detection" below: a file under 10 MiB is hashed - in full, while a larger file is gated on cheap 64 KiB end windows - first, and gets a content hash only when its size and both end - windows match another file's. That hash reads the whole file below - 50 MiB but only gigabyte-spaced 1 MiB samples at or above it, so the - very largest files are still never read in full. Scale target: tens of - millions of files, ~150 TB filesystem, possibly slow or busy disks - (ZFS pool under resilver). Holding one small record (path, size, - mtime) per file in memory during a scan is acceptable; holding - every file's hashes is not (they stay in the database). The - reporting commands do not hold every file's hashes either: `report` - lets SQLite group and order the records and writes each row as it - reads it, so its memory does not grow with the database, and - `trees` reads the records in path order and keeps each directory's - path, digest and totals, plus the hashes of only the files in the - directories holding the record being read, so its memory grows with - the number of directories and with the size of the largest - directory. -3. **Scan incrementally, analyze offline.** The expensive filesystem - scan maintains a persistent database; an unchanged file is never - read again on a rescan, except to compute its content hash once a - file of 10 MiB or more comes to match another on size and both end - windows. All analysis (`report`, `trees`) works from - the database alone and must never touch the scanned filesystem - again. `scan` is designed to be cronned; the reports run at any +1. **Find whole duplicate trees, not just files.** The end goal is to identify + places where the exact same set of files and directories exists at two or + more paths (duplicate extractions, duplicate downloads, copied project + trees), so the operator can consider removing an entire subtree at once. + File-level duplicate detection is the foundation; tree-level detection is + built on top of it. +2. **Spend I/O in proportion to duplicate likelihood.** Only files whose size + at least one other file shares are read at all — a size-unique file cannot + be a duplicate. Those are compared by the ladder in "Duplicate detection" + below: a file under 10 MiB is hashed in full, while a larger file is gated + on cheap 64 KiB end windows first, and gets a content hash only when its + size and both end windows match another file's. That hash reads the whole + file below 50 MiB but only gigabyte-spaced 1 MiB samples at or above it, so + the very largest files are still never read in full. Scale target: tens of + millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool + under resilver). Holding one small record (path, size, mtime) per file in + memory during a scan is acceptable; holding every file's hashes is not (they + stay in the database). The reporting commands do not hold every file's + hashes either: `report` lets SQLite group and order the records and writes + each row as it reads it, so its memory does not grow with the database, and + `trees` reads the records in path order and keeps each directory's path, + digest and totals, plus the hashes of only the files in the directories + holding the record being read, so its memory grows with the number of + directories and with the size of the largest directory. +3. **Scan incrementally, analyze offline.** The expensive filesystem scan + maintains a persistent database; an unchanged file is never read again on a + rescan, except to compute its content hash once a file of 10 MiB or more + comes to match another on size and both end windows. All analysis (`report`, + `trees`) works from the database alone and must never touch the scanned + filesystem again. `scan` is designed to be cronned; the reports run at any time against the last completed scan. -4. **Clean stream separation.** Everything on stdout is machine-readable - data. All progress, warnings, summaries, and help and usage text go - to stderr. Never mix them. +4. **Clean stream separation.** Everything on stdout is machine-readable data. + All progress, warnings, summaries, and help and usage text go to stderr. + Never mix them. ### Constraints -- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: - `sfdupes`. -- Dependencies: standard library, `github.com/spf13/cobra` for the - CLI, **one progress-bar library** - (`github.com/schollz/progressbar/v3`), `golang.org/x/term` to tell - whether stderr is a terminal, **one SQLite driver** - (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and - `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). - `github.com/spf13/viper` is permitted if configuration-file support - is ever needed, but is not currently used. No other third-party - deps. -- Cross-compilation is not a concern. Builds run with cgo disabled (the - `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. -- Analysis modes (`report`, `trees`) must be deterministic: identical - database contents, identical output, regardless of the order in - which records were inserted. +- Language: Go (module `sneak.berlin/go/sfdupes`). Binary name: `sfdupes`. +- Dependencies: standard library, `github.com/spf13/cobra` for the CLI, **one + progress-bar library** (`github.com/schollz/progressbar/v3`), + `golang.org/x/term` to tell whether stderr is a terminal, **one SQLite + driver** (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled), and + `golang.org/x/sys` for `flock(2)` (the scan lock, see "Database"). + `github.com/spf13/viper` is permitted if configuration-file support is ever + needed, but is not currently used. No other third-party deps. +- Cross-compilation is not a concern. Builds run with cgo disabled (the + `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go. +- Analysis modes (`report`, `trees`) must be deterministic: identical database + contents, identical output, regardless of the order in which records were + inserted. ### Subcommands Three subcommands, all implemented: -1. `scan` — walk the filesystem and synchronize the database: one - signature record per regular file. +1. `scan` — walk the filesystem and synchronize the database: one signature + record per regular file. 2. `report` — file-level duplicate report from the database. -3. `trees` — tree-level duplicate report: reconstruct the directory - hierarchy from the database records, compute a Merkle-style digest - per directory, and report maximal groups of identical trees. +3. `trees` — tree-level duplicate report: reconstruct the directory hierarchy + from the database records, compute a Merkle-style digest per directory, and + report maximal groups of identical trees. ``` sfdupes scan [--workers N] [-x] PATH... @@ -269,74 +249,66 @@ sfdupes --version sfdupes [command] --help ``` -`--workers N` sets the size of each `scan` worker pool (default: the -number of CPUs), and `-x` (`--one-file-system`) keeps the walk of each -operand on that operand's filesystem; both are described under -"`scan` mode". `sfdupes --version` (or `-v`) prints one line, -`sfdupes VERSION`, to stdout and exits 0, writing nothing to stderr. -`-h` or `--help`, alone or after a subcommand, prints the help text to -stderr and exits 0, writing nothing to stdout. +`--workers N` sets the size of each `scan` worker pool (default: the number of +CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that +operand's filesystem; both are described under "`scan` mode". +`sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and +exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a +subcommand, prints the help text to stderr and exits 0, writing nothing to +stdout. ### Database All three subcommands operate on a single SQLite database file: -- Location: the value of the `SFDUPES_DATABASE` environment variable - when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. - There is no command-line flag. The path names the file exactly, - whatever characters it holds (`?`, `#` and `%` included); a - relative path is relative to the working directory. -- `scan` creates the database (and its parent directory) on first - use. `report` and `trees` require an existing database; a missing - database file is a fatal error (exit 1) telling the user to run - `scan` first. -- Only one `scan` runs against a database at a time. For its whole - run, `scan` holds an exclusive `flock(2)` lock on a lock file - beside the database, named by appending `.lock` to the database - path (`/var/lib/sfdupes/db.sqlite.lock` by default), taken before - it walks the filesystem or opens the database. A second `scan` - against the same database does not wait: it fails at once with a - one-line error naming the lock file and exits 1, without walking - anything or opening the database, and the running scan carries on. - The lock file is created on first use, open to its owner only, and - left in place: a leftover file blocks nothing, because the lock - ends with the process holding it however it ends, a fatal error or - an interrupt included, and deleting the file while a scan runs - would let a second scan start. `report` and `trees` never take the - lock, so they run during a scan. -- While `scan` runs, the database is in WAL journal mode with a busy - timeout, so running a report while a cron `scan` is in progress is - safe. The filesystem is authoritative; the database is an - eventually-consistent reflection of it. Hashed records are - committed in batched transactions while the scan is still running - (keeping the WAL small and letting concurrent reports observe - progress), so a report may see a scan's changes partially applied, - and a scan that dies partway leaves a valid database holding - every batch committed so far (an interrupted scan also commits the - batch in progress, see "Error handling and exit codes"); the next - scan skips those records and converges toward the filesystem. -- `scan` switches the database back to rollback-journal mode when it - closes it, so between scans the database file alone holds the whole - database. Each switch needs the database to itself: a `scan` that - starts while a report is still reading waits for it up to the - 10-second busy timeout, then fails; a `scan` that ends while a - report has the database open warns and leaves the database in WAL - mode until the next scan. `report` writes each row as it reads it, - so it is still reading while its output is paused (a pager, a - stalled pipe), and a `scan` started then fails after the busy - timeout. -- `report` and `trees` open the database read-only and need only read - access to the database file, and no write access to its directory. - While the database is in WAL mode they also read the `-wal` and - `-shm` files beside it, which SQLite creates with the database - file's permissions. -- Schema (`PRAGMA user_version` is the schema version, currently 1; a - database with any other version is a fatal error. `scan` creates - the schema and sets the version in one transaction, so a first scan - stopped while doing so leaves an empty database the next scan sets - up. A database at version 0 that already has a `files` table was - therefore not made by sfdupes; every subcommand refuses it with an - error telling the user to remove the file and rescan): +- Location: the value of the `SFDUPES_DATABASE` environment variable when set + and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`. There is no + command-line flag. The path names the file exactly, whatever characters it + holds (`?`, `#` and `%` included); a relative path is relative to the working + directory. +- `scan` creates the database (and its parent directory) on first use. `report` + and `trees` require an existing database; a missing database file is a fatal + error (exit 1) telling the user to run `scan` first. +- Only one `scan` runs against a database at a time. For its whole run, `scan` + holds an exclusive `flock(2)` lock on a lock file beside the database, named + by appending `.lock` to the database path (`/var/lib/sfdupes/db.sqlite.lock` + by default), taken before it walks the filesystem or opens the database. A + second `scan` against the same database does not wait: it fails at once with a + one-line error naming the lock file and exits 1, without walking anything or + opening the database, and the running scan carries on. The lock file is + created on first use, open to its owner only, and left in place: a leftover + file blocks nothing, because the lock ends with the process holding it however + it ends, a fatal error or an interrupt included, and deleting the file while a + scan runs would let a second scan start. `report` and `trees` never take the + lock, so they run during a scan. +- While `scan` runs, the database is in WAL journal mode with a busy timeout, so + running a report while a cron `scan` is in progress is safe. The filesystem is + authoritative; the database is an eventually-consistent reflection of it. + Hashed records are committed in batched transactions while the scan is still + running (keeping the WAL small and letting concurrent reports observe + progress), so a report may see a scan's changes partially applied, and a scan + that dies partway leaves a valid database holding every batch committed so far + (an interrupted scan also commits the batch in progress, see "Error handling + and exit codes"); the next scan skips those records and converges toward the + filesystem. +- `scan` switches the database back to rollback-journal mode when it closes it, + so between scans the database file alone holds the whole database. Each switch + needs the database to itself: a `scan` that starts while a report is still + reading waits for it up to the 10-second busy timeout, then fails; a `scan` + that ends while a report has the database open warns and leaves the database + in WAL mode until the next scan. `report` writes each row as it reads it, so + it is still reading while its output is paused (a pager, a stalled pipe), and + a `scan` started then fails after the busy timeout. +- `report` and `trees` open the database read-only and need only read access to + the database file, and no write access to its directory. While the database is + in WAL mode they also read the `-wal` and `-shm` files beside it, which SQLite + creates with the database file's permissions. +- Schema (`PRAGMA user_version` is the schema version, currently 1; a database + with any other version is a fatal error. `scan` creates the schema and sets + the version in one transaction, so a first scan stopped while doing so leaves + an empty database the next scan sets up. A database at version 0 that already + has a `files` table was therefore not made by sfdupes; every subcommand + refuses it with an error telling the user to remove the file and rescan): ```sql CREATE TABLE files ( @@ -350,274 +322,242 @@ All three subcommands operate on a single SQLite database file: CREATE INDEX files_signature ON files (size, head, tail, content); ``` - Paths are stored as BLOBs because Unix paths are raw bytes, not - guaranteed UTF-8. `mtime` is used only for change detection; it is - not part of the duplicate key. For a file under 10 MiB `head`, `tail`, - and `content` all hold the whole-file hash (that range is hashed in - full, with no end windows); for a larger file `head` and `tail` hold - the first- and last-64 KiB hashes and `content` the whole-file or - sampled hash. All three are empty strings when the file has never - been hashed because its size was unique as of the last scan that - covered it. For a file of 10 MiB or more, `content` stays empty - until the content phase of a scan (see "`scan` mode" below) has - read the file. A record with an empty `content` is never part of a - duplicate group, though it still defines the file for tree - reconstruction. The `files_signature` index lets SQLite group the - records by signature for `report` without sorting the whole table. + Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed + UTF-8. `mtime` is used only for change detection; it is not part of the + duplicate key. For a file under 10 MiB `head`, `tail`, and `content` all + hold the whole-file hash (that range is hashed in full, with no end + windows); for a larger file `head` and `tail` hold the first- and last-64 + KiB hashes and `content` the whole-file or sampled hash. All three are empty + strings when the file has never been hashed because its size was unique as + of the last scan that covered it. For a file of 10 MiB or more, `content` + stays empty until the content phase of a scan (see "`scan` mode" below) has + read the file. A record with an empty `content` is never part of a duplicate + group, though it still defines the file for tree reconstruction. The + `files_signature` index lets SQLite group the records by signature for + `report` without sorting the whole table. ### Duplicate detection -Two files are duplicates only when they agree on every rung of this -ladder; a mismatch at any rung means they are not duplicates. `scan` -stores each file's hashes, and `report` and `trees` group files by the -whole signature — size, `head`, `tail`, and `content` — so the grouping -is exactly this ladder applied across everything scanned into the -database, even across separate scans. +Two files are duplicates only when they agree on every rung of this ladder; a +mismatch at any rung means they are not duplicates. `scan` stores each file's +hashes, and `report` and `trees` group files by the whole signature — size, +`head`, `tail`, and `content` — so the grouping is exactly this ladder applied +across everything scanned into the database, even across separate scans. -1. **Size.** Files of different sizes are never compared. Only files - whose size at least one other file shares are hashed at all. -2. **Under 10 MiB: whole file.** A file smaller than 10 MiB is hashed - in full and compared directly, with no separate end-window step — - small files are cheap to read to the last byte, and doing so makes - the comparison exact. `head`, `tail`, and `content` all hold this - whole-file SHA-256, so such a file's signature is decided entirely - by its size and its content. -3. **10 MiB and above: head and tail.** For a larger file, the SHA-256 - of the first 64 KiB (`head`) and of the last 64 KiB (`tail`) are a - cheap gate that eliminates most same-size pairs before any bulk - reading: the content hash of the next two rungs is computed only for - a file whose size, `head`, and `tail` match another file's, whether - that file is scanned in the same run or stored by an earlier scan. - A stored file that first gains such a match in a later scan gets its - content hash then; until it has one, its `content` is empty and it - is not a duplicate. At 10 MiB and above the two windows never +1. **Size.** Files of different sizes are never compared. Only files whose size + at least one other file shares are hashed at all. +2. **Under 10 MiB: whole file.** A file smaller than 10 MiB is hashed in full + and compared directly, with no separate end-window step — small files are + cheap to read to the last byte, and doing so makes the comparison exact. + `head`, `tail`, and `content` all hold this whole-file SHA-256, so such a + file's signature is decided entirely by its size and its content. +3. **10 MiB and above: head and tail.** For a larger file, the SHA-256 of the + first 64 KiB (`head`) and of the last 64 KiB (`tail`) are a cheap gate that + eliminates most same-size pairs before any bulk reading: the content hash of + the next two rungs is computed only for a file whose size, `head`, and + `tail` match another file's, whether that file is scanned in the same run or + stored by an earlier scan. A stored file that first gains such a match in a + later scan gets its content hash then; until it has one, its `content` is + empty and it is not a duplicate. At 10 MiB and above the two windows never overlap. -4. **10 MiB and above, content below 50 MiB.** The SHA-256 of the - entire file. Agreement here is proof of identical content (barring a - SHA-256 collision). -5. **10 MiB and above, content 50 MiB and above.** A sampled SHA-256: - the 1 MiB window at each gigabyte-aligned offset (0, 1 GiB, 2 GiB, … - while inside the file, the final window truncated at end of file) is - fed, in order, into one hash. This is **deliberately probabilistic** - — the gaps between samples are never read, so two large files that - agree on every sample are reported as duplicates without being read - in full. It is the price of never reading a 150 GB file end to end. - Because size is already part of the signature, only equal-size files - reach this rung, so their sample boundaries always align. +4. **10 MiB and above, content below 50 MiB.** The SHA-256 of the entire file. + Agreement here is proof of identical content (barring a SHA-256 collision). +5. **10 MiB and above, content 50 MiB and above.** A sampled SHA-256: the 1 MiB + window at each gigabyte-aligned offset (0, 1 GiB, 2 GiB, … while inside the + file, the final window truncated at end of file) is fed, in order, into one + hash. This is **deliberately probabilistic** — the gaps between samples are + never read, so two large files that agree on every sample are reported as + duplicates without being read in full. It is the price of never reading a + 150 GB file end to end. Because size is already part of the signature, only + equal-size files reach this rung, so their sample boundaries always align. -`head`, `tail`, and `content` are one column each. A file below 10 MiB -and one at or above it never share a size, and neither do a file below -50 MiB and one at or above it, so a stored value is never ambiguous -between the whole-file, end-window, and sampled forms. +`head`, `tail`, and `content` are one column each. A file below 10 MiB and one +at or above it never share a size, and neither do a file below 50 MiB and one at +or above it, so a stored value is never ambiguous between the whole-file, +end-window, and sampled forms. ### `scan` mode -`scan` requires one or more `PATH` operands naming the trees to scan. -There is no default path; invoking `scan` with no operand is a usage -error (usage message on stderr, exit 2). An operand may be a directory -or a regular file; an operand that does not exist is a fatal error -(exit 1). Because database records persist between runs and are keyed -by absolute path, each operand is resolved to an absolute, lexically -cleaned path (symlinks are not resolved) before walking, so results do -not depend on the working directory. All operands belong to a single -scan and are enumerated concurrently: every operand seeds the shared -walk worker pool. Overlapping operands are harmless — an operand that -duplicates another or lies under another is dropped before walking, -so every file is reached exactly once and produces one database -record. +`scan` requires one or more `PATH` operands naming the trees to scan. There is +no default path; invoking `scan` with no operand is a usage error (usage message +on stderr, exit 2). An operand may be a directory or a regular file; an operand +that does not exist is a fatal error (exit 1). Because database records persist +between runs and are keyed by absolute path, each operand is resolved to an +absolute, lexically cleaned path (symlinks are not resolved) before walking, so +results do not depend on the working directory. All operands belong to a single +scan and are enumerated concurrently: every operand seeds the shared walk worker +pool. Overlapping operands are harmless — an operand that duplicates another or +lies under another is dropped before walking, so every file is reached exactly +once and produces one database record. -An operand that is a symlink (never followed, not even as an operand), -socket, FIFO, or device node, or a directory named `.zfs`, is not -scanned. `scan` prints a one-line warning naming the path and what it -is, counts it as skipped, and drops it from the scanned operands before -reading the database. Another operand beneath it is still scanned. The -records stored beneath it are not deleted: they are treated like any -other record outside the scanned operands, including the content-phase -exception below. If it lies under another operand, they are under that -operand instead, and are deleted like any other record there that this -scan did not verify. This is not an error: a scan whose every operand -is dropped walks nothing and exits 0. +An operand that is a symlink (never followed, not even as an operand), socket, +FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints +a one-line warning naming the path and what it is, counts it as skipped, and +drops it from the scanned operands before reading the database. Another operand +beneath it is still scanned. The records stored beneath it are not deleted: they +are treated like any other record outside the scanned operands, including the +content-phase exception below. If it lies under another operand, they are under +that operand instead, and are deleted like any other record there that this scan +did not verify. This is not an error: a scan whose every operand is dropped +walks nothing and exits 0. -`scan` synchronizes the database with the filesystem state under the -scanned operands: +`scan` synchronizes the database with the filesystem state under the scanned +operands: -- Only a file whose size at least one other file shares is ever - read: a size-unique file cannot be a duplicate, so it is recorded - without hashes (`head`, `tail`, and `content` empty). The size - census covers every file walked this scan plus every database - record outside the scanned operands, so a possible duplicate of a - separately scanned tree is still recognized. -- A file not yet in the database is inserted: hashed when its size - is shared, without hashes otherwise. -- A file already in the database is **skipped without reading its - contents** when its lstat size equals the recorded size and its - lstat mtime is not newer than the recorded mtime. This is what - makes a daily rescan cheap. Exception: an unchanged file whose - record lacks hashes is hashed — and its record updated — once its - size becomes shared, so hashing deferred by size-uniqueness - happens as soon as it could matter. Likewise, an unchanged file of - 10 MiB or more whose record has no `content` hash is read for one - by the content phase below once its size, `head`, and `tail` match - another record's. -- A file whose mtime is newer than recorded, or whose size differs, - is processed as if new: re-hashed, or recorded without hashes, - per the shared-size rule. -- A database record whose path lies under one of the scanned operands - but was not successfully processed this run is deleted. This - removes records for deleted files. It also removes records for - paths that failed to stat or hash this run: the database only ever - contains signatures verified by the most recent scan that covered - them (a subsequent successful scan re-adds such files). A failure - in the content phase below removes nothing: the record is left as - it is. -- Database records outside the scanned operands are untouched, so - disjoint trees can be scanned on different schedules into the same - database. The one exception is the content phase below: a stored - file of 10 MiB or more without a `content` hash is read for one, - wherever it lies, once its size, `head`, and `tail` match another - record's. If that file is gone or has changed since its record was - written, the record is left as it is. +- Only a file whose size at least one other file shares is ever read: a + size-unique file cannot be a duplicate, so it is recorded without hashes + (`head`, `tail`, and `content` empty). The size census covers every file + walked this scan plus every database record outside the scanned operands, so a + possible duplicate of a separately scanned tree is still recognized. +- A file not yet in the database is inserted: hashed when its size is shared, + without hashes otherwise. +- A file already in the database is **skipped without reading its contents** + when its lstat size equals the recorded size and its lstat mtime is not newer + than the recorded mtime. This is what makes a daily rescan cheap. Exception: + an unchanged file whose record lacks hashes is hashed — and its record updated + — once its size becomes shared, so hashing deferred by size-uniqueness happens + as soon as it could matter. Likewise, an unchanged file of 10 MiB or more + whose record has no `content` hash is read for one by the content phase below + once its size, `head`, and `tail` match another record's. +- A file whose mtime is newer than recorded, or whose size differs, is processed + as if new: re-hashed, or recorded without hashes, per the shared-size rule. +- A database record whose path lies under one of the scanned operands but was + not successfully processed this run is deleted. This removes records for + deleted files. It also removes records for paths that failed to stat or hash + this run: the database only ever contains signatures verified by the most + recent scan that covered them (a subsequent successful scan re-adds such + files). A failure in the content phase below removes nothing: the record is + left as it is. +- Database records outside the scanned operands are untouched, so disjoint trees + can be scanned on different schedules into the same database. The one + exception is the content phase below: a stored file of 10 MiB or more without + a `content` hash is read for one, wherever it lies, once its size, `head`, and + `tail` match another record's. If that file is gone or has changed since its + record was written, the record is left as it is. -`scan` runs **four sequential phases over the whole scan**. -Parallelism lives inside each phase; batched database writes begin -during the hash phase: +`scan` runs **four sequential phases over the whole scan**. Parallelism lives +inside each phase; batched database writes begin during the hash phase: -1. **walk + stat** — enumerate the trees under all `PATH` operands - concurrently with the walk worker pool: every operand seeds the - shared queue, and each worker reads one directory at a time, - handing discovered subdirectories back to the queue and running - `lstat` on each regular file as it is discovered (while the - directory's metadata is still hot). Sequential directory - enumeration is metadata-latency-bound and takes hours at tens of - millions of files; per-directory parallelism is what makes the - walk tractable on large or busy pools. The walk builds the size - census and resolves unchanged already-hashed files on the fly; - every other file is carried to the hash phase as a (path, size, - mtime) record. -2. **hash** — with the census complete, each carried file's size - decides its fate. Size-unique files are never read: new or - changed ones are recorded without hashes in the update phase, - unchanged unhashed ones simply keep their records. Every file - with a shared size is hashed by the worker pool as described in - "Duplicate detection" above: a file under 10 MiB in full, which - gives its `head`, `tail`, and `content` alike, and a larger file - only in its end windows, which give its `head` and `tail`; its - content hash is left to the content phase. Zero-length files have - constant hashes and are never opened. Files are hashed in **inode - order** (minimizing seeks on spinning disks), and paths that are hard - links to the same inode are **read once**, all sharing the one - result — a hard-link backup farm costs one read per inode, not - per path. The phase total counts actual reads, so progress and - ETA are meaningful. Completed records are committed in batched - transactions **while hashing runs**, so a scan interrupted after - hours keeps everything hashed so far and the next scan resumes - cheaply, skipping records already written. -3. **update** — commit the final partial batch, the hash-less - records for size-unique new and changed files, and the deletions - for records the scan did not verify (vanished files, plus paths - that failed to stat or hash). -4. **content** — find every record of 10 MiB or more without a - `content` hash whose size, `head`, and `tail` equal another - record's, anywhere in the database: records from this scan and - records stored by earlier scans, inside or outside the scanned - operands. SQLite finds them, so only the records to be read are - kept in memory, never every file's hashes. Every record sharing - their size, `head`, and `tail`, including one that already has a - `content` hash, has its file checked with `lstat` first. A file - that is gone, is no longer a regular file, or has changed (a - different size, or an mtime newer than recorded) keeps its record - as it is and does not count as a match for the others. Any other - `lstat` error is warned about and counted as skipped, with the same - result. If such a record has no `content` hash, it stays out of - duplicate groups; if it has one, it is still reported until a scan - covering its own tree updates or removes it. The files that pass - and have no `content` hash are read only if at least two of those - records pass, so a file whose only matches are stale costs no read; - a file that already has a `content` hash is never read again. - They are read by a worker pool as in the hash phase, in inode order - and once per inode, and their content hashes are committed in - batches. A failed read is warned about and counted as skipped; its - record keeps an empty `content`, so it is not a duplicate, and a - later scan tries again. +1. **walk + stat** — enumerate the trees under all `PATH` operands concurrently + with the walk worker pool: every operand seeds the shared queue, and each + worker reads one directory at a time, handing discovered subdirectories back + to the queue and running `lstat` on each regular file as it is discovered + (while the directory's metadata is still hot). Sequential directory + enumeration is metadata-latency-bound and takes hours at tens of millions of + files; per-directory parallelism is what makes the walk tractable on large + or busy pools. The walk builds the size census and resolves unchanged + already-hashed files on the fly; every other file is carried to the hash + phase as a (path, size, mtime) record. +2. **hash** — with the census complete, each carried file's size decides its + fate. Size-unique files are never read: new or changed ones are recorded + without hashes in the update phase, unchanged unhashed ones simply keep + their records. Every file with a shared size is hashed by the worker pool as + described in "Duplicate detection" above: a file under 10 MiB in full, which + gives its `head`, `tail`, and `content` alike, and a larger file only in its + end windows, which give its `head` and `tail`; its content hash is left to + the content phase. Zero-length files have constant hashes and are never + opened. Files are hashed in **inode order** (minimizing seeks on spinning + disks), and paths that are hard links to the same inode are **read once**, + all sharing the one result — a hard-link backup farm costs one read per + inode, not per path. The phase total counts actual reads, so progress and + ETA are meaningful. Completed records are committed in batched transactions + **while hashing runs**, so a scan interrupted after hours keeps everything + hashed so far and the next scan resumes cheaply, skipping records already + written. +3. **update** — commit the final partial batch, the hash-less records for + size-unique new and changed files, and the deletions for records the scan + did not verify (vanished files, plus paths that failed to stat or hash). +4. **content** — find every record of 10 MiB or more without a `content` hash + whose size, `head`, and `tail` equal another record's, anywhere in the + database: records from this scan and records stored by earlier scans, inside + or outside the scanned operands. SQLite finds them, so only the records to + be read are kept in memory, never every file's hashes. Every record sharing + their size, `head`, and `tail`, including one that already has a `content` + hash, has its file checked with `lstat` first. A file that is gone, is no + longer a regular file, or has changed (a different size, or an mtime newer + than recorded) keeps its record as it is and does not count as a match for + the others. Any other `lstat` error is warned about and counted as skipped, + with the same result. If such a record has no `content` hash, it stays out + of duplicate groups; if it has one, it is still reported until a scan + covering its own tree updates or removes it. The files that pass and have no + `content` hash are read only if at least two of those records pass, so a + file whose only matches are stale costs no read; a file that already has a + `content` hash is never read again. They are read by a worker pool as in the + hash phase, in inode order and once per inode, and their content hashes are + committed in batches. A failed read is warned about and counted as skipped; + its record keeps an empty `content`, so it is not a duplicate, and a later + scan tries again. Rules for the walk: -- Only regular files. Skip directories, symlinks (do not follow, - including symlink operands), sockets, FIFOs, and device nodes. An - operand that is a symlink, socket, FIFO, or device node is dropped - as described in "`scan` mode" above. -- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; - walking them would list every file once per snapshot), not even - when it is an operand; such an operand is dropped the same way. -- Filesystem boundaries are crossed by default. With `-x` - (long form `--one-file-system`, following the GNU `du`/`rsync` - convention), never descend into a directory on a different - filesystem than its `PATH` operand; each operand is bounded by its - own filesystem. -- On any per-path error (permission denied, file vanished between - passes, unreadable): print a one-line warning to stderr, skip the - path, and continue. Per-file errors never abort the run; the final - summary reports how many were skipped. As specified above, a - skipped path that has a database record from an earlier scan loses - that record, unless it failed only in the content phase, or is an - operand dropped before the database was read that lies under no - other operand; an unreadable directory subtree likewise loses its - records (accepted: the database mirrors what the latest scan could - actually verify). +- Only regular files. Skip directories, symlinks (do not follow, including + symlink operands), sockets, FIFOs, and device nodes. An operand that is a + symlink, socket, FIFO, or device node is dropped as described in "`scan` mode" + above. +- Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs; walking + them would list every file once per snapshot), not even when it is an operand; + such an operand is dropped the same way. +- Filesystem boundaries are crossed by default. With `-x` (long form + `--one-file-system`, following the GNU `du`/`rsync` convention), never descend + into a directory on a different filesystem than its `PATH` operand; each + operand is bounded by its own filesystem. +- On any per-path error (permission denied, file vanished between passes, + unreadable): print a one-line warning to stderr, skip the path, and continue. + Per-file errors never abort the run; the final summary reports how many were + skipped. As specified above, a skipped path that has a database record from an + earlier scan loses that record, unless it failed only in the content phase, or + is an operand dropped before the database was read that lies under no other + operand; an unreadable directory subtree likewise loses its records (accepted: + the database mirrors what the latest scan could actually verify). -Concurrency: the walk phase (which also stats files), the hash phase, -and the content phase each use a worker pool of `--workers` workers -(default `runtime.NumCPU()`); the walk parallelizes across -directories, hashing across files. `--workers` must be at least 1: a -smaller value is a usage error, reported in one line on stderr with -exit 2 before anything is scanned. All three phases are seek-bound on -spinning disks, so raising `--workers` well past the core count can -help on pools with many spindles. The main goroutine owns -partitioning, database writes, and progress rendering; progress -display must never block the workers. +Concurrency: the walk phase (which also stats files), the hash phase, and the +content phase each use a worker pool of `--workers` workers (default +`runtime.NumCPU()`); the walk parallelizes across directories, hashing across +files. `--workers` must be at least 1: a smaller value is a usage error, +reported in one line on stderr with exit 2 before anything is scanned. All three +phases are seek-bound on spinning disks, so raising `--workers` well past the +core count can help on pools with many spindles. The main goroutine owns +partitioning, database writes, and progress rendering; progress display must +never block the workers. -`scan` writes nothing to stdout. The summary line on stderr reports the -files seen this run broken down by disposition, plus skips: +`scan` writes nothing to stdout. The summary line on stderr reports the files +seen this run broken down by disposition, plus skips: ``` scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped ``` -(`removed` counts deleted database records, which are not part of the -files-seen total.) +(`removed` counts deleted database records, which are not part of the files-seen +total.) ### `report` mode -`report` reads every record from the database and takes no positional -arguments. +`report` reads every record from the database and takes no positional arguments. -**`report` must never touch the filesystem being analyzed.** It does not -stat, open, or otherwise access any path that appears in the records; its -only I/O is reading the database, writing stdout/stderr, and the -temporary file SQLite sorts in when the duplicate rows do not fit in -memory. SQLite puts that file in `$SQLITE_TMPDIR` or `$TMPDIR` when set, -otherwise in `/var/tmp` (or `/tmp`), and deletes it as soon as it has -opened it. `report` must produce identical output whether or not the -scanned filesystem is still mounted. +**`report` must never touch the filesystem being analyzed.** It does not stat, +open, or otherwise access any path that appears in the records; its only I/O is +reading the database, writing stdout/stderr, and the temporary file SQLite sorts +in when the duplicate rows do not fit in memory. SQLite puts that file in +`$SQLITE_TMPDIR` or `$TMPDIR` when set, otherwise in `/var/tmp` (or `/tmp`), and +deletes it as soon as it has opened it. `report` must produce identical output +whether or not the scanned filesystem is still mounted. Processing: -- Records without a `content` hash (see "Database" above) are - excluded: their content is unknown, so they are never reported as - duplicates. -- Group the remaining records by the key - `(size, head, tail, content)`. -- Every group with two or more paths is a duplicate group. -- Within each group, sort paths lexicographically (byte order). The - first path is the group's `first`; every other path is a `dupe`. -- Order groups by size descending (biggest reclaimable space first), - tie-broken by `first` path ascending. Output must be fully - deterministic for a given database state. +- Records without a `content` hash (see "Database" above) are excluded: their + content is unknown, so they are never reported as duplicates. +- Group the remaining records by the key `(size, head, tail, content)`. +- Every group with two or more paths is a duplicate group. +- Within each group, sort paths lexicographically (byte order). The first path + is the group's `first`; every other path is a `dupe`. +- Order groups by size descending (biggest reclaimable space first), tie-broken + by `first` path ascending. Output must be fully deterministic for a given + database state. #### Report output format -TSV on stdout: a header line, then one row per duplicate file (N-1 rows -for a group of N): +TSV on stdout: a header line, then one row per duplicate file (N-1 rows for a +group of N): ``` first dupe size @@ -625,114 +565,105 @@ first dupe size /srv/a/big.iso /srv/c/big-copy2.iso 4294967296 ``` -Paths are raw bytes and may hold any byte except NUL, so the path -columns (`first` and `dupe`) are escaped to keep every row one line of -tab-separated fields: a backslash is written as `\\`, a tab as `\t`, a -newline as `\n`, and a carriage return as `\r`. Every other byte is -written unchanged, including bytes that are not valid UTF-8. Undoing -those four escapes gives back the stored path. Grouping and ordering -use the stored path, not the escaped one. The warnings `scan` prints on -stderr are escaped the same way, so each warning is one line. +Paths are raw bytes and may hold any byte except NUL, so the path columns +(`first` and `dupe`) are escaped to keep every row one line of tab-separated +fields: a backslash is written as `\\`, a tab as `\t`, a newline as `\n`, and a +carriage return as `\r`. Every other byte is written unchanged, including bytes +that are not valid UTF-8. Undoing those four escapes gives back the stored path. +Grouping and ordering use the stored path, not the escaped one. The warnings +`scan` prints on stderr are escaped the same way, so each warning is one line. -Summary to stderr: records read, number of duplicate groups, number of -dupe files, and total reclaimable bytes (sum of `size` over all dupe -rows) in human units. +Summary to stderr: records read, number of duplicate groups, number of dupe +files, and total reclaimable bytes (sum of `size` over all dupe rows) in human +units. ### `trees` mode -`trees` reads the same database as `report` (no positional arguments) -and reports **entire duplicate directory trees**: directories under -which the exact same set of relative paths exists with the exact same -file signatures. +`trees` reads the same database as `report` (no positional arguments) and +reports **entire duplicate directory trees**: directories under which the exact +same set of relative paths exists with the exact same file signatures. -**`trees` must never touch the filesystem being analyzed** — the same -rule as `report`. The directory hierarchy is reconstructed purely from -the paths in the records, split on `/`. +**`trees` must never touch the filesystem being analyzed** — the same rule as +`report`. The directory hierarchy is reconstructed purely from the paths in the +records, split on `/`. Definitions: -- A file's **signature** is `(size, head, tail, content)` — mtime is - informational and excluded. A record without a `content` hash has - unknown content: its signature is treated as unique to that file, - so a tree containing such a file never compares equal to any other - tree. -- A directory's **digest** is a SHA-256 Merkle digest computed - bottom-up: serialize the directory's child entries — for a file - child, its name and signature; for a subdirectory child, its name - and that subdirectory's digest — sort the serialized entries - byte-lexicographically, and hash the concatenation. Names are part - of the digest: two trees whose files differ only in name are *not* - duplicates. -- Two directories are **duplicate trees** when their digests are - equal. Equal digests imply equal recursive file count and equal - total byte size. +- A file's **signature** is `(size, head, tail, content)` — mtime is + informational and excluded. A record without a `content` hash has unknown + content: its signature is treated as unique to that file, so a tree containing + such a file never compares equal to any other tree. +- A directory's **digest** is a SHA-256 Merkle digest computed bottom-up: + serialize the directory's child entries — for a file child, its name and + signature; for a subdirectory child, its name and that subdirectory's digest — + sort the serialized entries byte-lexicographically, and hash the + concatenation. Names are part of the digest: two trees whose files differ only + in name are _not_ duplicates. +- Two directories are **duplicate trees** when their digests are equal. Equal + digests imply equal recursive file count and equal total byte size. -Known limitation (accepted): hard-linked paths are reported as -duplicates by `report` and count toward duplicate trees — their -content is genuinely identical — even though they share storage, so -removing one reclaims no space. Inode identity is used during the -scan to avoid redundant reads but is not persisted in the database. +Known limitation (accepted): hard-linked paths are reported as duplicates by +`report` and count toward duplicate trees — their content is genuinely identical +— even though they share storage, so removing one reclaims no space. Inode +identity is used during the scan to avoid redundant reads but is not persisted +in the database. -Known limitation (accepted): only regular files that appear in the -database define a tree. Empty directories are invisible, and a file -skipped during the scan (e.g. permission error) in one copy but not the -other will make otherwise-identical trees compare as different. +Known limitation (accepted): only regular files that appear in the database +define a tree. Empty directories are invisible, and a file skipped during the +scan (e.g. permission error) in one copy but not the other will make +otherwise-identical trees compare as different. Processing: -- Build the hierarchy, compute every directory's digest, and group - directories by digest. Every group with two or more directories is a - duplicate-tree group. -- **Report only maximal trees.** A group is suppressed when its - members' parents are pairwise distinct directories that all share a - single digest — such a group is wholly implied by its parents' (or a - further ancestor's) group. Groups containing sibling directories, or - members whose parents differ, are always reported. -- Within each group, sort paths lexicographically (byte order); the - first path is `first`, every other path is a `dupe`. -- Order groups by total tree size descending, tie-broken by `first` - path ascending. Output must be fully deterministic for a given - input. +- Build the hierarchy, compute every directory's digest, and group directories + by digest. Every group with two or more directories is a duplicate-tree group. +- **Report only maximal trees.** A group is suppressed when its members' parents + are pairwise distinct directories that all share a single digest — such a + group is wholly implied by its parents' (or a further ancestor's) group. + Groups containing sibling directories, or members whose parents differ, are + always reported. +- Within each group, sort paths lexicographically (byte order); the first path + is `first`, every other path is a `dupe`. +- Order groups by total tree size descending, tie-broken by `first` path + ascending. Output must be fully deterministic for a given input. #### Trees output format -TSV on stdout: a header line, then one row per duplicate tree (N-1 rows -for a group of N). `files` is the recursive regular-file count of one -copy of the tree; `size` is the recursive total byte size of one copy: +TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a +group of N). `files` is the recursive regular-file count of one copy of the +tree; `size` is the recursive total byte size of one copy: ``` first dupe files size /srv/a/project /srv/backup/project 3417 104857600 ``` -The `first` and `dupe` paths are escaped as described under "Report -output format". The root directory's path is `/`. +The `first` and `dupe` paths are escaped as described under "Report output +format". The root directory's path is `/`. -Summary to stderr: records read, number of duplicate-tree groups, -number of dupe trees, and total reclaimable bytes (sum of `size` over -all dupe rows) in human units. +Summary to stderr: records read, number of duplicate-tree groups, number of dupe +trees, and total reclaimable bytes (sum of `size` over all dupe rows) in human +units. ### Progress -Use the progress-bar library for all scan progress; rendering in the -style of `pv` is the model. All progress goes to stderr. +Use the progress-bar library for all scan progress; rendering in the style of +`pv` is the model. All progress goes to stderr. -Each phase gets its own display, rendered the moment the phase -starts — a scan must never look hung. Loading the existing-record -index (`load`) and the walk have no known totals while running: show -a live count, rate, and elapsed time (spinner-style, no percentage or -ETA). The content phase's display (`content`) starts the same way, -counting the records checked while SQLite finds the files to read and -`lstat` checks them, then shows a bar once reading starts. The hash -and update phases, and the content phase's reads, have exact totals — -only files that actually need hashing appear in the hash and content -totals, so their ETAs are meaningful. Required elements for the bars -with known totals: +Each phase gets its own display, rendered the moment the phase starts — a scan +must never look hung. Loading the existing-record index (`load`) and the walk +have no known totals while running: show a live count, rate, and elapsed time +(spinner-style, no percentage or ETA). The content phase's display (`content`) +starts the same way, counting the records checked while SQLite finds the files +to read and `lstat` checks them, then shows a bar once reading starts. The hash +and update phases, and the content phase's reads, have exact totals — only files +that actually need hashing appear in the hash and content totals, so their ETAs +are meaningful. Required elements for the bars with known totals: -- elapsed time -- estimated time remaining -- a `[m/n] x%` display (items processed / total items, percent) -- current rate (items/s) +- elapsed time +- estimated time remaining +- a `[m/n] x%` display (items processed / total items, percent) +- current rate (items/s) Example shape (exact layout is flexible, content is not): @@ -742,199 +673,189 @@ hash: [12345/98765] 12% |████ | 92 files/s elapsed 2:32 eta 17:54 Additional requirements: -- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not - emit ANSI redraws: print a plain one-line progress update the moment - each phase starts, then no more often than every 5 seconds. -- Progress updates are driven from the main goroutine and must be - non-blocking with respect to the worker pool. On a terminal the - spinner-style displays also redraw on their own several times a - second, so their count and elapsed time stay current while a phase - waits for its next item. -- A warning printed during a phase always lands on a line of its own, - never inside the progress display. -- A bar whose phase stops short of its total, as an interrupted one - does, is left as last drawn rather than filled up. -- `report` and `trees` modes need no progress display, only their - stderr summaries. +- When stderr is not a terminal (a pipe, a file, `/dev/null`), do not emit ANSI + redraws: print a plain one-line progress update the moment each phase starts, + then no more often than every 5 seconds. +- Progress updates are driven from the main goroutine and must be non-blocking + with respect to the worker pool. On a terminal the spinner-style displays also + redraw on their own several times a second, so their count and elapsed time + stay current while a phase waits for its next item. +- A warning printed during a phase always lands on a line of its own, never + inside the progress display. +- A bar whose phase stops short of its total, as an interrupted one does, is + left as last drawn rather than filled up. +- `report` and `trees` modes need no progress display, only their stderr + summaries. ### Error handling and exit codes -- `0`: success, even if individual files were skipped with warnings. -- `1`: fatal error (e.g., a `PATH` operand does not exist, another - `scan` is already running against the same database, the database - cannot be created/opened/read/written, a missing database for - `report`/`trees`, stdout write failure), or a `scan` stopped by - `SIGINT` or `SIGTERM` (see below). -- `2`: usage error (including `scan` with no `PATH` operand, `scan` - with `--workers` below 1, and `report`/`trees` with any positional - argument). +- `0`: success, even if individual files were skipped with warnings. +- `1`: fatal error (e.g., a `PATH` operand does not exist, another `scan` is + already running against the same database, the database cannot be + created/opened/read/written, a missing database for `report`/`trees`, stdout + write failure), or a `scan` stopped by `SIGINT` or `SIGTERM` (see below). +- `2`: usage error (including `scan` with no `PATH` operand, `scan` with + `--workers` below 1, and `report`/`trees` with any positional argument). -A stdout write failure, such as a full disk, is reported in one line on -stderr and exits 1. Two cases never reach sfdupes as a failed write: +A stdout write failure, such as a full disk, is reported in one line on stderr +and exits 1. Two cases never reach sfdupes as a failed write: -- When the reader of a stdout pipe exits early, as in - `sfdupes report | head`, the next write ends sfdupes with `SIGPIPE`, - quietly and without a summary, the way `cat` or `sort` end. The - shell reports the signal (status 141 in most shells), not exit 1. -- When stdout is closed outright (`sfdupes report >&-`), the Go - runtime opens `/dev/null` in its place before sfdupes starts, so - the output is discarded and the run succeeds, as with - `> /dev/null`. +- When the reader of a stdout pipe exits early, as in `sfdupes report | head`, + the next write ends sfdupes with `SIGPIPE`, quietly and without a summary, the + way `cat` or `sort` end. The shell reports the signal (status 141 in most + shells), not exit 1. +- When stdout is closed outright (`sfdupes report >&-`), the Go runtime opens + `/dev/null` in its place before sfdupes starts, so the output is discarded and + the run succeeds, as with `> /dev/null`. -`scan` stops cleanly on `SIGINT` (Ctrl-C) or `SIGTERM`. Its workers -stop taking work, each finishing at most the directory listing or file -it is reading; the progress display is finished; and the records it has -hashed but not yet committed are committed, so the next scan does not -hash them again. Apart from that commit it starts no further writes or -deletions: records are deleted only after a complete walk, so those -under paths an interrupted walk never reached are kept. The database is -closed and the lock released as on any other exit, the line -`scan: interrupted after N files` goes to stderr, N being the number of -files the walk reached, and the exit code is 1. The next scan skips the -records already written and converges as usual. +`scan` stops cleanly on `SIGINT` (Ctrl-C) or `SIGTERM`. Its workers stop taking +work, each finishing at most the directory listing or file it is reading; the +progress display is finished; and the records it has hashed but not yet +committed are committed, so the next scan does not hash them again. Apart from +that commit it starts no further writes or deletions: records are deleted only +after a complete walk, so those under paths an interrupted walk never reached +are kept. The database is closed and the lock released as on any other exit, the +line `scan: interrupted after N files` goes to stderr, N being the number of +files the walk reached, and the exit code is 1. The next scan skips the records +already written and converges as usual. -After the first signal `scan` stops catching them, so a second one ends -it at once, as an uncaught signal does: the records not yet committed -are lost, and the database is left valid, as when any scan dies (see -"Database"). A `SIGINT` that `scan` inherits as ignored, as a script's -background job does, stays ignored. +After the first signal `scan` stops catching them, so a second one ends it at +once, as an uncaught signal does: the records not yet committed are lost, and +the database is left valid, as when any scan dies (see "Database"). A `SIGINT` +that `scan` inherits as ignored, as a script's background job does, stays +ignored. ## Entrypoints This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) -standard: the normalized executables in `script/` are the entrypoints -for the development workflow, and the `Makefile` targets are thin -shims that call them. Every script is POSIX `sh`, resolves the -repository root itself so it can be run from any working directory, -and may be invoked directly. The provided entrypoints are: +standard: the normalized executables in `script/` are the entrypoints for the +development workflow, and the `Makefile` targets are thin shims that call them. +Every script is POSIX `sh`, resolves the repository root itself so it can be run +from any working directory, and may be invoked directly. The provided +entrypoints are: -- `script/bootstrap` — install everything needed to build and - develop this repository, idempotently, assuming nothing is - present. `git`, `make`, and `go` come from the first of nix, apt, - brew, or apk found on the host, and are presence-checked only. - `golangci-lint` is deliberately **not** installed: it runs from a - digest-pinned image via `script/lint` and never from a host - install, so there is no host copy to drift from the pin. A missing - `docker` is warned about rather than installed or treated as - fatal — everything except linting works without it. Ends with - `go mod download`. -- `script/setup` — make a fresh clone ready for development: runs - `script/bootstrap`, then `script/install-precommit`. -- `script/projectname` — print this project's name (`sfdupes`). - Scripts that need the name call it, so they stay identical across - repositories. -- `script/test` — run the test suite with a 30-second timeout and - coverage enabled, rerunning verbosely on failure so the logs show - which test failed. -- `script/lint` — run the linter. It builds `Dockerfile.lint`, which - copies the repository into the digest-pinned - `golangci/golangci-lint` image and runs - `golangci-lint config verify` and `golangci-lint run` as build - steps, so a successful build is a clean lint. That exit status is - all it produces, so it runs with `--output=type=cacheonly` and - writes no image; a run leaves only build cache. The linter is never - run on the host, which makes a working `docker` the one - prerequisite for linting — and therefore for `make check` and the - pre-commit hook. Offline machines: the gate steps themselves make - no network calls. `golangci-lint run` does not, and neither does - `golangci-lint config verify` — it validates against a schema the - pinned binary embeds, measured under `--network none` to both - pass a valid config and reject an invalid one. The build around - them does. `Dockerfile.lint` runs `go mod download` before the - gates and this module has external dependencies, so a first lint - on a machine with a cold BuildKit cache reaches the network there - (as well as pulling the pinned image); under `--network none` it - fails at that step, before any gate. That layer sits above the - gates and stays cached, so once it is warm `script/lint` — and - with it `make check` — runs entirely offline, until `go.mod` or - `go.sum` changes and the download layer goes cold again. Because - the daemon only ever sees a build context, this works when the - docker daemon is remote and bind mounts are impossible. -- `script/fmt` — format the Go sources in place (`gofmt -s -w`). - Markdown is not formatted. -- `script/fmt-check` — the read-only counterpart of `script/fmt`: - prints any unformatted file and exits non-zero instead of writing. -- `script/check` — run `script/test`, `script/lint`, and - `script/fmt-check`, in that order. Modifies nothing. Needs - `docker`, because `script/lint` does. -- `script/docker` — build the Docker image, tagged with the name - from `script/projectname`. The `Dockerfile` runs the gates as - build steps, so this is also the check a developer or reviewer - runs by hand. -- `script/cibuild` — build the Docker image untagged. This is what - the Gitea workflow runs on push; because the gates run as build - steps, a successful build implies the repository is green. -- `script/precommit` — run by the git pre-commit hook: `go mod tidy` - must be a no-op (a resulting change to `go.mod` or `go.sum` fails - the commit), then `script/check`. -- `script/install-precommit` — install the git pre-commit hook that - runs `script/precommit`. The hook is written to the common git - directory, so the main checkout and every worktree share it. -- `script/verify-lint-image-pin` — fail unless the - `golangci/golangci-lint` reference in `Dockerfile.lint` and the - one in the `Dockerfile` lint stage are the same image at the same - digest, naming both if not. The linter is pinned in those two - files and nothing else keeps them in sync, so a bump applied to - one alone would leave `make lint` and the `Dockerfile`'s - fail-fast lint stage checking the same tree against different - rulesets, both green. The guard restates neither pin — a third - copy would be the same drift one file further out — and runs as a - gate in both files, so `make lint`, `make check` and `make docker` - all catch it. +- `script/bootstrap` — install everything needed to build and develop this + repository, idempotently, assuming nothing is present. `git`, `make`, and `go` + come from the first of nix, apt, brew, or apk found on the host, and are + presence-checked only. `golangci-lint` and prettier are deliberately **not** + installed: they run in Docker (see `script/lint` and `script/fmt`) and never + from a host install, so there is no host copy to drift from the pin. A missing + `docker` is warned about rather than installed or treated as fatal — + everything except linting and formatting works without it. Ends with + `go mod download`. +- `script/setup` — make a fresh clone ready for development: runs + `script/bootstrap`, then `script/install-precommit`. +- `script/projectname` — print this project's name (`sfdupes`). Scripts that + need the name call it, so they stay identical across repositories. +- `script/test` — run the test suite with a 30-second timeout and coverage + enabled, rerunning verbosely on failure so the logs show which test failed. +- `script/lint` — run the linter. It builds `Dockerfile.lint`, which copies the + repository into the digest-pinned `golangci/golangci-lint` image and runs + `golangci-lint config verify` and `golangci-lint run` as build steps, so a + successful build is a clean lint. That exit status is all it produces, so it + runs with `--output=type=cacheonly` and writes no image; a run leaves only + build cache. The linter is never run on the host, which makes a working + `docker` the one prerequisite for linting — and therefore for `make check` and + the pre-commit hook. Offline machines: the gate steps themselves make no + network calls. `golangci-lint run` does not, and neither does + `golangci-lint config verify` — it validates against a schema the pinned + binary embeds, measured under `--network none` to both pass a valid config and + reject an invalid one. The build around them does. `Dockerfile.lint` runs + `go mod download` before the gates and this module has external dependencies, + so a first lint on a machine with a cold BuildKit cache reaches the network + there (as well as pulling the pinned image); under `--network none` it fails + at that step, before any gate. That layer sits above the gates and stays + cached, so once it is warm `script/lint` — and with it `make check` — runs + entirely offline, until `go.mod` or `go.sum` changes and the download layer + goes cold again. Because the daemon only ever sees a build context, this works + when the docker daemon is remote and bind mounts are impossible. +- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every + Markdown file with prettier, at the settings in `.prettierrc` (4-space + indents, prose wrapped at 80 columns). prettier is pinned by hash through + `package.json` and `yarn.lock` and never installed on the host: this builds + the `Dockerfile`'s `prettier` stage, a digest-pinned node image into which + `yarn install --frozen-lockfile` installs it, tagged `sfdupes-prettier`, and + runs that with the repository mounted, as the calling user. Needs `docker`, + and because of the mount, unlike `script/lint`, a local docker daemon. +- `script/fmt-check` — the read-only counterpart of `script/fmt`, with the + repository mounted read-only: prints any unformatted file and exits non-zero + instead of writing. gofmt and prettier both run every time, and each names + itself when it fails. The `Dockerfile` runs the same two checks as gates: the + gofmt check in its lint stage, prettier in its `markdown` stage. +- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`, in + that order. Modifies nothing. Needs `docker`, because `script/lint` and + `script/fmt-check` do. +- `script/docker` — build the Docker image, tagged with the name from + `script/projectname`. The `Dockerfile` runs the gates as build steps, so this + is also the check a developer or reviewer runs by hand. +- `script/cibuild` — build the Docker image untagged. This is what the Gitea + workflow runs on push; because the gates run as build steps, a successful + build implies the repository is green. +- `script/precommit` — run by the git pre-commit hook: `go mod tidy` must be a + no-op (a resulting change to `go.mod` or `go.sum` fails the commit), then + `script/check`. +- `script/install-precommit` — install the git pre-commit hook that runs + `script/precommit`. The hook is written to the common git directory, so the + main checkout and every worktree share it. +- `script/verify-lint-image-pin` — fail unless the `golangci/golangci-lint` + reference in `Dockerfile.lint` and the one in the `Dockerfile` lint stage are + the same image at the same digest, naming both if not. The linter is pinned in + those two files and nothing else keeps them in sync, so a bump applied to one + alone would leave `make lint` and the `Dockerfile`'s fail-fast lint stage + checking the same tree against different rulesets, both green. The guard + restates neither pin — a third copy would be the same drift one file further + out — and runs as a gate in both files, so `make lint`, `make check` and + `make docker` all catch it. -`script/verify-linter-pin` used to live here. It compared a linter -binary against a version pin in `script/bootstrap`, and both of its -subjects are gone: no linter binary is copied between build stages any -more, and bootstrap pins no version because it installs no linter. The -drift it existed to catch has moved from binary-versus-pin to -pin-versus-pin, which is what `script/verify-lint-image-pin` above -checks. +`script/verify-linter-pin` used to live here. It compared a linter binary +against a version pin in `script/bootstrap`, and both of its subjects are gone: +no linter binary is copied between build stages any more, and bootstrap pins no +version because it installs no linter. The drift it existed to catch has moved +from binary-versus-pin to pin-versus-pin, which is what +`script/verify-lint-image-pin` above checks. -`script/lint`, `script/docker` and `script/cibuild` all pass a freshly -computed `CHECK_EPOCH` build argument, and the gate steps in -`Dockerfile.lint` and `Dockerfile` reference it. Without that, an -unchanged tree lets Docker serve the gate layers from cache and the -build exits 0 having executed no tests and no lint — a green it never -earned, and one this repository has produced twice. `CHECK_EPOCH` -invalidates the gate layers on every run while leaving the pinned base -images and the dependency layers cached. `script/lint`'s value carries -the process id as well as the epoch, because two lint runs land inside -the same second easily and a bare epoch would cache the second one. +`script/lint`, `script/docker` and `script/cibuild` all pass a freshly computed +`CHECK_EPOCH` build argument, and the gate steps in `Dockerfile.lint` and +`Dockerfile` reference it. Without that, an unchanged tree lets Docker serve the +gate layers from cache and the build exits 0 having executed no tests and no +lint — a green it never earned, and one this repository has produced twice. +`CHECK_EPOCH` invalidates the gate layers on every run while leaving the pinned +base images and the dependency layers cached. `script/lint`'s value carries the +process id as well as the epoch, because two lint runs land inside the same +second easily and a bare epoch would cache the second one. ## Build -The `script/` entrypoints above are where the implementations live; -the `Makefile` targets are shims onto them, except `build`, which -carries the compile recipe: +The `script/` entrypoints above are where the implementations live; the +`Makefile` targets are shims onto them, except `build`, which carries the +compile recipe: -- `make` / `make build` — build the `sfdupes` binary (cgo - disabled); building is the default target. -- `make bootstrap` — install the build and development - dependencies. -- `make setup` — prepare a fresh clone: `bootstrap` plus the - pre-commit hook. -- `make test` — run the test suite (30-second timeout; reruns with - `-v` on failure). -- `make lint` — run `golangci-lint` with the repo config, in Docker - (see `script/lint`); requires `docker`. -- `make fmt` / `make fmt-check` — format Go sources / verify - formatting without writing. -- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. - Requires `docker`, via `lint`. -- `make docker` — build the Docker image, which runs the gates as - build stages. -- `make hooks` — install the pre-commit hook. -- `make clean` — remove the binary. +- `make` / `make build` — build the `sfdupes` binary (cgo disabled); building is + the default target. +- `make bootstrap` — install the build and development dependencies. +- `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook. +- `make test` — run the test suite (30-second timeout; reruns with `-v` on + failure). +- `make lint` — run `golangci-lint` with the repo config, in Docker (see + `script/lint`); requires `docker`. +- `make fmt` / `make fmt-check` — format the Go sources and the Markdown / + verify formatting without writing; requires `docker`, for prettier (see + `script/fmt`). +- `make check` — `test`, `lint`, and `fmt-check`; modifies nothing. Requires + `docker`, via `lint` and `fmt-check`. +- `make docker` — build the Docker image, which runs the gates as build stages. +- `make hooks` — install the pre-commit hook. +- `make clean` — remove the binary. ### Definition of done All of the following, run in this directory, must pass: -1. `make check` passes (tests, lint, `gofmt`). +1. `make check` passes (tests, lint, `gofmt`, prettier). 2. `make docker` succeeds. -3. Smoke test — create a throwaway tree in a temp dir (never test - against real data): +3. Smoke test — create a throwaway tree in a temp dir (never test against real + data): ```sh d=$(mktemp -d) @@ -966,30 +887,30 @@ All of the following, run in this directory, must pass: ./sfdupes report ``` - (The database lives in a temp directory of its own: inside `$d`, - the scan would record it, and its empty lock file would join the - `empty1`/`empty2` group.) + (The database lives in a temp directory of its own: inside `$d`, the scan + would record it, and its empty lock file would join the `empty1`/`empty2` + group.) - Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` - form one group (two dupe rows, `first` is the lexicographically - smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group; - `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group; - `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and - `tiny3` appear nowhere; groups ordered by size descending. + Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin` form one + group (two dupe rows, `first` is the lexicographically smallest path); + `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/ + `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair; + `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere; groups + ordered by size descending. - Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` - `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is - suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3` - appears nowhere (its file set differs by name). + Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2 + files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal + (implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set + differs by name). Expected from the second `report` (after the modify/delete rescan): `one.bin` has left its group (its content changed), so - `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is - gone from the database. + `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is gone from the + database. - The test suite automates this scenario (see `scan_test.go`), plus a - negative check: `report` and `trees` operate on the database alone - and never touch the scanned filesystem. + The test suite automates this scenario (see `scan_test.go`), plus a negative + check: `report` and `trees` operate on the database alone and never touch + the scanned filesystem. ## TODO @@ -997,13 +918,13 @@ Tracked in [TODO.md](TODO.md). ## Non-goals -- No byte-for-byte compare, and no deletion or linking of - duplicates. Files that match are compared by a SHA-256 of the whole - file below 50 MiB, and only by samples at 50 MiB and over. The - reports are advisory; acting on them is the user's job. -- No persistence beyond the SQLite database described above; no - export/import formats. -- No daemon or filesystem watcher; scheduling rescans is cron's job. +- No byte-for-byte compare, and no deletion or linking of duplicates. Files that + match are compared by a SHA-256 of the whole file below 50 MiB, and only by + samples at 50 MiB and over. The reports are advisory; acting on them is the + user's job. +- No persistence beyond the SQLite database described above; no export/import + formats. +- No daemon or filesystem watcher; scheduling rescans is cron's job. ## License diff --git a/TODO.md b/TODO.md index 06d931a..5e65e74 100644 --- a/TODO.md +++ b/TODO.md @@ -1,60 +1,62 @@ # Workflow -- take an issue from the `1.0.0` milestone on the tracker; work not - yet on the tracker gets filed as an issue first +- take an issue from the `1.0.0` milestone on the tracker; work not yet on the + tracker gets filed as an issue first - branch (from `main`) - do the work, with tests, in small focused commits -- record it at the top of Completed Steps (`TODO.md` changes in the - same commit as the work) -- push the branch and open a PR whose title ends with - ` (closes #N)` -- an independent review gates the merge; every finding is addressed - or explicitly rebutted on the PR +- record it at the top of Completed Steps (`TODO.md` changes in the same commit + as the work) +- push the branch and open a PR whose title ends with ` (closes #N)` +- an independent review gates the merge; every finding is addressed or + explicitly rebutted on the PR - merge to `main` once the review passes # Status - pre-1.0 -- the Gitea tracker is authoritative for the pre-1.0 backlog: the - open issues under the `1.0.0` milestone are what remains before - the tag, and this file records history and process, not the queue +- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues + under the `1.0.0` milestone are what remains before the tag, and this file + records history and process, not the queue # Next Step - take the next issue from the `1.0.0` milestone on the tracker: - https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is - the source of truth for what is left before 1.0.0. Individual - issues are deliberately not restated here; a copy in this file - drifts out of date the moment the tracker moves + https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source + of truth for what is left before 1.0.0. Individual issues are deliberately not + restated here; a copy in this file drifts out of date the moment the tracker + moves # Completed Steps -- `script/lint` writes no image, so a run no longer leaves an untagged - one behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48) +- `make fmt` and `make fmt-check` run prettier over all Markdown, in Docker, and + CI checks it; all Markdown reformatted (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/19) -- tests cover a missing database, `scan` keeping stdout empty, its skip - warning, the `report` and `trees` summary lines, and every subcommand - going through `runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16) +- `script/lint` writes no image, so a run no longer leaves an untagged one + behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48) + +- tests cover a missing database, `scan` keeping stdout empty, its skip warning, + the `report` and `trees` summary lines, and every subcommand going through + `runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16) - `.golangci.yml` replaced with the current canonical copy, which uses - `gomodguard_v2`, so lint no longer prints a deprecation warning - (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/26) + `gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/26) -- a test fails when either `hashWorker` cancellation check in `scan.go` - is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) +- a test fails when either `hashWorker` cancellation check in `scan.go` is + removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) - a database path holding `?`, `#` or `%` opens exactly the file it names (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) -- `scan` rejects `--workers` below 1 as a usage error instead of - running single-threaded (2026-10-04, - https://git.eeqj.de/sneak/sfdupes/issues/10) +- `scan` rejects `--workers` below 1 as a usage error instead of running + single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10) -- a test fails when either walk cancellation check in `scan.go` is - removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) +- a test fails when either walk cancellation check in `scan.go` is removed + (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) -- test that `scan` refuses a database with another schema version - (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64) +- test that `scan` refuses a database with another schema version (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/64) - correct four inaccurate comments in `cancel_test.go` and rename `walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04, @@ -67,479 +69,428 @@ `files` table is refused with a clear schema-version error (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/11) -- README documents install, Docker, a daily cron scan and how to read - and check the reports (2026-10-04, - https://git.eeqj.de/sneak/sfdupes/issues/54) +- README documents install, Docker, a daily cron scan and how to read and check + the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) -- the `Dockerfile` build stage keeps the Go module cache out of `builder`'s - home and copies the sources with `--chown`, so no `chown -R` walks them +- the `Dockerfile` build stage keeps the Go module cache out of `builder`'s home + and copies the sources with `--chown`, so no `chown -R` walks them (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43) - `--version` prints `sfdupes VERSION` to stdout; README documents it and `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) -- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has - hashed, deletes nothing more, exits 1 (2026-10-04, +- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed, + deletes nothing more, exits 1 (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/5) -- `report` and `trees` stream the records instead of holding them all in - memory; the schema gains the `files_signature` index (2026-10-04, +- `report` and `trees` stream the records instead of holding them all in memory; + the schema gains the `files_signature` index (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/14) -- progress prints at once on a non-terminal, uses a real terminal test, - and prints warnings through a spinner instead of racing its redraw - (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/13) +- progress prints at once on a non-terminal, uses a real terminal test, and + prints warnings through a spinner instead of racing its redraw (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/13) -- warn about and skip symlink, socket, FIFO, device and `.zfs` - operands, keeping the records beneath them (2026-10-03, +- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping + the records beneath them (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/9) -- `scan` holds a lock on a lock file beside the database for its whole run, - so a second `scan` fails at once with exit 1 (2026-10-03, +- `scan` holds a lock on a lock file beside the database for its whole run, so a + second `scan` fails at once with exit 1 (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/53) - test stdout write failures in `report` and `trees`; README states that `| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null` (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30) -- `report` and `trees` open the database read-only, and `scan` leaves it - out of WAL mode, so reading needs only read access (2026-10-03, closes +- `report` and `trees` open the database read-only, and `scan` leaves it out of + WAL mode, so reading needs only read access (2026-10-03, closes https://git.eeqj.de/sneak/sfdupes/issues/8) -- escape tabs, newlines, carriage returns and backslashes in report, - trees and warning paths; the root directory's path is `/` - (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/7) +- escape tabs, newlines, carriage returns and backslashes in report, trees and + warning paths; the root directory's path is `/` (2026-10-03, + https://git.eeqj.de/sneak/sfdupes/issues/7) -- stamp the git tag or short commit in a plain `docker build .` - instead of `dev` (2026-10-02, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now - sends `.git`, without `.git/config`, and the `Dockerfile` build - stage takes the `VERSION` build argument when one is given, - otherwise `git describe --tags --always` of that `.git`. The build - fails if the context carries `.git` and the version still comes out - empty, `dev` or `unknown`. The CI checkout step fetches the full - history (`fetch-depth: 0`) so CI sees the tag and stamps the same - value as `make build`. +- stamp the git tag or short commit in a plain `docker build .` instead of `dev` + (2026-10-02, branch `next`, closes + https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now sends + `.git`, without `.git/config`, and the `Dockerfile` build stage takes the + `VERSION` build argument when one is given, otherwise + `git describe --tags --always` of that `.git`. The build fails if the context + carries `.git` and the version still comes out empty, `dev` or `unknown`. The + CI checkout step fetches the full history (`fetch-depth: 0`) so CI sees the + tag and stamps the same value as `make build`. -- replace the 1 KiB end-window sampling with the head/tail plus - content-hash ladder (2026-09-22, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is - hashed in full and compared directly, with no end-window step — its - `head`, `tail`, and `content` all hold the whole-file hash. A file at - 10 MiB or above gets only the 64 KiB `head` and `tail` in the hash - phase; a new content phase, after the update phase, reads it for its - `content` hash — the whole file below 50 MiB, gigabyte-spaced 1 MiB - samples at or above — only when its size, `head`, and `tail` match - another record's, from the same scan or stored by an earlier one, so - a stored file gains its content hash when it gains a match. A file - that is gone or has changed since its record was written is not - read. The `content` column is part of the version 1 schema. `report` - and `trees` group by the extended signature and leave out any record - without a `content` hash, so the ladder is applied across the whole - database. README "Duplicate detection" documents every rung including - the probabilistic large-file path. +- replace the 1 KiB end-window sampling with the head/tail plus content-hash + ladder (2026-09-22, branch `next`, closes + https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is hashed in + full and compared directly, with no end-window step — its `head`, `tail`, and + `content` all hold the whole-file hash. A file at 10 MiB or above gets only + the 64 KiB `head` and `tail` in the hash phase; a new content phase, after the + update phase, reads it for its `content` hash — the whole file below 50 MiB, + gigabyte-spaced 1 MiB samples at or above — only when its size, `head`, and + `tail` match another record's, from the same scan or stored by an earlier one, + so a stored file gains its content hash when it gains a match. A file that is + gone or has changed since its record was written is not read. The `content` + column is part of the version 1 schema. `report` and `trees` group by the + extended signature and leave out any record without a `content` hash, so the + ladder is applied across the whole database. README "Duplicate detection" + documents every rung including the probabilistic large-file path. -- remove the dead `files.dat` references from `Makefile`, `.gitignore` - and `.dockerignore` (2026-09-21, branch `next`, closes +- remove the dead `files.dat` references from `Makefile`, `.gitignore` and + `.dockerignore` (2026-09-21, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/22) - fix the lint-image pin comments and `FROM` form in `Dockerfile` and `Dockerfile.lint` (2026-08-10, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/25): dropped the false - `(Debian-based)` parenthetical (v2.12.1 was Debian too) and the - redundant tag, so both pins are the policy `# image:vX.Y.Z, - YYYY-MM-DD` comment over a bare `FROM image@sha256:...`. Digest - unchanged. `script/verify-lint-image-pin` parses those `FROM` lines - and still matches the tagless form; its advice line lost the now - meaningless "tag and digest". With no tag in either reference, a - tag-only disagreement no longer exists — a one-sided tag is caught as - a plain mismatch. + `(Debian-based)` parenthetical (v2.12.1 was Debian too) and the redundant tag, + so both pins are the policy `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare + `FROM image@sha256:...`. Digest unchanged. `script/verify-lint-image-pin` + parses those `FROM` lines and still matches the tagless form; its advice line + lost the now meaningless "tag and digest". With no tag in either reference, a + tag-only disagreement no longer exists — a one-sided tag is caught as a plain + mismatch. -- run all linting in Docker via `Dockerfile.lint` and `script/lint` - (2026-08-10, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/46): per the owner ruling, the - linter runs inside a container invoked through the `script/` - entrypoint and is never installed on a host. New root - `Dockerfile.lint` COPYs the repo into the digest-pinned - `golangci/golangci-lint:v2.12.2` image and runs - `golangci-lint config verify` and `golangci-lint run` as build - steps, so a successful build IS a clean lint; `script/lint` is - reduced to building it. `script/bootstrap` loses the `go install`, - the pin constants, the version parser and `verify_golangci_lint` - outright rather than hardening them — with nothing linting on the - host, the `$GOPATH/bin` versus `PATH` problem that motivated them has - no subject — and now warns rather than fails when `docker` is absent. - Two traps handled. A lint build on an unchanged tree returns success - in well under a second having run no linter, which is +- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, + branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the + owner ruling, the linter runs inside a container invoked through the `script/` + entrypoint and is never installed on a host. New root `Dockerfile.lint` COPYs + the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and + runs `golangci-lint config verify` and `golangci-lint run` as build steps, so + a successful build IS a clean lint; `script/lint` is reduced to building it. + `script/bootstrap` loses the `go install`, the pin constants, the version + parser and `verify_golangci_lint` outright rather than hardening them — with + nothing linting on the host, the `$GOPATH/bin` versus `PATH` problem that + motivated them has no subject — and now warns rather than fails when `docker` + is absent. Two traps handled. A lint build on an unchanged tree returns + success in well under a second having run no linter, which is https://git.eeqj.de/sneak/sfdupes/issues/32 and - https://git.eeqj.de/sneak/sfdupes/issues/39 again, so - `Dockerfile.lint` carries `ARG CHECK_EPOCH` referenced - inside every gate `RUN` (BuildKit hashes the expanded command, not - the declaration) and `script/lint` passes `"$(date +%s)-$$"` — the - PID matters because two lint runs land inside the same second easily. - And nothing inside an image build may shell out to docker, so the - main `Dockerfile`'s lint stage now invokes `golangci-lint` directly + https://git.eeqj.de/sneak/sfdupes/issues/39 again, so `Dockerfile.lint` + carries `ARG CHECK_EPOCH` referenced inside every gate `RUN` (BuildKit hashes + the expanded command, not the declaration) and `script/lint` passes + `"$(date +%s)-$$"` — the PID matters because two lint runs land inside the + same second easily. And nothing inside an image build may shell out to docker, + so the main `Dockerfile`'s lint stage now invokes `golangci-lint` directly instead of `make lint`, and its build stage runs `make test` and - `make fmt-check` instead of the `make check` aggregate (`make`, not - the scripts bare, because the Makefile's `export CGO_ENABLED = 0` - only reaches what it invokes). `COPY --from=lint` - `/usr/bin/golangci-lint` is replaced by - `COPY --from=lint /src/go.sum /dev/null`: the copied binary was the - only edge forcing BuildKit to finish linting before the build stage - starts, and dropping it without replacing the edge would have ended - fail-fast linting silently under a still-green build. That is - canonical `REPO_POLICIES.md:107`'s ordering edge, restored. - `ENV PATH=/home/builder/go/bin:$PATH` is gone with the `go install` - that justified it. `script/verify-linter-pin` is retired, deleted - along with its README entry, because both of its subjects ceased to - exist in the same change: it compared a linter binary against - `GOLANGCI_LINT_VERSION` in `script/bootstrap`, and there is now - neither a binary crossing between stages nor a version pin in - bootstrap. The drift it guarded has not gone away, it has moved — the - linter is still pinned twice, now as the `FROM` line of - `Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage, - with nothing syncing them, which is exactly what + `make fmt-check` instead of the `make check` aggregate (`make`, not the + scripts bare, because the Makefile's `export CGO_ENABLED = 0` only reaches + what it invokes). `COPY --from=lint` `/usr/bin/golangci-lint` is replaced by + `COPY --from=lint /src/go.sum /dev/null`: the copied binary was the only edge + forcing BuildKit to finish linting before the build stage starts, and dropping + it without replacing the edge would have ended fail-fast linting silently + under a still-green build. That is canonical `REPO_POLICIES.md:107`'s ordering + edge, restored. `ENV PATH=/home/builder/go/bin:$PATH` is gone with the + `go install` that justified it. `script/verify-linter-pin` is retired, deleted + along with its README entry, because both of its subjects ceased to exist in + the same change: it compared a linter binary against `GOLANGCI_LINT_VERSION` + in `script/bootstrap`, and there is now neither a binary crossing between + stages nor a version pin in bootstrap. The drift it guarded has not gone away, + it has moved — the linter is still pinned twice, now as the `FROM` line of + `Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage, with + nothing syncing them, which is exactly what https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its - replacement is one new `script/verify-lint-image-pin`, - run as a gate in both files, which compares the two references to - each other and deliberately restates neither: a hardcoded expected - digest would be a third copy and the same drift one file further out. - `golangci-lint config verify` is included per the ruling, and the - concern about its unpinned live HTTPS schema fetch was measured - rather than assumed — under `--network none` the pinned binary both - passes a valid config and rejects an invalid one with the jsonschema - error, so it validates from an embedded schema and makes no network - call of its own. The README scopes that to the gate steps rather - than to linting as a whole: `Dockerfile.lint` runs `go mod download` - above them, so a cold cache still needs the network and only a warm - one lints offline. Verified: `make lint` green with every `PATH` - directory containing a `golangci-lint` removed + replacement is one new `script/verify-lint-image-pin`, run as a gate in both + files, which compares the two references to each other and deliberately + restates neither: a hardcoded expected digest would be a third copy and the + same drift one file further out. `golangci-lint config verify` is included per + the ruling, and the concern about its unpinned live HTTPS schema fetch was + measured rather than assumed — under `--network none` the pinned binary both + passes a valid config and rejects an invalid one with the jsonschema error, so + it validates from an embedded schema and makes no network call of its own. The + README scopes that to the gate steps rather than to linting as a whole: + `Dockerfile.lint` runs `go mod download` above them, so a cold cache still + needs the network and only a warm one lints offline. Verified: `make lint` + green with every `PATH` directory containing a `golangci-lint` removed (`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`; - `command -v golangci-lint` empty); two consecutive `script/lint` runs - on an untouched tree both executed the linter, 27.7s and 28.7s in the - lint step under distinct epochs with the `COPY . .` layer `CACHED` - above them, at 42.2s and 41.8s wall clock — the no-cache rule was not - weakened to shorten that. Negative control: a planted - `var unusedIssue46Sentinel = 1` failed `script/lint` with - `report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and - failed `make docker` at `[lint 9/9]` with the build stage stopped at - `[builder 3/12]` — `COPY --from=lint`, `script/bootstrap`, the test - gate and `make build` all zero occurrences — then reverted clean. The - drift guard fails on a tag-only disagreement, on a digest-only - disagreement, and on an unreadable reference, naming both sides. - `make docker` green in 5m35s with all six gates executing under one - epoch (lint 37.6s, test 25.2s reporting - `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`). - The non-root quirk still holds: in the builder image with the Go test - cache off, `--user 0:0` fails `TestScanHardlinkRunFailsTogether` - (exit 1) where the unprivileged user passes (exit 0). Noted for - follow-up, not fixed here: `golangci-lint` warns that the - `gomodguard` linter is deprecated since v2.12.0 in favour of - `gomodguard_v2`. + `command -v golangci-lint` empty); two consecutive `script/lint` runs on an + untouched tree both executed the linter, 27.7s and 28.7s in the lint step + under distinct epochs with the `COPY . .` layer `CACHED` above them, at 42.2s + and 41.8s wall clock — the no-cache rule was not weakened to shorten that. + Negative control: a planted `var unusedIssue46Sentinel = 1` failed + `script/lint` with + `report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and failed + `make docker` at `[lint 9/9]` with the build stage stopped at `[builder 3/12]` + — `COPY --from=lint`, `script/bootstrap`, the test gate and `make build` all + zero occurrences — then reverted clean. The drift guard fails on a tag-only + disagreement, on a digest-only disagreement, and on an unreadable reference, + naming both sides. `make docker` green in 5m35s with all six gates executing + under one epoch (lint 37.6s, test 25.2s reporting + `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`). The + non-root quirk still holds: in the builder image with the Go test cache off, + `--user 0:0` fails `TestScanHardlinkRunFailsTogether` (exit 1) where the + unprivileged user passes (exit 0). Noted for follow-up, not fixed here: + `golangci-lint` warns that the `gomodguard` linter is deprecated since v2.12.0 + in favour of `gomodguard_v2`. -- install the Docker build stage's prerequisites by running - `script/bootstrap` instead of `apk add --no-cache make` inline - (2026-08-09, branch `dockerfile-bootstrap`, closes #42): canonical - `REPO_POLICIES.md:97` requires it, and the inline install left the - build stage maintaining its own notion of the toolchain — exactly - the divergence #24 exists to close, one layer down. The stage now - copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`, - which ends in `go mod download`, so the separate invocation of that - is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, and moves - above the bootstrap layer. It is the only edge making this stage - depend on the lint stage, so deleting it as redundant would end - fail-fast linting silently. Letting bootstrap install its own linter - here would have reintroduced the second toolchain and paid for a - from-source build of it. What makes the two stages provably one - toolchain rather than two that happen to agree is a new - `script/verify-linter-pin`, run in the build stage on the binary - that arrives from the lint stage, before bootstrap: it fails the - build naming both versions unless that binary is the version - `script/bootstrap` pins. Bootstrap's own check could not serve that - purpose — it reinstalls its pin from source and then verifies - whatever `PATH` resolves, so drift self-heals silently and a lint - stage image bumped on its own would lint at the new version while - `make check` ran at the old one, green. The linter version is pinned - in two independent places (the lint stage image digest and +- install the Docker build stage's prerequisites by running `script/bootstrap` + instead of `apk add --no-cache make` inline (2026-08-09, branch + `dockerfile-bootstrap`, closes #42): canonical `REPO_POLICIES.md:97` requires + it, and the inline install left the build stage maintaining its own notion of + the toolchain — exactly the divergence #24 exists to close, one layer down. + The stage now copies `script/` plus `go.mod`/`go.sum` and runs + `script/bootstrap`, which ends in `go mod download`, so the separate + invocation of that is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, + and moves above the bootstrap layer. It is the only edge making this stage + depend on the lint stage, so deleting it as redundant would end fail-fast + linting silently. Letting bootstrap install its own linter here would have + reintroduced the second toolchain and paid for a from-source build of it. What + makes the two stages provably one toolchain rather than two that happen to + agree is a new `script/verify-linter-pin`, run in the build stage on the + binary that arrives from the lint stage, before bootstrap: it fails the build + naming both versions unless that binary is the version `script/bootstrap` + pins. Bootstrap's own check could not serve that purpose — it reinstalls its + pin from source and then verifies whatever `PATH` resolves, so drift + self-heals silently and a lint stage image bumped on its own would lint at the + new version while `make check` ran at the old one, green. The linter version + is pinned in two independent places (the lint stage image digest and `GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a half-applied bump is now a build failure. The pin is read out of - `script/bootstrap`, which stays the single source of truth; a pin - that cannot be read is a hard failure, not a skip. The check needs - no `CHECK_EPOCH`: its only inputs are the copied binary and - `script/`, so Docker invalidates the layer exactly when a cached - result would stop being true, and it is documented with the other - entrypoints in the README. `$GOPATH/bin` joins `PATH` because - that is where bootstrap's `go install` lands and bootstrap verifies - its installs against what `PATH` resolves — nothing in the image is - shadowed by it, the directory does not exist until bootstrap runs. - Everything added sits above `ARG CHECK_EPOCH`, and the `chown` and - `USER builder` still precede `make check`. Verified: the guard fails - the build with both versions named when the lint stage's linter is - faked to a different version, and an unmodified build still passes - it; bootstrap runs clean under Alpine's `sh` and its `apk` branch, - installing `git` and `make` and finding the copied - linter already at the pin; a second build served the bootstrap and - dependency layers `CACHED` while both gates ran with a fresh epoch; - a planted `unused` finding failed the build at the lint gate in - 48.9s with the build stage's `make check` never starting; and the - suite run in the image as `--user 0:0` fails - `TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged - user is still load-bearing. That last check needs the Go test cache - disabled — the first attempt reported `ok ... (cached)` as root, - reusing the result the build-time run had left in the shared cache, - which would have read as a pass. Build wall time, on a shared host - running many concurrent builds and so noisy: 2m13s on an unchanged - tree, 2m17s and 4m29s for two builds after a source change, 5m14s - cold. Only the cold one breaches the policy ceiling, and not because - of this change — `chown -R builder:builder /src /home/builder` walks - the module cache and re-runs on every source change, and it alone - varied between 77s and 210s across those four builds, which is also - the whole spread in the totals. The same cold measurement against - `main` is 5m03s with a 209s `chown`. Filed as #43 -- bust the Docker layer cache for the gate steps, so `script/cibuild` - and `script/docker` cannot report a green they did not earn - (2026-08-09, branch `cibuild-cache-bust`, closes #32): both scripts - were bare `docker build` invocations with no cache control, and the - `Dockerfile` copies the tree before running its gates, so on an - unchanged tree Docker served those layers from cache and the build - exited 0 having executed nothing. That is not hypothetical here — - every merge this repo has done is a non-fast-forward merge of an - undiverged branch, so each merge commit's tree is byte-identical to - the branch head's and each merge CI run was almost certainly a full - cache hit; and PR #31's reviewer found `make docker` returning - success as a 17-layer cache hit, catching it only by being - suspicious. The fix is `ARG CHECK_EPOCH` with the scripts passing - `--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or break - it. `ARG` is scoped per stage and this `Dockerfile` has three gates - across two — `make fmt-check` and `make lint` in the lint stage, - `make check` in the build stage — so a single declaration would have - left one stage silently cacheable; it is declared in both. And - BuildKit hashes the expanded command, not the declaration, so a - declared-but-unreferenced `ARG` invalidates nothing: each gate `RUN` - echoes the epoch, which also puts the value in the build log as - evidence the layer really ran. Placement is below the dependency - layers on purpose — a build that goes cold every time would be a - different bug, not a fix. Verified by running each script twice back - to back on an unchanged tree under `BUILDKIT_PROGRESS=plain`: all - three gates executed on all four runs, each with a fresh epoch in - the log (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s - then 53.4s), and twelve steps were still served `CACHED` in the - steady state — both `go mod download`s, `apk add`, `adduser`, the - `chown`, every `go.mod`/`go.sum` and source copy, the linter copy - out of the lint stage, and the binary copy into the runtime stage. - The lint stage still gates the build stage: with a deliberate - `unused` finding planted in the tree, the build failed at - `make lint` in 36.1s and the build-stage `make check` never started. - The build stage also still drops to the unprivileged `builder` user - before `make check`, which the suite depends on rather than merely - prefers: forcing the same image to run the tests as root fails - `TestScanHardlinkRunFailsTogether`, because root reads straight - through the `chmod(0)` the test uses to prove hard links are read - once. This is the local fix only; propagating it to the canonical - templates is `prompts` #26 -- check the installed golangci-lint version in `script/bootstrap` - instead of only its presence (2026-08-09, branch - `bootstrap-version-check`, closes #24): `missing golangci-lint` meant - any linter already on `PATH` satisfied the check, so the pin was never - consulted and the v2.12.2 bump from #3 was inert on every host that - already had one — this host ran v2.10.1 against a v2.12.2 pin, - `make check` went green, and `make docker` then rejected the same - commit with findings the local gate never saw. The version now lives - in one place, `GOLANGCI_LINT_VERSION`, with the `go install` module - ref derived from it so a bump cannot half-apply; a - `golangci_lint_version` helper parses `golangci-lint --version` - (taking the field after the word `version` and tolerating an optional - leading `v`, which the module ref carries and the binary's output does - not), and any version that is not the pin — older, newer, absent or - unparseable — is reinstalled. The install is then verified against the - binary `PATH` actually resolves: `go install` writes into `GOBIN` (or - `GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes - first on `PATH`, so a wrong-version one sitting ahead of it — nix, - apt, brew, apk, or the `/usr/local/bin` copy the `Dockerfile` builder - stage makes — would swallow the install and leave the local gate - disagreeing with CI under an affirmative `bootstrap complete`. - Bootstrap now re-reads the effective version after installing and, on - a mismatch, prints both paths and both versions to stderr and exits - non-zero instead of claiming success; it does not reorder anyone's + `script/bootstrap`, which stays the single source of truth; a pin that cannot + be read is a hard failure, not a skip. The check needs no `CHECK_EPOCH`: its + only inputs are the copied binary and `script/`, so Docker invalidates the + layer exactly when a cached result would stop being true, and it is documented + with the other entrypoints in the README. `$GOPATH/bin` joins `PATH` because + that is where bootstrap's `go install` lands and bootstrap verifies its + installs against what `PATH` resolves — nothing in the image is shadowed by + it, the directory does not exist until bootstrap runs. Everything added sits + above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still precede + `make check`. Verified: the guard fails the build with both versions named + when the lint stage's linter is faked to a different version, and an + unmodified build still passes it; bootstrap runs clean under Alpine's `sh` and + its `apk` branch, installing `git` and `make` and finding the copied linter + already at the pin; a second build served the bootstrap and dependency layers + `CACHED` while both gates ran with a fresh epoch; a planted `unused` finding + failed the build at the lint gate in 48.9s with the build stage's `make check` + never starting; and the suite run in the image as `--user 0:0` fails + `TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged user is + still load-bearing. That last check needs the Go test cache disabled — the + first attempt reported `ok ... (cached)` as root, reusing the result the + build-time run had left in the shared cache, which would have read as a pass. + Build wall time, on a shared host running many concurrent builds and so noisy: + 2m13s on an unchanged tree, 2m17s and 4m29s for two builds after a source + change, 5m14s cold. Only the cold one breaches the policy ceiling, and not + because of this change — `chown -R builder:builder /src /home/builder` walks + the module cache and re-runs on every source change, and it alone varied + between 77s and 210s across those four builds, which is also the whole spread + in the totals. The same cold measurement against `main` is 5m03s with a 209s + `chown`. Filed as #43 +- bust the Docker layer cache for the gate steps, so `script/cibuild` and + `script/docker` cannot report a green they did not earn (2026-08-09, branch + `cibuild-cache-bust`, closes #32): both scripts were bare `docker build` + invocations with no cache control, and the `Dockerfile` copies the tree before + running its gates, so on an unchanged tree Docker served those layers from + cache and the build exited 0 having executed nothing. That is not hypothetical + here — every merge this repo has done is a non-fast-forward merge of an + undiverged branch, so each merge commit's tree is byte-identical to the branch + head's and each merge CI run was almost certainly a full cache hit; and PR + #31's reviewer found `make docker` returning success as a 17-layer cache hit, + catching it only by being suspicious. The fix is `ARG CHECK_EPOCH` with the + scripts passing `--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or + break it. `ARG` is scoped per stage and this `Dockerfile` has three gates + across two — `make fmt-check` and `make lint` in the lint stage, `make check` + in the build stage — so a single declaration would have left one stage + silently cacheable; it is declared in both. And BuildKit hashes the expanded + command, not the declaration, so a declared-but-unreferenced `ARG` invalidates + nothing: each gate `RUN` echoes the epoch, which also puts the value in the + build log as evidence the layer really ran. Placement is below the dependency + layers on purpose — a build that goes cold every time would be a different + bug, not a fix. Verified by running each script twice back to back on an + unchanged tree under `BUILDKIT_PROGRESS=plain`: all three gates executed on + all four runs, each with a fresh epoch in the log (`script/cibuild` 78.8s then + 61.1s; `script/docker` 61.1s then 53.4s), and twelve steps were still served + `CACHED` in the steady state — both `go mod download`s, `apk add`, `adduser`, + the `chown`, every `go.mod`/`go.sum` and source copy, the linter copy out of + the lint stage, and the binary copy into the runtime stage. The lint stage + still gates the build stage: with a deliberate `unused` finding planted in the + tree, the build failed at `make lint` in 36.1s and the build-stage + `make check` never started. The build stage also still drops to the + unprivileged `builder` user before `make check`, which the suite depends on + rather than merely prefers: forcing the same image to run the tests as root + fails `TestScanHardlinkRunFailsTogether`, because root reads straight through + the `chmod(0)` the test uses to prove hard links are read once. This is the + local fix only; propagating it to the canonical templates is `prompts` #26 +- check the installed golangci-lint version in `script/bootstrap` instead of + only its presence (2026-08-09, branch `bootstrap-version-check`, closes #24): + `missing golangci-lint` meant any linter already on `PATH` satisfied the + check, so the pin was never consulted and the v2.12.2 bump from #3 was inert + on every host that already had one — this host ran v2.10.1 against a v2.12.2 + pin, `make check` went green, and `make docker` then rejected the same commit + with findings the local gate never saw. The version now lives in one place, + `GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it so a + bump cannot half-apply; a `golangci_lint_version` helper parses + `golangci-lint --version` (taking the field after the word `version` and + tolerating an optional leading `v`, which the module ref carries and the + binary's output does not), and any version that is not the pin — older, newer, + absent or unparseable — is reinstalled. The install is then verified against + the binary `PATH` actually resolves: `go install` writes into `GOBIN` (or + `GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes first on + `PATH`, so a wrong-version one sitting ahead of it — nix, apt, brew, apk, or + the `/usr/local/bin` copy the `Dockerfile` builder stage makes — would swallow + the install and leave the local gate disagreeing with CI under an affirmative + `bootstrap complete`. Bootstrap now re-reads the effective version after + installing and, on a mismatch, prints both paths and both versions to stderr + and exits non-zero instead of claiming success; it does not reorder anyone's `PATH` or delete their binary. The `--version` call keeps its stderr - connected, so a present-but-broken binary says why rather than - reinstalling forever in silence, and is bounded by `timeout(1)` where - that exists, so a wedged binary cannot hang bootstrap. `git`, `make` - and `go` keep their presence-only checks and now say why in a - comment: they are host package-manager tools the repo deliberately - does not pin, with `go.mod` governing the language version and the - digest-pinned images covering reproducible builds. Verified on this - host by bootstrapping from v2.10.1 to v2.12.2 and running it again to - a no-op, plus stub runs of the real script under `dash` covering a - thirteen-input parse matrix (absent, older, newer, host-style, - image-style, leading-`v`, stderr-only, empty, non-zero exit, impostor - binary, `(devel)`, trailing `version`), a shadowed install that must - exit non-zero, an install destination not on `PATH` at all, `GOBIN` - set, and a wedged binary that must hit the timeout; `make check` and - `make lint` are clean at v2.12.2, so v2.10.1 was not hiding any - findings on `main` + connected, so a present-but-broken binary says why rather than reinstalling + forever in silence, and is bounded by `timeout(1)` where that exists, so a + wedged binary cannot hang bootstrap. `git`, `make` and `go` keep their + presence-only checks and now say why in a comment: they are host + package-manager tools the repo deliberately does not pin, with `go.mod` + governing the language version and the digest-pinned images covering + reproducible builds. Verified on this host by bootstrapping from v2.10.1 to + v2.12.2 and running it again to a no-op, plus stub runs of the real script + under `dash` covering a thirteen-input parse matrix (absent, older, newer, + host-style, image-style, leading-`v`, stderr-only, empty, non-zero exit, + impostor binary, `(devel)`, trailing `version`), a shadowed install that must + exit non-zero, an install destination not on `PATH` at all, `GOBIN` set, and a + wedged binary that must hit the timeout; `make check` and `make lint` are + clean at v2.12.2, so v2.10.1 was not hiding any findings on `main` - unwind the hash worker pool on the error path (2026-08-09, branch - `hash-pool-cleanup`, closes #6): `hashPhase` used to return the - moment `recordRun` failed and abandon the pool — the feeder parked - forever on a full `jobs` channel and every worker on a full - `results` channel. That only stopped being invisible when #4 landed - and `runScan` began unwinding instead of calling `os.Exit`. The - pool is now an owned, context-aware `hashPool`: every blocking send - in the feeder and the workers selects on `ctx.Done()`, `jobs` is - closed on every path out, and `hashPhase` defers `pool.stop()`, - which cancels and then drains `results` until the last goroutine - has exited — draining is what frees a worker already parked on a - send. `ctx` is threaded from `cmd.Context()` through `runScan`, - `syncScan`, both worker pools and the whole database layer (it is - the first parameter everywhere), so #5 can hand this path a signal - and needs to add nothing else. The walk pool never leaked, because - `walkPhase` always drains its events to close, but it has the same - unbounded-send shape and #5 will give it an early return, so it - gets the same treatment plus a `ctx.Err()` guard after the walk: a - cancelled walk yields a partial size census, and every file it never - reached looks vanished to the update phase. That phase's own - `BeginTx` fails on the same cancelled context before deleting - anything, so the guard is defence in depth rather than the only - barrier — but it is the one that survives #5 deciding an interrupted - scan may commit what it has. Tests drive `run(scan)` against a - database whose insert trigger aborts, and assert both that the scan - fails instead of hanging and that `runtime.NumGoroutine()` polls - back to its pre-scan baseline; a second set cancels a scan part-way - through the walk — deterministically, by counting the scan's own - consultations of `ctx.Done()` rather than racing a timer — and - asserts that it stops at the guard holding a partial census and a - still-populated record index, with every record intact. The - remaining cancellation branches of both pools are covered by direct - tests of `sendEvent`, the walk workers, `dispatchDirs`, - `feedHashJobs`, `hashWorker` and `hashPhase` -- guarantee the database is closed on every fatal exit path - (2026-08-09, branch `db-close-on-fatal`, closes #4): `fatalf` and - its `os.Exit(1)` are gone, so the deferred `db.Close()` — and with - it the SQLite WAL checkpoint — now actually runs when a subcommand - fails; `runScan`, `runReport`, `runTrees`, `loadRecords` and - `resolveRoots` return errors instead. The single exit point is `run` - in `main.go`: it maps a `fatalError` (anything a subcommand - returned) to exit 1 and cobra's own argument and flag errors to exit - 2, which keeps a runtime failure from being reported as a usage - error or printing the usage text. New `main_test.go` drives the CLI - in-process and asserts the exit codes from README §Error handling - plus the stdout/stderr split, including that a fatal error raised - after the database is open leaves no `-wal`/`-shm` sidecar behind - for `scan`, `report` or `trees` -- update golangci-lint to v2.12.2 with the canonical config - (2026-08-09, branch `golangci-v2.12.2`, merged as `38a01bd`, - closes #3): bumped the pinned linter in the `Dockerfile` lint - stage and `script/bootstrap` from v2.12.1 to v2.12.2, and replaced - `.golangci.yml` with the canonical file — the linter settings + `hash-pool-cleanup`, closes #6): `hashPhase` used to return the moment + `recordRun` failed and abandon the pool — the feeder parked forever on a full + `jobs` channel and every worker on a full `results` channel. That only stopped + being invisible when #4 landed and `runScan` began unwinding instead of + calling `os.Exit`. The pool is now an owned, context-aware `hashPool`: every + blocking send in the feeder and the workers selects on `ctx.Done()`, `jobs` is + closed on every path out, and `hashPhase` defers `pool.stop()`, which cancels + and then drains `results` until the last goroutine has exited — draining is + what frees a worker already parked on a send. `ctx` is threaded from + `cmd.Context()` through `runScan`, `syncScan`, both worker pools and the whole + database layer (it is the first parameter everywhere), so #5 can hand this + path a signal and needs to add nothing else. The walk pool never leaked, + because `walkPhase` always drains its events to close, but it has the same + unbounded-send shape and #5 will give it an early return, so it gets the same + treatment plus a `ctx.Err()` guard after the walk: a cancelled walk yields a + partial size census, and every file it never reached looks vanished to the + update phase. That phase's own `BeginTx` fails on the same cancelled context + before deleting anything, so the guard is defence in depth rather than the + only barrier — but it is the one that survives #5 deciding an interrupted scan + may commit what it has. Tests drive `run(scan)` against a database whose + insert trigger aborts, and assert both that the scan fails instead of hanging + and that `runtime.NumGoroutine()` polls back to its pre-scan baseline; a + second set cancels a scan part-way through the walk — deterministically, by + counting the scan's own consultations of `ctx.Done()` rather than racing a + timer — and asserts that it stops at the guard holding a partial census and a + still-populated record index, with every record intact. The remaining + cancellation branches of both pools are covered by direct tests of + `sendEvent`, the walk workers, `dispatchDirs`, `feedHashJobs`, `hashWorker` + and `hashPhase` +- guarantee the database is closed on every fatal exit path (2026-08-09, branch + `db-close-on-fatal`, closes #4): `fatalf` and its `os.Exit(1)` are gone, so + the deferred `db.Close()` — and with it the SQLite WAL checkpoint — now + actually runs when a subcommand fails; `runScan`, `runReport`, `runTrees`, + `loadRecords` and `resolveRoots` return errors instead. The single exit point + is `run` in `main.go`: it maps a `fatalError` (anything a subcommand returned) + to exit 1 and cobra's own argument and flag errors to exit 2, which keeps a + runtime failure from being reported as a usage error or printing the usage + text. New `main_test.go` drives the CLI in-process and asserts the exit codes + from README §Error handling plus the stdout/stderr split, including that a + fatal error raised after the database is open leaves no `-wal`/`-shm` sidecar + behind for `scan`, `report` or `trees` +- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch + `golangci-v2.12.2`, merged as `38a01bd`, closes #3): bumped the pinned linter + in the `Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, + and replaced `.golangci.yml` with the canonical file — the linter settings (`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under - `linters.settings` per the v2 schema, so they are actually - applied; no new lint findings surfaced -- convert Makefile targets to scripts-to-rule-them-all `script/` - entrypoints like the other managed repos (2026-07-26, commit - `3abeacf`, closes #1): all 12 `script/` entrypoints exist - (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, - `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, - `install-precommit`) and every Makefile target is now a thin shim - over them, matching the other managed repos + `linters.settings` per the v2 schema, so they are actually applied; no new + lint findings surfaced +- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints + like the other managed repos (2026-07-26, commit `3abeacf`, closes #1): all 12 + `script/` entrypoints exist (`bootstrap`, `setup`, `projectname`, `test`, + `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, + `install-precommit`) and every Makefile target is now a thin shim over them, + matching the other managed repos - make the binary the default Make target (2026-07-24, branch - `make-default-target`): plain `make` now builds `sfdupes` - (previously it ran `check` plus `build`); `make build` remains as - an alias -- scan-wide phases, concurrent operands, batched updates (2026-07-24, - branch `scan-wide-phases`): all operands seed the shared walk pool - and every pass runs once over the whole scan, so totals and ETAs - are scan-global; the per-operand walk/hash/update cycles and their - stderr announcements are gone; the update pass commits in batched - transactions — the filesystem is authoritative and the database an - eventually-consistent reflection, so scan-level atomicity is not - required + `make-default-target`): plain `make` now builds `sfdupes` (previously it ran + `check` plus `build`); `make build` remains as an alias +- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch + `scan-wide-phases`): all operands seed the shared walk pool and every pass + runs once over the whole scan, so totals and ETAs are scan-global; the + per-operand walk/hash/update cycles and their stderr announcements are gone; + the update pass commits in batched transactions — the filesystem is + authoritative and the database an eventually-consistent reflection, so + scan-level atomicity is not required - split the stat pass back out of the walk (2026-07-24, branch - `parallel-phases`): phases are strictly sequential again — walk, - stat, hash, update per operand — with parallelism only inside each - phase; the walk enumerates paths with per-directory workers and the - stat pass lstats them with per-file workers, restoring the exact - total/ETA stat bar -- announce each operand on stderr before its passes (2026-07-24, - branch `scan-operand-progress`): with per-operand walk/hash/update - cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals - that looked like the whole run's — an operator watching operand 3 of - 14 hash 300k files concluded 20M files were being skipped -- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass - was a single goroutine and took hours at ~20M files on a busy pool - (observed: 22M files in 4h on a ZFS server); it is now a - per-directory worker-pool traversal that records size/mtime during - the walk (folding away the separate stat pass, halving metadata - I/O), and each `PATH` operand commits in its own transaction so an - interrupted scan keeps completed operands + `parallel-phases`): phases are strictly sequential again — walk, stat, hash, + update per operand — with parallelism only inside each phase; the walk + enumerates paths with per-directory workers and the stat pass lstats them with + per-file workers, restoring the exact total/ETA stat bar +- announce each operand on stderr before its passes (2026-07-24, branch + `scan-operand-progress`): with per-operand walk/hash/update cycles, a + multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the + whole run's — an operator watching operand 3 of 14 hash 300k files concluded + 20M files were being skipped +- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single + goroutine and took hours at ~20M files on a busy pool (observed: 22M files in + 4h on a ZFS server); it is now a per-directory worker-pool traversal that + records size/mtime during the walk (folding away the separate stat pass, + halving metadata I/O), and each `PATH` operand commits in its own transaction + so an interrupted scan keeps completed operands -- persistent scan database (2026-07-24, branch `persistent-database`): - `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure - Go, cgo stays disabled) keyed by absolute path that survives between - runs — a rescan hashes only new or changed files (by mtime/size), - deletes records for files vanished from under the scanned operands, - and leaves records outside them untouched, so `scan` can be cronned - daily; `report` and `trees` read the database (no positional - arguments) instead of a scan stream. Database at - `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; - WAL journaling plus a single-transaction update keep a report run - during a scan safe -- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag - `v0.0.1`, and push `main` plus tags (2026-07-23) +- persistent scan database (2026-07-24, branch `persistent-database`): `scan` + now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays + disabled) keyed by absolute path that survives between runs — a rescan hashes + only new or changed files (by mtime/size), deletes records for files vanished + from under the scanned operands, and leaves records outside them untouched, so + `scan` can be cronned daily; `report` and `trees` read the database (no + positional arguments) instead of a scan stream. Database at + `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL + journaling plus a single-transaction update keep a report run during a scan + safe +- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`, + and push `main` plus tags (2026-07-23) - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required - `PATH...` operands via cobra flags replacing the `/srv` `-root` - default; new `-x`/`--one-file-system` flag (GNU convention) to stop - at filesystem boundaries, which are crossed by default + `PATH...` operands via cobra flags replacing the `/srv` `-root` default; new + `-x`/`--one-file-system` flag (GNU convention) to stop at filesystem + boundaries, which are crossed by default - bring the repo into full policy compliance (2026-07-23, branch `repo-policy-compliance`; checklist below) -- `git init` with README-only first commit; code baseline committed on - `main` (2026-07-22) +- `git init` with README-only first commit; code baseline committed on `main` + (2026-07-22) - implement `scan`, `report`, and `trees` subcommands (pre-git history) # Future Steps -- possible later features (explicitly out of scope per README): - full-content verification of candidates, removal-script helpers +- possible later features (explicitly out of scope per README): full-content + verification of candidates, removal-script helpers # Repo Policy Compliance -Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing -repo checklist, and the Go styleguide. Code is already gofmt-clean, so no -standalone formatting commit is needed. +Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo +checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone +formatting commit is needed. -- [x] `.gitignore` missing — the compiled `sfdupes` binary and - `files.dat` sit untracked in the tree; needs OS/editor/Go - artifacts plus secrets patterns +- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit + untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns - [x] `.editorconfig` missing -- [x] `LICENSE` missing and README has no License section (MIT assumed - from house convention — user to confirm) +- [x] `LICENSE` missing and README has no License section (MIT assumed from + house convention — user to confirm) - [x] `REPO_POLICIES.md` missing from repo root -- [x] `.golangci.yml` missing (install canonical copy); code must then - pass `make lint` (150 findings fixed; `make lint` is clean) -- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, - `fmt-check`, `docker`, `hooks`; `check` currently depends on - `build`, which writes the binary (`make check` must not modify - files) -- [x] no tests — `go test ./...` has nothing to run; policy requires - real tests with a 30-second timeout and the conditional `-v` - rerun pattern (suite covers parsing, grouping, digests, - suppression, hashing, and the scan pipeline; 64% coverage) -- [x] `Dockerfile` missing — Go multistage with hash-pinned images: - fail-fast lint stage, build stage running `make check` +- [x] `.golangci.yml` missing (install canonical copy); code must then pass + `make lint` (150 findings fixed; `make lint` is clean) +- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`, + `docker`, `hooks`; `check` currently depends on `build`, which writes the + binary (`make check` must not modify files) +- [x] no tests — `go test ./...` has nothing to run; policy requires real tests + with a 30-second timeout and the conditional `-v` rerun pattern (suite + covers parsing, grouping, digests, suppression, hashing, and the scan + pipeline; 64% coverage) +- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast + lint stage, build stage running `make check` - [x] `.dockerignore` missing -- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, - checkout action pinned by commit SHA) +- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout + action pinned by commit SHA) - [x] README lacks required sections: Description first line - (name/purpose/category/license/author), Getting Started, - Rationale, TODO, License, Author -- [x] README non-goal "no git repository setup and no CI" is stale now - that the repo is under git with CI -- [x] pre-commit hook not installed (`make hooks` once the target - exists) + (name/purpose/category/license/author), Getting Started, Rationale, TODO, + License, Author +- [x] README non-goal "no git repository setup and no CI" is stale now that the + repo is under git with CI +- [x] pre-commit hook not installed (`make hooks` once the target exists) Accepted divergences (no action): -- flat single-package layout with `.go` files in the repo root — fine - for a small single-binary tool per the Go styleguide; the tracker - audit agrees -- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0` - (pure-Go builds) and the race detector requires cgo +- flat single-package layout with `.go` files in the repo root — fine for a + small single-binary tool per the Go styleguide; the tracker audit agrees +- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go + builds) and the race detector requires cgo diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc05cde --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/script/bootstrap b/script/bootstrap index 936ea8b..380191d 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,10 +3,12 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes nothing is present (not git, -# make, or go). The linter is NOT installed: golangci-lint runs via -# docker only (script/lint), pinned by image digest, so the only lint +# make, or go). Neither the linter nor the Markdown formatter is +# installed: golangci-lint (script/lint) and prettier (script/fmt, +# script/fmt-check) run via docker only, pinned by hash, so their only # prerequisite is a working docker — which is warned about, not -# installed, because everything except linting works without it. +# installed, because everything except linting and formatting works +# without it. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -71,15 +73,15 @@ main() { if missing make; then pkg_install gnumake make make make; fi if missing go; then pkg_install go golang go go; fi - # Linting runs via docker only (script/lint), so docker is a lint - # prerequisite rather than something bootstrap installs. Warn, do - # not fail: everything except `make lint` — and, through it, - # `make check`, `make docker` and the pre-commit hook — works - # without it. + # Linting and Markdown formatting run via docker only, so docker is + # their prerequisite rather than something bootstrap installs. Warn, + # do not fail: everything except `make lint`, `make fmt` and + # `make fmt-check` — and, through them, `make check`, `make docker` + # and the pre-commit hook — works without it. if missing docker; then - echo "bootstrap: WARNING: docker not found; make lint, make check" >&2 - echo "bootstrap: and make docker require it. Install docker to" >&2 - echo "bootstrap: run the linter." >&2 + echo "bootstrap: WARNING: docker not found; make lint, make fmt," >&2 + echo "bootstrap: make fmt-check, make check and make docker" >&2 + echo "bootstrap: require it." >&2 fi go mod download diff --git a/script/cibuild b/script/cibuild index 55a90ba..7283a88 100755 --- a/script/cibuild +++ b/script/cibuild @@ -3,17 +3,17 @@ # push. # # The Dockerfile runs the gates individually as build steps, not the -# make check aggregate: the lint stage runs make fmt-check, +# make check aggregate: the lint stage runs the gofmt check, # script/verify-lint-image-pin, golangci-lint config verify and -# golangci-lint run; the build stage, dropped to an unprivileged user, -# runs make test and make fmt-check. Neither make lint nor make check -# appears, because both reach script/lint, which is itself a docker -# build, and a docker build cannot run inside one. Lint is not skipped -# by that — the linter is invoked directly in the lint stage, and the -# build stage's COPY --from=lint makes that stage a prerequisite, so -# BuildKit must finish it first. Between the two stages everything -# make check would run has run, which is why a successful build here -# implies the repo is green. +# golangci-lint run; the markdown stage runs the prettier check; the +# build stage, dropped to an unprivileged user, runs make test. None of +# make lint, make fmt-check or make check appears, because each runs +# docker, and docker cannot run inside a docker build. Nothing is +# skipped by that — the linter, gofmt and prettier are invoked directly +# in their stages, and the build stage's COPY --from lines make those +# stages prerequisites, so BuildKit must finish them first. Between the +# three stages everything make check would run has run, which is why a +# successful build here implies the repo is green. # # That implication holds only because of CHECK_EPOCH. A COPY layer is # invalidated only by changed content, and a rebuild of an unchanged diff --git a/script/docker b/script/docker index 9f52294..3c7f7bc 100755 --- a/script/docker +++ b/script/docker @@ -4,11 +4,11 @@ # # CHECK_EPOCH is passed for the same reason script/cibuild passes it: # without it Docker serves the Dockerfile's gate layers from cache on an -# unchanged tree and this exits 0 having run neither the lint stage's -# gates nor the builder stage's test and fmt-check gates. This is the -# set of gates a developer or reviewer runs by hand, so a cached pass -# here is the most misleading result the repo can produce. Dependency -# layers sit above the ARG and stay cached. +# unchanged tree and this exits 0 having run none of the lint stage's +# gates, the markdown stage's prettier gate or the builder stage's test +# gate. This is the set of gates a developer or reviewer runs by hand, +# so a cached pass here is the most misleading result the repo can +# produce. Dependency layers sit above the ARG and stay cached. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/fmt b/script/fmt index 28af13b..8994c8b 100755 --- a/script/fmt +++ b/script/fmt @@ -1,12 +1,22 @@ #!/bin/sh -# script/fmt: format all files (writes). +# script/fmt: format all files (writes): the Go sources with gofmt, the +# Markdown with prettier. prettier is never installed on the host: it +# runs from the Dockerfile's prettier stage with the repository mounted, +# as the calling user so the files it rewrites keep their owner. The tag +# makes each build replace the previous image instead of leaving another +# one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" gofmt -s -w . + image="$("$SCRIPT_DIR/projectname")-prettier" + docker build -q --target prettier -t "$image" . >/dev/null + docker run --rm --user "$(id -u):$(id -g)" -v "$ROOT:/src" "$image" \ + prettier --write '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 96fffa0..788f585 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -1,18 +1,39 @@ #!/bin/sh # script/fmt-check: check formatting (read-only). Same scope as -# script/fmt, but fails instead of writing. +# script/fmt, but fails instead of writing. gofmt and prettier both run +# every time and each reports its own failure, so the output says which +# one failed. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - files="$(gofmt -s -l .)" + status=0 + + # Under set -e a bare assignment would end the script when gofmt + # fails (a Go file it cannot parse), and prettier would never run. + if ! files="$(gofmt -s -l .)"; then + echo "gofmt: failed; see its errors above" >&2 + status=1 + fi if [ -n "$files" ]; then echo "gofmt: files not formatted:" >&2 echo "$files" >&2 - exit 1 + status=1 fi + + # Same image as script/fmt; see there. + image="$("$SCRIPT_DIR/projectname")-prettier" + docker build -q --target prettier -t "$image" . >/dev/null + if ! docker run --rm -v "$ROOT:/src:ro" "$image" \ + prettier --check '**/*.md' --tab-width 4 --prose-wrap always; then + echo "prettier: Markdown not formatted; run make fmt" >&2 + status=1 + fi + + exit "$status" } main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg== From bebfac1dcbf97ed6262eea96d5bd59d326a4f91c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 19:30:20 +0200 Subject: [PATCH 25/33] Fail a bare docker build instead of serving cached gates (closes #39) Each Dockerfile stage that runs gates now checks, right after its ARG CHECK_EPOCH, that the value is not empty, and stops with a message naming script/cibuild and script/docker. A plain `docker build .` can no longer report a green from cached gate layers. script/cibuild and script/docker now append the process id to the epoch, the form script/lint already uses, so two runs started in the same second still get different values. README says both. TODO.md corrects the steady-state CACHED count recorded for issue 32 from twelve to thirteen. Model: opus-5-5 --- Dockerfile | 27 +++++++++++++++++++++------ README.md | 9 ++++++--- TODO.md | 29 +++++++++++++++++------------ script/cibuild | 7 +++++-- script/docker | 4 ++-- 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index b911abe..1f2dbd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,9 @@ COPY . . # invalidates COPY only when the copied content changes, so on an # unchanged tree the gates below would be served from cache and the # build would exit 0 having run nothing. script/cibuild and -# script/docker pass a fresh CHECK_EPOCH on every invocation. +# script/docker pass a fresh CHECK_EPOCH on every invocation. A build +# that passes none, such as a bare `docker build .`, fails at the check +# right after the ARG instead of quietly serving the gates from cache. # # Two properties this depends on. ARG is per-stage, so the markdown and # build stages below declare it again; one declaration here would leave @@ -22,6 +24,10 @@ COPY . . # (the pinned base image, go mod download) keeps its cache; only the # gates go cold. ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi # The linter is invoked directly here, not through `make lint`. That # target now runs `docker build -f Dockerfile.lint`, and a docker build @@ -71,9 +77,13 @@ WORKDIR /src # Markdown stage: the Markdown half of `make fmt-check`, as a gate. FROM prettier AS markdown COPY . . -# Second per-stage declaration of the gate cache-buster; see the lint -# stage above. +# Second per-stage declaration of the gate cache-buster and its check; +# see the lint stage above. ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi RUN echo "gate prettier, epoch ${CHECK_EPOCH}" && \ prettier --check '**/*.md' --tab-width 4 --prose-wrap always @@ -153,10 +163,15 @@ USER builder # prerequisites. `make`, not the script directly, because the Makefile's # `export CGO_ENABLED = 0` applies only to what it invokes. # -# Third per-stage declaration of the gate cache-buster; see the lint -# stage above for why one is not enough. It is placed after USER so the -# drop to the unprivileged user still happens before the checks run. +# Third per-stage declaration of the gate cache-buster and its check; +# see the lint stage above for why one is not enough. It is placed after +# USER so the drop to the unprivileged user still happens before the +# checks run. ARG CHECK_EPOCH +RUN if [ -z "${CHECK_EPOCH}" ]; then \ + echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ + exit 1; \ + fi RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test # The version stamped into the binary: the VERSION build argument when diff --git a/README.md b/README.md index 8e6c030..b4078a2 100644 --- a/README.md +++ b/README.md @@ -821,9 +821,12 @@ from binary-versus-pin to pin-versus-pin, which is what gate layers from cache and the build exits 0 having executed no tests and no lint — a green it never earned, and one this repository has produced twice. `CHECK_EPOCH` invalidates the gate layers on every run while leaving the pinned -base images and the dependency layers cached. `script/lint`'s value carries the -process id as well as the epoch, because two lint runs land inside the same -second easily and a bare epoch would cache the second one. +base images and the dependency layers cached. Each script's value carries the +process id as well as the epoch, because two runs land inside the same second +easily and a bare epoch would cache the second one. Each `Dockerfile` stage with +gates fails when the value is empty, so a bare `docker build .` stops with +`CHECK_EPOCH is unset; build via script/cibuild or script/docker` instead of +serving the gates from cache. ## Build diff --git a/TODO.md b/TODO.md index 5e65e74..a962fe4 100644 --- a/TODO.md +++ b/TODO.md @@ -28,6 +28,10 @@ # Completed Steps +- a bare `docker build .` fails with a message naming `script/cibuild` and + `script/docker` instead of serving the gates from cache (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/39) + - `make fmt` and `make fmt-check` run prettier over all Markdown, in Docker, and CI checks it; all Markdown reformatted (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/19) @@ -298,18 +302,19 @@ bug, not a fix. Verified by running each script twice back to back on an unchanged tree under `BUILDKIT_PROGRESS=plain`: all three gates executed on all four runs, each with a fresh epoch in the log (`script/cibuild` 78.8s then - 61.1s; `script/docker` 61.1s then 53.4s), and twelve steps were still served - `CACHED` in the steady state — both `go mod download`s, `apk add`, `adduser`, - the `chown`, every `go.mod`/`go.sum` and source copy, the linter copy out of - the lint stage, and the binary copy into the runtime stage. The lint stage - still gates the build stage: with a deliberate `unused` finding planted in the - tree, the build failed at `make lint` in 36.1s and the build-stage - `make check` never started. The build stage also still drops to the - unprivileged `builder` user before `make check`, which the suite depends on - rather than merely prefers: forcing the same image to run the tests as root - fails `TestScanHardlinkRunFailsTogether`, because root reads straight through - the `chmod(0)` the test uses to prove hard links are read once. This is the - local fix only; propagating it to the canonical templates is `prompts` #26 + 61.1s; `script/docker` 61.1s then 53.4s), and thirteen steps were still served + `CACHED` in the steady state — the lint stage's `WORKDIR /src`, both + `go mod download`s, `apk add`, `adduser`, the `chown`, every `go.mod`/`go.sum` + and source copy, the linter copy out of the lint stage, and the binary copy + into the runtime stage. The lint stage still gates the build stage: with a + deliberate `unused` finding planted in the tree, the build failed at + `make lint` in 36.1s and the build-stage `make check` never started. The build + stage also still drops to the unprivileged `builder` user before `make check`, + which the suite depends on rather than merely prefers: forcing the same image + to run the tests as root fails `TestScanHardlinkRunFailsTogether`, because + root reads straight through the `chmod(0)` the test uses to prove hard links + are read once. This is the local fix only; propagating it to the canonical + templates is `prompts` #26 - check the installed golangci-lint version in `script/bootstrap` instead of only its presence (2026-08-09, branch `bootstrap-version-check`, closes #24): `missing golangci-lint` meant any linter already on `PATH` satisfied the diff --git a/script/cibuild b/script/cibuild index 7283a88..f9a12b5 100755 --- a/script/cibuild +++ b/script/cibuild @@ -21,14 +21,17 @@ # serves the gate layers from cache and the build reports a green it # never earned. Passing the current epoch invalidates the gate # layers on every run while leaving the pinned base images and -# go mod download cached; see the Dockerfile for the placement. +# go mod download cached; see the Dockerfile for the placement. The +# process id goes in with the epoch so that two runs started in the +# same second still get different values, the same form script/lint +# uses. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - docker build --build-arg CHECK_EPOCH="$(date +%s)" . + docker build --build-arg CHECK_EPOCH="$(date +%s)-$$" . } main "$@" diff --git a/script/docker b/script/docker index 3c7f7bc..3245f26 100755 --- a/script/docker +++ b/script/docker @@ -3,7 +3,7 @@ # The tag comes from script/projectname. # # CHECK_EPOCH is passed for the same reason script/cibuild passes it: -# without it Docker serves the Dockerfile's gate layers from cache on an +# without a fresh value Docker serves the gate layers from cache on an # unchanged tree and this exits 0 having run none of the lint stage's # gates, the markdown stage's prettier gate or the builder stage's test # gate. This is the set of gates a developer or reviewer runs by hand, @@ -17,7 +17,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" docker build \ - --build-arg CHECK_EPOCH="$(date +%s)" \ + --build-arg CHECK_EPOCH="$(date +%s)-$$" \ -t "$("$SCRIPT_DIR/projectname")" \ . } From 546203afe5e5f61e811e1c21a4a1ba10b78c9689 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 20:01:26 +0200 Subject: [PATCH 26/33] Run the tests under the race detector with make test-race (closes #18) script/test-race runs go test -race in a digest-pinned Debian golang image that has gcc, since the detector needs cgo and the build keeps it off. The checkout is mounted read-only and the container is removed afterwards. The tests run as the calling user, or as nobody when that is root, so the tests that make a file unreadable still see the read fail. It is not part of make check. The detector found no races. Model: opus-5-5 --- Makefile | 5 ++++- README.md | 15 +++++++++++++-- TODO.md | 9 +++++++-- script/bootstrap | 15 ++++++++------- script/test-race | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) create mode 100755 script/test-race diff --git a/Makefile b/Makefile index 67c7260..5adfcec 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ BINARY := sfdupes VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -X main.Version=$(VERSION) -.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean +.PHONY: sfdupes build bootstrap setup test test-race lint fmt fmt-check check docker hooks clean # Standard targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern. @@ -27,6 +27,9 @@ setup: test: @script/test +test-race: + @script/test-race + lint: @script/lint diff --git a/README.md b/README.md index b4078a2..41afbc8 100644 --- a/README.md +++ b/README.md @@ -742,14 +742,23 @@ entrypoints are: installed: they run in Docker (see `script/lint` and `script/fmt`) and never from a host install, so there is no host copy to drift from the pin. A missing `docker` is warned about rather than installed or treated as fatal — - everything except linting and formatting works without it. Ends with - `go mod download`. + everything except linting, formatting and `make test-race` works without it. + Ends with `go mod download`. - `script/setup` — make a fresh clone ready for development: runs `script/bootstrap`, then `script/install-precommit`. - `script/projectname` — print this project's name (`sfdupes`). Scripts that need the name call it, so they stay identical across repositories. - `script/test` — run the test suite with a 30-second timeout and coverage enabled, rerunning verbosely on failure so the logs show which test failed. +- `script/test-race` — run the test suite under the race detector with a + 60-second timeout. The detector needs cgo and a C compiler, which the build + never uses, so the tests run in a digest-pinned Debian `golang` image that has + `gcc`, with the checkout mounted read-only; the container is removed when it + exits. They run as the calling user, or as `nobody` when that is root, because + several tests make a file unreadable and root reads it anyway; only then must + the checkout be readable by other users. Not part of `script/check`. Every run + starts with empty caches, so it needs the network and takes minutes, and the + mount needs a local docker daemon. - `script/lint` — run the linter. It builds `Dockerfile.lint`, which copies the repository into the digest-pinned `golangci/golangci-lint` image and runs `golangci-lint config verify` and `golangci-lint run` as build steps, so a @@ -840,6 +849,8 @@ compile recipe: - `make setup` — prepare a fresh clone: `bootstrap` plus the pre-commit hook. - `make test` — run the test suite (30-second timeout; reruns with `-v` on failure). +- `make test-race` — run the test suite under the race detector, in Docker (see + `script/test-race`); requires `docker`. Not part of `make check`. - `make lint` — run `golangci-lint` with the repo config, in Docker (see `script/lint`); requires `docker`. - `make fmt` / `make fmt-check` — format the Go sources and the Markdown / diff --git a/TODO.md b/TODO.md index a962fe4..6b7118a 100644 --- a/TODO.md +++ b/TODO.md @@ -28,6 +28,10 @@ # Completed Steps +- `make test-race` runs the test suite under the race detector in a cgo-enabled + container, outside `make check` (2026-10-04, + https://git.eeqj.de/sneak/sfdupes/issues/18) + - a bare `docker build .` fails with a message naming `script/cibuild` and `script/docker` instead of serving the gates from cache (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/39) @@ -497,5 +501,6 @@ Accepted divergences (no action): - flat single-package layout with `.go` files in the repo root — fine for a small single-binary tool per the Go styleguide; the tracker audit agrees -- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go - builds) and the race detector requires cgo +- `make test` runs without `-race` — the repo mandates `CGO_ENABLED=0` (pure-Go + builds) and the race detector requires cgo, so the detector runs in a separate + cgo-enabled container, `make test-race`, which is not part of `make check` diff --git a/script/bootstrap b/script/bootstrap index 380191d..224e8dd 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -7,8 +7,8 @@ # installed: golangci-lint (script/lint) and prettier (script/fmt, # script/fmt-check) run via docker only, pinned by hash, so their only # prerequisite is a working docker — which is warned about, not -# installed, because everything except linting and formatting works -# without it. +# installed, because everything except linting, formatting and +# make test-race works without it. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -75,13 +75,14 @@ main() { # Linting and Markdown formatting run via docker only, so docker is # their prerequisite rather than something bootstrap installs. Warn, - # do not fail: everything except `make lint`, `make fmt` and - # `make fmt-check` — and, through them, `make check`, `make docker` - # and the pre-commit hook — works without it. + # do not fail: everything except `make lint`, `make fmt`, + # `make fmt-check` and `make test-race` — and, through them, + # `make check`, `make docker` and the pre-commit hook — works + # without it. if missing docker; then echo "bootstrap: WARNING: docker not found; make lint, make fmt," >&2 - echo "bootstrap: make fmt-check, make check and make docker" >&2 - echo "bootstrap: require it." >&2 + echo "bootstrap: make fmt-check, make check, make docker and" >&2 + echo "bootstrap: make test-race require it." >&2 fi go mod download diff --git a/script/test-race b/script/test-race new file mode 100755 index 0000000..8e85862 --- /dev/null +++ b/script/test-race @@ -0,0 +1,40 @@ +#!/bin/sh +# script/test-race: run the test suite under the race detector. Not part +# of script/check. +# +# The race detector needs cgo and a C compiler, which the host build +# never uses, so the tests run in a golang image that has gcc. The +# checkout is mounted read-only, so the docker daemon must be local. The +# container starts with empty caches every time: each run downloads the +# dependencies and compiles them with the detector, which needs the +# network and takes minutes. +# +# The tests run as the calling user, never as root: several of them make +# a file unreadable and expect reading it to fail, and root reads it +# anyway. When the caller is root they run as nobody, and then the +# checkout must be readable by other users. Neither user has a home +# directory in the image, so HOME is /tmp, where Go puts its build cache. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# golang:1.25-trixie, 2026-10-04. Debian rather than the Alpine image the +# Dockerfile builds with, because this one includes gcc. +IMAGE="golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73" + +main() { + user="$(id -u):$(id -g)" + if [ "$(id -u)" -eq 0 ]; then + user=65534:65534 + fi + docker run --rm \ + --user "$user" \ + --env HOME=/tmp \ + --env CGO_ENABLED=1 \ + --volume "$ROOT:/src:ro" \ + --workdir /src \ + "$IMAGE" \ + go test -race -timeout 60s ./... +} + +main "$@" From 0064eba542dcc756ffce317d77b662b9ffa32c5b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 20:47:21 +0200 Subject: [PATCH 27/33] Cut the narration from TODO.md and the script and Dockerfile comments (closes #49) Completed Steps entries keep what landed, the traps, every disclosure and every record that a check ran; the argument and history go, with bare issue numbers turned into full links. Comment blocks in script/, Dockerfile and Dockerfile.lint keep the trap and drop the defence of past decisions. TODO.md Workflow now branches from next, targets next, and leaves merging next to main to the owner. Only comments and Markdown change. Model: opus-5-5 --- Dockerfile | 101 +++----- Dockerfile.lint | 56 ++--- TODO.md | 440 +++++++++++++---------------------- script/bootstrap | 30 +-- script/cibuild | 30 +-- script/docker | 13 +- script/lint | 26 +-- script/verify-lint-image-pin | 23 +- 8 files changed, 263 insertions(+), 456 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f2dbd8..b2d7424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,47 +6,36 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -# Cache-buster for the gate layers, and only for them. Docker -# invalidates COPY only when the copied content changes, so on an -# unchanged tree the gates below would be served from cache and the -# build would exit 0 having run nothing. script/cibuild and -# script/docker pass a fresh CHECK_EPOCH on every invocation. A build -# that passes none, such as a bare `docker build .`, fails at the check -# right after the ARG instead of quietly serving the gates from cache. +# Cache-buster for the gate layers, and only for them: on an unchanged +# tree Docker would serve the gates below from cache and the build would +# exit 0 having run nothing. script/cibuild and script/docker pass a +# fresh CHECK_EPOCH; a build without one, such as a bare +# `docker build .`, fails at the check right after the ARG. # -# Two properties this depends on. ARG is per-stage, so the markdown and -# build stages below declare it again; one declaration here would leave -# their gates cacheable. And each gate RUN must reference the value, -# because BuildKit hashes the expanded command: a declared but -# unreferenced ARG invalidates nothing. -# -# It sits below the dependency layers deliberately. Everything above it -# (the pinned base image, go mod download) keeps its cache; only the -# gates go cold. +# ARG is per-stage, so the markdown and build stages declare it again. +# Each gate RUN must reference the value: BuildKit hashes the expanded +# command, so a declared but unreferenced ARG invalidates nothing. Keep +# it below the dependency layers so they stay cached. ARG CHECK_EPOCH RUN if [ -z "${CHECK_EPOCH}" ]; then \ echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ exit 1; \ fi -# The linter is invoked directly here, not through `make lint`. That -# target now runs `docker build -f Dockerfile.lint`, and a docker build -# cannot run a docker build: routing the gate through make would mean -# nesting docker inside this image. Same reason `make check` is gone -# from the build stage below, and `make fmt-check` from both stages: it -# runs prettier through docker too. Its gofmt half is the step below, -# its Markdown half the markdown stage further down. gofmt's output is -# assigned to a variable first so that its own exit status, as when it -# cannot parse a file, still fails the step. +# These gates call the tools directly, not through `make lint` or +# `make fmt-check`: both run docker, which cannot run inside a docker +# build. This step is the gofmt half of `make fmt-check`; the markdown +# stage is its prettier half. gofmt's output is assigned to a variable +# first so that its own exit status, as when it cannot parse a file, +# still fails the step. RUN echo "gate gofmt, epoch ${CHECK_EPOCH}" && \ files="$(gofmt -s -l .)" && \ if [ -n "$files" ]; then \ echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \ fi -# The FROM above and the one in Dockerfile.lint pin the same linter -# twice, and nothing else keeps them in sync; this fails the build when -# they disagree. See the script for why it restates neither pin. +# Fails the build when the FROM above and the one in Dockerfile.lint pin +# different linter images. RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \ script/verify-lint-image-pin @@ -93,16 +82,13 @@ FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c # We never build or run as root. Create an unprivileged user and point # HOME and the build cache at its home so go build and go test can write -# it when we drop to it below. $GOPATH/bin is deliberately not on PATH: -# script/bootstrap no longer `go install`s anything (the linter runs -# from a pinned image, never from a host install), so nothing lands -# there and adding it would only widen what this image resolves. +# it when we drop to it below. # -# The module cache is kept outside that home, at the base image's -# default /go/pkg/mod, and belongs to root: script/bootstrap fills it as -# root. Do not move it into the home and hand it over with `chown -R`: -# that walks every file in it, which took from about 80 s to over ten -# minutes on a shared host, depending on load. +# The module cache stays at the base image's default /go/pkg/mod and +# belongs to root: script/bootstrap fills it as root. Do not move it +# into the home and hand it over with `chown -R`: that walks every file +# in it, which took from about 80 s to over ten minutes on a shared +# host, depending on load. RUN adduser -D -u 1000 builder ENV HOME=/home/builder ENV GOPATH=/home/builder/go @@ -111,24 +97,16 @@ ENV GOCACHE=/home/builder/.cache/go-build WORKDIR /src -# No-op file copies whose only purpose is the build-graph edge: they are -# what make this stage depend on the lint and markdown stages, and so -# what forces BuildKit to finish gofmt, the pin guard, lint and prettier -# before compilation and tests start. Remove one and the fail-fast -# design dies silently — the build stops gating on that stage and still -# exits 0. The first replaces a copy of the linter binary itself, which -# is no longer wanted here: nothing in this stage runs the linter, -# because `make lint` is now a docker build and a docker build cannot -# run inside one. +# No-op file copies whose only purpose is the build-graph edge: they +# make this stage depend on the lint and markdown stages, so BuildKit +# finishes those gates before compilation and tests start. Remove one +# and the build silently stops gating on that stage and still exits 0. COPY --from=lint /src/go.sum /dev/null COPY --from=markdown /src/go.sum /dev/null -# Install development prerequisites the same way a developer does, -# rather than duplicating the installs inline. Only script/ and the -# dependency manifests are copied first, nothing else, so this layer -# stays cached until the scripts or the dependencies change — bootstrap -# ends in `go mod download`, which is why there is no separate -# invocation of it here. +# Install development prerequisites the same way a developer does. Only +# script/ and the dependency manifests are copied first, so this layer +# stays cached until they change. Bootstrap ends in `go mod download`. COPY script/ script/ COPY go.mod go.sum ./ RUN script/bootstrap @@ -151,22 +129,17 @@ RUN chown builder:builder /src /go/pkg/mod/cache/download && \ COPY --chown=builder:builder . . USER builder -# Fail the build unless the branch is green. Runs as non-root so the -# permission-denied test paths are exercised legitimately (root would -# bypass the chmod(0) the tests rely on). +# Fail the build unless the branch is green. Runs as non-root: root +# would bypass the chmod(0) the permission-denied tests rely on. # -# The gate is `make test`, not `make check`: that aggregate runs -# `script/lint` and `script/fmt-check`, which both run docker, and -# nothing inside an image build may shell out to docker. Lint and the -# format checks are not skipped by this — they ran in the lint and -# markdown stages above, which this stage's COPY --from lines make -# prerequisites. `make`, not the script directly, because the Makefile's +# The gate is `make test`, not `make check`, which runs docker; lint and +# the format checks ran in the lint and markdown stages above. `make`, +# not the script directly, because the Makefile's # `export CGO_ENABLED = 0` applies only to what it invokes. # # Third per-stage declaration of the gate cache-buster and its check; -# see the lint stage above for why one is not enough. It is placed after -# USER so the drop to the unprivileged user still happens before the -# checks run. +# see the lint stage above. It is placed after USER so the drop to the +# unprivileged user still happens before the checks run. ARG CHECK_EPOCH RUN if [ -z "${CHECK_EPOCH}" ]; then \ echo "CHECK_EPOCH is unset; build via script/cibuild or script/docker" >&2; \ diff --git a/Dockerfile.lint b/Dockerfile.lint index aac1cb9..4d6707f 100644 --- a/Dockerfile.lint +++ b/Dockerfile.lint @@ -1,14 +1,12 @@ -# Lint-only image: this is how the linter runs, everywhere. The repo is -# COPYed into the pinned golangci-lint image and the linter runs as a -# build step, so a successful build IS a clean lint. golangci-lint is -# never installed on a host — one toolchain, pinned by digest, identical -# on a laptop and in CI — and this works even when the docker daemon is -# remote and bind mounts are impossible. +# Lint-only image, built by script/lint: the repo is copied into the +# pinned golangci-lint image and the linter runs as a build step, so a +# successful build is a clean lint. No bind mount, so it works when the +# docker daemon is remote. # -# script/lint builds this file. It is a separate image from the lint -# stage of the main Dockerfile because script/lint must not depend on -# the rest of that build; the two FROM lines are kept identical by -# script/verify-lint-image-pin, run as a gate below. +# It is separate from the main Dockerfile's lint stage because +# script/lint must not depend on the rest of that build; the two FROM +# lines are kept identical by script/verify-lint-image-pin, run as a +# gate below. # golangci/golangci-lint:v2.12.2, 2026-08-07 FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 @@ -20,38 +18,26 @@ RUN go mod download COPY . . -# Cache-buster for the gate layers, and only for them. Caching of the -# lint run is waived by ruling: COPY is invalidated only by changed -# content, so on an unchanged tree the gates below would be served from -# cache and this build would exit 0 in under a second having run no -# linter at all. That exact false green has bitten this repo twice -# already (#32, #39). script/lint passes a fresh value on every +# Cache-buster for the gate layers, and only for them; caching of the +# lint run is waived by ruling. On an unchanged tree the gates below +# would be served from cache and this build would exit 0 in under a +# second having run no linter. script/lint passes a fresh value on every # invocation. # -# Each gate RUN must reference the value, because BuildKit hashes the -# expanded command and not the ARG declaration: a declared but -# unreferenced ARG invalidates nothing. The ARG sits below the -# dependency layers deliberately — everything above it keeps its cache, -# only the gates go cold. +# Each gate RUN must reference the value: BuildKit hashes the expanded +# command, so a declared but unreferenced ARG invalidates nothing. Keep +# it below the dependency layers so they stay cached. ARG CHECK_EPOCH -# The linter version is pinned in two places, here and in the main -# Dockerfile's lint stage. Nothing else keeps them in sync, so a -# half-applied bump is a build failure; see the script. +# Fails the build when the FROM above and the main Dockerfile's lint +# stage pin different linter images. RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \ script/verify-lint-image-pin -# Validates .golangci.yml against golangci-lint's JSON schema. The -# concern about this step was that it fetches that schema over a live, -# unpinned HTTPS call; measured on the pinned image, it does not. The -# binary carries the schema for its own version, so under -# `--network none` this both passes on a valid config and still rejects -# an invalid one with the jsonschema error. That holds for the gate -# steps generally — none of them makes a network call — but not for -# this build as a whole: `go mod download` above needs the network on a -# cold cache, and under `--network none` a first build fails there -# before reaching any gate. That layer stays cached, so only a warm -# cache lints offline, until go.mod or go.sum changes. +# Validates .golangci.yml against golangci-lint's JSON schema, which the +# pinned binary embeds: measured under `--network none`, it passes a +# valid config and rejects an invalid one. No gate step makes a network +# call, but `go mod download` above needs the network on a cold cache. RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \ golangci-lint config verify --config .golangci.yml diff --git a/TODO.md b/TODO.md index 6b7118a..a538990 100644 --- a/TODO.md +++ b/TODO.md @@ -2,14 +2,15 @@ - take an issue from the `1.0.0` milestone on the tracker; work not yet on the tracker gets filed as an issue first -- branch (from `main`) +- branch from `next` - do the work, with tests, in small focused commits - record it at the top of Completed Steps (`TODO.md` changes in the same commit as the work) -- push the branch and open a PR whose title ends with ` (closes #N)` -- an independent review gates the merge; every finding is addressed or - explicitly rebutted on the PR -- merge to `main` once the review passes +- push the branch and open a PR against `next` whose title ends with + ` (closes #N)` +- an independent review gates each merge to `next`; every finding is addressed + or explicitly rebutted on the PR +- only the owner merges `next` to `main` # Status @@ -28,6 +29,10 @@ # Completed Steps +- cut the narration from `TODO.md` Completed Steps and from the comments in + `script/` and both Dockerfiles; §Workflow now branches from and merges to + `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49) + - `make test-race` runs the test suite under the race detector in a cgo-enabled container, outside `make check` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/18) @@ -121,29 +126,24 @@ - stamp the git tag or short commit in a plain `docker build .` instead of `dev` (2026-10-02, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now sends - `.git`, without `.git/config`, and the `Dockerfile` build stage takes the - `VERSION` build argument when one is given, otherwise - `git describe --tags --always` of that `.git`. The build fails if the context - carries `.git` and the version still comes out empty, `dev` or `unknown`. The - CI checkout step fetches the full history (`fetch-depth: 0`) so CI sees the - tag and stamps the same value as `make build`. + https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git` + without `.git/config`; the build stage stamps the `VERSION` build argument, + else `git describe --tags --always`, and fails if the context carries `.git` + and the version is still empty, `dev` or `unknown`. CI checks out the full + history (`fetch-depth: 0`) so it stamps the same value as `make build`. - replace the 1 KiB end-window sampling with the head/tail plus content-hash ladder (2026-09-22, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is hashed in - full and compared directly, with no end-window step — its `head`, `tail`, and - `content` all hold the whole-file hash. A file at 10 MiB or above gets only - the 64 KiB `head` and `tail` in the hash phase; a new content phase, after the - update phase, reads it for its `content` hash — the whole file below 50 MiB, - gigabyte-spaced 1 MiB samples at or above — only when its size, `head`, and - `tail` match another record's, from the same scan or stored by an earlier one, - so a stored file gains its content hash when it gains a match. A file that is - gone or has changed since its record was written is not read. The `content` - column is part of the version 1 schema. `report` and `trees` group by the - extended signature and leave out any record without a `content` hash, so the - ladder is applied across the whole database. README "Duplicate detection" - documents every rung including the probabilistic large-file path. + https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection" + documents every rung. A file under 10 MiB is hashed in full, and its `head`, + `tail` and `content` all hold that hash. A larger file gets only its 64 KiB + `head` and `tail` in the hash phase; the content phase, after the update + phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1 + MiB samples at or above) only when its size, `head` and `tail` match another + record's from this scan or an earlier one, and never reads a file gone or + changed since its record was written. `report` and `trees` leave out any + record without a `content` hash. The `content` column is part of the version 1 + schema. - remove the dead `files.dat` references from `Makefile`, `.gitignore` and `.dockerignore` (2026-09-21, branch `next`, closes @@ -151,265 +151,162 @@ - fix the lint-image pin comments and `FROM` form in `Dockerfile` and `Dockerfile.lint` (2026-08-10, branch `next`, closes - https://git.eeqj.de/sneak/sfdupes/issues/25): dropped the false - `(Debian-based)` parenthetical (v2.12.1 was Debian too) and the redundant tag, - so both pins are the policy `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare - `FROM image@sha256:...`. Digest unchanged. `script/verify-lint-image-pin` - parses those `FROM` lines and still matches the tagless form; its advice line - lost the now meaningless "tag and digest". With no tag in either reference, a - tag-only disagreement no longer exists — a one-sided tag is caught as a plain - mismatch. + https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy + `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`, + without the false `(Debian-based)` note or the tag; digest unchanged. + `script/verify-lint-image-pin` still matches the tagless form, and a tag on + one side only is caught as a plain mismatch. - run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the - owner ruling, the linter runs inside a container invoked through the `script/` - entrypoint and is never installed on a host. New root `Dockerfile.lint` COPYs + owner ruling the linter is never installed on a host. `Dockerfile.lint` copies the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and - runs `golangci-lint config verify` and `golangci-lint run` as build steps, so - a successful build IS a clean lint; `script/lint` is reduced to building it. - `script/bootstrap` loses the `go install`, the pin constants, the version - parser and `verify_golangci_lint` outright rather than hardening them — with - nothing linting on the host, the `$GOPATH/bin` versus `PATH` problem that - motivated them has no subject — and now warns rather than fails when `docker` - is absent. Two traps handled. A lint build on an unchanged tree returns - success in well under a second having run no linter, which is - https://git.eeqj.de/sneak/sfdupes/issues/32 and - https://git.eeqj.de/sneak/sfdupes/issues/39 again, so `Dockerfile.lint` - carries `ARG CHECK_EPOCH` referenced inside every gate `RUN` (BuildKit hashes - the expanded command, not the declaration) and `script/lint` passes - `"$(date +%s)-$$"` — the PID matters because two lint runs land inside the - same second easily. And nothing inside an image build may shell out to docker, - so the main `Dockerfile`'s lint stage now invokes `golangci-lint` directly - instead of `make lint`, and its build stage runs `make test` and - `make fmt-check` instead of the `make check` aggregate (`make`, not the - scripts bare, because the Makefile's `export CGO_ENABLED = 0` only reaches - what it invokes). `COPY --from=lint` `/usr/bin/golangci-lint` is replaced by - `COPY --from=lint /src/go.sum /dev/null`: the copied binary was the only edge - forcing BuildKit to finish linting before the build stage starts, and dropping - it without replacing the edge would have ended fail-fast linting silently - under a still-green build. That is canonical `REPO_POLICIES.md:107`'s ordering - edge, restored. `ENV PATH=/home/builder/go/bin:$PATH` is gone with the - `go install` that justified it. `script/verify-linter-pin` is retired, deleted - along with its README entry, because both of its subjects ceased to exist in - the same change: it compared a linter binary against `GOLANGCI_LINT_VERSION` - in `script/bootstrap`, and there is now neither a binary crossing between - stages nor a version pin in bootstrap. The drift it guarded has not gone away, - it has moved — the linter is still pinned twice, now as the `FROM` line of - `Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage, with - nothing syncing them, which is exactly what - https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its - replacement is one new `script/verify-lint-image-pin`, run as a gate in both - files, which compares the two references to each other and deliberately - restates neither: a hardcoded expected digest would be a third copy and the - same drift one file further out. `golangci-lint config verify` is included per - the ruling, and the concern about its unpinned live HTTPS schema fetch was - measured rather than assumed — under `--network none` the pinned binary both - passes a valid config and rejects an invalid one with the jsonschema error, so - it validates from an embedded schema and makes no network call of its own. The - README scopes that to the gate steps rather than to linting as a whole: - `Dockerfile.lint` runs `go mod download` above them, so a cold cache still - needs the network and only a warm one lints offline. Verified: `make lint` - green with every `PATH` directory containing a `golangci-lint` removed - (`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`; - `command -v golangci-lint` empty); two consecutive `script/lint` runs on an - untouched tree both executed the linter, 27.7s and 28.7s in the lint step - under distinct epochs with the `COPY . .` layer `CACHED` above them, at 42.2s - and 41.8s wall clock — the no-cache rule was not weakened to shorten that. - Negative control: a planted `var unusedIssue46Sentinel = 1` failed - `script/lint` with - `report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and failed - `make docker` at `[lint 9/9]` with the build stage stopped at `[builder 3/12]` - — `COPY --from=lint`, `script/bootstrap`, the test gate and `make build` all - zero occurrences — then reverted clean. The drift guard fails on a tag-only - disagreement, on a digest-only disagreement, and on an unreadable reference, - naming both sides. `make docker` green in 5m35s with all six gates executing - under one epoch (lint 37.6s, test 25.2s reporting - `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`). The - non-root quirk still holds: in the builder image with the Go test cache off, - `--user 0:0` fails `TestScanHardlinkRunFailsTogether` (exit 1) where the - unprivileged user passes (exit 0). Noted for follow-up, not fixed here: - `golangci-lint` warns that the `gomodguard` linter is deprecated since v2.12.0 - in favour of `gomodguard_v2`. + runs `golangci-lint config verify` and `golangci-lint run` as build steps; + `script/lint` builds it. `script/bootstrap` no longer installs or pins the + linter, and warns rather than fails when `docker` is absent; + `ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`. + `script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate + in both files, compares their two `FROM` lines and restates neither pin. + Traps: an unchanged tree lets a lint build pass in under a second having run + no linter, so every gate `RUN` references `ARG CHECK_EPOCH` (BuildKit hashes + the expanded command) and `script/lint` passes `"$(date +%s)-$$"`, the PID + because two runs land in the same second easily. Nothing inside an image build + may shell out to docker, so the `Dockerfile` lint stage calls `golangci-lint` + directly and the build stage runs `make test` and `make fmt-check` instead of + `make check`, through `make` because the Makefile's `export CGO_ENABLED = 0` + only reaches what it invokes. `COPY --from=lint /src/go.sum /dev/null` + replaces the copied linter binary as the only edge making the build stage wait + for lint; dropping it would end fail-fast linting under a still-green build. + `golangci-lint config verify`, included per the ruling, validates from an + embedded schema with no network call, but `go mod download` above the gates + still needs the network on a cold cache. Verified: `make lint` green with no + `golangci-lint` on `PATH`; two back-to-back `script/lint` runs on an untouched + tree both ran the linter (27.7s and 28.7s in the lint step, `COPY . .` + `CACHED` above); a planted unused variable failed `script/lint`, and failed + `make docker` at `[lint 9/9]` with the build stage stopped at + `[builder 3/12]`; the drift guard fails on a tag-only, a digest-only and an + unreadable reference, naming both sides; under `--network none` config verify + passes a valid config and rejects an invalid one; `make docker` green in 5m35s + with all six gates run under one epoch (lint 37.6s, test 25.2s reporting + `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`); in the + builder image with the Go test cache off, `--user 0:0` still fails + `TestScanHardlinkRunFailsTogether` where the unprivileged user passes. Noted + for follow-up, not fixed here: `golangci-lint` warns that `gomodguard` is + deprecated since v2.12.0 in favour of `gomodguard_v2`. - install the Docker build stage's prerequisites by running `script/bootstrap` instead of `apk add --no-cache make` inline (2026-08-09, branch - `dockerfile-bootstrap`, closes #42): canonical `REPO_POLICIES.md:97` requires - it, and the inline install left the build stage maintaining its own notion of - the toolchain — exactly the divergence #24 exists to close, one layer down. - The stage now copies `script/` plus `go.mod`/`go.sum` and runs - `script/bootstrap`, which ends in `go mod download`, so the separate - invocation of that is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, - and moves above the bootstrap layer. It is the only edge making this stage - depend on the lint stage, so deleting it as redundant would end fail-fast - linting silently. Letting bootstrap install its own linter here would have - reintroduced the second toolchain and paid for a from-source build of it. What - makes the two stages provably one toolchain rather than two that happen to - agree is a new `script/verify-linter-pin`, run in the build stage on the - binary that arrives from the lint stage, before bootstrap: it fails the build - naming both versions unless that binary is the version `script/bootstrap` - pins. Bootstrap's own check could not serve that purpose — it reinstalls its - pin from source and then verifies whatever `PATH` resolves, so drift - self-heals silently and a lint stage image bumped on its own would lint at the - new version while `make check` ran at the old one, green. The linter version - is pinned in two independent places (the lint stage image digest and - `GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a - half-applied bump is now a build failure. The pin is read out of - `script/bootstrap`, which stays the single source of truth; a pin that cannot - be read is a hard failure, not a skip. The check needs no `CHECK_EPOCH`: its - only inputs are the copied binary and `script/`, so Docker invalidates the - layer exactly when a cached result would stop being true, and it is documented - with the other entrypoints in the README. `$GOPATH/bin` joins `PATH` because - that is where bootstrap's `go install` lands and bootstrap verifies its - installs against what `PATH` resolves — nothing in the image is shadowed by - it, the directory does not exist until bootstrap runs. Everything added sits - above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still precede - `make check`. Verified: the guard fails the build with both versions named - when the lint stage's linter is faked to a different version, and an - unmodified build still passes it; bootstrap runs clean under Alpine's `sh` and - its `apk` branch, installing `git` and `make` and finding the copied linter - already at the pin; a second build served the bootstrap and dependency layers - `CACHED` while both gates ran with a fresh epoch; a planted `unused` finding - failed the build at the lint gate in 48.9s with the build stage's `make check` - never starting; and the suite run in the image as `--user 0:0` fails - `TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged user is - still load-bearing. That last check needs the Go test cache disabled — the - first attempt reported `ok ... (cached)` as root, reusing the result the - build-time run had left in the shared cache, which would have read as a pass. - Build wall time, on a shared host running many concurrent builds and so noisy: - 2m13s on an unchanged tree, 2m17s and 4m29s for two builds after a source - change, 5m14s cold. Only the cold one breaches the policy ceiling, and not - because of this change — `chown -R builder:builder /src /home/builder` walks - the module cache and re-runs on every source change, and it alone varied - between 77s and 210s across those four builds, which is also the whole spread - in the totals. The same cold measurement against `main` is 5m03s with a 209s - `chown`. Filed as #43 + `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42): + the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`, + which ends in `go mod download`, so the separate call to it is gone. + `COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap + layer: it is the only edge making this stage depend on the lint stage, so + deleting it would end fail-fast linting silently. A new + `script/verify-linter-pin`, run in the build stage before bootstrap, fails the + build naming both versions unless that copied binary is the version + `script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip. + `$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. Everything + added sits above `ARG CHECK_EPOCH`, and the `chown` and `USER builder` still + precede `make check`. Verified: the guard fails the build with both versions + named when the lint stage's linter is faked to another version, and passes an + unmodified build; bootstrap runs clean under Alpine's `sh` and `apk`, finding + the copied linter already at the pin; a second build served the bootstrap and + dependency layers `CACHED` while both gates ran with a fresh epoch; a planted + `unused` finding failed the build at the lint gate in 48.9s with the build + stage's `make check` never starting; and the suite run in the image as + `--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the drop to the + unprivileged user is still needed. That last check needs the Go test cache + off: as root it first reported `ok ... (cached)`, reusing the build-time + result. Build times on a noisy shared host: 2m13s on an unchanged tree, 2m17s + and 4m29s after a source change, 5m14s cold, which breaches the policy + ceiling; `chown -R builder:builder /src /home/builder` walks the module cache + and alone varied from 77s to 210s across those builds, and `main` measured + 5m03s cold with a 209s `chown`. Filed as + https://git.eeqj.de/sneak/sfdupes/issues/43 - bust the Docker layer cache for the gate steps, so `script/cibuild` and `script/docker` cannot report a green they did not earn (2026-08-09, branch - `cibuild-cache-bust`, closes #32): both scripts were bare `docker build` - invocations with no cache control, and the `Dockerfile` copies the tree before - running its gates, so on an unchanged tree Docker served those layers from - cache and the build exited 0 having executed nothing. That is not hypothetical - here — every merge this repo has done is a non-fast-forward merge of an - undiverged branch, so each merge commit's tree is byte-identical to the branch - head's and each merge CI run was almost certainly a full cache hit; and PR - #31's reviewer found `make docker` returning success as a 17-layer cache hit, - catching it only by being suspicious. The fix is `ARG CHECK_EPOCH` with the - scripts passing `--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or - break it. `ARG` is scoped per stage and this `Dockerfile` has three gates - across two — `make fmt-check` and `make lint` in the lint stage, `make check` - in the build stage — so a single declaration would have left one stage - silently cacheable; it is declared in both. And BuildKit hashes the expanded - command, not the declaration, so a declared-but-unreferenced `ARG` invalidates - nothing: each gate `RUN` echoes the epoch, which also puts the value in the - build log as evidence the layer really ran. Placement is below the dependency - layers on purpose — a build that goes cold every time would be a different - bug, not a fix. Verified by running each script twice back to back on an - unchanged tree under `BUILDKIT_PROGRESS=plain`: all three gates executed on - all four runs, each with a fresh epoch in the log (`script/cibuild` 78.8s then - 61.1s; `script/docker` 61.1s then 53.4s), and thirteen steps were still served - `CACHED` in the steady state — the lint stage's `WORKDIR /src`, both - `go mod download`s, `apk add`, `adduser`, the `chown`, every `go.mod`/`go.sum` - and source copy, the linter copy out of the lint stage, and the binary copy - into the runtime stage. The lint stage still gates the build stage: with a - deliberate `unused` finding planted in the tree, the build failed at - `make lint` in 36.1s and the build-stage `make check` never started. The build - stage also still drops to the unprivileged `builder` user before `make check`, - which the suite depends on rather than merely prefers: forcing the same image - to run the tests as root fails `TestScanHardlinkRunFailsTogether`, because - root reads straight through the `chmod(0)` the test uses to prove hard links - are read once. This is the local fix only; propagating it to the canonical - templates is `prompts` #26 + `cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the + `Dockerfile` copies the tree before its gates, so on an unchanged tree Docker + served them from cache and the build exited 0 having run nothing. Both scripts + now pass `--build-arg CHECK_EPOCH="$(date +%s)"`. `ARG` is per stage and the + gates span two stages, so it is declared in both; BuildKit hashes the expanded + command, so each gate `RUN` echoes the epoch, which also logs it as evidence + the layer ran. It sits below the dependency layers so they stay cached. + Verified under `BUILDKIT_PROGRESS=plain`, each script run twice back to back + on an unchanged tree: all three gates ran on all four runs with a fresh epoch + (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s then 53.4s), and + thirteen steps were still served `CACHED`. With a planted `unused` finding the + build failed at `make lint` in 36.1s and the build-stage `make check` never + started. Run as root, the same image fails `TestScanHardlinkRunFailsTogether`, + because root reads through the `chmod(0)` the test relies on, so the build + stage must drop to the unprivileged `builder` user. Local fix only; + propagating it to the canonical templates is + https://git.eeqj.de/sneak/prompts/issues/26 - check the installed golangci-lint version in `script/bootstrap` instead of - only its presence (2026-08-09, branch `bootstrap-version-check`, closes #24): - `missing golangci-lint` meant any linter already on `PATH` satisfied the - check, so the pin was never consulted and the v2.12.2 bump from #3 was inert - on every host that already had one — this host ran v2.10.1 against a v2.12.2 - pin, `make check` went green, and `make docker` then rejected the same commit - with findings the local gate never saw. The version now lives in one place, - `GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it so a - bump cannot half-apply; a `golangci_lint_version` helper parses - `golangci-lint --version` (taking the field after the word `version` and - tolerating an optional leading `v`, which the module ref carries and the - binary's output does not), and any version that is not the pin — older, newer, - absent or unparseable — is reinstalled. The install is then verified against - the binary `PATH` actually resolves: `go install` writes into `GOBIN` (or - `GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes first on - `PATH`, so a wrong-version one sitting ahead of it — nix, apt, brew, apk, or - the `/usr/local/bin` copy the `Dockerfile` builder stage makes — would swallow - the install and leave the local gate disagreeing with CI under an affirmative - `bootstrap complete`. Bootstrap now re-reads the effective version after - installing and, on a mismatch, prints both paths and both versions to stderr - and exits non-zero instead of claiming success; it does not reorder anyone's - `PATH` or delete their binary. The `--version` call keeps its stderr - connected, so a present-but-broken binary says why rather than reinstalling - forever in silence, and is bounded by `timeout(1)` where that exists, so a - wedged binary cannot hang bootstrap. `git`, `make` and `go` keep their - presence-only checks and now say why in a comment: they are host - package-manager tools the repo deliberately does not pin, with `go.mod` - governing the language version and the digest-pinned images covering - reproducible builds. Verified on this host by bootstrapping from v2.10.1 to - v2.12.2 and running it again to a no-op, plus stub runs of the real script - under `dash` covering a thirteen-input parse matrix (absent, older, newer, - host-style, image-style, leading-`v`, stderr-only, empty, non-zero exit, - impostor binary, `(devel)`, trailing `version`), a shadowed install that must - exit non-zero, an install destination not on `PATH` at all, `GOBIN` set, and a - wedged binary that must hit the timeout; `make check` and `make lint` are - clean at v2.12.2, so v2.10.1 was not hiding any findings on `main` + only its presence (2026-08-09, branch `bootstrap-version-check`, closes + https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in + `GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and + any installed version that is not the pin — older, newer, absent or + unparseable — is reinstalled. `go install` writes into `GOBIN` (or + `GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so + bootstrap re-reads the effective version after installing and, on a mismatch, + prints both paths and both versions and exits non-zero; it does not reorder + `PATH` or delete anyone's binary. The `--version` call keeps its stderr and is + bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep + presence-only checks. Verified by bootstrapping this host from v2.10.1 to + v2.12.2 and again to a no-op, and by stub runs of the script under `dash` + covering a thirteen-input version-parse matrix, a shadowed install that must + exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged + binary that must hit the timeout; `make check` and `make lint` are clean at + v2.12.2, so v2.10.1 was not hiding any findings on `main` - unwind the hash worker pool on the error path (2026-08-09, branch - `hash-pool-cleanup`, closes #6): `hashPhase` used to return the moment - `recordRun` failed and abandon the pool — the feeder parked forever on a full - `jobs` channel and every worker on a full `results` channel. That only stopped - being invisible when #4 landed and `runScan` began unwinding instead of - calling `os.Exit`. The pool is now an owned, context-aware `hashPool`: every - blocking send in the feeder and the workers selects on `ctx.Done()`, `jobs` is - closed on every path out, and `hashPhase` defers `pool.stop()`, which cancels - and then drains `results` until the last goroutine has exited — draining is - what frees a worker already parked on a send. `ctx` is threaded from - `cmd.Context()` through `runScan`, `syncScan`, both worker pools and the whole - database layer (it is the first parameter everywhere), so #5 can hand this - path a signal and needs to add nothing else. The walk pool never leaked, - because `walkPhase` always drains its events to close, but it has the same - unbounded-send shape and #5 will give it an early return, so it gets the same - treatment plus a `ctx.Err()` guard after the walk: a cancelled walk yields a - partial size census, and every file it never reached looks vanished to the - update phase. That phase's own `BeginTx` fails on the same cancelled context - before deleting anything, so the guard is defence in depth rather than the - only barrier — but it is the one that survives #5 deciding an interrupted scan - may commit what it has. Tests drive `run(scan)` against a database whose - insert trigger aborts, and assert both that the scan fails instead of hanging - and that `runtime.NumGoroutine()` polls back to its pre-scan baseline; a - second set cancels a scan part-way through the walk — deterministically, by - counting the scan's own consultations of `ctx.Done()` rather than racing a - timer — and asserts that it stops at the guard holding a partial census and a - still-populated record index, with every record intact. The remaining - cancellation branches of both pools are covered by direct tests of - `sendEvent`, the walk workers, `dispatchDirs`, `feedHashJobs`, `hashWorker` - and `hashPhase` + `hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the + pool is now an owned, context-aware `hashPool`: every blocking send in the + feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path + out, and `hashPhase` defers `pool.stop()`, which cancels and then drains + `results` until the last goroutine has exited — draining is what frees a + worker already parked on a send. `ctx` is threaded from `cmd.Context()` + through `runScan`, `syncScan`, both worker pools and the whole database layer, + as the first parameter everywhere. The walk pool gets the same treatment plus + a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size + census, and every file it never reached looks vanished to the update phase. + That phase's own `BeginTx` also fails on the cancelled context before deleting + anything, but the guard is the barrier that still holds once an interrupted + scan may commit what it has. Tests drive `run(scan)` against a database whose + insert trigger aborts and assert that the scan fails instead of hanging and + that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others + cancel a scan part-way through the walk, deterministically, by counting its + own consultations of `ctx.Done()`, and assert that it stops at the guard + holding a partial census and a still-populated record index, with every record + intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`, + `feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation + branches of both pools - guarantee the database is closed on every fatal exit path (2026-08-09, branch - `db-close-on-fatal`, closes #4): `fatalf` and its `os.Exit(1)` are gone, so - the deferred `db.Close()` — and with it the SQLite WAL checkpoint — now - actually runs when a subcommand fails; `runScan`, `runReport`, `runTrees`, - `loadRecords` and `resolveRoots` return errors instead. The single exit point - is `run` in `main.go`: it maps a `fatalError` (anything a subcommand returned) - to exit 1 and cobra's own argument and flag errors to exit 2, which keeps a - runtime failure from being reported as a usage error or printing the usage - text. New `main_test.go` drives the CLI in-process and asserts the exit codes - from README §Error handling plus the stdout/stderr split, including that a - fatal error raised after the database is open leaves no `-wal`/`-shm` sidecar - behind for `scan`, `report` or `trees` + `db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4): + `fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and + with it the SQLite WAL checkpoint — now actually runs when a subcommand fails; + `runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return + errors instead. The single exit point is `run` in `main.go`: it maps a + `fatalError` (anything a subcommand returned) to exit 1 and cobra's own + argument and flag errors to exit 2, which keeps a runtime failure from being + reported as a usage error or printing the usage text. New `main_test.go` + drives the CLI in-process and asserts the exit codes from README §Error + handling plus the stdout/stderr split, including that a fatal error raised + after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`, + `report` or `trees` - update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch - `golangci-v2.12.2`, merged as `38a01bd`, closes #3): bumped the pinned linter - in the `Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, - and replaced `.golangci.yml` with the canonical file — the linter settings - (`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under - `linters.settings` per the v2 schema, so they are actually applied; no new - lint findings surfaced + `golangci-v2.12.2`, merged as `38a01bd`, closes + https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the + `Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and + replaced `.golangci.yml` with the canonical file — the linter settings (`lll`, + `funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per + the v2 schema, so they are actually applied; no new lint findings surfaced - convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints - like the other managed repos (2026-07-26, commit `3abeacf`, closes #1): all 12 - `script/` entrypoints exist (`bootstrap`, `setup`, `projectname`, `test`, - `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, - `install-precommit`) and every Makefile target is now a thin shim over them, - matching the other managed repos + like the other managed repos (2026-07-26, commit `3abeacf`, closes + https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints + exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, + `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`) + and every Makefile target is now a thin shim over them - make the binary the default Make target (2026-07-24, branch `make-default-target`): plain `make` now builds `sfdupes` (previously it ran `check` plus `build`); `make build` remains as an alias @@ -428,8 +325,7 @@ - announce each operand on stderr before its passes (2026-07-24, branch `scan-operand-progress`): with per-operand walk/hash/update cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the - whole run's — an operator watching operand 3 of 14 hash 300k files concluded - 20M files were being skipped + whole run's - parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single goroutine and took hours at ~20M files on a busy pool (observed: 22M files in 4h on a ZFS server); it is now a per-directory worker-pool traversal that diff --git a/script/bootstrap b/script/bootstrap index 224e8dd..3e4c106 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -1,14 +1,9 @@ #!/bin/sh # script/bootstrap: install all dependencies needed to build and develop -# this repo. Idempotent: every install is guarded by a check so already -# installed tools are skipped. Base tooling comes from nix, apt, brew, -# or apk (detected in that order); assumes nothing is present (not git, -# make, or go). Neither the linter nor the Markdown formatter is -# installed: golangci-lint (script/lint) and prettier (script/fmt, -# script/fmt-check) run via docker only, pinned by hash, so their only -# prerequisite is a working docker — which is warned about, not -# installed, because everything except linting, formatting and -# make test-race works without it. +# this repo. Idempotent; assumes nothing is present (not git, make, or +# go). Base tooling comes from nix, apt, brew, or apk (detected in that +# order). golangci-lint and prettier are never installed: they run via +# docker only (script/lint, script/fmt, script/fmt-check). set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -63,22 +58,15 @@ missing() { main() { cd "$ROOT" - # System tooling, deliberately unpinned: these come from the host - # package manager and whatever version it ships is what the host - # gets, so a presence check is the right check. The repo pins no - # system toolchain versions — the Go language version is governed by - # go.mod, and builds that must be reproducible run in the Docker - # image, whose base images are pinned by digest. + # Deliberately unpinned, so presence is the whole check: go.mod + # governs the Go version, and reproducible builds run in the + # digest-pinned Docker images. if missing git; then pkg_install git git git git; fi if missing make; then pkg_install gnumake make make make; fi if missing go; then pkg_install go golang go go; fi - # Linting and Markdown formatting run via docker only, so docker is - # their prerequisite rather than something bootstrap installs. Warn, - # do not fail: everything except `make lint`, `make fmt`, - # `make fmt-check` and `make test-race` — and, through them, - # `make check`, `make docker` and the pre-commit hook — works - # without it. + # Warn, do not fail: only the targets named below, and the + # pre-commit hook, need docker. if missing docker; then echo "bootstrap: WARNING: docker not found; make lint, make fmt," >&2 echo "bootstrap: make fmt-check, make check, make docker and" >&2 diff --git a/script/cibuild b/script/cibuild index f9a12b5..5eca17f 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,30 +1,12 @@ #!/bin/sh # script/cibuild: run the CI build. The Gitea workflow runs this on -# push. +# push. The Dockerfile runs every gate make check runs, as build steps, +# so a successful build means the repo is green. # -# The Dockerfile runs the gates individually as build steps, not the -# make check aggregate: the lint stage runs the gofmt check, -# script/verify-lint-image-pin, golangci-lint config verify and -# golangci-lint run; the markdown stage runs the prettier check; the -# build stage, dropped to an unprivileged user, runs make test. None of -# make lint, make fmt-check or make check appears, because each runs -# docker, and docker cannot run inside a docker build. Nothing is -# skipped by that — the linter, gofmt and prettier are invoked directly -# in their stages, and the build stage's COPY --from lines make those -# stages prerequisites, so BuildKit must finish them first. Between the -# three stages everything make check would run has run, which is why a -# successful build here implies the repo is green. -# -# That implication holds only because of CHECK_EPOCH. A COPY layer is -# invalidated only by changed content, and a rebuild of an unchanged -# checkout sends the same content, so without a fresh value here Docker -# serves the gate layers from cache and the build reports a green it -# never earned. Passing the current epoch invalidates the gate -# layers on every run while leaving the pinned base images and -# go mod download cached; see the Dockerfile for the placement. The -# process id goes in with the epoch so that two runs started in the -# same second still get different values, the same form script/lint -# uses. +# Without a fresh CHECK_EPOCH, a rebuild of an unchanged checkout serves +# the gate layers from cache and passes having run none of them. The +# process id goes in with the epoch so two runs started in the same +# second still differ. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/docker b/script/docker index 3245f26..7ebc548 100755 --- a/script/docker +++ b/script/docker @@ -1,14 +1,9 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. -# The tag comes from script/projectname. -# -# CHECK_EPOCH is passed for the same reason script/cibuild passes it: -# without a fresh value Docker serves the gate layers from cache on an -# unchanged tree and this exits 0 having run none of the lint stage's -# gates, the markdown stage's prettier gate or the builder stage's test -# gate. This is the set of gates a developer or reviewer runs by hand, -# so a cached pass here is the most misleading result the repo can -# produce. Dependency layers sit above the ARG and stay cached. +# The tag comes from script/projectname. CHECK_EPOCH is passed for the +# same reason script/cibuild passes it: without a fresh value an +# unchanged tree is served from cache and this exits 0 having run no +# gate. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/lint b/script/lint index 20b021a..d942a1c 100755 --- a/script/lint +++ b/script/lint @@ -1,23 +1,17 @@ #!/bin/sh # script/lint: run the linter. golangci-lint is never installed on a -# host: it runs via docker only, one way, everywhere — this builds -# Dockerfile.lint, which COPYs the repo into the digest-pinned -# golangci-lint image and lints as a build step, so a successful build -# is a clean lint. The only prerequisite is a working docker. The gate -# steps make no network calls of their own, but Dockerfile.lint runs -# `go mod download` above them, so a cold cache does reach the network -# (as does pulling the pinned image); that layer stays cached, and once -# it is warm this runs offline until go.mod or go.sum changes. +# host: this builds Dockerfile.lint, which copies the repo into the +# digest-pinned golangci-lint image and lints as a build step, so a +# successful build is a clean lint. A cold cache needs the network to +# pull the image and for `go mod download`; once warm this runs offline +# until go.mod or go.sum changes. # -# CHECK_EPOCH is what makes the result mean anything. Without it docker -# serves the gate layers from cache on an unchanged tree and this exits -# 0 in well under a second having run no linter. The PID is in the value -# as well as the epoch because two lint runs land inside the same second -# easily, and `date +%s` alone would cache the second one. +# Without a fresh CHECK_EPOCH docker serves the gate layers from cache +# on an unchanged tree and this exits 0 having run no linter. The PID is +# in the value because two lint runs land inside the same second easily. # -# The result is the build's exit status and the image is never used, so -# --output=type=cacheonly writes none. Without it every run spends -# seconds exporting an image and leaves it behind untagged. +# The image is never used, so --output=type=cacheonly writes none; +# without it every run leaves an untagged image behind. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/verify-lint-image-pin b/script/verify-lint-image-pin index fa30722..1a9fb69 100755 --- a/script/verify-lint-image-pin +++ b/script/verify-lint-image-pin @@ -2,23 +2,16 @@ # script/verify-lint-image-pin: fail unless the golangci-lint image # referenced by Dockerfile.lint and the one referenced by the main # Dockerfile's lint stage are the same image at the same digest. Our own -# extension to scripts-to-rule-them-all, not one of its entrypoints. +# extension to scripts-to-rule-them-all, not one of its entrypoints; run +# as a gate in both files. Nothing else keeps the two pins in sync, and +# a bump applied to one alone would lint the same tree against different +# rulesets, both green. # -# The linter version is pinned in two independent files. That is the -# shape #42 turned into a build failure rather than tolerate: nothing -# else keeps the two in sync, and a bump applied to one file alone would -# leave `make lint` and the fail-fast lint stage of `make docker` -# linting the same tree against different rulesets, both green. This is -# the single guard that stops it, run as a gate in both files. +# Do not hardcode the expected digest here: that is a third copy to keep +# in sync. # -# It deliberately restates neither pin. A hardcoded expected digest here -# would be a third copy — one more thing to bump, and the same drift one -# file further out. It compares the two files to each other and knows -# nothing about which version is correct. -# -# A reference that cannot be read is a hard failure, not a skip: a -# comparison of two empty strings succeeds, which would turn this guard -# into exactly the unearned green it exists to prevent. +# A reference that cannot be read is a hard failure, not a skip: two +# empty strings compare equal. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" From 5900feb51540e98421fb38bb22378ebadb55e11c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 02:51:53 +0200 Subject: [PATCH 28/33] Store mtime to the nanosecond so a same-second rewrite is re-hashed (closes #12) scan recorded mtime in whole seconds, so a file rewritten in place at the same size within the same second as its recorded mtime was classed unchanged and kept its old hashes. The files table keeps mtime as whole Unix seconds and gains mtime_nsec, the nanoseconds within that second. scan holds the mtime as a time.Time and decides "newer" by comparing Unix() and then Nanosecond(), so any time a filesystem can record compares in the right order; After would misorder one too late for a time.Time to hold without wrapping. The walk, a file given as an operand, and the content phase's recheck all move over. PRAGMA user_version stays 1, per the owner's ruling. README states what both columns hold. Model: opus-5-5 --- README.md | 44 ++++++---- TODO.md | 5 ++ db.go | 66 ++++++++------ db_test.go | 15 ++-- report.go | 3 +- report_test.go | 11 ++- scan.go | 25 ++++-- scan_test.go | 234 +++++++++++++++++++++++++++++++++++++++++++++++-- 8 files changed, 335 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 41afbc8..255bb36 100644 --- a/README.md +++ b/README.md @@ -312,29 +312,34 @@ All three subcommands operate on a single SQLite database file: ```sql CREATE TABLE files ( - path BLOB PRIMARY KEY, -- absolute path, raw bytes - size INTEGER NOT NULL, -- bytes, from lstat - mtime INTEGER NOT NULL, -- Unix seconds, from lstat - head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB - tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB - content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples + path BLOB PRIMARY KEY, -- absolute path, raw bytes + size INTEGER NOT NULL, -- bytes, from lstat + mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat + mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 + head TEXT NOT NULL, -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB + tail TEXT NOT NULL, -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB + content TEXT NOT NULL -- lowercase-hex SHA-256, whole file or samples ) WITHOUT ROWID; CREATE INDEX files_signature ON files (size, head, tail, content); ``` Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed - UTF-8. `mtime` is used only for change detection; it is not part of the - duplicate key. For a file under 10 MiB `head`, `tail`, and `content` all - hold the whole-file hash (that range is hashed in full, with no end - windows); for a larger file `head` and `tail` hold the first- and last-64 - KiB hashes and `content` the whole-file or sampled hash. All three are empty - strings when the file has never been hashed because its size was unique as - of the last scan that covered it. For a file of 10 MiB or more, `content` - stays empty until the content phase of a scan (see "`scan` mode" below) has - read the file. A record with an empty `content` is never part of a duplicate - group, though it still defines the file for tree reconstruction. The - `files_signature` index lets SQLite group the records by signature for - `report` without sorting the whole table. + UTF-8. `mtime` and `mtime_nsec` hold the file's mtime to the nanosecond: + `mtime` the whole Unix seconds, rounded down, and `mtime_nsec` the + nanoseconds past that second. Split this way they hold any mtime a + filesystem can record, one before 1678 or after 2262 included, which a + single 64-bit count of nanoseconds cannot. They are used only for change + detection and are not part of the duplicate key. For a file under 10 MiB + `head`, `tail`, and `content` all hold the whole-file hash (that range is + hashed in full, with no end windows); for a larger file `head` and `tail` + hold the first- and last-64 KiB hashes and `content` the whole-file or + sampled hash. All three are empty strings when the file has never been + hashed because its size was unique as of the last scan that covered it. For + a file of 10 MiB or more, `content` stays empty until the content phase of a + scan (see "`scan` mode" below) has read the file. A record with an empty + `content` is never part of a duplicate group, though it still defines the + file for tree reconstruction. The `files_signature` index lets SQLite group + the records by signature for `report` without sorting the whole table. ### Duplicate detection @@ -421,6 +426,9 @@ operands: once its size, `head`, and `tail` match another record's. - A file whose mtime is newer than recorded, or whose size differs, is processed as if new: re-hashed, or recorded without hashes, per the shared-size rule. + Change detection compares the mtime to the nanosecond, as finely as the + filesystem records it, so a same-size rewrite counts as a change whenever the + filesystem gives it a later mtime than recorded, even within the same second. - A database record whose path lies under one of the scanned operands but was not successfully processed this run is deleted. This removes records for deleted files. It also removes records for paths that failed to stat or hash diff --git a/TODO.md b/TODO.md index a538990..547648f 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,11 @@ # Completed Steps +- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus + `mtime_nsec`, and compares it at that resolution, so a same-size rewrite + within the same second is re-hashed (2026-10-07, + https://git.eeqj.de/sneak/sfdupes/issues/12) + - cut the narration from `TODO.md` Completed Steps and from the comments in `script/` and both Dockerfiles; §Workflow now branches from and merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49) diff --git a/db.go b/db.go index ca42037..07b39cf 100644 --- a/db.go +++ b/db.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "strconv" + "time" "golang.org/x/sys/unix" // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo @@ -40,15 +41,18 @@ const dbDirPerm = 0o755 const lockFilePerm = 0o600 // createTableSQL is the schema applied to a fresh database. Paths are -// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. +// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime +// holds whole Unix seconds and mtime_nsec the nanoseconds within that +// second. const createTableSQL = ` CREATE TABLE files ( - path BLOB PRIMARY KEY, - size INTEGER NOT NULL, - mtime INTEGER NOT NULL, - head TEXT NOT NULL, - tail TEXT NOT NULL, - content TEXT NOT NULL + path BLOB PRIMARY KEY, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL, + mtime_nsec INTEGER NOT NULL, + head TEXT NOT NULL, + tail TEXT NOT NULL, + content TEXT NOT NULL ) WITHOUT ROWID ` @@ -61,10 +65,11 @@ CREATE INDEX files_signature ON files (size, head, tail, content) // upsertSQL inserts one file record, replacing any existing record for // the same path. const upsertSQL = ` -INSERT INTO files (path, size, mtime, head, tail, content) -VALUES (?, ?, ?, ?, ?, ?) +INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content) +VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (path) DO UPDATE SET size = excluded.size, mtime = excluded.mtime, + mtime_nsec = excluded.mtime_nsec, head = excluded.head, tail = excluded.tail, content = excluded.content ` @@ -356,8 +361,8 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) { // which is the order of the primary key, so SQLite does not sort. func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error { rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, head, tail, content FROM files "+ - "ORDER BY path") + "SELECT path, size, mtime, mtime_nsec, head, tail, content "+ + "FROM files ORDER BY path") if err != nil { return fmt.Errorf("read records: %w", err) } @@ -366,17 +371,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error { for rows.Next() { var ( - path []byte - r scanRec + path []byte + sec, nsec int64 + r scanRec ) - err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, + err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, &r.content) if err != nil { return fmt.Errorf("read record: %w", err) } r.path = string(path) + r.mtime = time.Unix(sec, nsec) fn(r) } @@ -466,10 +473,10 @@ func loadDupeRows(ctx context.Context, db *sql.DB, // values, and skipping the hash columns keeps the scan's in-memory // index small on multi-million-file databases. func loadFileMeta(ctx context.Context, db *sql.DB, - fn func(path string, size, mtime int64, hashed bool), + fn func(path string, size int64, mtime time.Time, hashed bool), ) error { rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, head <> '' FROM files") + "SELECT path, size, mtime, mtime_nsec, head <> '' FROM files") if err != nil { return fmt.Errorf("read records: %w", err) } @@ -478,17 +485,17 @@ func loadFileMeta(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - size, mtime int64 - hashed int64 + path []byte + size, sec, nsec int64 + hashed int64 ) - err = rows.Scan(&path, &size, &mtime, &hashed) + err = rows.Scan(&path, &size, &sec, &nsec, &hashed) if err != nil { return fmt.Errorf("read record: %w", err) } - fn(string(path), size, mtime, hashed != 0) + fn(string(path), size, time.Unix(sec, nsec), hashed != 0) } err = rows.Err() @@ -507,7 +514,7 @@ func loadFileMeta(ctx context.Context, db *sql.DB, // memory; the rows come ordered by size, head, and tail, so each // group's rows arrive together. const contentCandidatesSQL = ` -SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> '' +SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> '' FROM files AS f JOIN ( SELECT size, head, tail @@ -533,17 +540,20 @@ func loadContentCandidates(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - r scanRec - hashed int64 + path []byte + sec, nsec int64 + r scanRec + hashed int64 ) - err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed) + err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, + &hashed) if err != nil { return fmt.Errorf("read record: %w", err) } r.path = string(path) + r.mtime = time.Unix(sec, nsec) fn(r, hashed != 0) } @@ -627,8 +637,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, defer func() { _ = st.Close() }() for _, r := range upserts { - _, err = st.ExecContext(ctx, - []byte(r.path), r.size, r.mtime, r.head, r.tail, r.content) + _, err = st.ExecContext(ctx, []byte(r.path), r.size, + r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content) if err != nil { return fmt.Errorf("upsert %s: %w", r.path, err) } diff --git a/db_test.go b/db_test.go index ef7e833..6920d6e 100644 --- a/db_test.go +++ b/db_test.go @@ -10,6 +10,7 @@ import ( "slices" "strings" "testing" + "time" ) // testDBPath returns a database path inside a fresh temp dir. @@ -260,10 +261,13 @@ func TestApplyChangesRoundTrip(t *testing.T) { // written. recs := []scanRec{ { - size: 2, mtime: 20, head: "h2", tail: "t2", content: "c2", - path: "/a/tab\tnew\nline", + size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2", + content: "c2", path: "/a/tab\tnew\nline", + }, + { + size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1", + content: "c1", path: "/a/x", }, - {size: 1, mtime: 10, head: "h1", tail: "t1", content: "c1", path: "/a/x"}, } err := applyChanges(t.Context(), db, recs, nil, @@ -281,7 +285,8 @@ func TestApplyChangesRoundTrip(t *testing.T) { // An upsert for an existing path updates in place; a delete // removes exactly its path. upd := scanRec{ - size: 3, mtime: 30, head: "h3", tail: "t3", content: "c3", path: "/a/x", + size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3", + path: "/a/x", } err = applyChanges(t.Context(), db, []scanRec{upd}, @@ -308,7 +313,7 @@ func TestApplyChangesBatching(t *testing.T) { recs := make([]scanRec, 0, n) for i := range n { recs = append(recs, scanRec{ - size: int64(i), mtime: 1, head: "h", tail: "t", + size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t", path: fmt.Sprintf("/batch/%07d", i), }) } diff --git a/report.go b/report.go index d2a0c7b..425f412 100644 --- a/report.go +++ b/report.go @@ -7,6 +7,7 @@ import ( "io" "os" "strings" + "time" ) // ioBufSize is the buffer size for the buffered stdout writers. @@ -21,7 +22,7 @@ const minGroupSize = 2 // only and used by scan for change detection. type scanRec struct { size int64 - mtime int64 + mtime time.Time head string tail string content string diff --git a/report_test.go b/report_test.go index 796dbbf..82f8b03 100644 --- a/report_test.go +++ b/report_test.go @@ -11,6 +11,7 @@ import ( "slices" "strings" "testing" + "time" ) // awkwardDir is a directory name holding every byte the reports escape. @@ -313,8 +314,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) { // mtime is informational only; records differing only in mtime // still group together. recs := []scanRec{ - {size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"}, - {size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"}, + { + size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", + content: "c", path: "/m/1", + }, + { + size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t", + content: "c", path: "/m/2", + }, } groups := dupeGroupsOf(t, recs) diff --git a/scan.go b/scan.go index f181f0a..5514ab1 100644 --- a/scan.go +++ b/scan.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "syscall" + "time" ) // The duplicate ladder (see hashSignature and README "Duplicate @@ -68,7 +69,7 @@ var errInterrupted = errors.New("scan interrupted") type fileRec struct { path string size int64 - mtime int64 + mtime time.Time dev uint64 ino uint64 } @@ -79,7 +80,7 @@ type fileRec struct { // they would dominate the scan's memory. type fileMeta struct { size int64 - mtime int64 + mtime time.Time hashed bool } @@ -369,7 +370,7 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { s.existing = make(map[string]fileMeta) return loadFileMeta(ctx, s.db, - func(path string, size, mtime int64, hashed bool) { + func(path string, size int64, mtime time.Time, hashed bool) { prog.increment() if underAnyRoot(path, roots) { @@ -415,7 +416,7 @@ func (s *scanState) walkPhase( old, ok := s.existing[ev.rec.path] switch { - case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime: + case !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime): changed = append(changed, ev.rec) case old.hashed: delete(s.existing, ev.rec.path) @@ -807,7 +808,7 @@ func unchangedFile(r scanRec) (fileRec, bool, error) { } if !fi.Mode().IsRegular() || fi.Size() != r.size || - fi.ModTime().Unix() > r.mtime { + mtimeAfter(fi.ModTime(), r.mtime) { return fileRec{}, false, nil } @@ -818,6 +819,16 @@ func unchangedFile(r scanRec) (fileRec, bool, error) { }, true, nil } +// mtimeAfter reports whether mtime a is later than mtime b. +// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it. +func mtimeAfter(a, b time.Time) bool { + if a.Unix() != b.Unix() { + return a.Unix() > b.Unix() + } + + return a.Nanosecond() > b.Nanosecond() +} + // underAnyRoot reports whether path is any of the roots or lies under // one of them. func underAnyRoot(path string, roots []string) bool { @@ -966,7 +977,7 @@ func seedRoot(ctx context.Context, root string, sendEvent(ctx, events, walkEvent{rec: fileRec{ path: root, size: fi.Size(), - mtime: fi.ModTime().Unix(), + mtime: fi.ModTime(), dev: dev, ino: ino, }}) @@ -1124,7 +1135,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, sendEvent(ctx, events, walkEvent{rec: fileRec{ path: p, size: info.Size(), - mtime: info.ModTime().Unix(), + mtime: info.ModTime(), dev: dev, ino: ino, }}) diff --git a/scan_test.go b/scan_test.go index d2560b5..079f7b9 100644 --- a/scan_test.go +++ b/scan_test.go @@ -20,6 +20,8 @@ import ( "syscall" "testing" "time" + + "golang.org/x/sys/unix" ) // writeFile creates a file with the given content and returns its path. @@ -581,6 +583,65 @@ func TestScanContentHashedStalePartners(t *testing.T) { } } +// TestScanContentSameSecondRewrite is TestScanContentStalePartners for +// a stored file rewritten in place at the same size with an mtime later +// in the same second than recorded: the file counts as changed, so +// neither it nor its match inside the operand is read. +func TestScanContentSameSecondRewrite(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + dirA := t.TempDir() + changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(changed, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, dirA) + + before := dbRecords(t, db) + + // Rewrite one byte in place, keeping the size. + pokeAt(t, changed, headTailMin/2, []byte{1}) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(changed, later, later) + if err != nil { + t.Fatal(err) + } + + dirB := t.TempDir() + sparseFile(t, dirB, "changed-copy", headTailMin) + + st := syncTree(t, db, dirB) + if st != (scanStats{walked: 1, added: 1}) { + t.Errorf("stats = %+v, want 1 added and nothing skipped", st) + } + + recs := dbRecords(t, db) + for _, r := range recs { + if r.content != "" { + t.Errorf("%s: content = %q, want none: its only match is stale", + r.path, r.content) + } + } + + for _, old := range before { + if r := recordByPath(t, recs, old.path); r != old { + t.Errorf("record = %+v, want it left as %+v", r, old) + } + } + + if groups := dupeGroups(t, db); len(groups) != 0 { + t.Errorf("groups = %+v, want none", groups) + } +} + // TestScanContentReadFailure checks that a failed content read is // counted as skipped and leaves the record without a content hash, and // that a later scan tries the read again. @@ -783,8 +844,8 @@ func TestWalk(t *testing.T) { t.Errorf("%s: size = %d, want 1..3", r.path, r.size) } - if r.mtime <= 0 { - t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime) + if r.mtime.Unix() <= 0 { + t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime) } } } @@ -1285,11 +1346,172 @@ func TestSyncScanMtimeBump(t *testing.T) { t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) } - if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() { - t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix()) + if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) { + t.Fatalf("mtime = %v, want %v", r.mtime, future) } } +// assertWholeFileHashed fails unless the record for path holds the +// whole-file hash of data as its head, tail, and content. +func assertWholeFileHashed(t *testing.T, db *sql.DB, path string, + data []byte, +) { + t.Helper() + + r := recordByPath(t, dbRecords(t, db), path) + if want := hexSum(data); r.head != want || r.tail != want || + r.content != want { + t.Fatalf("head, tail, content = %q, %q, %q, want %q for each", + r.head, r.tail, r.content, want) + } +} + +// TestSyncScanSameSecondRewrite rewrites a file in place at the same +// size with an mtime later in the same second as the recorded one: the +// next scan must notice the change and re-hash the file. +func TestSyncScanSameSecondRewrite(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + writeFile(t, dir, "b.bin", pattern(2, 500)) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(a, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, dir) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(a, later, later) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, dir) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + +// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite +// for files given to scan as operands, which scan stats without reading +// their directory. +func TestSyncScanOperandSameSecondRewrite(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + b := writeFile(t, dir, "b.bin", pattern(2, 500)) + + first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC) + + err := os.Chtimes(a, first, first) + if err != nil { + t.Fatal(err) + } + + syncTree(t, db, a, b) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + later := first.Add(500 * time.Millisecond) + + err = os.Chtimes(a, later, later) + if err != nil { + t.Fatal(err) + } + + st := syncTree(t, db, a, b) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + +// TestSyncScanRewriteAfter2262 runs assertLateRewriteRehashed with an +// mtime after 2262, a time too late to count in nanoseconds in an int64. +func TestSyncScanRewriteAfter2262(t *testing.T) { + t.Parallel() + + assertLateRewriteRehashed(t, time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC)) +} + +// TestSyncScanRewritePastTimeLimit runs assertLateRewriteRehashed with an +// mtime one second past the latest a time.Time holds without wrapping it +// to a time far in the past. +func TestSyncScanRewritePastTimeLimit(t *testing.T) { + t.Parallel() + + assertLateRewriteRehashed(t, time.Unix(9223371974719179008, 0)) +} + +// assertLateRewriteRehashed scans a directory, rewrites a file in it in +// place at the same size, sets its mtime to late, and fails unless the +// next scan re-hashes the file. It skips where late does not fit the +// platform's timespec or the filesystem does not store it. +func assertLateRewriteRehashed(t *testing.T, late time.Time) { + t.Helper() + + dir := t.TempDir() + db := openTestDB(t) + a := writeFile(t, dir, "a.bin", pattern(1, 500)) + + // b.bin shares the size of a.bin, so a.bin is hashed. + writeFile(t, dir, "b.bin", pattern(2, 500)) + + syncTree(t, db, dir) + + rewritten := pattern(3, 500) + writeFile(t, dir, "a.bin", rewritten) + + // os.Chtimes cannot set such a time: it converts through UnixNano. + ts, err := unix.TimeToTimespec(late) + if err != nil { + t.Skipf("an mtime %d seconds after 1970 does not fit this platform's "+ + "timespec: %v", late.Unix(), err) + } + + err = unix.UtimesNano(a, []unix.Timespec{ts, ts}) + if err != nil { + t.Fatal(err) + } + + fi, err := os.Lstat(a) + if err != nil { + t.Fatal(err) + } + + if fi.ModTime().Unix() != late.Unix() { + t.Skipf("the filesystem stored the mtime as %d seconds after 1970, "+ + "not %d", fi.ModTime().Unix(), late.Unix()) + } + + st := syncTree(t, db, dir) + if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st) + } + + assertWholeFileHashed(t, db, a, rewritten) +} + func TestSyncScanAddRemove(t *testing.T) { t.Parallel() @@ -1335,9 +1557,7 @@ func TestSyncScanSizeChange(t *testing.T) { writeFile(t, dir, "f", pattern(1, 200)) - mt := time.Unix(old.mtime, 0) - - err := os.Chtimes(p, mt, mt) + err := os.Chtimes(p, old.mtime, old.mtime) if err != nil { t.Fatal(err) } From a4d76af8feaee7aaf09fffbb1ecd71ac1b9afeaa Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 19:55:58 +0200 Subject: [PATCH 29/33] Redraw a finished bar below a warning for its last file (closes #102) On a terminal, warnf clears the bar's line before it prints a warning. When the warning is for the last item of a phase, the bar has already drawn its finished frame and the progress library draws nothing more for a finished bar, so the warning took the bar's place for good. warnf now draws the finished bar's last frame again below the warning. Output when stderr is not a terminal does not change. Model: opus-5-5 --- TODO.md | 4 ++++ progress.go | 7 +++++++ progress_test.go | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/TODO.md b/TODO.md index 547648f..2e0aeda 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- on a terminal, a warning for the last file of a phase no longer erases the + phase's finished bar: the bar is drawn again below the warning (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/102) + - `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus `mtime_nsec`, and compares it at that resolution, so a same-size rewrite within the same second is re-hashed (2026-10-07, diff --git a/progress.go b/progress.go index bc042e8..79b05bb 100644 --- a/progress.go +++ b/progress.go @@ -139,6 +139,13 @@ func (p *progress) warnf(format string, args ...any) { } fmt.Fprintln(os.Stderr, msg) + + // The library never draws a finished bar again, so the warning for + // a pass's last item would erase it for good: draw its last frame + // again below the warning. + if p.bar != nil && p.bar.IsFinished() { + fmt.Fprint(os.Stderr, p.bar.String()) + } } // finish terminates the pass's display. A bar whose pass stopped short diff --git a/progress_test.go b/progress_test.go index 2d36e5f..453fa94 100644 --- a/progress_test.go +++ b/progress_test.go @@ -187,3 +187,25 @@ func TestBarStoppedShortKeepsCount(t *testing.T) { t.Errorf("terminal shows %q, want a count of 2 of 10", shown) } } + +// TestWarningAfterLastItemKeepsBar checks that a warning printed after +// a pass's last item, as for a failed read of the last file, leaves the +// pass's finished bar on the terminal below the warning. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestWarningAfterLastItemKeepsBar(t *testing.T) { + stderr := captureStderr(t) + p := &progress{ + label: "hash", total: 2, start: time.Now(), + bar: newBar("hash", 2), + } + + p.increment() + p.increment() + p.warnf("warning") + p.finish() + + if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") { + t.Errorf("terminal shows %q, want a count of 2 of 2", shown) + } +} From 1b26b5ce770163650da0c77b655f9496cb0eb359 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 20:11:54 +0200 Subject: [PATCH 30/33] Name the scan lock file from the database's resolved path (closes #101) The lock file was the configured database path with .lock appended, so a scan reaching the database through a symlink to it locked a different file and ran alongside a scan holding the lock through the real path. Before it takes the lock, scan now creates the database file, empty, if it is missing, following a symlink to its target as SQLite would, and names the lock file from that file's path with symlinks resolved. A scan started before the database exists therefore takes the same lock file as every later scan through the same path. testDBPath resolves its temp dir, so lock errors name the expected path where the temp dir is itself reached through a symlink. Model: opus-5-5 --- README.md | 25 ++++++++------ TODO.md | 5 +++ db.go | 36 +++++++++++++++---- db_test.go | 12 +++++-- main_test.go | 97 +++++++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 152 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 255bb36..27f10b4 100644 --- a/README.md +++ b/README.md @@ -271,16 +271,21 @@ All three subcommands operate on a single SQLite database file: error (exit 1) telling the user to run `scan` first. - Only one `scan` runs against a database at a time. For its whole run, `scan` holds an exclusive `flock(2)` lock on a lock file beside the database, named - by appending `.lock` to the database path (`/var/lib/sfdupes/db.sqlite.lock` - by default), taken before it walks the filesystem or opens the database. A - second `scan` against the same database does not wait: it fails at once with a - one-line error naming the lock file and exits 1, without walking anything or - opening the database, and the running scan carries on. The lock file is - created on first use, open to its owner only, and left in place: a leftover - file blocks nothing, because the lock ends with the process holding it however - it ends, a fatal error or an interrupt included, and deleting the file while a - scan runs would let a second scan start. `report` and `trees` never take the - lock, so they run during a scan. + by appending `.lock` to the database's path with symlinks resolved + (`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the + filesystem or opens the database. Every path to one database, through a + symlink to the file or to a directory above it, names the same lock file. Just + before it takes the lock, `scan` creates the database file, empty, if it is + missing (through a symlink at the database path, it creates the symlink's + target), so the lock file is named the same way before the first scan sets up + the database as after. A second `scan` against the same database does not + wait: it fails at once with a one-line error naming the lock file and exits 1, + without walking anything or opening the database, and the running scan carries + on. The lock file is created on first use, open to its owner only, and left in + place: a leftover file blocks nothing, because the lock ends with the process + holding it however it ends, a fatal error or an interrupt included, and + deleting the file while a scan runs would let a second scan start. `report` + and `trees` never take the lock, so they run during a scan. - While `scan` runs, the database is in WAL journal mode with a busy timeout, so running a report while a cron `scan` is in progress is safe. The filesystem is authoritative; the database is an eventually-consistent reflection of it. diff --git a/TODO.md b/TODO.md index 2e0aeda..ef00742 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,11 @@ # Completed Steps +- the scan lock file is named from the database's path with symlinks resolved, + after `scan` creates the database file if it is missing, so a scan through a + symlink to the database fails while another scan holds the lock (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/101) + - on a terminal, a warning for the last file of a phase no longer erases the phase's finished bar: the bar is drawn again below the warning (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/102) diff --git a/db.go b/db.go index 07b39cf..e0dff42 100644 --- a/db.go +++ b/db.go @@ -35,6 +35,10 @@ const schemaVersion = 1 // scan. const dbDirPerm = 0o755 +// dbFilePerm is the mode for a database file created by scan, the mode +// SQLite gives a database file it creates. +const dbFilePerm = 0o644 + // lockFilePerm is the mode for the scan lock file. Anyone who can open // the file can hold the lock and keep every scan from running, so it // is open to its owner only. @@ -140,19 +144,37 @@ func openDB(path, params string) (*sql.DB, error) { // lockScanDatabase takes the lock that keeps a second scan off the // database at path: an exclusive flock(2) on the file beside it named -// path with ".lock" appended, created along with the database's parent -// directory if missing. A lock held by another scan fails at once -// instead of waiting. The lock lasts until the returned file is closed -// or the process ends. The file is never deleted: a scan that deleted -// it would let the next scan lock a new file while another still holds -// the old one. +// by the database's path with symlinks resolved and ".lock" appended, +// so every path to one database names the same lock file. If missing, +// the database's parent directory and an empty database file are +// created first, so the lock file is named the same way before the +// database is set up as after. A lock held by another scan fails at +// once instead of waiting. The lock lasts until the returned file is +// closed or the process ends. The lock file is never deleted: a scan +// that deleted it would let the next scan lock a new file while +// another still holds the old one. func lockScanDatabase(path string) (*os.File, error) { err := os.MkdirAll(filepath.Dir(path), dbDirPerm) if err != nil { return nil, fmt.Errorf("create database directory: %w", err) } - lockPath := path + ".lock" + // Like SQLite, this follows a symlink at path and creates its + // target. SQLite opens an empty file as a new database. + //nolint:gosec // the operator chooses the database path + dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm) + if err != nil { + return nil, fmt.Errorf("create database: %w", err) + } + + _ = dbFile.Close() + + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, fmt.Errorf("resolve database path: %w", err) + } + + lockPath := resolved + ".lock" //nolint:gosec // the operator chooses the database path f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm) diff --git a/db_test.go b/db_test.go index 6920d6e..e4b8600 100644 --- a/db_test.go +++ b/db_test.go @@ -13,11 +13,19 @@ import ( "time" ) -// testDBPath returns a database path inside a fresh temp dir. +// testDBPath returns a database path inside a fresh temp dir, with +// symlinks resolved as the scan lock resolves them, so the lock file +// is this path plus ".lock" even where the temp dir is reached through +// a symlink. func testDBPath(t *testing.T) string { t.Helper() - return filepath.Join(t.TempDir(), "db.sqlite") + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + + return filepath.Join(dir, "db.sqlite") } // openTestDB creates a fresh scan database in a temp dir. diff --git a/main_test.go b/main_test.go index 7742365..ca6faf7 100644 --- a/main_test.go +++ b/main_test.go @@ -822,7 +822,7 @@ func holdScanLock(t *testing.T, path string) { func TestRunSecondScanFails(t *testing.T) { // README §Database: while one scan holds the lock, a second scan - // fails at once, naming the lock file, without creating the + // fails at once, naming the lock file, without opening the // database. path := testDBPath(t) t.Setenv(databaseEnv, path) @@ -848,10 +848,99 @@ func TestRunSecondScanFails(t *testing.T) { t.Errorf("stdout = %q, want nothing (data only)", got) } - _, err := os.Stat(path) - if !errors.Is(err, fs.ErrNotExist) { - t.Errorf("stat %s = %v, want the database not created", path, err) + // Taking the lock created the database file, empty; opening it as + // a database would have written to it. + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) } + + if info.Size() != 0 { + t.Errorf("database size = %d, want 0 (not opened)", info.Size()) + } +} + +// assertSecondScanFails runs a scan against the database +// SFDUPES_DATABASE names while the test holds the lock on it. It fails +// the test unless that scan fails at once, naming the lock file beside +// path, the database's real path. +func assertSecondScanFails(t *testing.T, path string) { + t.Helper() + + var stderr bytes.Buffer + + code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr) + if code != exitFatal { + t.Errorf("run(scan) = %d, want %d", code, exitFatal) + } + + want := "sfdupes: another scan is running (lock held on " + + path + ".lock)\n" + if got := stderr.String(); got != want { + t.Errorf("stderr = %q, want %q", got, want) + } +} + +func TestRunSecondScanThroughSymlinkFails(t *testing.T) { + // README §Database: every path to one database names the same lock + // file, so while a scan holds it through one path, a scan through + // another fails, naming the lock file beside the database's real + // path. + t.Run("symlink to the database", func(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + scanFixture(t) + + link := filepath.Join(t.TempDir(), "link.sqlite") + + err := os.Symlink(path, link) + if err != nil { + t.Fatal(err) + } + + holdScanLock(t, path) + t.Setenv(databaseEnv, link) + assertSecondScanFails(t, path) + }) + + t.Run("symlink to its directory, no database yet", func(t *testing.T) { + path := testDBPath(t) + link := filepath.Join(t.TempDir(), "link") + + err := os.Symlink(filepath.Dir(path), link) + if err != nil { + t.Fatal(err) + } + + holdScanLock(t, path) + t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path))) + assertSecondScanFails(t, path) + }) + + t.Run("symlink whose target does not exist yet", func(t *testing.T) { + path := testDBPath(t) + link := filepath.Join(t.TempDir(), "link.sqlite") + + err := os.Symlink(path, link) + if err != nil { + t.Fatal(err) + } + + // The first scan takes the lock through the link before the + // database exists, then creates the database through it. + holdScanLock(t, link) + + db, err := openScanDatabase(t.Context(), link) + if err != nil { + t.Fatal(err) + } + + closeScanDatabase(t.Context(), db, link) + + t.Setenv(databaseEnv, link) + assertSecondScanFails(t, path) + }) } func TestRunScanReleasesLock(t *testing.T) { From 5dc6bfb6b79b3d28f4954fc444d22076b4d9d4f2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 20:18:32 +0200 Subject: [PATCH 31/33] Open files to hash without blocking and reject non-regular ones (closes #100) A regular file replaced by a FIFO after the walk statted it made the hash phase's open wait forever for a writer, so the scan never ended and kept the lock. The hash and content phases now open each file with O_NONBLOCK and check with fstat that it is still a regular file; anything else is a per-path error, warned about and skipped. The new test replaces a walked file with a FIFO before the hash phase reads it. Model: opus-5-5 --- TODO.md | 5 +++++ scan.go | 38 ++++++++++++++++++++++++++++++++++---- scan_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index ef00742..423e692 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,11 @@ # Completed Steps +- the hash and content phases open each file without blocking and check that it + is still a regular file, so one replaced by a FIFO or device node after it was + statted is warned about and skipped instead of hanging the scan (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/100) + - the scan lock file is named from the database's path with symlinks resolved, after `scan` creates the database file if it is missing, so a scan through a symlink to the database fails while another scan holds the lock (2026-10-14, diff --git a/scan.go b/scan.go index 5514ab1..1b8eb68 100644 --- a/scan.go +++ b/scan.go @@ -1309,6 +1309,38 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec, } } +// errNotRegular reports a path that is no longer a regular file when it +// is opened to be read. +var errNotRegular = errors.New("not a regular file") + +// openRegularFile opens path for reading and returns it only if it is +// still a regular file, since the path may have been replaced after it +// was statted. O_NONBLOCK keeps the open of a FIFO or a device node +// from waiting forever, and fstat then rejects it. Reads of a regular +// file ignore O_NONBLOCK. +func openRegularFile(path string) (*os.File, error) { + //nolint:gosec // hashing operator-supplied paths is the tool's purpose + f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + + fi, err := f.Stat() + if err != nil { + _ = f.Close() + + return nil, err + } + + if !fi.Mode().IsRegular() { + _ = f.Close() + + return nil, errNotRegular + } + + return f, nil +} + // emptyHash is the lowercase-hex SHA-256 of the empty input: the head, // tail, and content hash of every zero-length file. const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + @@ -1331,8 +1363,7 @@ func hashSignature(path string, size int64) (string, string, string, error) { return emptyHash, emptyHash, emptyHash, nil } - //nolint:gosec // hashing operator-supplied paths is the tool's purpose - f, err := os.Open(path) + f, err := openRegularFile(path) if err != nil { return "", "", "", err } @@ -1363,8 +1394,7 @@ func hashSignature(path string, size int64) (string, string, string, error) { // tail are returned empty, because the content phase keeps the ones its // records already hold. func hashContentOnly(path string, size int64) (string, string, string, error) { - //nolint:gosec // hashing operator-supplied paths is the tool's purpose - f, err := os.Open(path) + f, err := openRegularFile(path) if err != nil { return "", "", "", err } diff --git a/scan_test.go b/scan_test.go index 079f7b9..7d3c705 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1797,6 +1797,54 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) { } } +// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file +// with a FIFO before the hash phase reads it. Opening a FIFO that has +// no writer waits for one, so a blocking open would hang the phase. The +// path must be warned about once and skipped, with no record written. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + p := writeFile(t, dir, "a.bin", pattern(1, 300)) + + walked, _ := collectWalk(t, []string{dir}, false, 1) + + err := os.Remove(p) + if err != nil { + t.Fatal(err) + } + + err = unix.Mkfifo(p, 0o600) + if err != nil { + t.Fatal(err) + } + + s := &scanState{db: openTestDB(t), toHash: walked} + + err = s.hashPhase(t.Context(), 1) + if err != nil { + t.Fatalf("hashPhase = %v, want nil", err) + } + + if s.st.skipped != 1 || len(s.batch) != 0 { + t.Errorf("skipped = %d, records = %d, want 1 skipped and none", + s.st.skipped, len(s.batch)) + } + + warning := "hash " + p + ": not a regular file\n" + if got := stderr(); strings.Count(got, warning) != 1 { + t.Errorf("stderr = %q, want %q once", got, warning) + } + + // The content phase opens files the same way. + _, _, content, err := hashContentOnly(p, headTailMin) + if content != "" || !errors.Is(err, errNotRegular) { + t.Errorf("hashContentOnly on a FIFO = %q, %v; want no hash and %v", + content, err, errNotRegular) + } +} + // injectedWriteFailure is the message the injected database trigger // aborts with, so the test can recognize its own failure in the error // the scan reports. From 80bd7c099935e564f8b3938ba84440d95e913f45 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 20:22:50 +0200 Subject: [PATCH 32/33] Keep records on other filesystems under an operand with -x (closes #99) With -x the walk does not enter a directory on another filesystem than its operand, but the update phase still deleted every record below it. A separate scan of a filesystem mounted under the operand therefore lost its records on every -x scan and re-hashed its whole tree. The walk now sends an event naming each directory it declines this way, and the walk phase treats the records below those directories like records outside the operands: they are not deleted, and their sizes join the size census. README "scan mode" and "Rules for the walk" state the rule. Model: opus-5-5 --- README.md | 9 +++++- TODO.md | 4 +++ scan.go | 42 +++++++++++++++++++----- scan_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 134 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 27f10b4..40d47e1 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,11 @@ operands: a `content` hash is read for one, wherever it lies, once its size, `head`, and `tail` match another record's. If that file is gone or has changed since its record was written, the record is left as it is. +- With `-x`, a record below a directory the walk did not enter because that + directory is on another filesystem than its operand counts as outside the + scanned operands: it is not deleted, and its size joins the size census. A + filesystem mounted under an operand can therefore be scanned on its own + schedule into the same database. `scan` runs **four sequential phases over the whole scan**. Parallelism lives inside each phase; batched database writes begin during the hash phase: @@ -513,7 +518,9 @@ Rules for the walk: - Filesystem boundaries are crossed by default. With `-x` (long form `--one-file-system`, following the GNU `du`/`rsync` convention), never descend into a directory on a different filesystem than its `PATH` operand; each - operand is bounded by its own filesystem. + operand is bounded by its own filesystem. The records below such a directory + count as outside the scanned operands, as described in "`scan` mode" above: + they are not deleted, and their sizes join the size census. - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were diff --git a/TODO.md b/TODO.md index 423e692..c588fef 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- with `-x`, `scan` keeps the records below a directory on another filesystem + under its operand, and counts their sizes in the size census, instead of + deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99) + - the hash and content phases open each file without blocking and check that it is still a regular file, so one replaced by a FIFO or device node after it was statted is warned about and skipped instead of hanging the scan (2026-10-14, diff --git a/scan.go b/scan.go index 1b8eb68..f4a031c 100644 --- a/scan.go +++ b/scan.go @@ -391,15 +391,26 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. +// their sizes are shared. A record below a directory -x did not enter +// counts as outside the roots: it is not deleted, and its size joins +// the census. func (s *scanState) walkPhase( events <-chan walkEvent, ) ([]fileRec, []fileRec) { prog := newProgress("walk", -1) - var changed, unhashed []fileRec + var ( + changed, unhashed []fileRec + otherFSDirs []string + ) for ev := range events { + if ev.otherFSDir != "" { + otherFSDirs = append(otherFSDirs, ev.otherFSDir) + + continue + } + if ev.fail { s.st.skipped++ @@ -429,6 +440,14 @@ func (s *scanState) walkPhase( prog.finish() + for path, old := range s.existing { + if underAnyRoot(path, otherFSDirs) { + delete(s.existing, path) + + s.sizes = append(s.sizes, old.size) + } + } + return changed, unhashed } @@ -866,16 +885,20 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, or a warning when fail is set. +// file's statted record, a warning when fail is set, or, when +// otherFSDir is set, a directory -x did not enter because it is on +// another filesystem than its operand. type walkEvent struct { - rec fileRec - warn string - fail bool + rec fileRec + warn string + fail bool + otherFSDir string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error. The channel is closed when the walk +// event per per-path error, and one event per directory -x did not +// enter. The channel is closed when the walk // completes, and also when ctx is cancelled — every goroutine in the // pool abandons its blocking send in that case, so the consumer sees a // truncated but properly terminated stream instead of a stalled one. @@ -1144,7 +1167,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per // snapshot), and with -x never enter a directory on a different -// filesystem than its operand. +// filesystem than its operand, sending an event that names it so the +// scan keeps the records below it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, events chan<- walkEvent, ) (dirJob, bool) { @@ -1168,6 +1192,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, } if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { + sendEvent(ctx, events, walkEvent{otherFSDir: p}) + return dirJob{}, false } diff --git a/scan_test.go b/scan_test.go index 7d3c705..56880ab 100644 --- a/scan_test.go +++ b/scan_test.go @@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry { } // callSubdirJob runs subdirJob against e under parent, collecting any -// warning events it emits (subdirJob emits at most one). +// events it emits (subdirJob emits at most one). func callSubdirJob(t *testing.T, p string, e fs.DirEntry, parent dirJob, oneFS bool, ) (dirJob, bool, []walkEvent) { @@ -1075,8 +1075,15 @@ func TestSubdirJobOneFilesystem(t *testing.T) { t.Fatalf("accepted = %v, want %v", ok, tc.wantOK) } - if len(evs) != 0 { - t.Fatalf("unexpected events: %+v", evs) + // A refused descent names the directory, so the scan keeps + // the records below it. + var wantEvs []walkEvent + if !ok { + wantEvs = []walkEvent{{otherFSDir: sub}} + } + + if !slices.Equal(evs, wantEvs) { + t.Fatalf("events = %+v, want %+v", evs, wantEvs) } // The accepted job must carry the operand's device down, or -x @@ -1128,6 +1135,84 @@ func TestSubdirJobStatError(t *testing.T) { } } +// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a +// record below a directory the walk does not enter, because it is on +// another filesystem than its operand, counts as outside the operands: +// it is not deleted, and its size joins the size census. The record of +// a gone file that is not below that directory is still deleted, even +// when its path begins with the directory's name. The walk starts from +// an operand device that no directory here is on, so no second real +// filesystem is needed; the phases after it run as runPhases runs them. +func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records the file below mnt, and mnt2, which is + // beside mnt, not below it. + mounted := writeFile(t, dir, "mnt/f", pattern(1, 100)) + gone := writeFile(t, dir, "mnt2", pattern(3, 50)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + // The same size as mnt/f, so it is hashed only if mnt/f's size is + // in the census. + added := writeFile(t, dir, "added", pattern(2, 100)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + ctx := t.Context() + s := &scanState{db: db} + + err = s.loadIndex(ctx, []string{dir}) + if err != nil { + t.Fatal(err) + } + + jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true} + dispatchDirs(ctx, []dirJob{root}, jobs, subdirs) + + changed, unhashed := s.walkPhase(events) + s.partition(changed, unhashed) + + err = s.hashPhase(ctx, 2) + if err != nil { + t.Fatal(err) + } + + err = s.updatePhase(ctx) + if err != nil { + t.Fatal(err) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, mounted) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, mounted) + } + + if slices.Contains(recordPaths(recs), gone) { + t.Errorf("record for %s kept: the file is gone and not below mnt", + gone) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { From 92c02c3bd177b8e49a5a5521bc37253917ebd93f Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 21:40:02 +0200 Subject: [PATCH 33/33] Walk an operand nested on another filesystem with -x (closes #106) With -x, an operand under another operand is kept and walked on its own when the walk of the deepest kept operand above it cannot reach it: some directory from it (from its directory, for a regular file) up to but not including that operand is on a different filesystem than that operand. Below a kept operand, the rule keeping records under a directory -x did not enter covers only directories its own walk did not enter, so records of files gone from it are deleted. The new tests stub the devices and share their scan steps with the test for https://git.eeqj.de/sneak/sfdupes/issues/99. Comments and README sentences on operands dropped before the walk now also hold with -x. Model: opus-5-5 --- README.md | 28 +++-- TODO.md | 7 ++ scan.go | 117 ++++++++++++++++---- scan_test.go | 297 ++++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 388 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 40d47e1..2a1bb62 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,11 @@ results do not depend on the working directory. All operands belong to a single scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. +once and produces one database record. With `-x`, though, an operand under +another one is kept and walked on its own when any directory from it (from its +directory, for a regular file) up to but not including the operand covering it +is on a different filesystem than that operand, because that operand's walk does +not enter such a directory and so does not reach it. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints @@ -407,9 +411,8 @@ drops it from the scanned operands before reading the database. Another operand beneath it is still scanned. The records stored beneath it are not deleted: they are treated like any other record outside the scanned operands, including the content-phase exception below. If it lies under another operand, they are under -that operand instead, and are deleted like any other record there that this scan -did not verify. This is not an error: a scan whose every operand is dropped -walks nothing and exits 0. +that operand instead, and are treated like any other record there. This is not +an error: a scan whose every operand is dropped walks nothing and exits 0. `scan` synchronizes the database with the filesystem state under the scanned operands: @@ -451,7 +454,8 @@ operands: directory is on another filesystem than its operand counts as outside the scanned operands: it is not deleted, and its size joins the size census. A filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. + schedule into the same database. Records below an operand at or inside such a + directory are not kept this way, since that operand's own walk verifies them. `scan` runs **four sequential phases over the whole scan**. Parallelism lives inside each phase; batched database writes begin during the hash phase: @@ -518,17 +522,19 @@ Rules for the walk: - Filesystem boundaries are crossed by default. With `-x` (long form `--one-file-system`, following the GNU `du`/`rsync` convention), never descend into a directory on a different filesystem than its `PATH` operand; each - operand is bounded by its own filesystem. The records below such a directory - count as outside the scanned operands, as described in "`scan` mode" above: - they are not deleted, and their sizes join the size census. + operand is bounded by its own filesystem. The records below such a directory, + unless they lie below an operand at or inside that directory, count as outside + the scanned operands, as described in "`scan` mode" above: they are not + deleted, and their sizes join the size census. - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were skipped. As specified above, a skipped path that has a database record from an earlier scan loses that record, unless it failed only in the content phase, or - is an operand dropped before the database was read that lies under no other - operand; an unreadable directory subtree likewise loses its records (accepted: - the database mirrors what the latest scan could actually verify). + is an operand dropped before the database was read whose record counts as + outside the scanned operands; an unreadable directory subtree likewise loses + its records (accepted: the database mirrors what the latest scan could + actually verify). Concurrency: the walk phase (which also stats files), the hash phase, and the content phase each use a worker pool of `--workers` workers (default diff --git a/TODO.md b/TODO.md index c588fef..185157f 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ # Completed Steps +- with `-x`, an operand under another operand is walked on its own instead of + being dropped when a directory from it (from its directory, for a regular + file) up to but not including that operand is on a different filesystem than + that operand, whose walk then cannot reach it, and the records of files gone + from below it are deleted (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/106) + - with `-x`, `scan` keeps the records below a directory on another filesystem under its operand, and counts their sizes in the size census, instead of deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99) diff --git a/scan.go b/scan.go index f4a031c..0f70378 100644 --- a/scan.go +++ b/scan.go @@ -203,10 +203,16 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. Every remaining -// file is then reachable through exactly one root, so walked paths are -// unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string) []string { +// duplicates and any operand lying under another one. With -x, an +// operand under another one is kept when the walk of the deepest kept +// operand above it cannot reach it: when device puts any directory from +// it (from its directory, for a regular file) up to but not including +// that operand on a different filesystem than that operand. Every +// remaining file is then reachable through exactly one root, so walked +// paths are unique without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, + device func(path string) (uint64, bool), +) []string { // Shorter-first ordering guarantees an ancestor is kept before any // operand under it is considered. sorted := slices.Clone(roots) @@ -221,14 +227,58 @@ func pruneRoots(roots []string) []string { kept := make([]string, 0, len(sorted)) for _, r := range sorted { - if !underAnyRoot(r, kept) { + above := deepestRoot(r, kept) + if above == "" { kept = append(kept, r) + + continue + } + + // Without -x, and always for a duplicate, r is dropped. + if !oneFS || above == r { + continue + } + + aboveDev, aboveOK := device(above) + if !aboveOK { + continue + } + + // The walk of above enters a directory only when it is on + // above's filesystem, and then reaches every regular file in it + // whatever device the file reports, so for a regular-file + // operand the directories checked start at its directory. + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + dev, ok := device(dir) + if ok && dev != aboveDev { + kept = append(kept, r) + + break + } } } return kept } +// deviceOfPath returns the filesystem device ID of the file at path, +// when lstat succeeds and the platform exposes one. +func deviceOfPath(path string) (uint64, bool) { + fi, err := os.Lstat(path) + if err != nil { + return 0, false + } + + return deviceOfInfo(fi) +} + // scanStats summarizes one scan's database synchronization for the // final stderr summary, or for the line an interrupted scan prints. type scanStats struct { @@ -288,21 +338,22 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, // size, head, and tail match another record's). Records outside the // roots are never touched, except that the content phase fills in // their content hash. Operands the walk cannot start from are dropped -// first, so the records beneath them count as outside the roots unless -// they lie under another root. +// first, so the records beneath them are treated as if those operands +// had not been given. func (s *scanState) runPhases(ctx context.Context, roots []string, workers int, oneFS bool, ) error { // Types are checked before pruning so that an operand under a // dropped one is still scanned, not dropped as lying under it. - roots = pruneRoots(s.walkableRoots(roots)) + roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { return err } - changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers)) + changed, unhashed := s.walkPhase( + startWalk(ctx, roots, oneFS, workers), roots) // A cancelled walk stops early, so its size census covers only part // of the roots, and every file it never reached would look vanished @@ -330,10 +381,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, // walkableRoots returns the operands the walk can start from: regular // files, and directories not named .zfs. Every other operand is warned // about, counted as skipped, and dropped. A dropped operand is no -// longer a root, so the records stored beneath it count as outside the -// roots and are not deleted as unverified, unless it lies under another -// root. An operand that fails lstat here is kept, and the walk warns -// about it. +// longer a root, so the records stored beneath it are treated as if it +// had not been given: those under no other root count as outside the +// roots and are not deleted as unverified. An operand that fails lstat +// here is kept, and the walk warns about it. func (s *scanState) walkableRoots(roots []string) []string { kept := make([]string, 0, len(roots)) @@ -393,9 +444,10 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // hashes; both remain candidates until the census decides whether // their sizes are shared. A record below a directory -x did not enter // counts as outside the roots: it is not deleted, and its size joins -// the census. +// the census. That does not hold below a root at or inside that +// directory, since that root's own walk verifies the records below it. func (s *scanState) walkPhase( - events <-chan walkEvent, + events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { prog := newProgress("walk", -1) @@ -441,7 +493,12 @@ func (s *scanState) walkPhase( prog.finish() for path, old := range s.existing { - if underAnyRoot(path, otherFSDirs) { + // Kept only when a directory -x did not enter lies below the + // deepest root above the record. A root at or below that + // directory was walked on its own and did not find the file. + // Both contain the record, so the longer path is the deeper. + otherFSDir := deepestRoot(path, otherFSDirs) + if len(otherFSDir) > len(deepestRoot(path, roots)) { delete(s.existing, path) s.sizes = append(s.sizes, old.size) @@ -875,6 +932,20 @@ func underRoot(path, root string) bool { return strings.HasPrefix(path, prefix) } +// deepestRoot returns the longest of the roots that path is or lies +// under, or "" when it lies under none. +func deepestRoot(path string, roots []string) string { + deepest := "" + + for _, root := range roots { + if underRoot(path, root) && len(root) > len(deepest) { + deepest = root + } + } + + return deepest +} + // dirJob is one directory awaiting traversal by the walk workers. It // carries its operand's filesystem device so -x can stop at // filesystem boundaries. @@ -898,10 +969,10 @@ type walkEvent struct { // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning // event per per-path error, and one event per directory -x did not -// enter. The channel is closed when the walk -// completes, and also when ctx is cancelled — every goroutine in the -// pool abandons its blocking send in that case, so the consumer sees a -// truncated but properly terminated stream instead of a stalled one. +// enter. The channel is closed when the walk completes, and also when +// ctx is cancelled — every goroutine in the pool abandons its blocking +// send in that case, so the consumer sees a truncated but properly +// terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, workers int, ) <-chan walkEvent { @@ -968,7 +1039,8 @@ func operandWarning(root string, fi fs.FileInfo) string { // operand becomes an initial job. walkableRoots has already dropped // every other operand. One that has changed into something else since // is warned about and skipped here; it is still a root, so the records -// stored beneath it are deleted as unverified. +// stored beneath it, other than those under another root, are deleted +// as unverified. func seedRoot(ctx context.Context, root string, events chan<- walkEvent, ) []dirJob { @@ -1168,7 +1240,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // .zfs (ZFS snapshot pseudo-dirs would list every file once per // snapshot), and with -x never enter a directory on a different // filesystem than its operand, sending an event that names it so the -// scan keeps the records below it. +// scan keeps the records below it, except those below an operand at or +// inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, events chan<- walkEvent, ) (dirJob, bool) { diff --git a/scan_test.go b/scan_test.go index 56880ab..778fe64 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1135,14 +1135,72 @@ func TestSubdirJobStatError(t *testing.T) { } } +// syncOneFilesystem scans roots into db with -x and returns the scan +// stats. Every device pruneRoots looks up, and the device each +// operand's walk compares its subdirectories with, comes from devices +// instead of from lstat, so a test can put a path on a filesystem that +// no directory here is on. Unlike runPhases, it skips walkableRoots, +// the check for a cancelled walk, and the content phase. +func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, + devices map[string]uint64, +) scanStats { + t.Helper() + + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + + roots = pruneRoots(roots, true, device) + + ctx := t.Context() + s := &scanState{db: db} + + err := s.loadIndex(ctx, roots) + if err != nil { + t.Fatal(err) + } + + jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + + // seedRoot emits a regular-file operand itself and returns a job + // only for a directory operand. + var initial []dirJob + + for _, root := range roots { + for _, job := range seedRoot(ctx, root, events) { + job.rootDev = devices[root] + initial = append(initial, job) + } + } + + dispatchDirs(ctx, initial, jobs, subdirs) + + changed, unhashed := s.walkPhase(events, roots) + s.partition(changed, unhashed) + + err = s.hashPhase(ctx, 2) + if err != nil { + t.Fatal(err) + } + + err = s.updatePhase(ctx) + if err != nil { + t.Fatal(err) + } + + return s.st +} + // TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a // record below a directory the walk does not enter, because it is on // another filesystem than its operand, counts as outside the operands: // it is not deleted, and its size joins the size census. The record of // a gone file that is not below that directory is still deleted, even -// when its path begins with the directory's name. The walk starts from -// an operand device that no directory here is on, so no second real -// filesystem is needed; the phases after it run as runPhases runs them. +// when its path begins with the directory's name. The operand is given +// a device that no directory here is on, so no second real filesystem +// is needed. func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { t.Parallel() @@ -1174,30 +1232,7 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { t.Skip("platform exposes no device id") } - ctx := t.Context() - s := &scanState{db: db} - - err = s.loadIndex(ctx, []string{dir}) - if err != nil { - t.Fatal(err) - } - - jobs, subdirs, events := startWalkWorkers(ctx, 2, true) - root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true} - dispatchDirs(ctx, []dirJob{root}, jobs, subdirs) - - changed, unhashed := s.walkPhase(events) - s.partition(changed, unhashed) - - err = s.hashPhase(ctx, 2) - if err != nil { - t.Fatal(err) - } - - err = s.updatePhase(ctx) - if err != nil { - t.Fatal(err) - } + syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1}) recs := dbRecords(t, db) recordByPath(t, recs, mounted) @@ -1213,6 +1248,183 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { } } +// TestScanOneFilesystemWalksNestedOperand checks that with -x an +// operand under another one, on a different filesystem than it, is +// walked on its own: a file added below it is recorded, and the record +// of a file gone from below it is deleted. A record below a directory +// that its own walk does not enter is still kept. Each operand is given +// a device that no directory here is on, so neither walk enters a +// subdirectory. +func TestScanOneFilesystemWalksNestedOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + syncOneFilesystem(t, db, []string{dir, mnt}, + map[string]uint64{dir: dev + 1, mnt: dev + 2}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, mnt) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, mnt, filepath.Dir(kept)) + } +} + +// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that +// with -x an operand on the same filesystem as the operand covering it +// is walked on its own when a directory between them is on another +// filesystem, since the walk of the covering operand does not enter +// that directory: a file added below the nested operand is recorded, +// and the record of a file gone from below it is deleted. A record +// below that directory but outside the nested operand is still kept. +// Both operands are given one device that no directory here is on, +// while mnt keeps its own. +func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/back/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + back := filepath.Join(mnt, "back") + syncOneFilesystem(t, db, []string{dir, back}, + map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, back) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, dir, mnt) + } +} + +// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a +// regular-file operand under another operand is dropped when the file +// reports a different device than that operand but its directory does +// not, as a file bind-mounted into the tree does: the walk of the other +// operand enters the directory and reaches the file, so the file is +// recorded once. +func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev, filepath.Dir(file): dev, file: dev + 1, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + +// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a +// regular-file operand under another operand is kept when its directory +// is on a different filesystem than that operand: the walk of the other +// operand does not enter the directory, so the file is recorded once, +// as its own operand. The other operand is given a device that no +// directory here is on. +func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev + 1, filepath.Dir(file): dev, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { @@ -2123,14 +2335,43 @@ func TestHashRuns(t *testing.T) { func TestPruneRoots(t *testing.T) { t.Parallel() + const pqRoot = "/p/q" + + // /p/q is on another filesystem than /p; /p/q/r is on the same one as + // /p/q, /p/s as /p, and /p/t has no known device. + devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1} + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). - got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}) + got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, + false, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { t.Fatalf("pruneRoots = %q, want %q", got, want) } + + nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} + + got = pruneRoots(nested, false, device) + + want = []string{"/p"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots without -x = %q, want %q", got, want) + } + + // With -x, /p/q, on another filesystem than /p, is kept as well. + got = pruneRoots(nested, true, device) + + want = []string{"/p", pqRoot} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with -x = %q, want %q", got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) {