Compare commits

..
Author SHA1 Message Date
sneak 4ae7274bc0 Record each file's size on disk and print it in trees (closes #147)
check / check (push) Canceled after 0s
The files table gets a disk_size column: the file's allocated blocks,
st_blocks × 512, from the lstat the walk already does. scan writes it
with every record it writes. When a rescan finds a file with its size
and mtime unchanged but a different disk_size, it updates disk_size
alone, without reading the file. trees adds disk_size up per directory
and prints it as a column after size. On a compressed or sparse file it
is less than size, so two copies of one tree can differ; each row shows
the dupe copy's.

Judgement calls: a row's disk_size is the dupe copy's, not the first's.
A file whose record gets only a new disk_size counts as unchanged.

Model: opus-5-5
2026-10-27 18:41:24 +00:00
27 changed files with 1583 additions and 2448 deletions

No files matched your search

+2 -2
View File
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
BINARY := sfdupes BINARY := sfdupes
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION) LDFLAGS := -X main.Version=$(VERSION)
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean .PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
@@ -14,7 +14,7 @@ LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
# Default target: build the binary. Phony so go build (which has its # Default target: build the binary. Phony so go build (which has its
# own build cache) always decides what to recompile. # own build cache) always decides what to recompile.
sfdupes: sfdupes:
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes go build -ldflags "$(LDFLAGS)" -o $(BINARY)
build: sfdupes build: sfdupes
+923 -1108
View File
File diff suppressed because it is too large. Load diff
+434 -638
View File
File diff suppressed because it is too large. Load diff
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker package main
import ( import (
"context" "context"
-29
View File
@@ -1,29 +0,0 @@
// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
// KiB, and of the content. The content hash covers the whole file under
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
// file is never read in full. scan maintains a persistent SQLite
// database of file signatures (SFDUPES_DATABASE, default
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes dwim
// sfdupes --version
//
// See README.md for the complete specification.
package main
import (
"os"
"sneak.berlin/go/sfdupes/internal/sfdupes"
)
func main() { os.Exit(sfdupes.Main()) }
+17 -28
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"context" "context"
@@ -146,44 +146,33 @@ var errHashSize = errors.New("hash is not 32 bytes")
// databasePath resolves the database location: SFDUPES_DATABASE when // databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the // set and non-empty, the compiled-in default otherwise. It returns the
// path, made absolute by absolutePath, and where the path came from, // path, with the working directory, its symlinks resolved, put in front
// "default" or "from SFDUPES_DATABASE", for the line each subcommand // of it when it is relative, and where the path came from, "default" or
// prints first on stderr. // "from SFDUPES_DATABASE", for the line each subcommand prints first on
// stderr.
func databasePath() (string, string, error) { func databasePath() (string, string, error) {
path, source := defaultDatabasePath, "default" path, source := defaultDatabasePath, "default"
if p := os.Getenv(databaseEnv); p != "" { if p := os.Getenv(databaseEnv); p != "" {
path, source = p, "from "+databaseEnv path, source = p, "from "+databaseEnv
} }
abs, err := absolutePath(path) if filepath.IsAbs(path) {
return path, source, nil
}
// The working directory as the kernel reports it, with no symlink
// in it, as README "Database" says. os.Getwd may return PWD
// instead, which can name it through a symlink.
wd, err := unix.Getwd()
if err != nil { if err != nil {
return "", "", fmt.Errorf("resolve database path %s: %w", path, err) return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
} }
return abs, source, nil // Not filepath.Abs or filepath.Join: they clean the path, which
} // drops each ".." together with the directory name before it. When
// that directory is a symlink, the cleaned path names another file.
// absolutePath returns path unchanged when it is absolute, and
// otherwise the working directory, a "/" and path. The working
// directory is the one the kernel reports, with no symlink in it;
// os.Getwd may return PWD instead, which can name it through a symlink.
// The result is not cleaned, as filepath.Abs and filepath.Join would
// clean it. Cleaning drops each ".." together with the directory name
// before it. When that name is a symlink, the cleaned path names
// another file than the one the system reaches by following the symlink
// first.
func absolutePath(path string) (string, error) {
if filepath.IsAbs(path) {
return path, nil
}
wd, err := unix.Getwd()
if err != nil {
return "", err
}
// The working directory ends in "/" only when it is the root. // The working directory ends in "/" only when it is the root.
return strings.TrimSuffix(wd, "/") + "/" + path, nil return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
} }
// createDatabaseDirectory creates the directory the database at path // createDatabaseDirectory creates the directory the database at path
+1 -1
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // calls the unexported database openers package main
import ( import (
"context" "context"
+1 -1
View File
@@ -1,4 +1,4 @@
package sfdupes package main
// This file holds two rules for the walk (README "Rules for the walk"): // This file holds two rules for the walk (README "Rules for the walk"):
// the default excludes, the paths a scan's walk does not enter below an // the default excludes, the paths a scan's walk does not enter below an
@@ -1,6 +1,6 @@
//go:build darwin //go:build darwin
package sfdupes package main
import ( import (
"slices" "slices"
@@ -1,6 +1,6 @@
//go:build !darwin //go:build !darwin
package sfdupes package main
import ( import (
"fmt" "fmt"
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes package main
import ( import (
"bytes" "bytes"
@@ -17,9 +17,7 @@ import (
func TestDefaultExcludes(t *testing.T) { func TestDefaultExcludes(t *testing.T) {
t.Parallel() t.Parallel()
// README "Rules for the walk": the Linux and macOS lists, and none // README "Rules for the walk": each system's list.
// on any other system. The Windows list exists in the code only: the
// binary does not build for Windows, so the README leaves it out.
cases := map[string][]string{ cases := map[string][]string{
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
goosDarwin: { goosDarwin: {
@@ -286,7 +284,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
t.Skip(dir + " is not inside a default exclude") t.Skip(dir + " is not inside a default exclude")
} }
+1 -1
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"context" "context"
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec package main
import ( import (
"fmt" "fmt"
-37
View File
@@ -1,37 +0,0 @@
package sfdupes
import (
"context"
"fmt"
"io"
"os"
"runtime"
)
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
// root" runs with no flags, then what "sfdupes trees" runs, and ends
// with a line on stderr saying how to print the report again. The dwim
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
// does, but only during the scan. An error from the scan, an interrupt
// included, ends it before the report. During the report, a signal
// ends it at once, as it ends trees.
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
scanCtx, stop := interruptContext(ctx)
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false)
stop()
if err != nil {
return err
}
err = runTrees(ctx, stdout)
if err != nil {
return err
}
fmt.Fprintln(os.Stderr,
`dwim: run "sfdupes trees" to print this report again`)
return nil
}
-83
View File
@@ -1,83 +0,0 @@
package sfdupes //nolint:testpackage // calls the unexported runDwim
import (
"bytes"
"path/filepath"
"strings"
"testing"
)
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
// dwim scans its root, then prints the trees report, the larger of
// two duplicate-tree pairs first, and ends by saying how to print
// the report again.
t.Setenv(databaseEnv, testDBPath(t))
root := realTempDir(t)
small := []string{
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
}
large := []string{
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
}
var stdout bytes.Buffer
stderr := captureStderr(t)
err := runDwim(t.Context(), root, &stdout)
if err != nil {
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
}
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
if len(lines) != 3 {
t.Fatalf("stdout = %q, want a header and two groups",
stdout.String())
}
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
if !strings.HasPrefix(lines[1], wantFirst) {
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
}
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
if !strings.HasPrefix(lines[2], wantSecond) {
t.Errorf("second group = %q, want it to start %q",
lines[2], wantSecond)
}
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
if got := stderr(); !strings.HasSuffix(got, last) {
t.Errorf("stderr = %q, want it to end %q", got, last)
}
}
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
// A fatal scan error ends dwim before the report: a root that does
// not exist fails the scan as it fails scan itself.
t.Setenv(databaseEnv, testDBPath(t))
missing := filepath.Join(realTempDir(t), "missing")
var stdout bytes.Buffer
stderr := captureStderr(t)
err := runDwim(t.Context(), missing, &stdout)
if err == nil {
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
}
if got := stdout.String(); got != "" {
t.Errorf("stdout = %q, want no report", got)
}
if got := stderr(); strings.Contains(got, "trees:") ||
strings.Contains(got, "dwim:") {
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
}
}
+25 -20
View File
@@ -1,6 +1,23 @@
// Package sfdupes implements the sfdupes command and the database its // Command sfdupes quickly identifies candidate duplicate files across
// subcommands share. // very large filesystems without reading every byte of every file.
package sfdupes // Files are considered duplicates when their sizes are equal and they
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
// KiB, and of the content. The content hash covers the whole file under
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
// file is never read in full. scan maintains a persistent SQLite
// database of file signatures (SFDUPES_DATABASE, default
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
// sfdupes info
// sfdupes --version
//
// See README.md for the complete specification.
package main
import ( import (
"context" "context"
@@ -28,7 +45,6 @@ const (
cmdReport = "report" cmdReport = "report"
cmdTrees = "trees" cmdTrees = "trees"
cmdInfo = "info" cmdInfo = "info"
cmdDwim = "dwim"
) )
// errNoSubcommand is returned by the root command when it is invoked // errNoSubcommand is returned by the root command when it is invoked
@@ -68,22 +84,20 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
return info.Main.Version return info.Main.Version
} }
// Main runs the command line in os.Args and returns the process exit func main() {
// code.
func Main() int {
// Once the reader of a stdout pipe has gone, as in "sfdupes report | // Once the reader of a stdout pipe has gone, as in "sfdupes report |
// head", the Go runtime ends the process with SIGPIPE on the next // head", the Go runtime ends the process with SIGPIPE on the next
// write instead of returning an error (README "Error handling"). // write instead of returning an error (README "Error handling").
// Registering for SIGPIPE with os/signal would change that. // Registering for SIGPIPE with os/signal would change that.
return run(os.Args[1:], os.Stdout, os.Stderr) os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
} }
// run executes args against the command tree and returns the process // run executes args against the command tree and returns the process
// exit code. It is the program's single exit point: the subcommands // exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup — // return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL — // above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees, info and dwim // runs before the process ends. The report, trees and info subcommands
// subcommands write their data to stdout. // write their data to stdout.
func run(args []string, stdout, stderr io.Writer) int { func run(args []string, stdout, stderr io.Writer) int {
// A nil slice makes cobra fall back to os.Args, which would let a // A nil slice makes cobra fall back to os.Args, which would let a
// test binary's own flags reach the command tree. // test binary's own flags reach the command tree.
@@ -187,16 +201,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
}), }),
} }
dwimCmd := &cobra.Command{ root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
Use: cmdDwim,
Short: "Scan / with the default excludes, then print the duplicate-tree report",
Args: cobra.NoArgs,
RunE: runE(func(ctx context.Context, _ []string) error {
return runDwim(ctx, "/", stdout)
}),
}
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
return root return root
} }
+68 -276
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported run package main
import ( import (
"bytes" "bytes"
@@ -14,8 +14,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"golang.org/x/sys/unix"
) )
// usageMarker is the first line of cobra's usage text, which a usage // usageMarker is the first line of cobra's usage text, which a usage
@@ -396,7 +394,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
} }
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) { func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
// README §Settings: the path names the file exactly. With link a // README §Database: the path names the file exactly. With link a
// symlink to the directory real/sub, link/../db.sqlite names // symlink to the directory real/sub, link/../db.sqlite names
// real/db.sqlite, not db.sqlite beside link. Every subcommand must // real/db.sqlite, not db.sqlite beside link. Every subcommand must
// use that file, relative or absolute, and its first line must name // use that file, relative or absolute, and its first line must name
@@ -622,7 +620,6 @@ func TestRunUsageErrors(t *testing.T) {
"report with args": {[]string{cmdReport, "x"}, usageMarker}, "report with args": {[]string{cmdReport, "x"}, usageMarker},
"trees with args": {[]string{cmdTrees, "x"}, usageMarker}, "trees with args": {[]string{cmdTrees, "x"}, usageMarker},
"info with args": {[]string{cmdInfo, "x"}, usageMarker}, "info with args": {[]string{cmdInfo, "x"}, usageMarker},
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker}, "unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
"unknown subcommand": {[]string{"nope"}, "unknown command"}, "unknown subcommand": {[]string{"nope"}, "unknown command"},
} }
@@ -652,7 +649,7 @@ func TestRunUsageErrors(t *testing.T) {
} }
func TestRunScanRejectsWorkersBelowOne(t *testing.T) { func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
// README §Settings: --workers below 1 is a usage error reported in // README §scan mode: --workers below 1 is a usage error reported in
// one line on stderr, before the scan opens the database. // one line on stderr, before the scan opens the database.
for _, workers := range []string{"0", "-1"} { for _, workers := range []string{"0", "-1"} {
t.Run(workers, func(t *testing.T) { t.Run(workers, func(t *testing.T) {
@@ -690,8 +687,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
func TestRunHelp(t *testing.T) { func TestRunHelp(t *testing.T) {
t.Parallel() t.Parallel()
// README §Settings: help goes to stderr, exits 0, and leaves stdout // README §Subcommands: help goes to stderr, exits 0, and leaves
// empty. // stdout empty.
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}} cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
for _, args := range cases { for _, args := range cases {
@@ -717,8 +714,8 @@ func TestRunHelp(t *testing.T) {
func TestRunVersion(t *testing.T) { func TestRunVersion(t *testing.T) {
t.Parallel() t.Parallel()
// README §Settings: the version is one line on stdout, with nothing // README §Subcommands: the version is one line on stdout, with
// on stderr, and exits 0. // nothing on stderr, and exits 0.
info, _ := debug.ReadBuildInfo() info, _ := debug.ReadBuildInfo()
for _, arg := range []string{"--version", "-v"} { for _, arg := range []string{"--version", "-v"} {
@@ -777,8 +774,8 @@ func TestBuildVersion(t *testing.T) {
func TestRunVersionWriteFailureIsFatal(t *testing.T) { func TestRunVersionWriteFailureIsFatal(t *testing.T) {
t.Parallel() t.Parallel()
// README §Error handling: a failed write to stdout is a fatal error, // README §Error handling: a stdout write failure exits 1, reported
// exit 1, reported on stderr starting with "sfdupes:". // in one line on stderr.
var stderr bytes.Buffer var stderr bytes.Buffer
code := run([]string{"--version"}, failingWriter{}, &stderr) code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -800,7 +797,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
func scanFixture(t *testing.T) []string { func scanFixture(t *testing.T) []string {
t.Helper() t.Helper()
dir := realTempDir(t) dir := t.TempDir()
dupes := []string{ dupes := []string{
writeFile(t, dir, "one/a.bin", pattern(1, 300)), writeFile(t, dir, "one/a.bin", pattern(1, 300)),
@@ -870,7 +867,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
writeFile(t, dir, "a.bin", pattern(1, 300)) writeFile(t, dir, "a.bin", pattern(1, 300))
// Same size as a.bin, so the scan reads it, and the read fails. // Same size as a.bin, so the scan reads it, and the read fails.
@@ -898,278 +895,58 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
assertNoSidecars(t, path) assertNoSidecars(t, path)
} }
// realTempDir returns a new temporary directory by its real path, so func TestRunScanSkipsSymlinkOperand(t *testing.T) {
// tests hold where the temporary directory is reached through a
// symlink.
func realTempDir(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return dir
}
// symlink creates link pointing at target.
func symlink(t *testing.T, target, link string) {
t.Helper()
err := os.Symlink(target, link)
if err != nil {
t.Fatal(err)
}
}
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
// The operand is scanned at its real path, and its file is recorded
// there.
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
f := writeFile(t, dir, "target/sub/f", pattern(1, 10)) writeFile(t, dir, "target/sub/f", pattern(1, 10))
link := filepath.Join(dir, "link") link := filepath.Join(dir, "link")
symlink(t, filepath.Join(dir, "target"), link)
stderr := scanOK(t, link) err := os.Symlink(filepath.Join(dir, "target"), link)
if err != nil {
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " + t.Fatal(err)
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
} }
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) { // Scanning a directory through the symlink stores a record beneath
t.Errorf("records = %q, want %q", got, []string{f}) // the symlink's own path for a file beneath its target.
} scanOK(t, filepath.Join(link, "sub"))
assertOperandSkipped(t, path, link, "symlink",
filepath.Join(link, "sub", "f"))
} }
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
// with link a symlink to target. Both resolve under target, so
// target/sub is walked once, as part of target.
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) { func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
f := writeFile(t, dir, "target/sub/f", pattern(1, 10)) writeFile(t, dir, "target/sub/f", pattern(1, 10))
link := filepath.Join(dir, "link") link := filepath.Join(dir, "link")
symlink(t, filepath.Join(dir, "target"), link)
stderr := scanOK(t, link, filepath.Join(link, "sub")) err := os.Symlink(filepath.Join(dir, "target"), link)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
// with dir/link a symlink to a directory outside dir. The walk of dir
// does not follow link, so dir/link/sub is walked on its own at its
// real path, and the record an earlier scan stored there survives.
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
a := writeFile(t, dir, "a", pattern(1, 10))
outside := realTempDir(t)
f := writeFile(t, outside, "sub/f", pattern(2, 10))
symlink(t, outside, filepath.Join(dir, "link"))
scanOK(t, filepath.Join(outside, "sub"))
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
"1 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
want := []string{a, f}
slices.Sort(want)
if got := storedPaths(t, path); !slices.Equal(got, want) {
t.Errorf("records = %q, want %q", got, want)
}
}
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
// link, outside dir, a symlink to dir/real. link resolves inside dir,
// so it is dropped as covered by dir, and its file is recorded once.
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "real/f", pattern(1, 10))
link := filepath.Join(realTempDir(t), "link")
symlink(t, filepath.Join(dir, "real"), link)
stderr := scanOK(t, dir, link)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
// a symlink whose target does not exist. The scan fails with exit 1 and
// an error naming the symlink as given, before it walks anything, so no
// database is created.
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
writeFile(t, dir, "f", pattern(1, 10))
dangling := filepath.Join(realTempDir(t), "dangling")
symlink(t, filepath.Join(dir, "missing"), dangling)
stdout := captureStdout(t)
var stderr bytes.Buffer
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
if code != exitFatal {
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
exitFatal)
}
assertFatalOutput(t, stderr.String(), stdout())
want := "sfdupes: resolve " + dangling + ": "
if !strings.Contains(stderr.String(), want) {
t.Errorf("stderr = %q, want %q", stderr.String(), want)
}
_, err := os.Stat(path)
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("database: stat error %v, want it missing", err)
}
}
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
// after a cd through home/proj, a symlink to data/proj. The file is
// recorded under data/proj, its real path.
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
symlink(t, filepath.Join(dir, "data", "proj"), // link is dropped as a symlink, but link/sub must still be scanned,
filepath.Join(dir, "home", "proj")) // not dropped as lying under link.
scanOK(t, link, filepath.Join(link, "sub"))
// t.Chdir also sets PWD to the path through the symlink, as a db, err := openDB(path, reportParams)
// shell does after cd.
t.Chdir(filepath.Join(dir, "home", "proj"))
scanOK(t, ".")
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
t.Errorf("records = %q, want %q", got, []string{f})
}
}
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
// a cd through home/proj. As with realpath, the ".." counts from
// data/proj, so data is scanned and home is not.
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
for _, tc := range []struct{ wd, operand string }{
{"", "home/proj/.."},
{"home/proj", ".."},
} {
t.Run(tc.operand, func(t *testing.T) {
path := testDBPath(t)
t.Setenv(databaseEnv, path)
dir := realTempDir(t)
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
g := writeFile(t, dir, "data/g", pattern(2, 10))
writeFile(t, dir, "home/h", pattern(3, 10))
symlink(t, filepath.Join(dir, "data", "proj"),
filepath.Join(dir, "home", "proj"))
// t.Chdir also sets PWD to the path given, as a shell does
// after cd.
t.Chdir(filepath.Join(dir, tc.wd))
scanOK(t, tc.operand)
want := []string{f, g}
slices.Sort(want)
if got := storedPaths(t, path); !slices.Equal(got, want) {
t.Errorf("records = %q, want %q", got, want)
}
})
}
}
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
// The scan warns about the FIFO by its real path, counts it as skipped,
// and exits 0.
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t))
dir := realTempDir(t)
fifo := filepath.Join(dir, "fifo")
err := unix.Mkfifo(fifo, 0o600)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
link := filepath.Join(dir, "link") t.Cleanup(func() { _ = db.Close() })
symlink(t, fifo, link)
stderr := scanOK(t, link) recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
warning := "walk " + fifo + ": skipping FIFO operand\n"
if !strings.Contains(stderr, warning) {
t.Errorf("stderr = %q, want %q", stderr, warning)
}
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n"
if !strings.Contains(stderr, summary) {
t.Errorf("stderr = %q, want %q", stderr, summary)
}
} }
func TestRunScanSkipsZFSOperand(t *testing.T) { func TestRunScanSkipsZFSOperand(t *testing.T) {
zfs := filepath.Join(realTempDir(t), ".zfs") zfs := filepath.Join(t.TempDir(), ".zfs")
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10)) f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
// No scan records a file inside a .zfs directory, so the record // No scan records a file inside a .zfs directory, so the record
@@ -1220,7 +997,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly") hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
writeFile(t, hourly, "f", pattern(1, 10)) writeFile(t, hourly, "f", pattern(1, 10))
f := writeFile(t, dir, "f", pattern(2, 10)) f := writeFile(t, dir, "f", pattern(2, 10))
@@ -1264,19 +1041,23 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where // TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about // snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
// by its real path, counted as skipped, and not walked. // by the path given, counted as skipped, and not walked.
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) { func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t)) t.Setenv(databaseEnv, testDBPath(t))
dir := realTempDir(t) dir := t.TempDir()
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10)) writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
snaps := filepath.Join(dir, "snaps") snaps := filepath.Join(dir, "snaps")
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
stderr := scanOK(t, filepath.Join(snaps, "daily")) err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(snaps, "daily")
stderr := scanOK(t, daily)
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n" warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " + summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
"0 unchanged), 1 skipped\n" "0 unchanged), 1 skipped\n"
@@ -1288,19 +1069,24 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
} }
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is // TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
// a symlink to pool/data. The operand is scanned as usual, and its file // a symlink to a directory outside any .zfs directory. The operand is
// is recorded by its real path. // scanned as usual, and its file is recorded by the path given.
func TestRunScanWalksOperandThroughSymlink(t *testing.T) { func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
dir := realTempDir(t) dir := t.TempDir()
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10)) writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
data := filepath.Join(dir, "data") data := filepath.Join(dir, "data")
symlink(t, filepath.Join(dir, "pool", "data"), data)
stderr := scanOK(t, filepath.Join(data, "daily")) err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
if err != nil {
t.Fatal(err)
}
daily := filepath.Join(data, "daily")
stderr := scanOK(t, daily)
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " + summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
"0 unchanged), 0 skipped\n" "0 unchanged), 0 skipped\n"
@@ -1308,9 +1094,14 @@ func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
t.Errorf("stderr = %q, want %q", stderr, summary) t.Errorf("stderr = %q, want %q", stderr, summary)
} }
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) { db, err := openDB(path, reportParams)
t.Errorf("records = %q, want %q", got, []string{f}) if err != nil {
t.Fatal(err)
} }
t.Cleanup(func() { _ = db.Close() })
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
} }
func TestRunReportSucceeds(t *testing.T) { func TestRunReportSucceeds(t *testing.T) {
@@ -1466,7 +1257,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
} }
func TestRunDatabasePathUsedAsGiven(t *testing.T) { func TestRunDatabasePathUsedAsGiven(t *testing.T) {
// README §Settings: the path names the database file exactly. In // README §Database: the path names the database file exactly. In
// SQLite's connection string an unescaped ? or # would end the file // SQLite's connection string an unescaped ? or # would end the file
// name and % would start an escape, and a path starting with // // name and % would start an escape, and a path starting with //
// could be read as a host name. %25 is a valid escape, so unescaped // could be read as a host name. %25 is a valid escape, so unescaped
@@ -1511,8 +1302,9 @@ func holdScanLock(t *testing.T, path string) {
} }
func TestRunSecondScanFails(t *testing.T) { func TestRunSecondScanFails(t *testing.T) {
// README §Database: a second scan that finds the lock held fails // README §Database: while one scan holds the lock, a second scan
// at once, naming the lock file, without opening the database. // fails at once, naming the lock file, without opening the
// database.
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
@@ -1571,7 +1363,7 @@ func assertSecondScanFails(t *testing.T, path string) {
} }
func TestRunSecondScanThroughSymlinkFails(t *testing.T) { func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
// README §Database: a path through a symlink names the same lock // README §Database: every path to one database names the same lock
// file, so while a scan holds it through one path, a scan through // file, so while a scan holds it through one path, a scan through
// another fails, naming the lock file beside the database's real // another fails, naming the lock file beside the database's real
// path. // path.
@@ -1676,8 +1468,8 @@ func TestRunReportsDuringScan(t *testing.T) {
} }
func TestRunStdoutClosedIsFatal(t *testing.T) { func TestRunStdoutClosedIsFatal(t *testing.T) {
// README §Error handling: a failed write to stdout is a fatal error, // README §Error handling: a stdout write failure exits 1, reported
// exit 1, reported on stderr starting with "sfdupes:". // in one line on stderr.
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} { for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Setenv(databaseEnv, testDBPath(t)) t.Setenv(databaseEnv, testDBPath(t))
+1 -1
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"fmt" "fmt"
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported startProgress package main
import ( import (
"fmt" "fmt"
+4 -5
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"context" "context"
@@ -195,11 +195,10 @@ func (w *reportWriter) add(r scanRec) error {
} }
// endGroup ends the paths read with w.sig: two or more are a duplicate // endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it. Empty // group, held with its paths sorted until writeGroups writes it.
// files form no group, since removing one frees nothing.
func (w *reportWriter) endGroup() { func (w *reportWriter) endGroup() {
if len(w.paths) < minGroupSize || w.sig.size == 0 { if len(w.paths) < minGroupSize {
// The paths are dropped, and their slice reused. // A path with no duplicate is dropped, and its slice reused.
w.paths = w.paths[:0] w.paths = w.paths[:0]
return return
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results package main
import ( import (
"bytes" "bytes"
@@ -434,27 +434,6 @@ func TestDupeGroups(t *testing.T) {
} }
} }
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
// duplicate group, while 1-byte files still do.
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
t.Parallel()
e, x := sha256.Sum256(nil), testHash("x")
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
}
groups := dupeGroupsOf(t, recs)
if len(groups) != 1 || groups[0].size != 1 ||
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
}
}
func TestDupeGroupsContentSeparates(t *testing.T) { func TestDupeGroupsContentSeparates(t *testing.T) {
t.Parallel() t.Parallel()
+32 -29
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"cmp" "cmp"
@@ -209,30 +209,29 @@ func interrupted(walked int) error {
return errInterrupted return errInterrupted
} }
// resolveRoots converts each PATH operand to its real path: absolute, // resolveRoots converts each PATH operand to an absolute, lexically
// with every symlink in it followed. Database records are keyed by // cleaned path (symlinks are not resolved) and verifies that it
// that path, so scan results depend neither on the working directory // exists. Database records are keyed by absolute path, so scan results
// nor on the symlinks an operand was given through. An operand that // must not depend on the working directory.
// cannot be resolved, because it or a symlink's target does not exist
// or cannot be read, is a fatal error before any scanning.
func resolveRoots(roots []string) ([]string, error) { func resolveRoots(roots []string) ([]string, error) {
resolved := make([]string, 0, len(roots)) abs := make([]string, 0, len(roots))
for _, root := range roots { for _, root := range roots {
a, err := absolutePath(root) a, err := filepath.Abs(root)
if err != nil { if err != nil {
return nil, fmt.Errorf("resolve %s: %w", root, err) return nil, fmt.Errorf("resolve %s: %w", root, err)
} }
r, err := filepath.EvalSymlinks(a) // A nonexistent operand is a fatal error before any scanning.
_, err = os.Lstat(a)
if err != nil { if err != nil {
return nil, fmt.Errorf("resolve %s: %w", root, err) return nil, err
} }
resolved = append(resolved, r) abs = append(abs, a)
} }
return resolved, nil return abs, nil
} }
// pruneRoots drops operands already covered by another operand: // pruneRoots drops operands already covered by another operand:
@@ -418,16 +417,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
// walkableRoots returns the operands the walk can start from: regular // walkableRoots returns the operands the walk can start from: regular
// files, and directories not named .zfs, none of them inside a // files, and directories not named .zfs, none of them inside a
// directory named .zfs. The operands are real paths from resolveRoots, // directory named .zfs. Every other operand is warned about, counted as
// so these checks apply to what a symlink operand points at. Every // skipped, and dropped. A dropped operand is no longer a root, so the
// other operand is warned about, counted as skipped, and dropped. A // records stored beneath it are treated as if it had not been given:
// dropped operand is no longer a root, so the records stored beneath it // those under no other root count as outside the roots and are not
// are treated as if it had not been given: those under no other root // deleted as unverified. An operand on a filesystem in rules that is not
// count as outside the roots and are not deleted as unverified. An // persistent is dropped the same way, with a line saying so that is not
// operand on a filesystem in rules that is not persistent is dropped // a warning, and is not counted as skipped. An operand that fails lstat
// the same way, with a line saying so that is not a warning, and is not // here is kept, and the walk warns about it unless it no longer exists.
// counted as skipped. An operand that fails lstat here is kept, and the
// walk warns about it unless it no longer exists.
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string { func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
kept := make([]string, 0, len(roots)) kept := make([]string, 0, len(roots))
@@ -557,7 +554,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
// The space a file takes on disk can change while its size and // The space a file takes on disk can change while its size and
// mtime stay the same. ZFS, for one, counts a new file's blocks // mtime stay the same. ZFS, for one, counts a new file's blocks
// up to seconds after the write. // only seconds after the write.
if old.diskSize != ev.rec.diskSize { if old.diskSize != ev.rec.diskSize {
s.diskSizes = append(s.diskSizes, ev.rec) s.diskSizes = append(s.diskSizes, ev.rec)
} }
@@ -897,13 +894,19 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
// operandWarning returns the one-line warning for an operand the walk // operandWarning returns the one-line warning for an operand the walk
// does not start from, naming the path and what it is, or "" for one it // does not start from, naming the path and what it is, or "" for one it
// does: a regular file, or a directory not named .zfs, that is not // does: a regular file, or a directory not named .zfs, that is not
// inside a directory named .zfs. resolveRoots has already followed the // inside a directory named .zfs, neither as given nor with its symlinks
// operand's symlinks, so a symlink here is an operand replaced by one // resolved. Symlinks are never followed, including as operands.
// after it was resolved, and is not followed.
func operandWarning(root string, fi fs.FileInfo) string { func operandWarning(root string, fi fs.FileInfo) string {
// The walk never enters a .zfs directory, so it starts from nothing // The walk never enters a .zfs directory, so it starts from nothing
// inside one either. // inside one either, not even through a symlink. An operand whose
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") { // symlinks cannot be resolved is checked as given.
resolved, err := filepath.EvalSymlinks(root)
if err != nil {
resolved = root
}
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory", return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
root) root)
} }
@@ -1,6 +1,6 @@
//go:build darwin //go:build darwin
package sfdupes package main
import "syscall" import "syscall"
@@ -1,6 +1,6 @@
//go:build !darwin //go:build !darwin
package sfdupes package main
import "syscall" import "syscall"
+56 -114
View File
@@ -1,4 +1,4 @@
package sfdupes //nolint:testpackage // drives the unexported syncTree package main
import ( import (
"bytes" "bytes"
@@ -43,16 +43,10 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
} }
// diskSizeOf returns the bytes the file at path takes on disk, from its // diskSizeOf returns the bytes the file at path takes on disk, from its
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's // own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
// blocks only when it commits the write, up to seconds later, and one // blocks only seconds after the write, and fsync does not hasten that.
// commit can take the file through several growing sizes. So this value // A test therefore calls unix.Sync, which waits for it, before the scan
// can change at any moment. Neither fsync nor unix.Sync hastens that, // whose disk_size it compares with this.
// but a unix.Sync call often takes long enough that the blocks are
// counted when it returns. A test that needs the value to change waits
// for the change. A test that compares it with a scan's disk_size reads
// it before and after the scan. The rewrite test rescans until the two
// reads match, then compares exactly. The other two tests fail only on
// a disk_size outside the two reads.
func diskSizeOf(t *testing.T, path string) int64 { func diskSizeOf(t *testing.T, path string) int64 {
t.Helper() t.Helper()
@@ -1088,13 +1082,12 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
} }
} }
// buildSmokeTree recreates the tree of the test under README // buildSmokeTree recreates the README smoke-test filesystem layout
// "Definition of done" with deterministic content and returns the tree // with deterministic content and returns the tree root.
// root.
func buildSmokeTree(t *testing.T) string { func buildSmokeTree(t *testing.T) string {
t.Helper() t.Helper()
dir := realTempDir(t) dir := t.TempDir()
one := pattern(10, 2000) one := pattern(10, 2000)
f1 := pattern(30, 3000) f1 := pattern(30, 3000)
f2 := pattern(40, 100) f2 := pattern(40, 100)
@@ -1185,12 +1178,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
t.Helper() t.Helper()
groups := dupeGroups(t, db) groups := dupeGroups(t, db)
if len(groups) != 4 { if len(groups) != 5 {
t.Fatalf("len(groups) = %d, want 4", len(groups)) t.Fatalf("len(groups) = %d, want 5", len(groups))
} }
// The empty files empty1 and empty2 form no group. wantSizes := []int64{3000, 2000, 100, 1, 0}
wantSizes := []int64{3000, 2000, 100, 1}
for i, g := range groups { for i, g := range groups {
if g.size != wantSizes[i] { if g.size != wantSizes[i] {
t.Errorf("groups[%d].size = %d, want %d", t.Errorf("groups[%d].size = %d, want %d",
@@ -1455,26 +1447,20 @@ func TestScanStoresDiskSize(t *testing.T) {
unix.Sync() // see diskSizeOf unix.Sync() // see diskSizeOf
size := diskSizeOf(t, p) want := diskSizeOf(t, p)
if size == 0 || size >= length { if want == 0 || want >= length {
t.Fatalf("the file takes %d bytes on disk, want more than none "+ t.Fatalf("the file takes %d bytes on disk, want more than none "+
"and fewer than its length %d", size, length) "and fewer than its length %d", want, length)
} }
for _, operand := range []string{filepath.Dir(p), p} { for _, operand := range []string{filepath.Dir(p), p} {
db := openTestDB(t) db := openTestDB(t)
// ZFS may count the blocks during the scan; see diskSizeOf.
before := diskSizeOf(t, p)
syncTree(t, db, operand) syncTree(t, db, operand)
after := diskSizeOf(t, p)
got := recordByPath(t, dbRecords(t, db), p).diskSize got := recordByPath(t, dbRecords(t, db), p).diskSize
if got < before || got > after { if got != want {
t.Errorf("scan of %s: disk_size = %d, want %d to %d", t.Errorf("scan of %s: disk_size = %d, want %d",
operand, got, before, after) operand, got, want)
} }
} }
} }
@@ -1492,7 +1478,7 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
syncTree(t, db, dir) syncTree(t, db, dir)
recorded := recordByPath(t, dbRecords(t, db), p).diskSize before := recordByPath(t, dbRecords(t, db), p).diskSize
pokeAt(t, p, 512*1024, pattern(7, 64*1024)) pokeAt(t, p, 512*1024, pattern(7, 64*1024))
@@ -1505,53 +1491,22 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Wait for ZFS to count the write; see diskSizeOf. unix.Sync() // see diskSizeOf
deadline := time.Now().Add(30 * time.Second)
for diskSizeOf(t, p) == recorded {
if time.Now().After(deadline) {
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
"after the write into its hole", recorded)
}
unix.Sync() want := diskSizeOf(t, p)
if want == before {
t.Fatalf("the file still takes %d bytes on disk after the write "+
"into its hole", want)
} }
// ZFS may still be counting, so the size may change during a rescan.
// Only a rescan with the same size on both sides of it can be compared
// exactly. A rescan that keeps the old size still fails, because the
// wait made before differ from it.
for {
before := diskSizeOf(t, p)
st := syncTree(t, db, dir) st := syncTree(t, db, dir)
after := diskSizeOf(t, p)
if st != (scanStats{walked: 1, updated: 1}) { if st != (scanStats{walked: 1, updated: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated", st) t.Fatalf("rescan stats = %+v, want 1 updated", st)
} }
if before == after {
got := recordByPath(t, dbRecords(t, db), p).diskSize got := recordByPath(t, dbRecords(t, db), p).diskSize
if got != after { if got != want {
t.Errorf("disk_size = %d, want %d", got, after) t.Errorf("disk_size = %d, want %d", got, want)
}
return
}
if time.Now().After(deadline) {
t.Fatalf("the size on disk was still changing 30 seconds after "+
"the write into its hole: %d, then %d", before, after)
}
// A later mtime makes the next rescan read the file again.
later = later.Add(time.Second)
err = os.Chtimes(p, later, later)
if err != nil {
t.Fatal(err)
}
} }
} }
@@ -1577,21 +1532,15 @@ func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
unix.Sync() // see diskSizeOf unix.Sync() // see diskSizeOf
// ZFS may count the blocks during the rescan; see diskSizeOf.
before := diskSizeOf(t, p)
// A file read again would count as updated. // A file read again would count as updated.
st := syncTree(t, db, dir) st := syncTree(t, db, dir)
after := diskSizeOf(t, p)
if st != (scanStats{walked: 1, unchanged: 1}) { if st != (scanStats{walked: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 unchanged", st) t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
} }
got := recordByPath(t, dbRecords(t, db), p).diskSize got := recordByPath(t, dbRecords(t, db), p).diskSize
if got < before || got > after { if want := diskSizeOf(t, p); got != want {
t.Errorf("disk_size = %d, want %d to %d", got, before, after) t.Errorf("disk_size = %d, want %d", got, want)
} }
} }
@@ -2311,24 +2260,33 @@ func TestUnderRoot(t *testing.T) {
} }
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own // TestRunScanPassesOverDatabase scans a tree that holds the scan's own
// database, and in one case also a hard link to the database file. The // database, reached directly and through a symlinked directory. The
// walk knows the database's files by device ID and inode number, so it // walk passes over the database file, its -wal and -shm files and its
// passes over the database file, its -wal and -shm files, its .lock // .lock file with no line and no count, and the record an earlier scan
// file and the hard link with no line and no count. The record an // left for the .lock file is deleted. A rescan of the unchanged tree
// earlier scan left for the .lock file is deleted. A rescan of the // updates nothing, and report lists none of the database's files.
// unchanged tree updates nothing, and report lists none of the
// database's files.
func TestRunScanPassesOverDatabase(t *testing.T) { func TestRunScanPassesOverDatabase(t *testing.T) {
for _, viaHardLink := range []bool{false, true} { for _, viaSymlink := range []bool{false, true} {
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) { t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
tree := realTempDir(t) base := t.TempDir()
hardLink := filepath.Join(tree, "hardlink") tree := filepath.Join(base, "real", "tree")
operand := tree
writeFile(t, tree, "a", pattern(1, 100)) writeFile(t, tree, "a", pattern(1, 100))
writeFile(t, tree, "b", pattern(1, 100)) writeFile(t, tree, "b", pattern(1, 100))
writeFile(t, tree, "empty1", nil) writeFile(t, tree, "empty1", nil)
writeFile(t, tree, "empty2", nil) writeFile(t, tree, "empty2", nil)
if viaSymlink {
err := os.Symlink(filepath.Join(base, "real"),
filepath.Join(base, "link"))
if err != nil {
t.Fatal(err)
}
operand = filepath.Join(base, "link", "tree")
}
dbPath := filepath.Join(tree, "db.sqlite") dbPath := filepath.Join(tree, "db.sqlite")
t.Setenv(databaseEnv, dbPath) t.Setenv(databaseEnv, dbPath)
@@ -2338,7 +2296,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
} }
err = applyChanges(t.Context(), db, []scanRec{ err = applyChanges(t.Context(), db, []scanRec{
{path: filepath.Join(tree, "db.sqlite.lock")}, {path: filepath.Join(operand, "db.sqlite.lock")},
}, nil, nil) }, nil, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -2346,30 +2304,17 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
closeScanDatabase(t.Context(), db, dbPath) closeScanDatabase(t.Context(), db, dbPath)
if viaHardLink { assertScanSummary(t,
err = os.Link(dbPath, hardLink)
if err != nil {
t.Fatal(err)
}
}
stderr := assertScanSummary(t,
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)", "4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
tree) operand)
stderr += assertScanSummary(t, assertScanSummary(t,
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)", "4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
tree) operand)
if strings.Contains(stderr, hardLink) {
t.Errorf("scan stderr = %q, want no line naming %s",
stderr, hardLink)
}
report := runStdout(t, cmdReport) report := runStdout(t, cmdReport)
if strings.Contains(report, "db.sqlite") || if strings.Contains(report, "db.sqlite") ||
strings.Contains(report, hardLink) || !strings.Contains(report, filepath.Join(operand, "empty2")) {
!strings.Contains(report, filepath.Join(tree, "b")) { t.Errorf("report = %q, want the empty files and none of "+
t.Errorf("report = %q, want a and b and none of "+
"the database's files", report) "the database's files", report)
} }
}) })
@@ -2393,9 +2338,8 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
// assertScanSummary scans operands and fails the test unless the // assertScanSummary scans operands and fails the test unless the
// summary line gives counts with nothing skipped, and the line naming // summary line gives counts with nothing skipped, and the line naming
// the database is the only one that names one of its files. It returns // the database is the only one that names one of its files.
// everything the scan printed to stderr. func assertScanSummary(t *testing.T, counts string, operands ...string) {
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
t.Helper() t.Helper()
stderr := scanOK(t, operands...) stderr := scanOK(t, operands...)
@@ -2409,6 +2353,4 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) string {
t.Errorf("scan %q: stderr = %q, want only the line naming the "+ t.Errorf("scan %q: stderr = %q, want only the line naming the "+
"database to name one of its files", operands, stderr) "database to name one of its files", operands, stderr)
} }
return stderr
} }
+3 -7
View File
@@ -1,4 +1,4 @@
package sfdupes package main
import ( import (
"context" "context"
@@ -301,9 +301,7 @@ func fileEntry(name string, r scanRec) string {
// collectTreeGroups groups directories by digest, counting each one on // collectTreeGroups groups directories by digest, counting each one on
// prog as it is grouped, and returns every maximal group with two or // prog as it is grouped, and returns every maximal group with two or
// more members, each group's members sorted by path, groups ordered by // more members, each group's members sorted by path, groups ordered by
// tree size descending then by first path ascending. A group whose // tree size descending then by first path ascending.
// trees hold 0 bytes is left out, since removing one frees nothing. The
// groups it implies hold 0 bytes too, so leaving it out hides nothing.
func collectTreeGroups(allDirs []*treeNode, super *treeNode, func collectTreeGroups(allDirs []*treeNode, super *treeNode,
prog *progress, prog *progress,
) [][]*treeNode { ) [][]*treeNode {
@@ -317,9 +315,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
var dupes [][]*treeNode var dupes [][]*treeNode
for _, g := range groups { for _, g := range groups {
// The members share a digest, so they share a total size. if len(g) < minGroupSize || suppressed(g, super) {
if len(g) < minGroupSize || g[0].totalSize == 0 ||
suppressed(g, super) {
continue continue
} }
@@ -1,8 +1,7 @@
package sfdupes //nolint:testpackage // checks the unexported treeNode results package main
import ( import (
"bytes" "bytes"
"crypto/sha256"
"database/sql" "database/sql"
"os" "os"
"slices" "slices"
@@ -16,9 +15,9 @@ import (
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " + const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
"remain unknown" "remain unknown"
// smokeTreeRecs mirrors the tree of the test under README "Definition // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only // /d/t2 are identical, /d/t3 differs from them only by one filename.
// by one filename. Each file takes one 4 KiB block on disk. // Each file takes one 4 KiB block on disk.
func smokeTreeRecs() []scanRec { func smokeTreeRecs() []scanRec {
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c") f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c") f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
@@ -436,35 +435,6 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
} }
} }
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
// holding only empty files form no group, and that an empty file still
// counts toward the digest of a tree that holds other files.
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
t.Parallel()
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
sha256.Sum256(nil)
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
// Without the empty file, /p/x3 is no copy of /p/x1.
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
}
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil))
want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("groups = %v, want %v", got, want)
}
}
func TestCollectTreeGroupsDifferingParents(t *testing.T) { func TestCollectTreeGroupsDifferingParents(t *testing.T) {
t.Parallel() t.Parallel()