Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97a02da837 | ||
|
|
6c529fcc6d | ||
|
|
202f083cc0 | ||
|
|
1017332d68 | ||
|
|
5c7284c351 | ||
|
|
7cb10ed303 | ||
|
|
da75b713aa | ||
|
|
24b0fc486d | ||
|
|
f52bbe5d95 | ||
|
|
30dae35c51 | ||
|
|
4ecf8dc484 | ||
|
|
2322f8bb02 | ||
|
|
eebf579f11 | ||
|
|
8a4616039b | ||
|
|
c2804d088b |
No files matched your search
@@ -4,7 +4,7 @@ export CGO_ENABLED = 0
|
|||||||
|
|
||||||
BINARY := sfdupes
|
BINARY := sfdupes
|
||||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||||
LDFLAGS := -X main.Version=$(VERSION)
|
LDFLAGS := -X sneak.berlin/go/sfdupes/internal/sfdupes.Version=$(VERSION)
|
||||||
|
|
||||||
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ LDFLAGS := -X main.Version=$(VERSION)
|
|||||||
# Default target: build the binary. Phony so go build (which has its
|
# Default target: build the binary. Phony so go build (which has its
|
||||||
# own build cache) always decides what to recompile.
|
# own build cache) always decides what to recompile.
|
||||||
sfdupes:
|
sfdupes:
|
||||||
go build -ldflags "$(LDFLAGS)" -o $(BINARY)
|
go build -ldflags "$(LDFLAGS)" -o $(BINARY) ./cmd/sfdupes
|
||||||
|
|
||||||
build: sfdupes
|
build: sfdupes
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
// Command sfdupes quickly identifies candidate duplicate files across
|
||||||
|
// very large filesystems without reading every byte of every file.
|
||||||
|
// Files are considered duplicates when their sizes are equal and they
|
||||||
|
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
||||||
|
// KiB, and of the content. The content hash covers the whole file under
|
||||||
|
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
||||||
|
// file is never read in full. scan maintains a persistent SQLite
|
||||||
|
// database of file signatures (SFDUPES_DATABASE, default
|
||||||
|
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
||||||
|
// sfdupes report > dupes.tsv
|
||||||
|
// sfdupes trees > dupetrees.tsv
|
||||||
|
// sfdupes info
|
||||||
|
// sfdupes dwim
|
||||||
|
// sfdupes --version
|
||||||
|
//
|
||||||
|
// See README.md for the complete specification.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"sneak.berlin/go/sfdupes/internal/sfdupes"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() { os.Exit(sfdupes.Main()) }
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // cancels the unexported runScan and hashWorker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,23 +1,6 @@
|
|||||||
// Command sfdupes quickly identifies candidate duplicate files across
|
// Package sfdupes implements the sfdupes command and the database its
|
||||||
// very large filesystems without reading every byte of every file.
|
// subcommands share.
|
||||||
// Files are considered duplicates when their sizes are equal and they
|
package sfdupes
|
||||||
// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
|
|
||||||
// KiB, and of the content. The content hash covers the whole file under
|
|
||||||
// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
|
|
||||||
// file is never read in full. scan maintains a persistent SQLite
|
|
||||||
// database of file signatures (SFDUPES_DATABASE, default
|
|
||||||
// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
|
|
||||||
//
|
|
||||||
// Usage:
|
|
||||||
//
|
|
||||||
// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
|
|
||||||
// sfdupes report > dupes.tsv
|
|
||||||
// sfdupes trees > dupetrees.tsv
|
|
||||||
// sfdupes info
|
|
||||||
// sfdupes --version
|
|
||||||
//
|
|
||||||
// See README.md for the complete specification.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -45,8 +28,16 @@ const (
|
|||||||
cmdReport = "report"
|
cmdReport = "report"
|
||||||
cmdTrees = "trees"
|
cmdTrees = "trees"
|
||||||
cmdInfo = "info"
|
cmdInfo = "info"
|
||||||
|
cmdDwim = "dwim"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// reportPathsHelp explains the PATH operands of report and trees in
|
||||||
|
// their help.
|
||||||
|
const reportPathsHelp = "With PATH operands, only the records at or " +
|
||||||
|
"below them are used. A relative PATH is joined to the working " +
|
||||||
|
"directory. No PATH is looked up on the filesystem, so a symlink in " +
|
||||||
|
"one is not followed."
|
||||||
|
|
||||||
// errNoSubcommand is returned by the root command when it is invoked
|
// errNoSubcommand is returned by the root command when it is invoked
|
||||||
// without a subcommand. That is a usage error, and the usage text
|
// without a subcommand. That is a usage error, and the usage text
|
||||||
// cobra prints for it is the whole message.
|
// cobra prints for it is the whole message.
|
||||||
@@ -84,20 +75,22 @@ func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
|
|||||||
return info.Main.Version
|
return info.Main.Version
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
// Main runs the command line in os.Args and returns the process exit
|
||||||
|
// code.
|
||||||
|
func Main() int {
|
||||||
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
// Once the reader of a stdout pipe has gone, as in "sfdupes report |
|
||||||
// head", the Go runtime ends the process with SIGPIPE on the next
|
// head", the Go runtime ends the process with SIGPIPE on the next
|
||||||
// write instead of returning an error (README "Error handling").
|
// write instead of returning an error (README "Error handling").
|
||||||
// Registering for SIGPIPE with os/signal would change that.
|
// Registering for SIGPIPE with os/signal would change that.
|
||||||
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
return run(os.Args[1:], os.Stdout, os.Stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// run executes args against the command tree and returns the process
|
// run executes args against the command tree and returns the process
|
||||||
// exit code. It is the program's single exit point: the subcommands
|
// exit code. It is the program's single exit point: the subcommands
|
||||||
// return their errors instead of exiting, so every deferred cleanup —
|
// return their errors instead of exiting, so every deferred cleanup —
|
||||||
// above all closing the database, which checkpoints the SQLite WAL —
|
// above all closing the database, which checkpoints the SQLite WAL —
|
||||||
// runs before the process ends. The report, trees and info subcommands
|
// runs before the process ends. The report, trees, info and dwim
|
||||||
// write their data to stdout.
|
// subcommands write their data to stdout.
|
||||||
func run(args []string, stdout, stderr io.Writer) int {
|
func run(args []string, stdout, stderr io.Writer) int {
|
||||||
// A nil slice makes cobra fall back to os.Args, which would let a
|
// A nil slice makes cobra fall back to os.Args, which would let a
|
||||||
// test binary's own flags reach the command tree.
|
// test binary's own flags reach the command tree.
|
||||||
@@ -175,20 +168,24 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
"print the version to stdout")
|
"print the version to stdout")
|
||||||
|
|
||||||
reportCmd := &cobra.Command{
|
reportCmd := &cobra.Command{
|
||||||
Use: cmdReport,
|
Use: cmdReport + " [PATH...]",
|
||||||
Short: "Read the scan database and print the file-level duplicates report",
|
Short: "Read the scan database and print the file-level duplicates report",
|
||||||
Args: cobra.NoArgs,
|
Long: "Read the scan database and print the file-level duplicates " +
|
||||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
"report.\n\n" + reportPathsHelp,
|
||||||
return runReport(ctx, stdout)
|
Args: cobra.ArbitraryArgs,
|
||||||
|
RunE: runE(func(ctx context.Context, args []string) error {
|
||||||
|
return runReport(ctx, args, stdout)
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
treesCmd := &cobra.Command{
|
treesCmd := &cobra.Command{
|
||||||
Use: cmdTrees,
|
Use: cmdTrees + " [PATH...]",
|
||||||
Short: "Read the scan database and print the duplicate-tree report",
|
Short: "Read the scan database and print the duplicate-tree report",
|
||||||
Args: cobra.NoArgs,
|
Long: "Read the scan database and print the duplicate-tree " +
|
||||||
RunE: runE(func(ctx context.Context, _ []string) error {
|
"report.\n\n" + reportPathsHelp,
|
||||||
return runTrees(ctx, stdout)
|
Args: cobra.ArbitraryArgs,
|
||||||
|
RunE: runE(func(ctx context.Context, args []string) error {
|
||||||
|
return runTrees(ctx, args, stdout)
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -201,7 +198,16 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
|
dwimCmd := &cobra.Command{
|
||||||
|
Use: cmdDwim,
|
||||||
|
Short: "Scan / with the default excludes, then print the duplicate-tree report",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: runE(func(ctx context.Context, _ []string) error {
|
||||||
|
return runDwim(ctx, "/", stdout)
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd, dwimCmd)
|
||||||
|
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported run
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -14,6 +14,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
// usageMarker is the first line of cobra's usage text, which a usage
|
// usageMarker is the first line of cobra's usage text, which a usage
|
||||||
@@ -394,7 +396,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
||||||
// README §Database: the path names the file exactly. With link a
|
// README §Settings: the path names the file exactly. With link a
|
||||||
// symlink to the directory real/sub, link/../db.sqlite names
|
// symlink to the directory real/sub, link/../db.sqlite names
|
||||||
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
||||||
// use that file, relative or absolute, and its first line must name
|
// use that file, relative or absolute, and its first line must name
|
||||||
@@ -574,14 +576,14 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
|
|||||||
},
|
},
|
||||||
cmdReport: {
|
cmdReport: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runReport(cancelledContext(t), io.Discard)
|
return runReport(cancelledContext(t), nil, io.Discard)
|
||||||
},
|
},
|
||||||
want: "report: database " + defaultDatabasePath + " (default)\n" +
|
want: "report: database " + defaultDatabasePath + " (default)\n" +
|
||||||
reportOpenLine + "\n" + reportOpenLine + "\n",
|
reportOpenLine + "\n" + reportOpenLine + "\n",
|
||||||
},
|
},
|
||||||
cmdTrees: {
|
cmdTrees: {
|
||||||
run: func() error {
|
run: func() error {
|
||||||
return runTrees(cancelledContext(t), io.Discard)
|
return runTrees(cancelledContext(t), nil, io.Discard)
|
||||||
},
|
},
|
||||||
want: "trees: database " + defaultDatabasePath + " (default)\n" +
|
want: "trees: database " + defaultDatabasePath + " (default)\n" +
|
||||||
treesOpenLine + "\n" + treesOpenLine + "\n",
|
treesOpenLine + "\n" + treesOpenLine + "\n",
|
||||||
@@ -617,9 +619,8 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
"no subcommand": {[]string{}, usageMarker},
|
"no subcommand": {[]string{}, usageMarker},
|
||||||
"scan without paths": {[]string{cmdScan}, usageMarker},
|
"scan without paths": {[]string{cmdScan}, usageMarker},
|
||||||
"report with args": {[]string{cmdReport, "x"}, usageMarker},
|
|
||||||
"trees with args": {[]string{cmdTrees, "x"}, usageMarker},
|
|
||||||
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
"info with args": {[]string{cmdInfo, "x"}, usageMarker},
|
||||||
|
"dwim with args": {[]string{cmdDwim, "x"}, usageMarker},
|
||||||
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
"unknown flag": {[]string{cmdScan, "--nope", "/"}, usageMarker},
|
||||||
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
"unknown subcommand": {[]string{"nope"}, "unknown command"},
|
||||||
}
|
}
|
||||||
@@ -649,7 +650,7 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||||
// README §scan mode: --workers below 1 is a usage error reported in
|
// README §Settings: --workers below 1 is a usage error reported in
|
||||||
// one line on stderr, before the scan opens the database.
|
// one line on stderr, before the scan opens the database.
|
||||||
for _, workers := range []string{"0", "-1"} {
|
for _, workers := range []string{"0", "-1"} {
|
||||||
t.Run(workers, func(t *testing.T) {
|
t.Run(workers, func(t *testing.T) {
|
||||||
@@ -687,8 +688,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
func TestRunHelp(t *testing.T) {
|
func TestRunHelp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Subcommands: help goes to stderr, exits 0, and leaves
|
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
||||||
// stdout empty.
|
// empty.
|
||||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||||
|
|
||||||
for _, args := range cases {
|
for _, args := range cases {
|
||||||
@@ -714,8 +715,8 @@ func TestRunHelp(t *testing.T) {
|
|||||||
func TestRunVersion(t *testing.T) {
|
func TestRunVersion(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Subcommands: the version is one line on stdout, with
|
// README §Settings: the version is one line on stdout, with nothing
|
||||||
// nothing on stderr, and exits 0.
|
// on stderr, and exits 0.
|
||||||
info, _ := debug.ReadBuildInfo()
|
info, _ := debug.ReadBuildInfo()
|
||||||
|
|
||||||
for _, arg := range []string{"--version", "-v"} {
|
for _, arg := range []string{"--version", "-v"} {
|
||||||
@@ -774,8 +775,8 @@ func TestBuildVersion(t *testing.T) {
|
|||||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Error handling: a stdout write failure exits 1, reported
|
// README §Error handling: a failed write to stdout is a fatal error,
|
||||||
// in one line on stderr.
|
// exit 1, reported on stderr starting with "sfdupes:".
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
||||||
@@ -797,7 +798,7 @@ func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
|||||||
func scanFixture(t *testing.T) []string {
|
func scanFixture(t *testing.T) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
|
|
||||||
dupes := []string{
|
dupes := []string{
|
||||||
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
writeFile(t, dir, "one/a.bin", pattern(1, 300)),
|
||||||
@@ -867,7 +868,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "a.bin", pattern(1, 300))
|
writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||||
|
|
||||||
// Same size as a.bin, so the scan reads it, and the read fails.
|
// Same size as a.bin, so the scan reads it, and the read fails.
|
||||||
@@ -895,58 +896,278 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
assertNoSidecars(t, path)
|
assertNoSidecars(t, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
|
// realTempDir returns a new temporary directory by its real path, so
|
||||||
path := testDBPath(t)
|
// tests hold where the temporary directory is reached through a
|
||||||
t.Setenv(databaseEnv, path)
|
// symlink.
|
||||||
|
func realTempDir(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scanning a directory through the symlink stores a record beneath
|
return dir
|
||||||
// the symlink's own path for a file beneath its target.
|
|
||||||
scanOK(t, filepath.Join(link, "sub"))
|
|
||||||
|
|
||||||
assertOperandSkipped(t, path, link, "symlink",
|
|
||||||
filepath.Join(link, "sub", "f"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// symlink creates link pointing at target.
|
||||||
|
func symlink(t *testing.T, target, link string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.Symlink(target, link)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
|
||||||
|
// The operand is scanned at its real path, and its file is recorded
|
||||||
|
// there.
|
||||||
|
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, filepath.Join(dir, "target"), link)
|
||||||
|
|
||||||
|
stderr := scanOK(t, link)
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
|
||||||
|
// with link a symlink to target. Both resolve under target, so
|
||||||
|
// target/sub is walked once, as part of target.
|
||||||
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||||
|
|
||||||
link := filepath.Join(dir, "link")
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, filepath.Join(dir, "target"), link)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
stderr := scanOK(t, link, filepath.Join(link, "sub"))
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
|
||||||
|
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
|
||||||
|
// with dir/link a symlink to a directory outside dir. The walk of dir
|
||||||
|
// does not follow link, so dir/link/sub is walked on its own at its
|
||||||
|
// real path, and the record an earlier scan stored there survives.
|
||||||
|
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
a := writeFile(t, dir, "a", pattern(1, 10))
|
||||||
|
|
||||||
|
outside := realTempDir(t)
|
||||||
|
f := writeFile(t, outside, "sub/f", pattern(2, 10))
|
||||||
|
|
||||||
|
symlink(t, outside, filepath.Join(dir, "link"))
|
||||||
|
|
||||||
|
scanOK(t, filepath.Join(outside, "sub"))
|
||||||
|
|
||||||
|
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
|
||||||
|
|
||||||
|
summary := "scan: 2 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"1 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{a, f}
|
||||||
|
slices.Sort(want)
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||||
|
t.Errorf("records = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
|
||||||
|
// link, outside dir, a symlink to dir/real. link resolves inside dir,
|
||||||
|
// so it is dropped as covered by dir, and its file is recorded once.
|
||||||
|
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "real/f", pattern(1, 10))
|
||||||
|
|
||||||
|
link := filepath.Join(realTempDir(t), "link")
|
||||||
|
symlink(t, filepath.Join(dir, "real"), link)
|
||||||
|
|
||||||
|
stderr := scanOK(t, dir, link)
|
||||||
|
|
||||||
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 0 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
|
||||||
|
// a symlink whose target does not exist. The scan fails with exit 1 and
|
||||||
|
// an error naming the symlink as given, before it walks anything, so no
|
||||||
|
// database is created.
|
||||||
|
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
writeFile(t, dir, "f", pattern(1, 10))
|
||||||
|
|
||||||
|
dangling := filepath.Join(realTempDir(t), "dangling")
|
||||||
|
symlink(t, filepath.Join(dir, "missing"), dangling)
|
||||||
|
|
||||||
|
stdout := captureStdout(t)
|
||||||
|
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
|
||||||
|
if code != exitFatal {
|
||||||
|
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
|
||||||
|
exitFatal)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertFatalOutput(t, stderr.String(), stdout())
|
||||||
|
|
||||||
|
want := "sfdupes: resolve " + dangling + ": "
|
||||||
|
if !strings.Contains(stderr.String(), want) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
if !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
t.Errorf("database: stat error %v, want it missing", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
|
||||||
|
// after a cd through home/proj, a symlink to data/proj. The file is
|
||||||
|
// recorded under data/proj, its real path.
|
||||||
|
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||||
|
|
||||||
|
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// link is dropped as a symlink, but link/sub must still be scanned,
|
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||||
// not dropped as lying under link.
|
filepath.Join(dir, "home", "proj"))
|
||||||
scanOK(t, link, filepath.Join(link, "sub"))
|
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
// t.Chdir also sets PWD to the path through the symlink, as a
|
||||||
|
// shell does after cd.
|
||||||
|
t.Chdir(filepath.Join(dir, "home", "proj"))
|
||||||
|
|
||||||
|
scanOK(t, ".")
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
|
||||||
|
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
|
||||||
|
// a cd through home/proj. As with realpath, the ".." counts from
|
||||||
|
// data/proj, so data is scanned and home is not.
|
||||||
|
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ wd, operand string }{
|
||||||
|
{"", "home/proj/.."},
|
||||||
|
{"home/proj", ".."},
|
||||||
|
} {
|
||||||
|
t.Run(tc.operand, func(t *testing.T) {
|
||||||
|
path := testDBPath(t)
|
||||||
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||||
|
g := writeFile(t, dir, "data/g", pattern(2, 10))
|
||||||
|
writeFile(t, dir, "home/h", pattern(3, 10))
|
||||||
|
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||||
|
filepath.Join(dir, "home", "proj"))
|
||||||
|
|
||||||
|
// t.Chdir also sets PWD to the path given, as a shell does
|
||||||
|
// after cd.
|
||||||
|
t.Chdir(filepath.Join(dir, tc.wd))
|
||||||
|
|
||||||
|
scanOK(t, tc.operand)
|
||||||
|
|
||||||
|
want := []string{f, g}
|
||||||
|
slices.Sort(want)
|
||||||
|
|
||||||
|
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||||
|
t.Errorf("records = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
|
||||||
|
// The scan warns about the FIFO by its real path, counts it as skipped,
|
||||||
|
// and exits 0.
|
||||||
|
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
dir := realTempDir(t)
|
||||||
|
fifo := filepath.Join(dir, "fifo")
|
||||||
|
|
||||||
|
err := unix.Mkfifo(fifo, 0o600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
link := filepath.Join(dir, "link")
|
||||||
|
symlink(t, fifo, link)
|
||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
|
stderr := scanOK(t, link)
|
||||||
|
|
||||||
|
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
||||||
|
if !strings.Contains(stderr, warning) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||||
|
"0 unchanged), 1 skipped\n"
|
||||||
|
if !strings.Contains(stderr, summary) {
|
||||||
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||||
zfs := filepath.Join(t.TempDir(), ".zfs")
|
zfs := filepath.Join(realTempDir(t), ".zfs")
|
||||||
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
||||||
|
|
||||||
// No scan records a file inside a .zfs directory, so the record
|
// No scan records a file inside a .zfs directory, so the record
|
||||||
@@ -997,7 +1218,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
||||||
writeFile(t, hourly, "f", pattern(1, 10))
|
writeFile(t, hourly, "f", pattern(1, 10))
|
||||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
f := writeFile(t, dir, "f", pattern(2, 10))
|
||||||
@@ -1041,23 +1262,19 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|||||||
|
|
||||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
||||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
||||||
// by the path given, counted as skipped, and not walked.
|
// by its real path, counted as skipped, and not walked.
|
||||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
||||||
|
|
||||||
snaps := filepath.Join(dir, "snaps")
|
snaps := filepath.Join(dir, "snaps")
|
||||||
|
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(snaps, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
|
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
||||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||||
"0 unchanged), 1 skipped\n"
|
"0 unchanged), 1 skipped\n"
|
||||||
@@ -1069,24 +1286,19 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
||||||
// a symlink to a directory outside any .zfs directory. The operand is
|
// a symlink to pool/data. The operand is scanned as usual, and its file
|
||||||
// scanned as usual, and its file is recorded by the path given.
|
// is recorded by its real path.
|
||||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||||
|
|
||||||
data := filepath.Join(dir, "data")
|
data := filepath.Join(dir, "data")
|
||||||
|
symlink(t, filepath.Join(dir, "pool", "data"), data)
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
|
stderr := scanOK(t, filepath.Join(data, "daily"))
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(data, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||||
"0 unchanged), 0 skipped\n"
|
"0 unchanged), 0 skipped\n"
|
||||||
@@ -1094,14 +1306,9 @@ func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
|||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||||
}
|
}
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||||
if err != nil {
|
t.Errorf("records = %q, want %q", got, []string{f})
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunReportSucceeds(t *testing.T) {
|
func TestRunReportSucceeds(t *testing.T) {
|
||||||
@@ -1257,7 +1464,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
||||||
// README §Database: the path names the database file exactly. In
|
// README §Settings: the path names the database file exactly. In
|
||||||
// SQLite's connection string an unescaped ? or # would end the file
|
// SQLite's connection string an unescaped ? or # would end the file
|
||||||
// name and % would start an escape, and a path starting with //
|
// name and % would start an escape, and a path starting with //
|
||||||
// could be read as a host name. %25 is a valid escape, so unescaped
|
// could be read as a host name. %25 is a valid escape, so unescaped
|
||||||
@@ -1302,9 +1509,8 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: while one scan holds the lock, a second scan
|
// README §Database: a second scan that finds the lock held fails
|
||||||
// fails at once, naming the lock file, without opening the
|
// at once, naming the lock file, without opening the database.
|
||||||
// database.
|
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
@@ -1363,7 +1569,7 @@ func assertSecondScanFails(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||||
// README §Database: every path to one database names the same lock
|
// README §Database: a path through a symlink names the same lock
|
||||||
// file, so while a scan holds it through one path, a scan through
|
// file, so while a scan holds it through one path, a scan through
|
||||||
// another fails, naming the lock file beside the database's real
|
// another fails, naming the lock file beside the database's real
|
||||||
// path.
|
// path.
|
||||||
@@ -1468,8 +1674,8 @@ func TestRunReportsDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
||||||
// README §Error handling: a stdout write failure exits 1, reported
|
// README §Error handling: a failed write to stdout is a fatal error,
|
||||||
// in one line on stderr.
|
// exit 1, reported on stderr starting with "sfdupes:".
|
||||||
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
@@ -1518,9 +1724,13 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
|
|||||||
scanFixture(t)
|
scanFixture(t)
|
||||||
|
|
||||||
cases := map[string]func(context.Context, io.Writer) error{
|
cases := map[string]func(context.Context, io.Writer) error{
|
||||||
cmdReport: runReport,
|
cmdReport: func(ctx context.Context, w io.Writer) error {
|
||||||
cmdTrees: runTrees,
|
return runReport(ctx, nil, w)
|
||||||
cmdInfo: runInfo,
|
},
|
||||||
|
cmdTrees: func(ctx context.Context, w io.Writer) error {
|
||||||
|
return runTrees(ctx, nil, w)
|
||||||
|
},
|
||||||
|
cmdInfo: runInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, fn := range cases {
|
for name, fn := range cases {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -146,33 +146,44 @@ var errHashSize = errors.New("hash is not 32 bytes")
|
|||||||
|
|
||||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||||
// path, with the working directory, its symlinks resolved, put in front
|
// path, made absolute by absolutePath, and where the path came from,
|
||||||
// of it when it is relative, and where the path came from, "default" or
|
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
||||||
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
|
// prints first on stderr.
|
||||||
// stderr.
|
|
||||||
func databasePath() (string, string, error) {
|
func databasePath() (string, string, error) {
|
||||||
path, source := defaultDatabasePath, "default"
|
path, source := defaultDatabasePath, "default"
|
||||||
if p := os.Getenv(databaseEnv); p != "" {
|
if p := os.Getenv(databaseEnv); p != "" {
|
||||||
path, source = p, "from "+databaseEnv
|
path, source = p, "from "+databaseEnv
|
||||||
}
|
}
|
||||||
|
|
||||||
if filepath.IsAbs(path) {
|
abs, err := absolutePath(path)
|
||||||
return path, source, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The working directory as the kernel reports it, with no symlink
|
|
||||||
// in it, as README "Database" says. os.Getwd may return PWD
|
|
||||||
// instead, which can name it through a symlink.
|
|
||||||
wd, err := unix.Getwd()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Not filepath.Abs or filepath.Join: they clean the path, which
|
return abs, source, nil
|
||||||
// drops each ".." together with the directory name before it. When
|
}
|
||||||
// that directory is a symlink, the cleaned path names another file.
|
|
||||||
|
// absolutePath returns path unchanged when it is absolute, and
|
||||||
|
// otherwise the working directory, a "/" and path. The working
|
||||||
|
// directory is the one the kernel reports, with no symlink in it;
|
||||||
|
// os.Getwd may return PWD instead, which can name it through a symlink.
|
||||||
|
// The result is not cleaned, as filepath.Abs and filepath.Join would
|
||||||
|
// clean it. Cleaning drops each ".." together with the directory name
|
||||||
|
// before it. When that name is a symlink, the cleaned path names
|
||||||
|
// another file than the one the system reaches by following the symlink
|
||||||
|
// first.
|
||||||
|
func absolutePath(path string) (string, error) {
|
||||||
|
if filepath.IsAbs(path) {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
wd, err := unix.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
// The working directory ends in "/" only when it is the root.
|
// The working directory ends in "/" only when it is the root.
|
||||||
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
return strings.TrimSuffix(wd, "/") + "/" + path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createDatabaseDirectory creates the directory the database at path
|
// createDatabaseDirectory creates the directory the database at path
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // calls the unexported database openers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package sfdupes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runDwim implements the dwim subcommand: it runs what "sfdupes scan
|
||||||
|
// root" runs with no flags, then what "sfdupes trees" runs, and ends
|
||||||
|
// with a line on stderr saying how to print the report again. The dwim
|
||||||
|
// command passes "/" as root. It catches SIGINT and SIGTERM as scan
|
||||||
|
// does, but only during the scan. An error from the scan, an interrupt
|
||||||
|
// included, ends it before the report. During the report, a signal
|
||||||
|
// ends it at once, as it ends trees.
|
||||||
|
func runDwim(ctx context.Context, root string, stdout io.Writer) error {
|
||||||
|
scanCtx, stop := interruptContext(ctx)
|
||||||
|
err := runScan(scanCtx, []string{root}, runtime.NumCPU(), false, false)
|
||||||
|
|
||||||
|
stop()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = runTrees(ctx, nil, stdout)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(os.Stderr,
|
||||||
|
`dwim: run "sfdupes trees" to print this report again`)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package sfdupes //nolint:testpackage // calls the unexported runDwim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunDwimPrintsLargestTreesFirst(t *testing.T) {
|
||||||
|
// dwim scans its root, then prints the trees report, the larger of
|
||||||
|
// two duplicate-tree pairs first, and ends by saying how to print
|
||||||
|
// the report again.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
root := realTempDir(t)
|
||||||
|
|
||||||
|
small := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "a1/f.bin", pattern(1, 300))),
|
||||||
|
filepath.Dir(writeFile(t, root, "a2/f.bin", pattern(1, 300))),
|
||||||
|
}
|
||||||
|
large := []string{
|
||||||
|
filepath.Dir(writeFile(t, root, "b1/f.bin", pattern(2, 5000))),
|
||||||
|
filepath.Dir(writeFile(t, root, "b2/f.bin", pattern(2, 5000))),
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), root, &stdout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runDwim: %v; stderr: %s", err, stderr())
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(strings.TrimSuffix(stdout.String(), "\n"), "\n")
|
||||||
|
if len(lines) != 3 {
|
||||||
|
t.Fatalf("stdout = %q, want a header and two groups",
|
||||||
|
stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFirst := large[0] + "\t" + large[1] + "\t1\t5000\t"
|
||||||
|
if !strings.HasPrefix(lines[1], wantFirst) {
|
||||||
|
t.Errorf("first group = %q, want it to start %q", lines[1], wantFirst)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantSecond := small[0] + "\t" + small[1] + "\t1\t300\t"
|
||||||
|
if !strings.HasPrefix(lines[2], wantSecond) {
|
||||||
|
t.Errorf("second group = %q, want it to start %q",
|
||||||
|
lines[2], wantSecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
last := "\ndwim: run \"sfdupes trees\" to print this report again\n"
|
||||||
|
if got := stderr(); !strings.HasSuffix(got, last) {
|
||||||
|
t.Errorf("stderr = %q, want it to end %q", got, last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDwimScanErrorPrintsNoReport(t *testing.T) {
|
||||||
|
// A fatal scan error ends dwim before the report: a root that does
|
||||||
|
// not exist fails the scan as it fails scan itself.
|
||||||
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
|
missing := filepath.Join(realTempDir(t), "missing")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
err := runDwim(t.Context(), missing, &stdout)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("runDwim on a missing root: error = nil, want it fatal")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stdout.String(); got != "" {
|
||||||
|
t.Errorf("stdout = %q, want no report", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stderr(); strings.Contains(got, "trees:") ||
|
||||||
|
strings.Contains(got, "dwim:") {
|
||||||
|
t.Errorf("stderr = %q, want nothing from trees or dwim", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
// This file holds two rules for the walk (README "Rules for the walk"):
|
// This file holds two rules for the walk (README "Rules for the walk"):
|
||||||
// the default excludes, the paths a scan's walk does not enter below an
|
// the default excludes, the paths a scan's walk does not enter below an
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"slices"
|
"slices"
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported defaultExcludes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -17,7 +17,9 @@ import (
|
|||||||
func TestDefaultExcludes(t *testing.T) {
|
func TestDefaultExcludes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README "Rules for the walk": each system's list.
|
// README "Rules for the walk": the Linux and macOS lists, and none
|
||||||
|
// on any other system. The Windows list exists in the code only: the
|
||||||
|
// binary does not build for Windows, so the README leaves it out.
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
@@ -284,7 +286,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
||||||
t.Skip(dir + " is not inside a default exclude")
|
t.Skip(dir + " is not inside a default exclude")
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // seeds the database with unexported scanRec
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported startProgress
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -41,10 +42,11 @@ func reportPhase(name string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runReport implements the report subcommand: it prints the file-level
|
// runReport implements the report subcommand: it prints the file-level
|
||||||
// duplicates report as TSV on stdout. It never touches the scanned
|
// duplicates report as TSV on stdout, from the records at or below the
|
||||||
// filesystem; its only I/O is the database, stdout, and stderr. It
|
// given paths, or from every record when none is given. It never
|
||||||
// names the database on stderr before it opens it. Any database
|
// touches the scanned filesystem; its only I/O is the database,
|
||||||
// problem, including a missing database, is fatal.
|
// stdout, and stderr. It names the database on stderr before it opens
|
||||||
|
// it. Any database problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in two phases, each under a progress display of its
|
// The work runs in two phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database) and read (read the number of
|
// own on stderr: open (open the database) and read (read the number of
|
||||||
@@ -52,7 +54,12 @@ func reportPhase(name string) string {
|
|||||||
// order, writing each size's duplicate groups to stdout once that size
|
// order, writing each size's duplicate groups to stdout once that size
|
||||||
// is read; see readReport). Each display ends before the next phase
|
// is read; see readReport). Each display ends before the next phase
|
||||||
// starts, and before an error is returned to be printed.
|
// starts, and before an error is returned to be printed.
|
||||||
func runReport(ctx context.Context, stdout io.Writer) error {
|
func runReport(ctx context.Context, args []string, stdout io.Writer) error {
|
||||||
|
paths, err := reportPaths(args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -83,7 +90,7 @@ func runReport(ctx context.Context, stdout io.Writer) error {
|
|||||||
label: reportPhase("read"), unit: "records", total: -1,
|
label: reportPhase("read"), unit: "records", total: -1,
|
||||||
waitingFor: countingRecords, stalledOn: "db/out",
|
waitingFor: countingRecords, stalledOn: "db/out",
|
||||||
})
|
})
|
||||||
report, err := readReport(ctx, db, dbPath, stdout, prog)
|
report, err := readReport(ctx, db, dbPath, paths, stdout, prog)
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
@@ -104,11 +111,12 @@ func runReport(ctx context.Context, stdout io.Writer) error {
|
|||||||
// dbPath, as its total, then reads every record in signature order,
|
// dbPath, as its total, then reads every record in signature order,
|
||||||
// counting each on prog, and writes the report to stdout: the header
|
// counting each on prog, and writes the report to stdout: the header
|
||||||
// line, then the rows of each size's duplicate groups once that size is
|
// line, then the rows of each size's duplicate groups once that size is
|
||||||
// read. stdout may be the terminal prog is drawn on, so the lines go
|
// read. Only the records that inReportPaths selects from paths join a
|
||||||
|
// group. stdout may be the terminal prog is drawn on, so the lines go
|
||||||
// through prog's writer. It returns what it wrote, with the totals for
|
// through prog's writer. It returns what it wrote, with the totals for
|
||||||
// the summary line.
|
// the summary line.
|
||||||
func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
||||||
stdout io.Writer, prog *progress,
|
paths []string, stdout io.Writer, prog *progress,
|
||||||
) (*reportWriter, error) {
|
) (*reportWriter, error) {
|
||||||
out := prog.writer(stdout)
|
out := prog.writer(stdout)
|
||||||
report := &reportWriter{out: out}
|
report := &reportWriter{out: out}
|
||||||
@@ -123,6 +131,12 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
|||||||
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
|
err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
|
report.records++
|
||||||
|
|
||||||
|
if !inReportPaths(r.path, paths) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
writeErr = report.add(r)
|
writeErr = report.add(r)
|
||||||
|
|
||||||
return writeErr
|
return writeErr
|
||||||
@@ -154,7 +168,7 @@ func readReport(ctx context.Context, db *sql.DB, dbPath string,
|
|||||||
// records of one size, so it holds only the duplicate groups of the
|
// records of one size, so it holds only the duplicate groups of the
|
||||||
// size being read, and writes them, ordered by first path, once a
|
// size being read, and writes them, ordered by first path, once a
|
||||||
// record of a smaller size comes. It counts what the summary line
|
// record of a smaller size comes. It counts what the summary line
|
||||||
// reports.
|
// reports; readReport counts the records read.
|
||||||
type reportWriter struct {
|
type reportWriter struct {
|
||||||
out io.Writer
|
out io.Writer
|
||||||
|
|
||||||
@@ -170,8 +184,6 @@ type reportWriter struct {
|
|||||||
// its paths; any other ends them, and one of a smaller size first
|
// its paths; any other ends them, and one of a smaller size first
|
||||||
// writes the groups of the size before.
|
// writes the groups of the size before.
|
||||||
func (w *reportWriter) add(r scanRec) error {
|
func (w *reportWriter) add(r scanRec) error {
|
||||||
w.records++
|
|
||||||
|
|
||||||
if r.size == w.sig.size && r.head == w.sig.head &&
|
if r.size == w.sig.size && r.head == w.sig.head &&
|
||||||
r.tail == w.sig.tail && r.content == w.sig.content {
|
r.tail == w.sig.tail && r.content == w.sig.content {
|
||||||
w.paths = append(w.paths, r.path)
|
w.paths = append(w.paths, r.path)
|
||||||
@@ -195,10 +207,11 @@ func (w *reportWriter) add(r scanRec) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// endGroup ends the paths read with w.sig: two or more are a duplicate
|
// endGroup ends the paths read with w.sig: two or more are a duplicate
|
||||||
// group, held with its paths sorted until writeGroups writes it.
|
// group, held with its paths sorted until writeGroups writes it. Empty
|
||||||
|
// files form no group, since removing one frees nothing.
|
||||||
func (w *reportWriter) endGroup() {
|
func (w *reportWriter) endGroup() {
|
||||||
if len(w.paths) < minGroupSize {
|
if len(w.paths) < minGroupSize || w.sig.size == 0 {
|
||||||
// A path with no duplicate is dropped, and its slice reused.
|
// The paths are dropped, and their slice reused.
|
||||||
w.paths = w.paths[:0]
|
w.paths = w.paths[:0]
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -245,6 +258,46 @@ func (w *reportWriter) finish() error {
|
|||||||
return w.writeGroups()
|
return w.writeGroups()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportPaths returns the paths given to report or trees, each made
|
||||||
|
// absolute by absolutePath and then cleaned. The records hold real
|
||||||
|
// paths, with no "." or ".." element and no repeated or trailing
|
||||||
|
// slash, so a given path is cleaned to match them. No path is looked up
|
||||||
|
// on the filesystem.
|
||||||
|
func reportPaths(args []string) ([]string, error) {
|
||||||
|
paths := make([]string, 0, len(args))
|
||||||
|
|
||||||
|
for _, a := range args {
|
||||||
|
abs, err := absolutePath(a)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve path %s: %w", escapePath(a), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
paths = append(paths, filepath.Clean(abs))
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// inReportPaths reports whether path is one of paths or lies below one
|
||||||
|
// of them. Each of paths is clean and absolute, as reportPaths returns
|
||||||
|
// it. With no paths, a report covers the whole database, so every path
|
||||||
|
// is in it.
|
||||||
|
func inReportPaths(path string, paths []string) bool {
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, p := range paths {
|
||||||
|
// Only the root directory's clean path ends in "/".
|
||||||
|
dir := strings.TrimSuffix(p, "/") + "/"
|
||||||
|
if path == p || strings.HasPrefix(path, dir) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// escapePath returns a path as it is written in a report column (README
|
// escapePath returns a path as it is written in a report column (README
|
||||||
// "Report output format"): a backslash, tab, newline or carriage return
|
// "Report output format"): a backslash, tab, newline or carriage return
|
||||||
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
|
// becomes \\, \t, \n or \r, and every other byte is kept as it is.
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported dupeGroup results
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -80,7 +80,7 @@ func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
|
|||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
|
|
||||||
_, err := readReport(t.Context(), db, "", &out, nil)
|
_, err := readReport(t.Context(), db, "", nil, &out, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -126,6 +126,98 @@ func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup {
|
|||||||
return dupeGroups(t, db)
|
return dupeGroups(t, db)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRunReportPaths runs report with paths on three copies of one
|
||||||
|
// file, two of them under /srv/photos, and a fourth copy in
|
||||||
|
// /srv/photos2. Only the records at or below the given paths form the
|
||||||
|
// groups and the totals, however each path is written.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
|
func TestRunReportPaths(t *testing.T) {
|
||||||
|
h := testHash("h")
|
||||||
|
a, x, y, z := "/a/copy", "/srv/photos/x", "/srv/photos/y", "/srv/photos2/z"
|
||||||
|
|
||||||
|
recs := make([]scanRec, 0, 4)
|
||||||
|
for _, p := range []string{a, x, y, z} {
|
||||||
|
recs = append(recs,
|
||||||
|
scanRec{size: 7, head: h, tail: h, content: h, path: p})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
||||||
|
|
||||||
|
header := "first\tdupe\tsize\n"
|
||||||
|
row := func(first, dupe string) string {
|
||||||
|
return first + "\t" + dupe + "\t7\n"
|
||||||
|
}
|
||||||
|
photos := header + row(x, y)
|
||||||
|
|
||||||
|
cases := map[string]struct {
|
||||||
|
paths []string
|
||||||
|
want string
|
||||||
|
summary string
|
||||||
|
}{
|
||||||
|
"directory": {[]string{"/srv/photos"}, photos,
|
||||||
|
"1 duplicate groups, 1 dupe files, 7 B reclaimable"},
|
||||||
|
"trailing slash": {[]string{"/srv/photos/"}, photos, ""},
|
||||||
|
"dot and dot-dot": {[]string{"/srv/./a/../photos"}, photos, ""},
|
||||||
|
"files": {[]string{a, z}, header + row(a, z), ""},
|
||||||
|
"overlapping": {[]string{"/srv", "/srv/photos"},
|
||||||
|
header + row(x, y) + row(x, z), ""},
|
||||||
|
"root": {[]string{"/"},
|
||||||
|
header + row(a, x) + row(a, y) + row(a, z), ""},
|
||||||
|
"no record": {[]string{"/srv/photo"}, header,
|
||||||
|
"0 duplicate groups, 0 dupe files, 0 B reclaimable"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
|
code := run(append([]string{cmdReport}, tc.paths...),
|
||||||
|
&stdout, os.Stderr)
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("run(report %q) = %d, want %d; stderr: %s",
|
||||||
|
tc.paths, code, exitOK, stderr())
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stdout.String(); got != tc.want {
|
||||||
|
t.Errorf("stdout = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The records read are all four, whatever the paths.
|
||||||
|
summary := "report: 4 records read, " + tc.summary + "\n"
|
||||||
|
if got := stderr(); tc.summary != "" &&
|
||||||
|
!strings.HasSuffix(got, summary) {
|
||||||
|
t.Errorf("stderr = %q, want it to end %q", got, summary)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//nolint:paralleltest // t.Chdir changes the process-wide working directory
|
||||||
|
func TestReportPathsMakesRelativePathsAbsolute(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Chdir(dir)
|
||||||
|
|
||||||
|
wd, err := absolutePath(".")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := reportPaths([]string{"photos/", "a/../b", "/c//d"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{
|
||||||
|
filepath.Join(wd, "photos"), filepath.Join(wd, "b"), "/c/d",
|
||||||
|
}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("reportPaths = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunReportEscapesPaths(t *testing.T) {
|
func TestRunReportEscapesPaths(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
|
t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs()))
|
||||||
|
|
||||||
@@ -172,7 +264,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
|
|||||||
// Counts the records read, and draws nothing.
|
// Counts the records read, and draws nothing.
|
||||||
prog := &progress{now: time.Now}
|
prog := &progress{now: time.Now}
|
||||||
|
|
||||||
_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
|
_, err = readReport(t.Context(), db, "", nil, failingWriter{}, prog)
|
||||||
if !errors.Is(err, errWriteFailed) ||
|
if !errors.Is(err, errWriteFailed) ||
|
||||||
!strings.HasPrefix(err.Error(), "write stdout: ") {
|
!strings.HasPrefix(err.Error(), "write stdout: ") {
|
||||||
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
|
t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
|
||||||
@@ -266,7 +358,7 @@ func TestReadReportOnTerminal(t *testing.T) {
|
|||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
}, terminalWidth, time.Hour, time.Now)
|
}, terminalWidth, time.Hour, time.Now)
|
||||||
|
|
||||||
_, err = readReport(t.Context(), db, "", os.Stderr, p)
|
_, err = readReport(t.Context(), db, "", nil, os.Stderr, p)
|
||||||
|
|
||||||
p.finish()
|
p.finish()
|
||||||
|
|
||||||
@@ -434,6 +526,27 @@ func TestDupeGroups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
|
||||||
|
// duplicate group, while 1-byte files still do.
|
||||||
|
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
e, x := sha256.Sum256(nil), testHash("x")
|
||||||
|
recs := []scanRec{
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
|
||||||
|
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
|
||||||
|
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
|
||||||
|
}
|
||||||
|
|
||||||
|
groups := dupeGroupsOf(t, recs)
|
||||||
|
if len(groups) != 1 || groups[0].size != 1 ||
|
||||||
|
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
|
||||||
|
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDupeGroupsContentSeparates(t *testing.T) {
|
func TestDupeGroupsContentSeparates(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
@@ -209,29 +209,30 @@ func interrupted(walked int) error {
|
|||||||
return errInterrupted
|
return errInterrupted
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveRoots converts each PATH operand to an absolute, lexically
|
// resolveRoots converts each PATH operand to its real path: absolute,
|
||||||
// cleaned path (symlinks are not resolved) and verifies that it
|
// with every symlink in it followed. Database records are keyed by
|
||||||
// exists. Database records are keyed by absolute path, so scan results
|
// that path, so scan results depend neither on the working directory
|
||||||
// must not depend on the working directory.
|
// nor on the symlinks an operand was given through. An operand that
|
||||||
|
// cannot be resolved, because it or a symlink's target does not exist
|
||||||
|
// or cannot be read, is a fatal error before any scanning.
|
||||||
func resolveRoots(roots []string) ([]string, error) {
|
func resolveRoots(roots []string) ([]string, error) {
|
||||||
abs := make([]string, 0, len(roots))
|
resolved := make([]string, 0, len(roots))
|
||||||
|
|
||||||
for _, root := range roots {
|
for _, root := range roots {
|
||||||
a, err := filepath.Abs(root)
|
a, err := absolutePath(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A nonexistent operand is a fatal error before any scanning.
|
r, err := filepath.EvalSymlinks(a)
|
||||||
_, err = os.Lstat(a)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
abs = append(abs, a)
|
resolved = append(resolved, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
return abs, nil
|
return resolved, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneRoots drops operands already covered by another operand:
|
// pruneRoots drops operands already covered by another operand:
|
||||||
@@ -417,14 +418,16 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
|
|
||||||
// walkableRoots returns the operands the walk can start from: regular
|
// walkableRoots returns the operands the walk can start from: regular
|
||||||
// files, and directories not named .zfs, none of them inside a
|
// files, and directories not named .zfs, none of them inside a
|
||||||
// directory named .zfs. Every other operand is warned about, counted as
|
// directory named .zfs. The operands are real paths from resolveRoots,
|
||||||
// skipped, and dropped. A dropped operand is no longer a root, so the
|
// so these checks apply to what a symlink operand points at. Every
|
||||||
// records stored beneath it are treated as if it had not been given:
|
// other operand is warned about, counted as skipped, and dropped. A
|
||||||
// those under no other root count as outside the roots and are not
|
// dropped operand is no longer a root, so the records stored beneath it
|
||||||
// deleted as unverified. An operand on a filesystem in rules that is not
|
// are treated as if it had not been given: those under no other root
|
||||||
// persistent is dropped the same way, with a line saying so that is not
|
// count as outside the roots and are not deleted as unverified. An
|
||||||
// a warning, and is not counted as skipped. An operand that fails lstat
|
// operand on a filesystem in rules that is not persistent is dropped
|
||||||
// here is kept, and the walk warns about it unless it no longer exists.
|
// the same way, with a line saying so that is not a warning, and is not
|
||||||
|
// counted as skipped. An operand that fails lstat here is kept, and the
|
||||||
|
// walk warns about it unless it no longer exists.
|
||||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||||
kept := make([]string, 0, len(roots))
|
kept := make([]string, 0, len(roots))
|
||||||
|
|
||||||
@@ -554,7 +557,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
// The space a file takes on disk can change while its size and
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
||||||
// only seconds after the write.
|
// up to seconds after the write.
|
||||||
if old.diskSize != ev.rec.diskSize {
|
if old.diskSize != ev.rec.diskSize {
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
s.diskSizes = append(s.diskSizes, ev.rec)
|
||||||
}
|
}
|
||||||
@@ -894,19 +897,13 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
|||||||
// operandWarning returns the one-line warning for an operand the walk
|
// operandWarning returns the one-line warning for an operand the walk
|
||||||
// does not start from, naming the path and what it is, or "" for one it
|
// does not start from, naming the path and what it is, or "" for one it
|
||||||
// does: a regular file, or a directory not named .zfs, that is not
|
// does: a regular file, or a directory not named .zfs, that is not
|
||||||
// inside a directory named .zfs, neither as given nor with its symlinks
|
// inside a directory named .zfs. resolveRoots has already followed the
|
||||||
// resolved. Symlinks are never followed, including as operands.
|
// operand's symlinks, so a symlink here is an operand replaced by one
|
||||||
|
// after it was resolved, and is not followed.
|
||||||
func operandWarning(root string, fi fs.FileInfo) string {
|
func operandWarning(root string, fi fs.FileInfo) string {
|
||||||
// The walk never enters a .zfs directory, so it starts from nothing
|
// The walk never enters a .zfs directory, so it starts from nothing
|
||||||
// inside one either, not even through a symlink. An operand whose
|
// inside one either.
|
||||||
// symlinks cannot be resolved is checked as given.
|
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
||||||
resolved, err := filepath.EvalSymlinks(root)
|
|
||||||
if err != nil {
|
|
||||||
resolved = root
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
|
|
||||||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
|
|
||||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
||||||
root)
|
root)
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build darwin
|
//go:build darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !darwin
|
//go:build !darwin
|
||||||
|
|
||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // drives the unexported syncTree
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -43,10 +43,16 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
||||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
|
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's
|
||||||
// blocks only seconds after the write, and fsync does not hasten that.
|
// blocks only when it commits the write, up to seconds later, and one
|
||||||
// A test therefore calls unix.Sync, which waits for it, before the scan
|
// commit can take the file through several growing sizes. So this value
|
||||||
// whose disk_size it compares with this.
|
// can change at any moment. Neither fsync nor unix.Sync hastens that,
|
||||||
|
// but a unix.Sync call often takes long enough that the blocks are
|
||||||
|
// counted when it returns. A test that needs the value to change waits
|
||||||
|
// for the change. A test that compares it with a scan's disk_size reads
|
||||||
|
// it before and after the scan. The rewrite test rescans until the two
|
||||||
|
// reads match, then compares exactly. The other two tests fail only on
|
||||||
|
// a disk_size outside the two reads.
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
func diskSizeOf(t *testing.T, path string) int64 {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1082,12 +1088,13 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
// buildSmokeTree recreates the tree of the test under README
|
||||||
// with deterministic content and returns the tree root.
|
// "Definition of done" with deterministic content and returns the tree
|
||||||
|
// root.
|
||||||
func buildSmokeTree(t *testing.T) string {
|
func buildSmokeTree(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := realTempDir(t)
|
||||||
one := pattern(10, 2000)
|
one := pattern(10, 2000)
|
||||||
f1 := pattern(30, 3000)
|
f1 := pattern(30, 3000)
|
||||||
f2 := pattern(40, 100)
|
f2 := pattern(40, 100)
|
||||||
@@ -1178,11 +1185,12 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
groups := dupeGroups(t, db)
|
groups := dupeGroups(t, db)
|
||||||
if len(groups) != 5 {
|
if len(groups) != 4 {
|
||||||
t.Fatalf("len(groups) = %d, want 5", len(groups))
|
t.Fatalf("len(groups) = %d, want 4", len(groups))
|
||||||
}
|
}
|
||||||
|
|
||||||
wantSizes := []int64{3000, 2000, 100, 1, 0}
|
// The empty files empty1 and empty2 form no group.
|
||||||
|
wantSizes := []int64{3000, 2000, 100, 1}
|
||||||
for i, g := range groups {
|
for i, g := range groups {
|
||||||
if g.size != wantSizes[i] {
|
if g.size != wantSizes[i] {
|
||||||
t.Errorf("groups[%d].size = %d, want %d",
|
t.Errorf("groups[%d].size = %d, want %d",
|
||||||
@@ -1447,20 +1455,26 @@ func TestScanStoresDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
unix.Sync() // see diskSizeOf
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
size := diskSizeOf(t, p)
|
||||||
if want == 0 || want >= length {
|
if size == 0 || size >= length {
|
||||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
||||||
"and fewer than its length %d", want, length)
|
"and fewer than its length %d", size, length)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, operand := range []string{filepath.Dir(p), p} {
|
for _, operand := range []string{filepath.Dir(p), p} {
|
||||||
db := openTestDB(t)
|
db := openTestDB(t)
|
||||||
|
|
||||||
|
// ZFS may count the blocks during the scan; see diskSizeOf.
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
syncTree(t, db, operand)
|
syncTree(t, db, operand)
|
||||||
|
|
||||||
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
if got != want {
|
if got < before || got > after {
|
||||||
t.Errorf("scan of %s: disk_size = %d, want %d",
|
t.Errorf("scan of %s: disk_size = %d, want %d to %d",
|
||||||
operand, got, want)
|
operand, got, before, after)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1478,7 +1492,7 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
syncTree(t, db, dir)
|
syncTree(t, db, dir)
|
||||||
|
|
||||||
before := recordByPath(t, dbRecords(t, db), p).diskSize
|
recorded := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
|
|
||||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
||||||
|
|
||||||
@@ -1491,22 +1505,53 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
// Wait for ZFS to count the write; see diskSizeOf.
|
||||||
|
deadline := time.Now().Add(30 * time.Second)
|
||||||
|
for diskSizeOf(t, p) == recorded {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
|
||||||
|
"after the write into its hole", recorded)
|
||||||
|
}
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
unix.Sync()
|
||||||
if want == before {
|
|
||||||
t.Fatalf("the file still takes %d bytes on disk after the write "+
|
|
||||||
"into its hole", want)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
// ZFS may still be counting, so the size may change during a rescan.
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
// Only a rescan with the same size on both sides of it can be compared
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
// exactly. A rescan that keeps the old size still fails, because the
|
||||||
}
|
// wait made before differ from it.
|
||||||
|
for {
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
st := syncTree(t, db, dir)
|
||||||
if got != want {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
|
if st != (scanStats{walked: 1, updated: 1}) {
|
||||||
|
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
||||||
|
}
|
||||||
|
|
||||||
|
if before == after {
|
||||||
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
|
if got != after {
|
||||||
|
t.Errorf("disk_size = %d, want %d", got, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("the size on disk was still changing 30 seconds after "+
|
||||||
|
"the write into its hole: %d, then %d", before, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later mtime makes the next rescan read the file again.
|
||||||
|
later = later.Add(time.Second)
|
||||||
|
|
||||||
|
err = os.Chtimes(p, later, later)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1532,15 +1577,21 @@ func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
|||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
unix.Sync() // see diskSizeOf
|
||||||
|
|
||||||
|
// ZFS may count the blocks during the rescan; see diskSizeOf.
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
// A file read again would count as updated.
|
// A file read again would count as updated.
|
||||||
st := syncTree(t, db, dir)
|
st := syncTree(t, db, dir)
|
||||||
|
|
||||||
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
if st != (scanStats{walked: 1, unchanged: 1}) {
|
||||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
if want := diskSizeOf(t, p); got != want {
|
if got < before || got > after {
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2260,33 +2311,24 @@ func TestUnderRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
||||||
// database, reached directly and through a symlinked directory. The
|
// database, and in one case also a hard link to the database file. The
|
||||||
// walk passes over the database file, its -wal and -shm files and its
|
// walk knows the database's files by device ID and inode number, so it
|
||||||
// .lock file with no line and no count, and the record an earlier scan
|
// passes over the database file, its -wal and -shm files, its .lock
|
||||||
// left for the .lock file is deleted. A rescan of the unchanged tree
|
// file and the hard link with no line and no count. The record an
|
||||||
// updates nothing, and report lists none of the database's files.
|
// earlier scan left for the .lock file is deleted. A rescan of the
|
||||||
|
// unchanged tree updates nothing, and report lists none of the
|
||||||
|
// database's files.
|
||||||
func TestRunScanPassesOverDatabase(t *testing.T) {
|
func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||||
for _, viaSymlink := range []bool{false, true} {
|
for _, viaHardLink := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
|
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
|
||||||
base := t.TempDir()
|
tree := realTempDir(t)
|
||||||
tree := filepath.Join(base, "real", "tree")
|
hardLink := filepath.Join(tree, "hardlink")
|
||||||
operand := tree
|
|
||||||
|
|
||||||
writeFile(t, tree, "a", pattern(1, 100))
|
writeFile(t, tree, "a", pattern(1, 100))
|
||||||
writeFile(t, tree, "b", pattern(1, 100))
|
writeFile(t, tree, "b", pattern(1, 100))
|
||||||
writeFile(t, tree, "empty1", nil)
|
writeFile(t, tree, "empty1", nil)
|
||||||
writeFile(t, tree, "empty2", nil)
|
writeFile(t, tree, "empty2", nil)
|
||||||
|
|
||||||
if viaSymlink {
|
|
||||||
err := os.Symlink(filepath.Join(base, "real"),
|
|
||||||
filepath.Join(base, "link"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
operand = filepath.Join(base, "link", "tree")
|
|
||||||
}
|
|
||||||
|
|
||||||
dbPath := filepath.Join(tree, "db.sqlite")
|
dbPath := filepath.Join(tree, "db.sqlite")
|
||||||
t.Setenv(databaseEnv, dbPath)
|
t.Setenv(databaseEnv, dbPath)
|
||||||
|
|
||||||
@@ -2296,7 +2338,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = applyChanges(t.Context(), db, []scanRec{
|
err = applyChanges(t.Context(), db, []scanRec{
|
||||||
{path: filepath.Join(operand, "db.sqlite.lock")},
|
{path: filepath.Join(tree, "db.sqlite.lock")},
|
||||||
}, nil, nil)
|
}, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -2304,17 +2346,30 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
|||||||
|
|
||||||
closeScanDatabase(t.Context(), db, dbPath)
|
closeScanDatabase(t.Context(), db, dbPath)
|
||||||
|
|
||||||
assertScanSummary(t,
|
if viaHardLink {
|
||||||
|
err = os.Link(dbPath, hardLink)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stderr := assertScanSummary(t,
|
||||||
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
||||||
operand)
|
tree)
|
||||||
assertScanSummary(t,
|
stderr += assertScanSummary(t,
|
||||||
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
||||||
operand)
|
tree)
|
||||||
|
|
||||||
|
if strings.Contains(stderr, hardLink) {
|
||||||
|
t.Errorf("scan stderr = %q, want no line naming %s",
|
||||||
|
stderr, hardLink)
|
||||||
|
}
|
||||||
|
|
||||||
report := runStdout(t, cmdReport)
|
report := runStdout(t, cmdReport)
|
||||||
if strings.Contains(report, "db.sqlite") ||
|
if strings.Contains(report, "db.sqlite") ||
|
||||||
!strings.Contains(report, filepath.Join(operand, "empty2")) {
|
strings.Contains(report, hardLink) ||
|
||||||
t.Errorf("report = %q, want the empty files and none of "+
|
!strings.Contains(report, filepath.Join(tree, "b")) {
|
||||||
|
t.Errorf("report = %q, want a and b and none of "+
|
||||||
"the database's files", report)
|
"the database's files", report)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -2338,8 +2393,9 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
|
|||||||
|
|
||||||
// assertScanSummary scans operands and fails the test unless the
|
// assertScanSummary scans operands and fails the test unless the
|
||||||
// summary line gives counts with nothing skipped, and the line naming
|
// summary line gives counts with nothing skipped, and the line naming
|
||||||
// the database is the only one that names one of its files.
|
// the database is the only one that names one of its files. It returns
|
||||||
func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
// everything the scan printed to stderr.
|
||||||
|
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
stderr := scanOK(t, operands...)
|
stderr := scanOK(t, operands...)
|
||||||
@@ -2353,4 +2409,6 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
|||||||
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
||||||
"database to name one of its files", operands, stderr)
|
"database to name one of its files", operands, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return stderr
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package main
|
package sfdupes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -37,8 +37,9 @@ func treesPhase(name string) string {
|
|||||||
|
|
||||||
// runTrees implements the trees subcommand: it reads every record from
|
// runTrees implements the trees subcommand: it reads every record from
|
||||||
// the database in path order, reconstructs the directory hierarchy from
|
// the database in path order, reconstructs the directory hierarchy from
|
||||||
// the record paths, computes a Merkle-style digest per directory, and
|
// the paths of the records at or below the given paths (of every record
|
||||||
// prints maximal duplicate-tree groups as TSV on stdout. It never
|
// when none is given), computes a Merkle-style digest per directory,
|
||||||
|
// and prints maximal duplicate-tree groups as TSV on stdout. It never
|
||||||
// touches the scanned filesystem; its only I/O is the database, stdout,
|
// touches the scanned filesystem; its only I/O is the database, stdout,
|
||||||
// and stderr. It names the database on stderr before it opens it. Any
|
// and stderr. It names the database on stderr before it opens it. Any
|
||||||
// database problem, including a missing database, is fatal.
|
// database problem, including a missing database, is fatal.
|
||||||
@@ -49,7 +50,12 @@ func treesPhase(name string) string {
|
|||||||
// hierarchy), group (group the directories by digest) and write (write
|
// hierarchy), group (group the directories by digest) and write (write
|
||||||
// the groups to stdout). Each display ends before the next phase
|
// the groups to stdout). Each display ends before the next phase
|
||||||
// starts, and before an error is returned to be printed.
|
// starts, and before an error is returned to be printed.
|
||||||
func runTrees(ctx context.Context, stdout io.Writer) error {
|
func runTrees(ctx context.Context, args []string, stdout io.Writer) error {
|
||||||
|
paths, err := reportPaths(args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -78,7 +84,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
|||||||
label: treesPhase("read"), unit: "records", total: -1,
|
label: treesPhase("read"), unit: "records", total: -1,
|
||||||
waitingFor: countingRecords, stalledOn: "db",
|
waitingFor: countingRecords, stalledOn: "db",
|
||||||
})
|
})
|
||||||
records, super, allDirs, err := readTree(ctx, db, prog)
|
records, super, allDirs, err := readTree(ctx, db, paths, prog)
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
@@ -117,10 +123,17 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// readTree gives prog the number of records in db as its total, then
|
// readTree gives prog the number of records in db as its total, then
|
||||||
// reads every record in path order into the directory hierarchy,
|
// reads every record in path order, counting each on prog. It adds the
|
||||||
// counting each on prog. It returns the number of records read, the
|
// records that inReportPaths selects from paths to the directory
|
||||||
// super-root and every directory.
|
// hierarchy. It returns the number of records read, the super-root and
|
||||||
func readTree(ctx context.Context, db *sql.DB,
|
// every directory at or below one of paths.
|
||||||
|
//
|
||||||
|
// A directory above every one of paths holds only the selected records
|
||||||
|
// under it, so its digest does not describe it. It is left out of the
|
||||||
|
// directories returned. A directory whose parent is left out gets the
|
||||||
|
// super-root as its parent, so suppressed never judges its group by
|
||||||
|
// that parent's digest.
|
||||||
|
func readTree(ctx context.Context, db *sql.DB, paths []string,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
) (int, *treeNode, []*treeNode, error) {
|
) (int, *treeNode, []*treeNode, error) {
|
||||||
records := 0
|
records := 0
|
||||||
@@ -130,7 +143,10 @@ func readTree(ctx context.Context, db *sql.DB,
|
|||||||
records++
|
records++
|
||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
tree.add(r)
|
|
||||||
|
if inReportPaths(r.path, paths) {
|
||||||
|
tree.add(r)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, err
|
||||||
@@ -138,7 +154,21 @@ func readTree(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
super, allDirs := tree.finish()
|
super, allDirs := tree.finish()
|
||||||
|
|
||||||
return records, super, allDirs, nil
|
dirs := allDirs[:0]
|
||||||
|
|
||||||
|
for _, d := range allDirs {
|
||||||
|
if !inReportPaths(d.path, paths) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !inReportPaths(d.parent.path, paths) {
|
||||||
|
d.parent = super
|
||||||
|
}
|
||||||
|
|
||||||
|
dirs = append(dirs, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
return records, super, dirs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
||||||
@@ -301,7 +331,9 @@ func fileEntry(name string, r scanRec) string {
|
|||||||
// collectTreeGroups groups directories by digest, counting each one on
|
// collectTreeGroups groups directories by digest, counting each one on
|
||||||
// prog as it is grouped, and returns every maximal group with two or
|
// prog as it is grouped, and returns every maximal group with two or
|
||||||
// more members, each group's members sorted by path, groups ordered by
|
// more members, each group's members sorted by path, groups ordered by
|
||||||
// tree size descending then by first path ascending.
|
// tree size descending then by first path ascending. A group whose
|
||||||
|
// trees hold 0 bytes is left out, since removing one frees nothing. The
|
||||||
|
// groups it implies hold 0 bytes too, so leaving it out hides nothing.
|
||||||
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
) [][]*treeNode {
|
) [][]*treeNode {
|
||||||
@@ -315,7 +347,9 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
|
|||||||
var dupes [][]*treeNode
|
var dupes [][]*treeNode
|
||||||
|
|
||||||
for _, g := range groups {
|
for _, g := range groups {
|
||||||
if len(g) < minGroupSize || suppressed(g, super) {
|
// The members share a digest, so they share a total size.
|
||||||
|
if len(g) < minGroupSize || g[0].totalSize == 0 ||
|
||||||
|
suppressed(g, super) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
package main
|
package sfdupes //nolint:testpackage // checks the unexported treeNode results
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -15,9 +16,9 @@ import (
|
|||||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"remain unknown"
|
||||||
|
|
||||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
// smokeTreeRecs mirrors the tree of the test under README "Definition
|
||||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
|
||||||
// Each file takes one 4 KiB block on disk.
|
// by one filename. Each file takes one 4 KiB block on disk.
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||||
@@ -211,6 +212,55 @@ func TestRunTreesEscapesPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRunTreesPaths runs trees with paths. /x/a and /y/a are copies,
|
||||||
|
// and so are /y/a and /srv/b. /x and /y also hold files that differ.
|
||||||
|
// Given /x/a and /y/a, trees reads /x and /y as holding one copy each
|
||||||
|
// of a, yet neither is reported: only the directories at or below a
|
||||||
|
// given path are.
|
||||||
|
func TestRunTreesPaths(t *testing.T) {
|
||||||
|
f, g, k := testHash("f"), testHash("g"), testHash("k")
|
||||||
|
recs := []scanRec{
|
||||||
|
{size: 10, head: f, tail: f, content: f, path: "/srv/b/f"},
|
||||||
|
{size: 10, head: f, tail: f, content: f, path: "/x/a/f"},
|
||||||
|
{size: 20, head: g, tail: g, content: g, path: "/x/g"},
|
||||||
|
{size: 10, head: f, tail: f, content: f, path: "/y/a/f"},
|
||||||
|
{size: 30, head: k, tail: k, content: k, path: "/y/k"},
|
||||||
|
}
|
||||||
|
|
||||||
|
header := "first\tdupe\tfiles\tsize\tdisk_size\n"
|
||||||
|
|
||||||
|
cases := map[string]struct {
|
||||||
|
paths []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
"none": {nil, header + "/srv/b\t/x/a\t1\t10\t0\n" +
|
||||||
|
"/srv/b\t/y/a\t1\t10\t0\n"},
|
||||||
|
"two copies": {[]string{"/x/a", "/y/a"},
|
||||||
|
header + "/x/a\t/y/a\t1\t10\t0\n"},
|
||||||
|
"parents": {[]string{"/x", "/y"}, header + "/x/a\t/y/a\t1\t10\t0\n"},
|
||||||
|
"one copy": {[]string{"/srv/b"}, header},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range cases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := run(append([]string{cmdTrees}, tc.paths...),
|
||||||
|
&stdout, &stderr)
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("run(trees %q) = %d, want %d; stderr: %s",
|
||||||
|
tc.paths, code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stdout.String(); got != tc.want {
|
||||||
|
t.Errorf("stdout = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
|
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
|
||||||
// take different space on disk. The row's disk_size is the sum over the
|
// take different space on disk. The row's disk_size is the sum over the
|
||||||
// dupe copy's files, those in its subdirectory included.
|
// dupe copy's files, those in its subdirectory included.
|
||||||
@@ -435,6 +485,35 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
|
||||||
|
// holding only empty files form no group, and that an empty file still
|
||||||
|
// counts toward the digest of a tree that holds other files.
|
||||||
|
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
|
||||||
|
sha256.Sum256(nil)
|
||||||
|
recs := []scanRec{
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
|
||||||
|
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
|
||||||
|
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
|
||||||
|
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
|
||||||
|
// Without the empty file, /p/x3 is no copy of /p/x1.
|
||||||
|
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
|
||||||
|
}
|
||||||
|
|
||||||
|
super, dirs := treeOf(t, recs)
|
||||||
|
|
||||||
|
got := groupPaths(collectTreeGroups(dirs, super, nil))
|
||||||
|
|
||||||
|
want := [][]string{{"/p/x1", "/p/x2"}}
|
||||||
|
if !slices.EqualFunc(got, want, slices.Equal) {
|
||||||
|
t.Fatalf("groups = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
func TestCollectTreeGroupsDifferingParents(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
Reference in new issue
Block a user