sneak
b62b4f297f
Restructure scan into three phases: walk+stat, hash, update. The stat pass is folded into the walk workers: each regular file is lstatted as its directory is read, while the metadata is hot. The walk builds a scan-wide size census (walked files plus records outside the scan roots), and unchanged already-hashed files resolve during the walk without further work. Only files whose size at least one other file shares are ever read: a size-unique file cannot be a duplicate, so it is recorded without hashes (head and tail empty). When a later scan makes its size shared, the file is hashed then, even if otherwise unchanged. report excludes unhashed records; trees gives them a never-matching signature so a tree containing one never compares equal to another. Hashed records are committed in batched transactions while the hash phase runs, so an interrupted scan keeps everything hashed so far and the next run resumes cheaply. The hash phase total is exact, giving a meaningful ETA. Memory drops accordingly: the existing-record index holds only path, size, mtime, and a hashed flag (no hash values); the walk carries one small record per candidate file; overlapping operands are pruned up front instead of deduplicating every walked path in a scan-wide set. Files no bigger than one chunk are hashed with a single read.
109 lines
2.9 KiB
Go
109 lines
2.9 KiB
Go
// Command sfdupes quickly identifies candidate duplicate files across
|
|
// very large filesystems without reading full file contents. Files are
|
|
// considered duplicates when they have identical size, identical SHA-256
|
|
// of their first 1024 bytes, and identical SHA-256 of their last 1024
|
|
// bytes. scan maintains a persistent SQLite database of file signatures
|
|
// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
|
|
// reporting subcommands read.
|
|
//
|
|
// Usage:
|
|
//
|
|
// sfdupes scan [--workers N] [-x] PATH...
|
|
// sfdupes report > dupes.tsv
|
|
// sfdupes trees > dupetrees.tsv
|
|
//
|
|
// See README.md for the complete specification.
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"runtime"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// Exit codes: 0 is success (even with per-file warnings), exitFatal is
|
|
// a fatal error, exitUsage is a usage error.
|
|
const (
|
|
exitFatal = 1
|
|
exitUsage = 2
|
|
)
|
|
|
|
// Version is the build version, injected at link time via -ldflags
|
|
// (see the Makefile); "dev" for a plain go build.
|
|
//
|
|
//nolint:gochecknoglobals // written only by the linker
|
|
var Version = "dev"
|
|
|
|
func main() {
|
|
root := &cobra.Command{
|
|
Use: "sfdupes",
|
|
Short: "Find candidate duplicate files by size and head/tail SHA-256",
|
|
Version: Version,
|
|
Args: cobra.NoArgs,
|
|
Run: func(cmd *cobra.Command, _ []string) {
|
|
// A missing subcommand prints usage and exits 2.
|
|
_ = cmd.Usage()
|
|
|
|
os.Exit(exitUsage)
|
|
},
|
|
}
|
|
// Everything on stdout is machine-readable data; all human-facing
|
|
// output (help, usage, errors) goes to stderr.
|
|
root.SetOut(os.Stderr)
|
|
root.SetErr(os.Stderr)
|
|
root.CompletionOptions.DisableDefaultCmd = true
|
|
|
|
var (
|
|
scanWorkers int
|
|
scanOneFS bool
|
|
)
|
|
|
|
scanCmd := &cobra.Command{
|
|
Use: "scan [--workers N] [-x] PATH...",
|
|
Short: "Walk trees and synchronize the scan database",
|
|
Args: cobra.MinimumNArgs(1),
|
|
Run: func(_ *cobra.Command, args []string) {
|
|
runScan(args, scanWorkers, scanOneFS)
|
|
},
|
|
}
|
|
scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
|
|
"concurrent workers for the walk and hash phases")
|
|
scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
|
|
"do not cross filesystem boundaries")
|
|
|
|
reportCmd := &cobra.Command{
|
|
Use: "report",
|
|
Short: "Read the scan database and print the file-level duplicates report",
|
|
Args: cobra.NoArgs,
|
|
Run: func(_ *cobra.Command, _ []string) {
|
|
runReport()
|
|
},
|
|
}
|
|
|
|
treesCmd := &cobra.Command{
|
|
Use: "trees",
|
|
Short: "Read the scan database and print the duplicate-tree report",
|
|
Args: cobra.NoArgs,
|
|
Run: func(_ *cobra.Command, _ []string) {
|
|
runTrees()
|
|
},
|
|
}
|
|
|
|
root.AddCommand(scanCmd, reportCmd, treesCmd)
|
|
|
|
err := root.Execute()
|
|
if err != nil {
|
|
// Cobra has already printed the error and usage to stderr;
|
|
// an invalid subcommand or bad arguments is a usage error.
|
|
os.Exit(exitUsage)
|
|
}
|
|
}
|
|
|
|
// fatalf reports a fatal error and exits 1.
|
|
func fatalf(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
|
|
os.Exit(exitFatal)
|
|
}
|