// Command sfdupes quickly identifies candidate duplicate files across // very large filesystems without reading full file contents. Files are // considered duplicates when they have identical size, identical SHA-256 // of their first 1024 bytes, and identical SHA-256 of their last 1024 // bytes. scan maintains a persistent SQLite database of file signatures // (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the // reporting subcommands read. // // Usage: // // sfdupes scan [--workers N] [-x] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv // // See README.md for the complete specification. package main import ( "context" "errors" "fmt" "io" "os" "runtime" "github.com/spf13/cobra" ) // Exit codes: exitOK is success (even with per-file warnings), // exitFatal is a fatal error, exitUsage is a usage error. const ( exitOK = 0 exitFatal = 1 exitUsage = 2 ) // The subcommand names, as typed on the command line. const ( cmdScan = "scan" cmdReport = "report" cmdTrees = "trees" ) // errNoSubcommand is returned by the root command when it is invoked // without a subcommand. That is a usage error, and the usage text // cobra prints for it is the whole message. var errNoSubcommand = errors.New("no subcommand") // Version is the build version, injected at link time via -ldflags // (see the Makefile); "dev" for a plain go build. // //nolint:gochecknoglobals // written only by the linker var Version = "dev" func main() { os.Exit(run(os.Args[1:], os.Stderr)) } // run executes args against the command tree and returns the process // exit code. It is the program's single exit point: the subcommands // return their errors instead of exiting, so every deferred cleanup — // above all closing the database, which checkpoints the SQLite WAL — // runs before the process ends. func run(args []string, stderr io.Writer) int { // A nil slice makes cobra fall back to os.Args, which would let a // test binary's own flags reach the command tree. if args == nil { args = []string{} } root := newRootCommand(stderr) root.SetArgs(args) err := root.Execute() var fatal fatalError switch { case err == nil: return exitOK case errors.As(err, &fatal): // The command ran and failed: a runtime error, reported // without the usage text that a usage error gets. _, _ = fmt.Fprintf(stderr, "sfdupes: %v\n", err) return exitFatal default: // A usage error: cobra has already printed the message and // the usage text. return exitUsage } } // newRootCommand builds the command tree. Everything on stdout is // machine-readable data; all human-facing output (help, usage, errors) // goes to stderr. func newRootCommand(stderr io.Writer) *cobra.Command { root := &cobra.Command{ Use: "sfdupes", Short: "Find candidate duplicate files by size and head/tail SHA-256", Version: Version, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { // A missing subcommand prints usage and exits 2: cobra // prints the usage text for the returned error, and run // maps everything that is not a fatal error to exit 2. cmd.SilenceErrors = true return errNoSubcommand }, } root.SetOut(stderr) root.SetErr(stderr) root.CompletionOptions.DisableDefaultCmd = true var ( scanWorkers int scanOneFS bool ) scanCmd := &cobra.Command{ Use: cmdScan + " [--workers N] [-x] PATH...", Short: "Walk trees and synchronize the scan database", Args: cobra.MinimumNArgs(1), RunE: runE(func(ctx context.Context, args []string) error { return runScan(ctx, args, scanWorkers, scanOneFS) }), } scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(), "concurrent workers for the walk and hash phases") scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, "do not cross filesystem boundaries") reportCmd := &cobra.Command{ Use: cmdReport, Short: "Read the scan database and print the file-level duplicates report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { return runReport(ctx) }), } treesCmd := &cobra.Command{ Use: cmdTrees, Short: "Read the scan database and print the duplicate-tree report", Args: cobra.NoArgs, RunE: runE(func(ctx context.Context, _ []string) error { return runTrees(ctx) }), } root.AddCommand(scanCmd, reportCmd, treesCmd) return root } // runE adapts a subcommand implementation to cobra's RunE. Cobra // prints the error and the command's usage text for every error RunE // returns, but a subcommand that ran and failed has no usage problem // to report: both are silenced here, and the error is marked fatal so // that run reports it on stderr and exits 1 rather than 2. The command's // context is handed to the implementation: cancelling it unwinds the // scan's worker pools. func runE( fn func(ctx context.Context, args []string) error, ) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true cmd.SilenceErrors = true err := fn(cmd.Context(), args) if err != nil { return fatalError{err: err} } return nil } } // fatalError marks a runtime failure, as opposed to the usage errors // cobra itself produces while parsing arguments and flags. Both come // out of Execute as plain errors, so the wrapper is what tells run to // report this one as "sfdupes: ..." and exit 1. type fatalError struct { err error } func (e fatalError) Error() string { return e.err.Error() } func (e fatalError) Unwrap() error { return e.err }