sfdupes/main.go

// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
// hashed in full. A larger file is compared on the hashes of its first
// and last 64 KiB, and only when those match another file's is its
// content hash computed and compared: of the whole file when it is
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
// or larger. scan maintains a persistent SQLite database of file
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
// that the reporting subcommands read.
//
// Usage:
//
//	sfdupes scan [--workers N] [-x] PATH...
//	sfdupes report > dupes.tsv
//	sfdupes trees > dupetrees.tsv
//
// See README.md for the complete specification.
package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"

	"github.com/spf13/cobra"
)

// Exit codes: exitOK is success (even with per-file warnings),
// exitFatal is a fatal error, exitUsage is a usage error.
const (
	exitOK    = 0
	exitFatal = 1
	exitUsage = 2
)

// The subcommand names, as typed on the command line.
const (
	cmdScan   = "scan"
	cmdReport = "report"
	cmdTrees  = "trees"
)

// errNoSubcommand is returned by the root command when it is invoked
// without a subcommand. That is a usage error, and the usage text
// cobra prints for it is the whole message.
var errNoSubcommand = errors.New("no subcommand")

// Version is the build version, injected at link time via -ldflags
// (see the Makefile); "dev" for a plain go build.
//
//nolint:gochecknoglobals // written only by the linker
var Version = "dev"

func main() {
	os.Exit(run(os.Args[1:], os.Stderr))
}

// run executes args against the command tree and returns the process
// exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends.
func run(args []string, stderr io.Writer) int {
	// A nil slice makes cobra fall back to os.Args, which would let a
	// test binary's own flags reach the command tree.
	if args == nil {
		args = []string{}
	}

	root := newRootCommand(stderr)
	root.SetArgs(args)

	err := root.Execute()

	var fatal fatalError

	switch {
	case err == nil:
		return exitOK
	case errors.As(err, &fatal):
		// The command ran and failed: a runtime error, reported
		// without the usage text that a usage error gets.
		_, _ = fmt.Fprintf(stderr, "sfdupes: %v\n", err)

		return exitFatal
	default:
		// A usage error: cobra has already printed the message and
		// the usage text.
		return exitUsage
	}
}

// newRootCommand builds the command tree. Everything on stdout is
// machine-readable data; all human-facing output (help, usage, errors)
// goes to stderr.
func newRootCommand(stderr io.Writer) *cobra.Command {
	root := &cobra.Command{
		Use:     "sfdupes",
		Short:   "Find candidate duplicate files by size and head/tail/content SHA-256",
		Version: Version,
		Args:    cobra.NoArgs,
		RunE: func(cmd *cobra.Command, _ []string) error {
			// A missing subcommand prints usage and exits 2: cobra
			// prints the usage text for the returned error, and run
			// maps everything that is not a fatal error to exit 2.
			cmd.SilenceErrors = true

			return errNoSubcommand
		},
	}
	root.SetOut(stderr)
	root.SetErr(stderr)
	root.CompletionOptions.DisableDefaultCmd = true

	var (
		scanWorkers int
		scanOneFS   bool
	)

	scanCmd := &cobra.Command{
		Use:   cmdScan + " [--workers N] [-x] PATH...",
		Short: "Walk trees and synchronize the scan database",
		Args:  cobra.MinimumNArgs(1),
		RunE: runE(func(ctx context.Context, args []string) error {
			return runScan(ctx, args, scanWorkers, scanOneFS)
		}),
	}
	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
		"concurrent workers for the walk, hash, and content phases")
	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
		"do not cross filesystem boundaries")

	reportCmd := &cobra.Command{
		Use:   cmdReport,
		Short: "Read the scan database and print the file-level duplicates report",
		Args:  cobra.NoArgs,
		RunE: runE(func(ctx context.Context, _ []string) error {
			return runReport(ctx)
		}),
	}

	treesCmd := &cobra.Command{
		Use:   cmdTrees,
		Short: "Read the scan database and print the duplicate-tree report",
		Args:  cobra.NoArgs,
		RunE: runE(func(ctx context.Context, _ []string) error {
			return runTrees(ctx)
		}),
	}

	root.AddCommand(scanCmd, reportCmd, treesCmd)

	return root
}

// runE adapts a subcommand implementation to cobra's RunE. Cobra
// prints the error and the command's usage text for every error RunE
// returns, but a subcommand that ran and failed has no usage problem
// to report: both are silenced here, and the error is marked fatal so
// that run reports it on stderr and exits 1 rather than 2. The command's
// context is handed to the implementation: cancelling it unwinds the
// scan's worker pools.
func runE(
	fn func(ctx context.Context, args []string) error,
) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		cmd.SilenceUsage = true
		cmd.SilenceErrors = true

		err := fn(cmd.Context(), args)
		if err != nil {
			return fatalError{err: err}
		}

		return nil
	}
}

// fatalError marks a runtime failure, as opposed to the usage errors
// cobra itself produces while parsing arguments and flags. Both come
// out of Execute as plain errors, so the wrapper is what tells run to
// report this one as "sfdupes: ..." and exit 1.
type fatalError struct {
	err error
}

func (e fatalError) Error() string { return e.err.Error() }

func (e fatalError) Unwrap() error { return e.err }