sfdupes/main.go

// Command sfdupes quickly identifies candidate duplicate files across
// very large filesystems without reading every byte of every file.
// Files are considered duplicates when their sizes are equal and they
// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
// hashed in full. A larger file is compared on the hashes of its first
// and last 64 KiB, and only when those match another file's is its
// content hash computed and compared: of the whole file when it is
// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
// or larger. scan maintains a persistent SQLite database of file
// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
// that the other subcommands read.
//
// Usage:
//
//	sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
//	sfdupes report > dupes.tsv
//	sfdupes trees > dupetrees.tsv
//	sfdupes info
//	sfdupes --version
//
// See README.md for the complete specification.
package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"runtime/debug"

	"github.com/spf13/cobra"
)

// Exit codes: exitOK is success (even with per-file warnings),
// exitFatal is a fatal error, exitUsage is a usage error.
const (
	exitOK    = 0
	exitFatal = 1
	exitUsage = 2
)

// The subcommand names, as typed on the command line.
const (
	cmdScan   = "scan"
	cmdReport = "report"
	cmdTrees  = "trees"
	cmdInfo   = "info"
)

// errNoSubcommand is returned by the root command when it is invoked
// without a subcommand. That is a usage error, and the usage text
// cobra prints for it is the whole message.
var errNoSubcommand = errors.New("no subcommand")

// errWorkersBelowOne is the usage error for a scan --workers value
// below 1.
var errWorkersBelowOne = errors.New("--workers must be at least 1")

// Version is the build version, injected at link time via -ldflags
// (see the Makefile); empty when the linker set none, as for a binary
// built by go install. buildVersion turns it into what --version prints.
//
//nolint:gochecknoglobals // written only by the linker
var Version string

// buildVersion returns the version that sfdupes --version prints
// (README "Install"): the version the linker set, when it set one;
// otherwise the main module's version that Go recorded in the binary,
// a tag such as v0.0.2 or the pseudo-version of an @main build; and
// "dev" when neither is known. info is what debug.ReadBuildInfo
// returns, nil when the binary carries none. Go records "(devel)" for
// a build that has no version, such as one inside a copy of the source
// without git.
func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
	if linkerVersion != "" {
		return linkerVersion
	}

	if info == nil || info.Main.Version == "" ||
		info.Main.Version == "(devel)" {
		return "dev"
	}

	return info.Main.Version
}

func main() {
	// Once the reader of a stdout pipe has gone, as in "sfdupes report |
	// head", the Go runtime ends the process with SIGPIPE on the next
	// write instead of returning an error (README "Error handling").
	// Registering for SIGPIPE with os/signal would change that.
	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
}

// run executes args against the command tree and returns the process
// exit code. It is the program's single exit point: the subcommands
// return their errors instead of exiting, so every deferred cleanup —
// above all closing the database, which checkpoints the SQLite WAL —
// runs before the process ends. The report, trees and info subcommands
// write their data to stdout.
func run(args []string, stdout, stderr io.Writer) int {
	// A nil slice makes cobra fall back to os.Args, which would let a
	// test binary's own flags reach the command tree.
	if args == nil {
		args = []string{}
	}

	root := newRootCommand(stdout, stderr)
	root.SetArgs(args)

	err := root.Execute()

	var fatal fatalError

	switch {
	case err == nil:
		return exitOK
	case errors.Is(err, errInterrupted):
		// The interrupted scan has printed its own line.
		return exitFatal
	case errors.As(err, &fatal):
		// The command ran and failed: a runtime error, reported
		// without the usage text that a usage error gets.
		_, _ = fmt.Fprintf(stderr, "sfdupes: %v\n", err)

		return exitFatal
	default:
		// A usage error, which cobra has already reported on stderr.
		return exitUsage
	}
}

// newRootCommand builds the command tree. Everything on stdout is
// machine-readable data, the version line included; all human-facing
// output (help, usage, errors) goes to stderr.
func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
	var showVersion bool

	printVersion := runE(func(context.Context, []string) error {
		info, _ := debug.ReadBuildInfo()

		_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
			buildVersion(Version, info))
		if err != nil {
			return fmt.Errorf("write stdout: %w", err)
		}

		return nil
	})

	root := &cobra.Command{
		Use:   "sfdupes",
		Short: "Find candidate duplicate files by size and head/tail/content SHA-256",
		Args:  cobra.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			if showVersion {
				return printVersion(cmd, args)
			}

			// A missing subcommand prints usage and exits 2: cobra
			// prints the usage text for the returned error, and run
			// maps everything that is not a fatal error to exit 2.
			cmd.SilenceErrors = true

			return errNoSubcommand
		},
	}
	root.SetOut(stderr)
	root.SetErr(stderr)
	root.CompletionOptions.DisableDefaultCmd = true

	// Cobra's built-in version flag prints through the help writer,
	// stderr; this one prints to stdout.
	root.Flags().BoolVarP(&showVersion, "version", "v", false,
		"print the version to stdout")

	reportCmd := &cobra.Command{
		Use:   cmdReport,
		Short: "Read the scan database and print the file-level duplicates report",
		Args:  cobra.NoArgs,
		RunE: runE(func(ctx context.Context, _ []string) error {
			return runReport(ctx, stdout)
		}),
	}

	treesCmd := &cobra.Command{
		Use:   cmdTrees,
		Short: "Read the scan database and print the duplicate-tree report",
		Args:  cobra.NoArgs,
		RunE: runE(func(ctx context.Context, _ []string) error {
			return runTrees(ctx, stdout)
		}),
	}

	infoCmd := &cobra.Command{
		Use:   cmdInfo,
		Short: "Read the scan database and print its details",
		Args:  cobra.NoArgs,
		RunE: runE(func(ctx context.Context, _ []string) error {
			return runInfo(ctx, stdout)
		}),
	}

	root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)

	return root
}

// newScanCommand builds the scan subcommand, with its --workers, -x and
// --no-default-excludes flags. Its help lists the default excludes of
// the system it runs on, and the filesystem types it never records
// anything on.
func newScanCommand() *cobra.Command {
	var (
		scanWorkers           int
		scanOneFS             bool
		scanNoDefaultExcludes bool
	)

	short := "Walk trees and synchronize the scan database"

	scanCmd := &cobra.Command{
		Use:   cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
		Short: short,
		Long:  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
		Args:  cobra.MinimumNArgs(1),
		PreRunE: func(cmd *cobra.Command, _ []string) error {
			return checkScanWorkers(cmd, scanWorkers)
		},
		RunE: runE(func(ctx context.Context, args []string) error {
			ctx, stop := interruptContext(ctx)
			defer stop()

			return runScan(ctx, args, scanWorkers, scanOneFS,
				scanNoDefaultExcludes)
		}),
	}
	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
		"concurrent workers for the walk, hash, and content phases")
	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
		"do not cross filesystem boundaries")
	scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
		false, "enter the default excludes, the paths listed above")

	return scanCmd
}

// checkScanWorkers rejects a scan --workers value below 1. That is a
// usage error reported in one line: cobra prints the returned message
// without the usage text, and run exits 2.
func checkScanWorkers(cmd *cobra.Command, workers int) error {
	if workers >= 1 {
		return nil
	}

	cmd.SilenceUsage = true

	return fmt.Errorf("%w, got %d", errWorkersBelowOne, workers)
}

// runE adapts a subcommand implementation, or the version print, to
// cobra's RunE. Cobra prints the error and the command's usage text for
// every error RunE returns, but a subcommand that ran and failed has no
// usage problem to report: both are silenced here, and the error is
// marked fatal so that run reports it on stderr and exits 1 rather than
// 2. The command's context is handed to the implementation: cancelling
// it unwinds the scan's worker pools.
func runE(
	fn func(ctx context.Context, args []string) error,
) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		cmd.SilenceUsage = true
		cmd.SilenceErrors = true

		err := fn(cmd.Context(), args)
		if err != nil {
			return fatalError{err: err}
		}

		return nil
	}
}

// fatalError marks a runtime failure, as opposed to the usage errors
// cobra itself produces while parsing arguments and flags. Both come
// out of Execute as plain errors, so the wrapper is what tells run to
// report this one as "sfdupes: ..." and exit 1.
type fatalError struct {
	err error
}

func (e fatalError) Error() string { return e.err.Error() }

func (e fatalError) Unwrap() error { return e.err }