mfer/internal/cli/gen.go

package cli

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/spf13/afero"
	"github.com/urfave/cli/v3"
	"sneak.berlin/go/mfer/internal/log"
	"sneak.berlin/go/mfer/mfer"
)

var (
	// errPathNotExist indicates an input path that does not exist.
	errPathNotExist = errors.New("path does not exist")
	// errOutputExists indicates the output file already exists and
	// --force was not given. It is wrapped mid-sentence so that the
	// rendered message stays exactly as mfer has always printed it.
	errOutputExists = errors.New(
		"already exists (use --force to overwrite)")
	// errEmptyOutput indicates --output given with an empty value.
	errEmptyOutput = errors.New("--output must not be empty")
	// errEmptySignKey indicates --sign-key given with an empty value.
	errEmptySignKey = errors.New("--sign-key must not be empty")
)

// reportEnumProgress renders enumeration progress until the channel
// closes.
func reportEnumProgress(progress <-chan mfer.EnumerateStatus, wg *sync.WaitGroup) {
	defer wg.Done()

	for status := range progress {
		log.Progressf("Enumerating: %d files, %s",
			status.FilesFound,
			humanize.IBytes(safeUint64(int64(status.BytesFound))))
	}

	log.ProgressDone()
}

// reportScanProgress renders scan progress until the channel closes.
func reportScanProgress(progress <-chan mfer.ScanStatus, wg *sync.WaitGroup) {
	defer wg.Done()

	for status := range progress {
		if status.ETA > 0 {
			log.Progressf("Scanning: %d/%d files, %s/s, ETA %s",
				status.ScannedFiles,
				status.TotalFiles,
				humanize.IBytes(safeRateUint64(status.BytesPerSec)),
				status.ETA.Round(time.Second))
		} else {
			log.Progressf("Scanning: %d/%d files, %s/s",
				status.ScannedFiles,
				status.TotalFiles,
				humanize.IBytes(safeRateUint64(status.BytesPerSec)))
		}
	}

	log.ProgressDone()
}

// collectInputPaths validates the input path arguments and returns them
// as absolute paths.
func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
	paths := make([]string, 0, args.Len())

	for i := range args.Len() {
		inputPath := args.Get(i)

		ap, err := filepath.Abs(inputPath)
		if err != nil {
			return nil, fmt.Errorf("invalid path %q: %w", inputPath, err)
		}
		// Validate path exists before adding to list
		if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
			return nil, fmt.Errorf("%w: %s", errPathNotExist, inputPath)
		}

		log.Debugf("enumerating path: %s", ap)
		paths = append(paths, ap)
	}

	return paths, nil
}

// outputPath returns the file gen writes the manifest to: the one --output
// names, or else index.mf in the directory the only argument names, or
// beside the file it names, or else in the current directory. An --output
// given with an empty value is refused.
func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
	if cmd.IsSet("output") {
		output := cmd.String("output")
		if output == "" {
			return "", errEmptyOutput
		}

		return output, nil
	}

	if cmd.Args().Len() != 1 {
		return defaultManifestName, nil
	}

	return mfa.topIndexMFPath(cmd.Args().First()), nil
}

// topIndexMFPath returns index.mf in the directory path names, or beside
// the file it names, which is the top of the tree gen lists for that path.
func (mfa *CLIApp) topIndexMFPath(path string) string {
	// A path that does not exist is refused when it is enumerated.
	info, err := mfa.Fs.Stat(path)
	if err == nil && !info.IsDir() {
		return filepath.Join(filepath.Dir(path), defaultManifestName)
	}

	return filepath.Join(path, defaultManifestName)
}

// buildScannerOptions constructs scanner options from the CLI flags and
// the path the manifest is written to. A --sign-key given with an empty
// value is refused.
func (mfa *CLIApp) buildScannerOptions(
	cmd *cli.Command, output string,
) (*mfer.ScannerOptions, error) {
	opts := &mfer.ScannerOptions{
		IncludeDotfiles:    cmd.Bool("include-dotfiles"),
		FollowSymLinks:     cmd.Bool("follow-symlinks"),
		IncludeTimestamps:  cmd.Bool("include-timestamps"),
		IncludePermissions: cmd.Bool(flagIncludePermissions),
		Fs:                 mfa.Fs,
		// Neither a manifest being replaced nor a temp file left by an
		// interrupted run belongs in the new manifest.
		ExcludePaths: []string{output, manifestTempPath(output)},
	}

	// Nor does index.mf at the top of the tree, since fetch saves the
	// manifest there and refuses a manifest that lists it. With no path
	// given, gen lists the current directory.
	paths := cmd.Args().Slice()
	if len(paths) == 0 {
		paths = []string{"."}
	}

	for _, path := range paths {
		opts.ExcludePaths = append(opts.ExcludePaths, mfa.topIndexMFPath(path))
	}

	// Set seed for deterministic UUID if provided
	if seed := cmd.String("seed"); seed != "" {
		opts.Seed = seed

		log.Infof("using deterministic seed for manifest UUID")
	}

	// Set up signing options if sign-key is provided
	if cmd.IsSet("sign-key") {
		signKey := cmd.String("sign-key")
		if signKey == "" {
			return nil, errEmptySignKey
		}

		signing, err := mfa.signingOptions(signKey)
		if err != nil {
			return nil, err
		}

		opts.SigningOptions = signing
	}

	return opts, nil
}

// enumerateInputs runs the enumeration phase over the argument paths,
// or the current directory when no arguments are given.
func (mfa *CLIApp) enumerateInputs(
	s *mfer.Scanner, args cli.Args, enumProgress chan mfer.EnumerateStatus,
) error {
	if args.Len() == 0 {
		// Default to current directory
		err := s.EnumeratePath(".", enumProgress)
		if err != nil {
			return fmt.Errorf("enumerate current directory: %w", err)
		}

		return nil
	}

	// Collect and validate all paths first
	paths, err := mfa.collectInputPaths(args)
	if err != nil {
		return err
	}

	err = s.EnumeratePaths(enumProgress, paths...)
	if err != nil {
		return fmt.Errorf("enumerate files: %w", err)
	}

	return nil
}

// cleanupOnSignal installs a handler that removes the temp output file
// and exits when the process is interrupted. It returns the signal
// channel so the caller can stop and close it when done.
func (mfa *CLIApp) cleanupOnSignal(outFile afero.File, tmpPath string) chan os.Signal {
	sigChan := make(chan os.Signal, 1)

	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)

	go func() {
		sig, ok := <-sigChan
		if !ok || sig == nil {
			return // Channel closed normally, not a signal
		}

		_ = outFile.Close()
		_ = mfa.Fs.Remove(tmpPath)

		os.Exit(1)
	}()

	return sigChan
}

// runEnumeratePhase enumerates all input paths with optional progress
// reporting and logs the totals.
func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
	// Set up enumeration progress reporting
	var (
		enumProgress chan mfer.EnumerateStatus
		enumWg       sync.WaitGroup
	)

	if cmd.Bool("progress") {
		enumProgress = make(chan mfer.EnumerateStatus, 1)

		enumWg.Add(1)

		go reportEnumProgress(enumProgress, &enumWg)
	}

	err := mfa.enumerateInputs(s, cmd.Args(), enumProgress)
	if err != nil {
		return err
	}

	enumWg.Wait()

	log.Infof("enumerated %d files, %s total", s.FileCount(),
		humanize.IBytes(safeUint64(int64(s.TotalBytes()))))

	return nil
}

// runScanPhase reads the enumerated files and writes the manifest to out,
// with optional progress reporting.
func (mfa *CLIApp) runScanPhase(
	ctx context.Context, cmd *cli.Command, s *mfer.Scanner, out io.Writer,
) error {
	var (
		scanProgress chan mfer.ScanStatus
		scanWg       sync.WaitGroup
	)

	if cmd.Bool("progress") {
		scanProgress = make(chan mfer.ScanStatus, 1)

		scanWg.Add(1)

		go reportScanProgress(scanProgress, &scanWg)
	}

	err := s.ToManifest(ctx, out, scanProgress)

	scanWg.Wait()

	if err != nil {
		return fmt.Errorf("generate manifest: %w", err)
	}

	return nil
}

func (mfa *CLIApp) generateManifestOperation(
	ctx context.Context, cmd *cli.Command,
) error {
	log.Debug("generateManifestOperation()")

	outputPath, err := mfa.outputPath(cmd)
	if err != nil {
		return err
	}

	opts, err := mfa.buildScannerOptions(cmd, outputPath)
	if err != nil {
		return err
	}

	s := mfer.NewScannerWithOptions(opts)

	// Phase 1: Enumeration - collect paths and stat files
	err = mfa.runEnumeratePhase(cmd, s)
	if err != nil {
		return err
	}

	// Check if output file exists
	if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") {
		return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
	}

	// Create temp file for atomic write
	tmpPath := manifestTempPath(outputPath)

	outFile, err := mfa.Fs.Create(tmpPath)
	if err != nil {
		return err
	}

	// Set up signal handler to clean up temp file on Ctrl-C
	sigChan := mfa.cleanupOnSignal(outFile, tmpPath)

	// Clean up temp file on any error or interruption
	success := false

	defer func() {
		signal.Stop(sigChan)
		close(sigChan)

		_ = outFile.Close()

		if !success {
			_ = mfa.Fs.Remove(tmpPath)
		}
	}()

	// Phase 2: Scan - read file contents and generate manifest
	err = mfa.runScanPhase(ctx, cmd, s, outFile)
	if err != nil {
		return err
	}

	// Close file before rename to ensure all data is flushed
	err = outFile.Close()
	if err != nil {
		return err
	}

	// Atomic rename
	err = mfa.Fs.Rename(tmpPath, outputPath)
	if err != nil {
		return err
	}

	success = true

	elapsed := time.Since(mfa.startupTime).Seconds()
	rate := float64(s.TotalBytes()) / elapsed
	log.Infof("wrote %d files (%s) to %s in %.1fs (%s/s)", s.FileCount(),
		humanize.IBytes(safeUint64(int64(s.TotalBytes()))), outputPath, elapsed,
		humanize.IBytes(safeRateUint64(rate)))

	return nil
}