mfer/internal/cli/freshen.go

package cli

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/multiformats/go-multihash"
	"github.com/spf13/afero"
	"github.com/urfave/cli/v3"
	"sneak.berlin/go/mfer/internal/log"
	"sneak.berlin/go/mfer/mfer"
)

const (
	// hashBufSize is the read buffer size used when hashing files.
	hashBufSize = 64 * 1024

	// scanProgressInterval is how many scanned files pass between
	// progress updates.
	scanProgressInterval = 100
)

var (
	// errEntryMissingMtime indicates a manifest entry that carries no
	// modification time where one is required to carry it forward unchanged.
	errEntryMissingMtime = errors.New("manifest entry has no mtime")

	// errEntryMissingHash indicates a manifest entry that carries no
	// SHA-256 hash to carry forward.
	errEntryMissingHash = errors.New("manifest entry has no SHA-256 hash")
)

// errSignKeyNeeded indicates a signed manifest given to freshen with no
// key to sign the rewritten one.
var errSignKeyNeeded = errors.New("give --sign-key to keep it signed")

// FreshenStatus contains progress information for the freshen operation.
type FreshenStatus struct {
	Phase        string        // "scan" or "hash"
	TotalFiles   int64         // Total files to process in current phase
	CurrentFiles int64         // Files processed so far
	TotalBytes   int64         // Total bytes to hash (hash phase only)
	CurrentBytes int64         // Bytes hashed so far
	BytesPerSec  float64       // Current throughput rate
	ETA          time.Duration // Estimated time to completion
}

// freshenEntry tracks a file's status during freshen
type freshenEntry struct {
	path      string
	size      int64
	mtime     time.Time
	mode      fs.FileMode      // mode to record, 0 for none
	needsHash bool             // true if new or changed
	existing  *mfer.MFFilePath // existing manifest entry if unchanged
}

// freshenScanner walks the filesystem and compares it against the
// entries of an existing manifest.
type freshenScanner struct {
	fs                 afero.Fs
	absBase            string
	excluded           []fs.FileInfo // files left out of the listing
	includeDotfiles    bool
	followSymlinks     bool
	includePermissions bool
	showProgress       bool
	existingByPath     map[string]*mfer.MFFilePath

	entries   []*freshenEntry
	scanCount int64
	changed   int64
	added     int64
	unchanged int64
}

// resolveSymlink resolves a symlink to its target's FileInfo. The
// second return value is false when the entry should be skipped.
func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) {
	if !s.followSymlinks {
		return nil, false
	}

	realPath, err := filepath.EvalSymlinks(path)
	if err != nil {
		return nil, false // Skip broken symlinks
	}

	realInfo, err := s.fs.Stat(realPath)
	if err != nil || realInfo.IsDir() {
		return nil, false
	}

	return realInfo, true
}

// recordEntry classifies a scanned file as changed, unchanged, or added
// relative to the existing manifest.
func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
	// A mode of 0 records 0000, which means none was recorded.
	var mode fs.FileMode
	if s.includePermissions {
		mode = info.Mode().Perm()
	}

	existing, inManifest := s.existingByPath[relPath]
	if !inManifest {
		s.added++

		log.Verbosef("A %s", relPath)
		s.entries = append(s.entries, &freshenEntry{
			path:      relPath,
			size:      info.Size(),
			mtime:     info.ModTime(),
			mode:      mode,
			needsHash: true,
		})

		return
	}

	// Check if changed (size, mtime, or the mode to record). An entry
	// with no recorded mtime cannot be compared, so it counts as changed
	// and gets re-hashed; silently treating the absent mtime as the Unix
	// epoch would classify every such entry as changed without saying why.
	existingMtime, haveMtime := entryMtime(existing)
	if !haveMtime {
		log.Debugf("%s: manifest entry has no mtime, treating as changed",
			relPath)
	}

	if !haveMtime || existing.GetSize() != info.Size() ||
		!existingMtime.Equal(info.ModTime()) ||
		fs.FileMode(existing.GetMode()) != mode {
		s.changed++

		log.Verbosef("M %s", relPath)
		s.entries = append(s.entries, &freshenEntry{
			path:      relPath,
			size:      info.Size(),
			mtime:     info.ModTime(),
			mode:      mode,
			needsHash: true,
		})
	} else {
		s.unchanged++

		s.entries = append(s.entries, &freshenEntry{
			path:      relPath,
			size:      info.Size(),
			mtime:     info.ModTime(),
			mode:      mode,
			needsHash: false,
			existing:  existing,
		})
	}
	// Mark as seen
	delete(s.existingByPath, relPath)
}

// walk is the afero.Walk callback for the scan phase.
func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) error {
	if walkErr != nil {
		return walkErr
	}

	// Get relative path
	relPath, err := filepath.Rel(s.absBase, path)
	if err != nil {
		return err
	}

	// Handle dotfiles
	if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
		if info.IsDir() {
			return filepath.SkipDir
		}

		return nil
	}

	// Skip directories
	if info.IsDir() {
		return nil
	}

	// Handle symlinks
	if info.Mode()&fs.ModeSymlink != 0 {
		realInfo, keep := s.resolveSymlink(path)
		if !keep {
			return nil
		}

		info = realInfo
	}

	// Opening a named pipe can block, and reading a device may never end.
	if !info.Mode().IsRegular() {
		log.Verbosef("skipping %s: not a regular file", relPath)

		return nil
	}

	for _, excluded := range s.excluded {
		if os.SameFile(info, excluded) {
			return nil
		}
	}

	s.scanCount++

	// Check against existing manifest
	s.recordEntry(relPath, info)

	// Report scan progress
	if s.showProgress && s.scanCount%scanProgressInterval == 0 {
		log.Progressf("Scanning: %d files found", s.scanCount)
	}

	return nil
}

// resolveFreshenManifestPath determines the manifest path from the CLI
// arguments, searching directories for a manifest where needed.
func (mfa *CLIApp) resolveFreshenManifestPath(cmd *cli.Command) (string, error) {
	if cmd.Args().Len() == 0 {
		return findManifest(mfa.Fs, ".")
	}

	arg := cmd.Args().Get(0)

	info, statErr := mfa.Fs.Stat(arg)
	if statErr == nil && info.IsDir() {
		return findManifest(mfa.Fs, arg)
	}

	return arg, nil
}

// freshenHasher hashes changed and added files and feeds all entries to
// a manifest builder.
type freshenHasher struct {
	fs             afero.Fs
	absBase        string
	showProgress   bool
	totalHashBytes int64
	filesToHash    int64
	startHash      time.Time
	builder        *mfer.Builder

	hashedFiles int64
	hashedBytes int64
}

// reportProgress renders hashing progress for the current byte count.
func (h *freshenHasher) reportProgress(n int64) {
	if !h.showProgress {
		return
	}

	currentBytes := h.hashedBytes + n
	elapsed := time.Since(h.startHash)

	var (
		rate float64
		eta  time.Duration
	)

	if elapsed > 0 && currentBytes > 0 {
		rate = float64(currentBytes) / elapsed.Seconds()

		remaining := h.totalHashBytes - currentBytes
		if rate > 0 {
			eta = time.Duration(float64(remaining)/rate) * time.Second
		}
	}

	if eta > 0 {
		log.Progressf("Hashing: %d/%d files, %s/s, ETA %s",
			h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)),
			eta.Round(time.Second))
	} else {
		log.Progressf("Hashing: %d/%d files, %s/s",
			h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)))
	}
}

// processEntry hashes the entry if needed and adds it to the builder.
func (h *freshenHasher) processEntry(e *freshenEntry) error {
	if !e.needsHash {
		// Use existing entry; the error names the entry
		return addExistingToBuilder(h.builder, e.existing)
	}

	// Need to read and hash the file
	absPath := filepath.Join(h.absBase, e.path)

	f, err := h.fs.Open(absPath)
	if err != nil {
		return err
	}

	hash, bytesRead, err := hashFile(f, h.reportProgress)
	_ = f.Close()

	if err != nil {
		return err
	}

	// The scan recorded the size; a file that grew or shrank since would
	// get an entry whose size and hash describe different contents.
	if bytesRead != e.size {
		return fmt.Errorf(
			"%w for %q: %d bytes when scanned but %d bytes when read; "+
				"the file changed while freshen ran",
			errSizeMismatch, e.path, e.size, bytesRead,
		)
	}

	h.hashedBytes += bytesRead
	h.hashedFiles++

	// Add to builder with computed hash; a refused path is named in the error
	return addFileToBuilder(h.builder, e.path, e.size, e.mtime, e.mode, hash)
}

// writeFreshenedManifest writes the manifest atomically (write to a
// temp file, then rename over the target).
func writeFreshenedManifest(
	ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
) error {
	tmpPath := manifestTempPath(manifestPath)

	outFile, err := afs.Create(tmpPath)
	if err != nil {
		return err
	}

	err = builder.Build(ctx, outFile)
	_ = outFile.Close()

	if err != nil {
		_ = afs.Remove(tmpPath)

		return fmt.Errorf("build manifest: %w", err)
	}

	// Rename temp to final
	err = afs.Rename(tmpPath, manifestPath)
	if err != nil {
		_ = afs.Remove(tmpPath)

		return err
	}

	return nil
}

// newFreshenBuilder constructs the manifest builder configured from CLI
// flags, and returns the fingerprint of the --sign-key key, or "" when
// none is given. A --sign-key given with an empty value is refused.
func (mfa *CLIApp) newFreshenBuilder(
	cmd *cli.Command,
) (*mfer.Builder, string, error) {
	builder := mfer.NewBuilder()
	if cmd.Bool("include-timestamps") {
		builder.SetIncludeTimestamps(true)
	}

	var signKeyFingerprint string

	// Set up signing options if sign-key is provided
	if cmd.IsSet("sign-key") {
		signKey := cmd.String("sign-key")
		if signKey == "" {
			return nil, "", errEmptySignKey
		}

		signing, err := mfa.signingOptions(signKey)
		if err != nil {
			return nil, "", err
		}

		signKeyFingerprint, err = mfer.SecretKeyFingerprint(signing.SecretKey)
		if err != nil {
			return nil, "", fmt.Errorf("%s: %w", signKey, err)
		}

		builder.SetSigningOptions(signing)
	}

	return builder, signKeyFingerprint, nil
}

// freshenScan runs the scan phase against the loaded manifest entries
// and returns the populated scanner and the count of removed files.
func (mfa *CLIApp) freshenScan(
	cmd *cli.Command, manifestPath, absBase string,
	existingByPath map[string]*mfer.MFFilePath,
) (*freshenScanner, int64, error) {
	log.Infof("scanning filesystem...")

	startScan := time.Now()
	showProgress := cmd.Bool("progress")

	// Leave out the manifest, a temp file left by an interrupted run, and
	// index.mf at the top of the base directory, where fetch saves the
	// manifest, as gen does. A path that cannot be stat'd, normally because
	// no file is there, needs no leaving out.
	var excluded []fs.FileInfo

	for _, p := range []string{
		manifestPath,
		manifestTempPath(manifestPath),
		filepath.Join(absBase, defaultManifestName),
	} {
		info, err := mfa.Fs.Stat(p)
		if err == nil {
			excluded = append(excluded, info)
		}
	}

	// The walk does not follow a symlink at its top, so a base directory
	// named through one is resolved first. If that fails, the base is
	// walked as named and the walk reports the problem.
	resolved, err := filepath.EvalSymlinks(absBase)
	if err == nil {
		absBase = resolved
	}

	scanner := &freshenScanner{
		fs:                 mfa.Fs,
		absBase:            absBase,
		excluded:           excluded,
		includeDotfiles:    cmd.Bool("include-dotfiles"),
		followSymlinks:     cmd.Bool("follow-symlinks"),
		includePermissions: cmd.Bool(flagIncludePermissions),
		showProgress:       showProgress,
		existingByPath:     existingByPath,
	}

	err = afero.Walk(mfa.Fs, absBase, scanner.walk)

	if showProgress {
		log.ProgressDone()
	}

	if err != nil {
		return nil, 0, fmt.Errorf("scan filesystem: %w", err)
	}

	// Remaining entries in existingByPath are removed files
	removed := int64(len(existingByPath))
	for path := range existingByPath {
		log.Verbosef("D %s", path)
	}

	scanDuration := time.Since(startScan)
	log.Infof("scan complete in %s: %d unchanged, %d changed, %d added, %d removed",
		scanDuration.Round(time.Millisecond), scanner.unchanged, scanner.changed,
		scanner.added, removed)

	return scanner, removed, nil
}

// hashTotals returns the total byte count and file count of entries
// that need hashing.
func hashTotals(entries []*freshenEntry) (int64, int64) {
	var (
		totalHashBytes int64
		filesToHash    int64
	)

	for _, e := range entries {
		if e.needsHash {
			totalHashBytes += e.size
			filesToHash++
		}
	}

	return totalHashBytes, filesToHash
}

// runFreshenHash processes every entry through the hasher, aborting if
// the context is canceled, and ends the hasher's progress line.
func runFreshenHash(
	ctx context.Context, hasher *freshenHasher, entries []*freshenEntry,
) error {
	for _, e := range entries {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		err := hasher.processEntry(e)
		if err != nil {
			return err
		}
	}

	if hasher.showProgress && hasher.filesToHash > 0 {
		log.ProgressDone()
	}

	return nil
}

// existingManifest is what freshen uses of the manifest it rewrites.
type existingManifest struct {
	entriesByPath map[string]*mfer.MFFilePath
	signer        string // fingerprint of the key it is signed with, "" if unsigned
	hasCreatedAt  bool
}

// loadExistingManifest loads the manifest and indexes its file entries
// by path. It refuses a signed manifest when no --sign-key is given, as
// the rewritten manifest would be unsigned.
func (mfa *CLIApp) loadExistingManifest(
	cmd *cli.Command, manifestPath string,
) (*existingManifest, error) {
	log.Infof("loading manifest from %s", manifestPath)

	// Load existing manifest
	manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
		Path: manifestPath,
		Fs:   mfa.Fs,
	})
	if err != nil {
		return nil, fmt.Errorf("load manifest: %w", err)
	}

	if signer := manifest.Signer(); signer != nil && cmd.String("sign-key") == "" {
		return nil, fmt.Errorf("manifest is signed by %s: %w", signer, errSignKeyNeeded)
	}

	existingFiles := manifest.Files()
	log.Infof("manifest contains %d files", len(existingFiles))

	// Build map of existing entries by path
	existingByPath := make(map[string]*mfer.MFFilePath, len(existingFiles))
	for _, f := range existingFiles {
		existingByPath[f.GetPath()] = f
	}

	return &existingManifest{
		entriesByPath: existingByPath,
		signer:        string(manifest.Signer()),
		hasCreatedAt:  manifest.CreatedAt() != nil,
	}, nil
}

func (mfa *CLIApp) freshenManifestOperation(
	ctx context.Context, cmd *cli.Command,
) error {
	log.Debug("freshenManifestOperation()")

	// Find manifest file
	manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
	if err != nil {
		return err
	}

	builder, signKeyFingerprint, err := mfa.newFreshenBuilder(cmd)
	if err != nil {
		return err
	}

	existing, err := mfa.loadExistingManifest(cmd, manifestPath)
	if err != nil {
		return err
	}

	absBase, err := filepath.Abs(resolveBasePath(cmd, manifestPath))
	if err != nil {
		return fmt.Errorf("invalid base path: %w", err)
	}

	// Phase 1: Scan filesystem
	scanner, removed, err := mfa.freshenScan(cmd, manifestPath, absBase,
		existing.entriesByPath)
	if err != nil {
		return err
	}

	// Calculate total bytes to hash
	totalHashBytes, filesToHash := hashTotals(scanner.entries)

	// Phase 2: Hash changed and new files
	if filesToHash > 0 {
		log.Infof("hashing %d files (%s)...", filesToHash,
			humanize.IBytes(safeUint64(totalHashBytes)))
	}

	hasher := &freshenHasher{
		fs:             mfa.Fs,
		absBase:        absBase,
		showProgress:   cmd.Bool("progress"),
		totalHashBytes: totalHashBytes,
		filesToHash:    filesToHash,
		startHash:      time.Now(),
		builder:        builder,
	}

	err = runFreshenHash(ctx, hasher, scanner.entries)
	if err != nil {
		return err
	}

	// Print summary
	log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
		scanner.unchanged, scanner.changed, scanner.added, removed)

	// Skip writing only when the manifest would come out the same: nothing
	// in the tree changed, and the manifest already holds what --sign-key,
	// --include-timestamps and --include-permissions ask for. The scan
	// counts a file whose recorded mode is not what --include-permissions
	// asks for as changed.
	if scanner.changed == 0 && scanner.added == 0 && removed == 0 &&
		existing.signer == signKeyFingerprint &&
		existing.hasCreatedAt == cmd.Bool("include-timestamps") {
		log.Infof("manifest unchanged, skipping write")

		return nil
	}

	// Write updated manifest atomically (write to temp, then rename)
	err = writeFreshenedManifest(ctx, mfa.Fs, hasher.builder, manifestPath)
	if err != nil {
		return err
	}

	totalDuration := time.Since(mfa.startupTime)
	if hasher.hashedBytes > 0 {
		hashDuration := time.Since(hasher.startHash)
		hashRate := float64(hasher.hashedBytes) / hashDuration.Seconds()
		log.Infof("hashed %s in %.1fs (%s/s)",
			humanize.IBytes(safeUint64(hasher.hashedBytes)),
			totalDuration.Seconds(), humanize.IBytes(safeRateUint64(hashRate)))
	}

	log.Infof("wrote %d files to %s", len(scanner.entries), manifestPath)

	return nil
}

// hashFile reads a file and computes its SHA256 multihash.
// Progress callback is called with bytes read so far.
func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
	h := sha256.New()
	buf := make([]byte, hashBufSize)

	var total int64

	for {
		n, err := r.Read(buf)
		if n > 0 {
			h.Write(buf[:n])

			total += int64(n)
			if progress != nil {
				progress(total)
			}
		}

		if err == io.EOF {
			break
		}

		// Returned unwrapped: a read error already names the file.
		if err != nil {
			return nil, total, err
		}
	}

	mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
	if err != nil {
		return nil, total, err
	}

	return mh, total, nil
}

// addFileToBuilder adds a new file entry to the builder
func addFileToBuilder(
	b *mfer.Builder, path string, size int64, mtime time.Time, mode fs.FileMode,
	hash []byte,
) error {
	return b.AddFileWithHash(
		mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), mode,
		hash)
}

// addExistingToBuilder adds an existing manifest entry to the builder.
//
// Entries reach this path only when recordEntry classified them as
// unchanged, which requires a recorded mtime, so an absent mtime here is
// an error rather than something to paper over with the Unix epoch.
// Only the entry's SHA-256 hash is carried over, wherever the entry lists
// it, since check and fetch compare with no other. Loading refuses an
// entry with no SHA-256 hash; one that got here anyway is an error too,
// not a file to leave out of the manifest.
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
	mtime, ok := entryMtime(entry)
	if !ok {
		return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
	}

	hash := mfer.SHA256Multihash(entry)
	if hash == nil {
		return fmt.Errorf("%w: %s", errEntryMissingHash, entry.GetPath())
	}

	err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
		mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
		fs.FileMode(entry.GetMode()), hash)
	if err != nil {
		return fmt.Errorf("manifest entry %s: %w", entry.GetPath(), err)
	}

	return nil
}