mfer/mfer/scanner.go

package mfer

import (
	"context"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/spf13/afero"
	"sneak.berlin/go/mfer/internal/log"
)

// Phase 1: Enumeration
// ---------------------
// Walking directories and calling stat() on files to collect metadata.
// Builds the list of files to be scanned. Relatively fast (metadata only).

// EnumerateStatus contains progress information for the enumeration phase.
type EnumerateStatus struct {
	FilesFound FileCount // Number of files discovered so far
	BytesFound FileSize  // Total size of discovered files (from stat)
}

// Phase 2: Scan (ToManifest)
// --------------------------
// Reading file contents and computing hashes for manifest generation.
// This is the expensive phase that reads all file data.

// ScanStatus contains progress information for the scan phase.
type ScanStatus struct {
	TotalFiles   FileCount     // Total number of files to scan
	ScannedFiles FileCount     // Number of files scanned so far
	TotalBytes   FileSize      // Total bytes to read (sum of all file sizes)
	ScannedBytes FileSize      // Bytes read so far
	BytesPerSec  float64       // Current throughput rate
	ETA          time.Duration // Estimated time to completion
}

// ScannerOptions configures scanner behavior.
type ScannerOptions struct {
	// IncludeDotfiles includes files and directories starting with a dot
	// (default: exclude).
	IncludeDotfiles bool
	// FollowSymLinks resolves symlinks instead of skipping them.
	FollowSymLinks bool
	// IncludeTimestamps includes a createdAt timestamp in the manifest
	// (default: omit for determinism).
	IncludeTimestamps bool
	// IncludePermissions records each file's permission bits, 0777 at
	// most, in the manifest (default: record 0000).
	IncludePermissions bool
	// Fs is the filesystem to use, defaults to OsFs if nil.
	Fs afero.Fs
	// SigningOptions holds the key to sign with (nil = no signing).
	SigningOptions *SigningOptions
	// Seed, if set, derives a deterministic UUID from this seed.
	Seed string
	// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
	ExcludePaths []string
}

// FileEntry represents a file that has been enumerated.
type FileEntry struct {
	Path    RelFilePath // Relative path (used in manifest)
	AbsPath AbsFilePath // Absolute path (used for reading file content)
	Size    FileSize    // File size in bytes
	Mtime   ModTime     // Last modification time
	Ctime   time.Time   // Creation time (platform-dependent)
	Mode    fs.FileMode // Permission bits (Perm() of the file's mode)
}

// Scanner accumulates files and generates manifests from them.
type Scanner struct {
	mu         sync.RWMutex
	files      []*FileEntry
	paths      map[RelFilePath]AbsFilePath // the file at each path in files
	totalBytes FileSize                    // cached sum of all file sizes
	options    *ScannerOptions
	fs         afero.Fs
	excluded   []fs.FileInfo // the files named in ExcludePaths that exist
}

// NewScanner creates a new Scanner with default options.
func NewScanner() *Scanner {
	return NewScannerWithOptions(nil)
}

// NewScannerWithOptions creates a new Scanner with the given options.
func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
	if opts == nil {
		opts = &ScannerOptions{}
	}

	fs := opts.Fs
	if fs == nil {
		fs = afero.NewOsFs()
	}

	s := &Scanner{
		files:   make([]*FileEntry, 0),
		paths:   make(map[RelFilePath]AbsFilePath),
		options: opts,
		fs:      fs,
	}

	// A path that cannot be stat'd, normally because no file is there,
	// needs no leaving out.
	for _, p := range opts.ExcludePaths {
		info, err := s.fs.Stat(p)
		if err == nil {
			s.excluded = append(s.excluded, info)
		}
	}

	return s
}

// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
func (s *Scanner) EnumerateFile(filePath string) error {
	abs, err := filepath.Abs(filePath)
	if err != nil {
		return err
	}

	info, err := s.fs.Stat(abs)
	if err != nil {
		return err
	}
	// For single files, use the filename as the relative path
	basePath := filepath.Dir(abs)

	return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil)
}

// EnumeratePath walks a directory path and adds all files to the scanner.
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
func (s *Scanner) EnumeratePath(
	inputPath string,
	progress chan<- EnumerateStatus,
) error {
	if progress != nil {
		defer close(progress)
	}

	abs, err := filepath.Abs(inputPath)
	if err != nil {
		return err
	}

	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))

	return s.enumerateFS(afs, abs, progress)
}

// EnumeratePaths adds to the scanner the files under each directory path,
// listed by their paths under it, and each file path, listed by its name
// as EnumerateFile lists it.
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
func (s *Scanner) EnumeratePaths(
	progress chan<- EnumerateStatus,
	inputPaths ...string,
) error {
	if progress != nil {
		defer close(progress)
	}

	for _, p := range inputPaths {
		abs, err := filepath.Abs(p)
		if err != nil {
			return err
		}

		info, err := s.fs.Stat(abs)
		if err != nil {
			return err
		}

		if info.IsDir() {
			afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
			err = s.enumerateFS(afs, abs, progress)
		} else {
			err = s.enumerateFileWithInfo(
				filepath.Base(abs), filepath.Dir(abs), info, progress)
		}

		if err != nil {
			return err
		}
	}

	return nil
}

// EnumerateFS walks an afero filesystem and adds all files to the scanner.
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
// basePath is used to compute absolute paths for file reading.
func (s *Scanner) EnumerateFS(
	afs afero.Fs,
	basePath string,
	progress chan<- EnumerateStatus,
) error {
	if progress != nil {
		defer close(progress)
	}

	return s.enumerateFS(afs, basePath, progress)
}

// Files returns a copy of all files added to the scanner.
func (s *Scanner) Files() []*FileEntry {
	s.mu.RLock()
	defer s.mu.RUnlock()

	out := make([]*FileEntry, len(s.files))
	copy(out, s.files)

	return out
}

// FileCount returns the number of files in the scanner.
func (s *Scanner) FileCount() FileCount {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return FileCount(len(s.files))
}

// TotalBytes returns the total size of all files in the scanner.
func (s *Scanner) TotalBytes() FileSize {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.totalBytes
}

// ToManifest reads all file contents, computes hashes, and generates a manifest.
// If progress is non-nil, status updates are sent approximately once per second.
// The progress channel is closed when the method returns.
// The manifest is written to the provided io.Writer.
func (s *Scanner) ToManifest(
	ctx context.Context, w io.Writer, progress chan<- ScanStatus,
) error {
	if progress != nil {
		defer close(progress)
	}

	s.mu.RLock()
	files := make([]*FileEntry, len(s.files))
	copy(files, s.files)
	totalFiles := FileCount(len(files))

	var totalBytes FileSize
	for _, f := range files {
		totalBytes += f.Size
	}

	s.mu.RUnlock()

	builder := s.configureBuilder()

	var (
		scannedFiles FileCount
		scannedBytes FileSize
	)

	lastProgressTime := time.Now()
	startTime := time.Now()

	pt := &scanProgressTracker{
		progress:     progress,
		totalFiles:   totalFiles,
		totalBytes:   totalBytes,
		startTime:    startTime,
		lastProgress: &lastProgressTime,
	}

	for _, entry := range files {
		// Check for cancellation
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		bytesRead, err := s.scanFile(builder, pt, entry, scannedFiles, scannedBytes)
		if err != nil {
			return err
		}

		scannedFiles++
		scannedBytes += bytesRead
	}

	// Send final progress (ETA is 0 at completion; remaining bytes are 0,
	// so computeRateETA yields eta 0 and the same average rate as before)
	if progress != nil {
		rate, _ := computeRateETA(time.Since(startTime), scannedBytes, totalBytes)

		sendScanStatus(progress, ScanStatus{
			TotalFiles:   totalFiles,
			ScannedFiles: scannedFiles,
			TotalBytes:   totalBytes,
			ScannedBytes: scannedBytes,
			BytesPerSec:  rate,
			ETA:          0,
		})
	}

	// Build and write manifest
	return builder.Build(ctx, w)
}

// configureBuilder constructs a manifest builder configured from the
// scanner options.
func (s *Scanner) configureBuilder() *Builder {
	builder := NewBuilder()
	if s.options.IncludeTimestamps {
		builder.SetIncludeTimestamps(true)
	}

	if s.options.SigningOptions != nil {
		builder.SetSigningOptions(s.options.SigningOptions)
	}

	if s.options.Seed != "" {
		builder.SetSeed(s.options.Seed)
	}

	return builder
}

// scanFile hashes a single file into the builder, forwarding per-file
// progress updates, and returns the number of bytes read.
func (s *Scanner) scanFile(
	builder *Builder,
	pt *scanProgressTracker,
	entry *FileEntry,
	scannedFiles FileCount,
	scannedBytes FileSize,
) (FileSize, error) {
	// Open file
	f, err := s.fs.Open(string(entry.AbsPath))
	if err != nil {
		return 0, err
	}

	// Create progress channel for this file
	var (
		fileProgress chan FileHashProgress
		wg           sync.WaitGroup
	)

	if pt.progress != nil {
		fileProgress = make(chan FileHashProgress, 1)

		wg.Add(1)

		go func(base FileSize, done FileCount) {
			defer wg.Done()

			pt.forward(fileProgress, done, base)
		}(scannedBytes, scannedFiles)
	}

	// A mode of 0 records 0000, which means none was recorded.
	var mode fs.FileMode
	if s.options.IncludePermissions {
		mode = entry.Mode
	}

	// Add to manifest with progress channel
	bytesRead, err := builder.AddFile(
		entry.Path,
		entry.Size,
		entry.Mtime,
		mode,
		f,
		fileProgress,
	)
	_ = f.Close()

	// Close channel and wait for goroutine to finish
	if fileProgress != nil {
		close(fileProgress)
		wg.Wait()
	}

	if err != nil {
		return 0, err
	}

	log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(sizeToUint64(bytesRead)))

	return bytesRead, nil
}

// enumerateFS is the internal implementation that doesn't close the
// progress channel.
func (s *Scanner) enumerateFS(
	afs afero.Fs,
	basePath string,
	progress chan<- EnumerateStatus,
) error {
	return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !s.options.IncludeDotfiles && IsHiddenPath(p) {
			if info.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		return s.enumerateFileWithInfo(p, basePath, info, progress)
	})
}

// enumerateFileWithInfo adds a file with pre-existing fs.FileInfo.
func (s *Scanner) enumerateFileWithInfo(
	filePath string,
	basePath string,
	info fs.FileInfo,
	progress chan<- EnumerateStatus,
) error {
	if info.IsDir() {
		// Manifests contain only files, directories are implied
		return nil
	}

	// Clean the path - remove leading slash if present
	cleanPath := filePath
	if len(cleanPath) > 0 && cleanPath[0] == '/' {
		cleanPath = cleanPath[1:]
	}

	// Compute absolute path for file reading
	absPath := filepath.Join(basePath, cleanPath)

	// Handle symlinks
	if info.Mode()&fs.ModeSymlink != 0 {
		if !s.options.FollowSymLinks {
			// Skip symlinks when not following them
			return nil
		}
		// Resolve symlink to get real file info
		realPath, err := filepath.EvalSymlinks(absPath)
		if err != nil {
			// Skip broken symlinks
			return nil //nolint:nilerr // broken symlinks are skipped by design
		}

		realInfo, err := s.fs.Stat(realPath)
		if err != nil {
			// Skip symlinks whose target cannot be stat'd
			return nil //nolint:nilerr // unreadable targets are skipped by design
		}
		// Skip if symlink points to a directory
		if realInfo.IsDir() {
			return nil
		}
		// Use resolved path for reading, but keep original path in manifest
		absPath = realPath
		info = realInfo
	}

	for _, excluded := range s.excluded {
		if os.SameFile(info, excluded) {
			return nil
		}
	}

	entry := &FileEntry{
		Path:    RelFilePath(cleanPath),
		AbsPath: AbsFilePath(absPath),
		Size:    FileSize(info.Size()),
		Mtime:   ModTime(info.ModTime()),
		Mode:    info.Mode().Perm(),
		// Note: Ctime not available from fs.FileInfo on all platforms
		// Will need platform-specific code to extract it
	}

	s.mu.Lock()

	// Each path is relative to the input path it was found under, so files
	// under two input paths can share one.
	first, ok := s.paths[entry.Path]
	if ok {
		s.mu.Unlock()

		return fmt.Errorf("%w %q: %s and %s",
			errDuplicatePath, entry.Path, first, entry.AbsPath)
	}

	s.paths[entry.Path] = entry.AbsPath
	s.files = append(s.files, entry)
	s.totalBytes += entry.Size
	filesFound := FileCount(len(s.files))
	bytesFound := s.totalBytes
	s.mu.Unlock()

	sendEnumerateStatus(progress, EnumerateStatus{
		FilesFound: filesFound,
		BytesFound: bytesFound,
	})

	return nil
}

// scanProgressTracker carries the shared state needed to report rate-limited
// scan progress updates.
type scanProgressTracker struct {
	progress     chan<- ScanStatus
	totalFiles   FileCount
	totalBytes   FileSize
	startTime    time.Time
	lastProgress *time.Time
}

// forward relays per-file hash progress to the scan progress channel,
// rate-limited to one update per second.
func (pt *scanProgressTracker) forward(
	fileProgress <-chan FileHashProgress,
	scannedFiles FileCount,
	baseBytes FileSize,
) {
	for p := range fileProgress {
		// Send progress at most once per second
		now := time.Now()
		if now.Sub(*pt.lastProgress) < time.Second {
			continue
		}

		currentBytes := baseBytes + p.BytesRead
		rate, eta := computeRateETA(now.Sub(pt.startTime), currentBytes, pt.totalBytes)

		sendScanStatus(pt.progress, ScanStatus{
			TotalFiles:   pt.totalFiles,
			ScannedFiles: scannedFiles,
			TotalBytes:   pt.totalBytes,
			ScannedBytes: currentBytes,
			BytesPerSec:  rate,
			ETA:          eta,
		})

		*pt.lastProgress = now
	}
}

// computeRateETA returns the average throughput over elapsed time and the
// estimated time to process the remaining bytes at that rate.
func computeRateETA(
	elapsed time.Duration,
	done FileSize,
	total FileSize,
) (float64, time.Duration) {
	var (
		rate float64
		eta  time.Duration
	)

	if elapsed > 0 && done > 0 {
		rate = float64(done) / elapsed.Seconds()

		remaining := total - done
		if rate > 0 {
			eta = time.Duration(float64(remaining)/rate) * time.Second
		}
	}

	return rate, eta
}

// sizeToUint64 converts a FileSize to uint64 for display, clamping
// negative values to zero so the conversion cannot overflow.
func sizeToUint64(v FileSize) uint64 {
	if v < 0 {
		return 0
	}

	return uint64(v)
}

// IsHiddenPath returns true if the path or any of its parent directories
// start with a dot (hidden files/directories).
// The path should use forward slashes.
func IsHiddenPath(p string) bool {
	tp := path.Clean(p)
	if tp == "." || tp == "/" {
		return false
	}

	if strings.HasPrefix(tp, ".") {
		return true
	}

	for {
		d, f := path.Split(tp)
		if strings.HasPrefix(f, ".") {
			return true
		}

		if d == "" {
			return false
		}

		tp = d[0 : len(d)-1] // trim trailing slash from dir
	}
}

// sendEnumerateStatus sends a status update without blocking.
// If the channel is full, the update is dropped.
func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) {
	if ch == nil {
		return
	}

	select {
	case ch <- status:
	default:
		// Channel full, drop this update
	}
}

// sendScanStatus sends a status update without blocking.
// If the channel is full, the update is dropped.
func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) {
	if ch == nil {
		return
	}

	select {
	case ch <- status:
	default:
		// Channel full, drop this update
	}
}