package mfer import ( "context" "io" "io/fs" "path" "path/filepath" "strings" "sync" "time" "github.com/dustin/go-humanize" "github.com/spf13/afero" "sneak.berlin/go/mfer/internal/log" ) // Phase 1: Enumeration // --------------------- // Walking directories and calling stat() on files to collect metadata. // Builds the list of files to be scanned. Relatively fast (metadata only). // EnumerateStatus contains progress information for the enumeration phase. type EnumerateStatus struct { FilesFound FileCount // Number of files discovered so far BytesFound FileSize // Total size of discovered files (from stat) } // Phase 2: Scan (ToManifest) // -------------------------- // Reading file contents and computing hashes for manifest generation. // This is the expensive phase that reads all file data. // ScanStatus contains progress information for the scan phase. type ScanStatus struct { TotalFiles FileCount // Total number of files to scan ScannedFiles FileCount // Number of files scanned so far TotalBytes FileSize // Total bytes to read (sum of all file sizes) ScannedBytes FileSize // Bytes read so far BytesPerSec float64 // Current throughput rate ETA time.Duration // Estimated time to completion } // ScannerOptions configures scanner behavior. type ScannerOptions struct { // IncludeDotfiles includes files and directories starting with a dot // (default: exclude). IncludeDotfiles bool // FollowSymLinks resolves symlinks instead of skipping them. FollowSymLinks bool // IncludeTimestamps includes a createdAt timestamp in the manifest // (default: omit for determinism). IncludeTimestamps bool // Fs is the filesystem to use, defaults to OsFs if nil. Fs afero.Fs // SigningOptions holds GPG signing options (nil = no signing). SigningOptions *SigningOptions // Seed, if set, derives a deterministic UUID from this seed. Seed string } // FileEntry represents a file that has been enumerated. type FileEntry struct { Path RelFilePath // Relative path (used in manifest) AbsPath AbsFilePath // Absolute path (used for reading file content) Size FileSize // File size in bytes Mtime ModTime // Last modification time Ctime time.Time // Creation time (platform-dependent) } // Scanner accumulates files and generates manifests from them. type Scanner struct { mu sync.RWMutex files []*FileEntry totalBytes FileSize // cached sum of all file sizes options *ScannerOptions fs afero.Fs } // NewScanner creates a new Scanner with default options. func NewScanner() *Scanner { return NewScannerWithOptions(nil) } // NewScannerWithOptions creates a new Scanner with the given options. func NewScannerWithOptions(opts *ScannerOptions) *Scanner { if opts == nil { opts = &ScannerOptions{} } fs := opts.Fs if fs == nil { fs = afero.NewOsFs() } return &Scanner{ files: make([]*FileEntry, 0), options: opts, fs: fs, } } // EnumerateFile adds a single file to the scanner, calling stat() to get metadata. func (s *Scanner) EnumerateFile(filePath string) error { abs, err := filepath.Abs(filePath) if err != nil { return err } info, err := s.fs.Stat(abs) if err != nil { return err } // For single files, use the filename as the relative path basePath := filepath.Dir(abs) return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil) } // EnumeratePath walks a directory path and adds all files to the scanner. // If progress is non-nil, status updates are sent as files are discovered. // The progress channel is closed when the method returns. func (s *Scanner) EnumeratePath( inputPath string, progress chan<- EnumerateStatus, ) error { if progress != nil { defer close(progress) } abs, err := filepath.Abs(inputPath) if err != nil { return err } afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs)) return s.enumerateFS(afs, abs, progress) } // EnumeratePaths walks multiple directory paths and adds all files to the scanner. // If progress is non-nil, status updates are sent as files are discovered. // The progress channel is closed when the method returns. func (s *Scanner) EnumeratePaths( progress chan<- EnumerateStatus, inputPaths ...string, ) error { if progress != nil { defer close(progress) } for _, p := range inputPaths { abs, err := filepath.Abs(p) if err != nil { return err } afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs)) err = s.enumerateFS(afs, abs, progress) if err != nil { return err } } return nil } // EnumerateFS walks an afero filesystem and adds all files to the scanner. // If progress is non-nil, status updates are sent as files are discovered. // The progress channel is closed when the method returns. // basePath is used to compute absolute paths for file reading. func (s *Scanner) EnumerateFS( afs afero.Fs, basePath string, progress chan<- EnumerateStatus, ) error { if progress != nil { defer close(progress) } return s.enumerateFS(afs, basePath, progress) } // Files returns a copy of all files added to the scanner. func (s *Scanner) Files() []*FileEntry { s.mu.RLock() defer s.mu.RUnlock() out := make([]*FileEntry, len(s.files)) copy(out, s.files) return out } // FileCount returns the number of files in the scanner. func (s *Scanner) FileCount() FileCount { s.mu.RLock() defer s.mu.RUnlock() return FileCount(len(s.files)) } // TotalBytes returns the total size of all files in the scanner. func (s *Scanner) TotalBytes() FileSize { s.mu.RLock() defer s.mu.RUnlock() return s.totalBytes } // ToManifest reads all file contents, computes hashes, and generates a manifest. // If progress is non-nil, status updates are sent approximately once per second. // The progress channel is closed when the method returns. // The manifest is written to the provided io.Writer. func (s *Scanner) ToManifest( ctx context.Context, w io.Writer, progress chan<- ScanStatus, ) error { if progress != nil { defer close(progress) } s.mu.RLock() files := make([]*FileEntry, len(s.files)) copy(files, s.files) totalFiles := FileCount(len(files)) var totalBytes FileSize for _, f := range files { totalBytes += f.Size } s.mu.RUnlock() builder := s.configureBuilder() var ( scannedFiles FileCount scannedBytes FileSize ) lastProgressTime := time.Now() startTime := time.Now() pt := &scanProgressTracker{ progress: progress, totalFiles: totalFiles, totalBytes: totalBytes, startTime: startTime, lastProgress: &lastProgressTime, } for _, entry := range files { // Check for cancellation select { case <-ctx.Done(): return ctx.Err() default: } bytesRead, err := s.scanFile(builder, pt, entry, scannedFiles, scannedBytes) if err != nil { return err } scannedFiles++ scannedBytes += bytesRead } // Send final progress (ETA is 0 at completion; remaining bytes are 0, // so computeRateETA yields eta 0 and the same average rate as before) if progress != nil { rate, _ := computeRateETA(time.Since(startTime), scannedBytes, totalBytes) sendScanStatus(progress, ScanStatus{ TotalFiles: totalFiles, ScannedFiles: scannedFiles, TotalBytes: totalBytes, ScannedBytes: scannedBytes, BytesPerSec: rate, ETA: 0, }) } // Build and write manifest //nolint:contextcheck // Build's GPG signing exec is not cancellable by design return builder.Build(w) } // configureBuilder constructs a manifest builder configured from the // scanner options. func (s *Scanner) configureBuilder() *Builder { builder := NewBuilder() if s.options.IncludeTimestamps { builder.SetIncludeTimestamps(true) } if s.options.SigningOptions != nil { builder.SetSigningOptions(s.options.SigningOptions) } if s.options.Seed != "" { builder.SetSeed(s.options.Seed) } return builder } // scanFile hashes a single file into the builder, forwarding per-file // progress updates, and returns the number of bytes read. func (s *Scanner) scanFile( builder *Builder, pt *scanProgressTracker, entry *FileEntry, scannedFiles FileCount, scannedBytes FileSize, ) (FileSize, error) { // Open file f, err := s.fs.Open(string(entry.AbsPath)) if err != nil { return 0, err } // Create progress channel for this file var ( fileProgress chan FileHashProgress wg sync.WaitGroup ) if pt.progress != nil { fileProgress = make(chan FileHashProgress, 1) wg.Add(1) go func(base FileSize, done FileCount) { defer wg.Done() pt.forward(fileProgress, done, base) }(scannedBytes, scannedFiles) } // Add to manifest with progress channel bytesRead, err := builder.AddFile( entry.Path, entry.Size, entry.Mtime, f, fileProgress, ) _ = f.Close() // Close channel and wait for goroutine to finish if fileProgress != nil { close(fileProgress) wg.Wait() } if err != nil { return 0, err } log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(sizeToUint64(bytesRead))) return bytesRead, nil } // enumerateFS is the internal implementation that doesn't close the // progress channel. func (s *Scanner) enumerateFS( afs afero.Fs, basePath string, progress chan<- EnumerateStatus, ) error { return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error { if err != nil { return err } if !s.options.IncludeDotfiles && IsHiddenPath(p) { if info.IsDir() { return filepath.SkipDir } return nil } return s.enumerateFileWithInfo(p, basePath, info, progress) }) } // enumerateFileWithInfo adds a file with pre-existing fs.FileInfo. func (s *Scanner) enumerateFileWithInfo( filePath string, basePath string, info fs.FileInfo, progress chan<- EnumerateStatus, ) error { if info.IsDir() { // Manifests contain only files, directories are implied return nil } // Clean the path - remove leading slash if present cleanPath := filePath if len(cleanPath) > 0 && cleanPath[0] == '/' { cleanPath = cleanPath[1:] } // Compute absolute path for file reading absPath := filepath.Join(basePath, cleanPath) // Handle symlinks if info.Mode()&fs.ModeSymlink != 0 { if !s.options.FollowSymLinks { // Skip symlinks when not following them return nil } // Resolve symlink to get real file info realPath, err := filepath.EvalSymlinks(absPath) if err != nil { // Skip broken symlinks return nil //nolint:nilerr // broken symlinks are skipped by design } realInfo, err := s.fs.Stat(realPath) if err != nil { // Skip symlinks whose target cannot be stat'd return nil //nolint:nilerr // unreadable targets are skipped by design } // Skip if symlink points to a directory if realInfo.IsDir() { return nil } // Use resolved path for reading, but keep original path in manifest absPath = realPath info = realInfo } entry := &FileEntry{ Path: RelFilePath(cleanPath), AbsPath: AbsFilePath(absPath), Size: FileSize(info.Size()), Mtime: ModTime(info.ModTime()), // Note: Ctime not available from fs.FileInfo on all platforms // Will need platform-specific code to extract it } s.mu.Lock() s.files = append(s.files, entry) s.totalBytes += entry.Size filesFound := FileCount(len(s.files)) bytesFound := s.totalBytes s.mu.Unlock() sendEnumerateStatus(progress, EnumerateStatus{ FilesFound: filesFound, BytesFound: bytesFound, }) return nil } // scanProgressTracker carries the shared state needed to report rate-limited // scan progress updates. type scanProgressTracker struct { progress chan<- ScanStatus totalFiles FileCount totalBytes FileSize startTime time.Time lastProgress *time.Time } // forward relays per-file hash progress to the scan progress channel, // rate-limited to one update per second. func (pt *scanProgressTracker) forward( fileProgress <-chan FileHashProgress, scannedFiles FileCount, baseBytes FileSize, ) { for p := range fileProgress { // Send progress at most once per second now := time.Now() if now.Sub(*pt.lastProgress) < time.Second { continue } currentBytes := baseBytes + p.BytesRead rate, eta := computeRateETA(now.Sub(pt.startTime), currentBytes, pt.totalBytes) sendScanStatus(pt.progress, ScanStatus{ TotalFiles: pt.totalFiles, ScannedFiles: scannedFiles, TotalBytes: pt.totalBytes, ScannedBytes: currentBytes, BytesPerSec: rate, ETA: eta, }) *pt.lastProgress = now } } // computeRateETA returns the average throughput over elapsed time and the // estimated time to process the remaining bytes at that rate. func computeRateETA( elapsed time.Duration, done FileSize, total FileSize, ) (float64, time.Duration) { var ( rate float64 eta time.Duration ) if elapsed > 0 && done > 0 { rate = float64(done) / elapsed.Seconds() remaining := total - done if rate > 0 { eta = time.Duration(float64(remaining)/rate) * time.Second } } return rate, eta } // sizeToUint64 converts a FileSize to uint64 for display, clamping // negative values to zero so the conversion cannot overflow. func sizeToUint64(v FileSize) uint64 { if v < 0 { return 0 } return uint64(v) } // IsHiddenPath returns true if the path or any of its parent directories // start with a dot (hidden files/directories). // The path should use forward slashes. func IsHiddenPath(p string) bool { tp := path.Clean(p) if tp == "." || tp == "/" { return false } if strings.HasPrefix(tp, ".") { return true } for { d, f := path.Split(tp) if strings.HasPrefix(f, ".") { return true } if d == "" { return false } tp = d[0 : len(d)-1] // trim trailing slash from dir } } // sendEnumerateStatus sends a status update without blocking. // If the channel is full, the update is dropped. func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) { if ch == nil { return } select { case ch <- status: default: // Channel full, drop this update } } // sendScanStatus sends a status update without blocking. // If the channel is full, the update is dropped. func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) { if ch == nil { return } select { case ch <- status: default: // Channel full, drop this update } }