package cli import ( "crypto/sha256" "errors" "fmt" "io" "io/fs" "path/filepath" "time" "github.com/dustin/go-humanize" "github.com/multiformats/go-multihash" "github.com/spf13/afero" "github.com/urfave/cli/v2" "sneak.berlin/go/mfer/internal/log" "sneak.berlin/go/mfer/mfer" ) const ( // hashBufSize is the read buffer size used when hashing files. hashBufSize = 64 * 1024 // scanProgressInterval is how many scanned files pass between // progress updates. scanProgressInterval = 100 ) // errEntryMissingMtime indicates a manifest entry that carries no // modification time where one is required to carry it forward unchanged. var errEntryMissingMtime = errors.New("manifest entry has no mtime") // FreshenStatus contains progress information for the freshen operation. type FreshenStatus struct { Phase string // "scan" or "hash" TotalFiles int64 // Total files to process in current phase CurrentFiles int64 // Files processed so far TotalBytes int64 // Total bytes to hash (hash phase only) CurrentBytes int64 // Bytes hashed so far BytesPerSec float64 // Current throughput rate ETA time.Duration // Estimated time to completion } // freshenEntry tracks a file's status during freshen type freshenEntry struct { path string size int64 mtime time.Time needsHash bool // true if new or changed existing *mfer.MFFilePath // existing manifest entry if unchanged } // freshenScanner walks the filesystem and compares it against the // entries of an existing manifest. type freshenScanner struct { fs afero.Fs absBase string manifestBase string includeDotfiles bool followSymlinks bool showProgress bool existingByPath map[string]*mfer.MFFilePath entries []*freshenEntry scanCount int64 changed int64 added int64 unchanged int64 } // resolveSymlink resolves a symlink to its target's FileInfo. The // second return value is false when the entry should be skipped. func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) { if !s.followSymlinks { return nil, false } realPath, err := filepath.EvalSymlinks(path) if err != nil { return nil, false // Skip broken symlinks } realInfo, err := s.fs.Stat(realPath) if err != nil || realInfo.IsDir() { return nil, false } return realInfo, true } // recordEntry classifies a scanned file as changed, unchanged, or added // relative to the existing manifest. func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) { existing, inManifest := s.existingByPath[relPath] if !inManifest { s.added++ log.Verbosef("A %s", relPath) s.entries = append(s.entries, &freshenEntry{ path: relPath, size: info.Size(), mtime: info.ModTime(), needsHash: true, }) return } // Check if changed (size or mtime). An entry with no recorded mtime // cannot be compared, so it counts as changed and gets re-hashed; // silently treating the absent mtime as the Unix epoch would classify // every such entry as changed without saying why. existingMtime, haveMtime := entryMtime(existing) if !haveMtime { log.Debugf("%s: manifest entry has no mtime, treating as changed", relPath) } if !haveMtime || existing.GetSize() != info.Size() || !existingMtime.Equal(info.ModTime()) { s.changed++ log.Verbosef("M %s", relPath) s.entries = append(s.entries, &freshenEntry{ path: relPath, size: info.Size(), mtime: info.ModTime(), needsHash: true, }) } else { s.unchanged++ s.entries = append(s.entries, &freshenEntry{ path: relPath, size: info.Size(), mtime: info.ModTime(), needsHash: false, existing: existing, }) } // Mark as seen delete(s.existingByPath, relPath) } // walk is the afero.Walk callback for the scan phase. func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } // Get relative path relPath, err := filepath.Rel(s.absBase, path) if err != nil { return fmt.Errorf( "freshen: failed to compute relative path for %s: %w", path, err) } // Skip the manifest file itself if relPath == s.manifestBase || relPath == "."+s.manifestBase { return nil } // Handle dotfiles if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) { if info.IsDir() { return filepath.SkipDir } return nil } // Skip directories if info.IsDir() { return nil } // Handle symlinks if info.Mode()&fs.ModeSymlink != 0 { realInfo, keep := s.resolveSymlink(path) if !keep { return nil } info = realInfo } s.scanCount++ // Check against existing manifest s.recordEntry(relPath, info) // Report scan progress if s.showProgress && s.scanCount%scanProgressInterval == 0 { log.Progressf("Scanning: %d files found", s.scanCount) } return nil } // resolveFreshenManifestPath determines the manifest path from the CLI // arguments, searching directories for a manifest where needed. func (mfa *CLIApp) resolveFreshenManifestPath(ctx *cli.Context) (string, error) { if ctx.Args().Len() == 0 { return findManifest(mfa.Fs, ".") } arg := ctx.Args().Get(0) info, statErr := mfa.Fs.Stat(arg) if statErr == nil && info.IsDir() { return findManifest(mfa.Fs, arg) } return arg, nil } // freshenHasher hashes changed and added files and feeds all entries to // a manifest builder. type freshenHasher struct { fs afero.Fs absBase string showProgress bool totalHashBytes int64 filesToHash int64 startHash time.Time builder *mfer.Builder hashedFiles int64 hashedBytes int64 } // reportProgress renders hashing progress for the current byte count. func (h *freshenHasher) reportProgress(n int64) { if !h.showProgress { return } currentBytes := h.hashedBytes + n elapsed := time.Since(h.startHash) var ( rate float64 eta time.Duration ) if elapsed > 0 && currentBytes > 0 { rate = float64(currentBytes) / elapsed.Seconds() remaining := h.totalHashBytes - currentBytes if rate > 0 { eta = time.Duration(float64(remaining)/rate) * time.Second } } if eta > 0 { log.Progressf("Hashing: %d/%d files, %s/s, ETA %s", h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)), eta.Round(time.Second)) } else { log.Progressf("Hashing: %d/%d files, %s/s", h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate))) } } // processEntry hashes the entry if needed and adds it to the builder. func (h *freshenHasher) processEntry(e *freshenEntry) error { if !e.needsHash { // Use existing entry err := addExistingToBuilder(h.builder, e.existing) if err != nil { return fmt.Errorf("failed to add %s: %w", e.path, err) } return nil } // Need to read and hash the file absPath := filepath.Join(h.absBase, e.path) f, err := h.fs.Open(absPath) if err != nil { return fmt.Errorf("failed to open %s: %w", e.path, err) } hash, bytesRead, err := hashFile(f, h.reportProgress) _ = f.Close() if err != nil { return fmt.Errorf("failed to hash %s: %w", e.path, err) } h.hashedBytes += bytesRead h.hashedFiles++ // Add to builder with computed hash err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, hash) if err != nil { return fmt.Errorf("failed to add %s: %w", e.path, err) } return nil } // writeFreshenedManifest writes the manifest atomically (write to a // temp file, then rename over the target). func writeFreshenedManifest( afs afero.Fs, builder *mfer.Builder, manifestPath string, ) error { tmpPath := manifestPath + ".tmp" outFile, err := afs.Create(tmpPath) if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } err = builder.Build(outFile) _ = outFile.Close() if err != nil { _ = afs.Remove(tmpPath) return fmt.Errorf("failed to write manifest: %w", err) } // Rename temp to final err = afs.Rename(tmpPath, manifestPath) if err != nil { _ = afs.Remove(tmpPath) return fmt.Errorf("failed to rename manifest: %w", err) } return nil } // newFreshenBuilder constructs the manifest builder configured from CLI // flags. func newFreshenBuilder(ctx *cli.Context) *mfer.Builder { builder := mfer.NewBuilder() if ctx.Bool("include-timestamps") { builder.SetIncludeTimestamps(true) } // Set up signing options if sign-key is provided if signKey := ctx.String("sign-key"); signKey != "" { builder.SetSigningOptions(&mfer.SigningOptions{ KeyID: mfer.GPGKeyID(signKey), }) log.Infof("signing manifest with GPG key: %s", signKey) } return builder } // freshenScan runs the scan phase against the loaded manifest entries // and returns the populated scanner and the count of removed files. func (mfa *CLIApp) freshenScan( ctx *cli.Context, manifestPath, absBase string, existingByPath map[string]*mfer.MFFilePath, ) (*freshenScanner, int64, error) { log.Infof("scanning filesystem...") startScan := time.Now() showProgress := ctx.Bool("progress") scanner := &freshenScanner{ fs: mfa.Fs, absBase: absBase, manifestBase: filepath.Base(manifestPath), includeDotfiles: ctx.Bool("include-dotfiles"), followSymlinks: ctx.Bool("follow-symlinks"), showProgress: showProgress, existingByPath: existingByPath, } err := afero.Walk(mfa.Fs, absBase, scanner.walk) if showProgress { log.ProgressDone() } if err != nil { return nil, 0, fmt.Errorf("failed to scan filesystem: %w", err) } // Remaining entries in existingByPath are removed files removed := int64(len(existingByPath)) for path := range existingByPath { log.Verbosef("D %s", path) } scanDuration := time.Since(startScan) log.Infof("scan complete in %s: %d unchanged, %d changed, %d added, %d removed", scanDuration.Round(time.Millisecond), scanner.unchanged, scanner.changed, scanner.added, removed) return scanner, removed, nil } // hashTotals returns the total byte count and file count of entries // that need hashing. func hashTotals(entries []*freshenEntry) (int64, int64) { var ( totalHashBytes int64 filesToHash int64 ) for _, e := range entries { if e.needsHash { totalHashBytes += e.size filesToHash++ } } return totalHashBytes, filesToHash } // runFreshenHash processes every entry through the hasher, aborting if // the context is canceled. func runFreshenHash( ctx *cli.Context, hasher *freshenHasher, entries []*freshenEntry, ) error { for _, e := range entries { select { case <-ctx.Done(): return ctx.Err() default: } err := hasher.processEntry(e) if err != nil { return err } } return nil } // loadExistingEntries loads the manifest and indexes its file entries // by path. func (mfa *CLIApp) loadExistingEntries( manifestPath string, ) (map[string]*mfer.MFFilePath, error) { log.Infof("loading manifest from %s", manifestPath) // Load existing manifest manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath) if err != nil { return nil, fmt.Errorf("failed to load manifest: %w", err) } existingFiles := manifest.Files() log.Infof("manifest contains %d files", len(existingFiles)) // Build map of existing entries by path existingByPath := make(map[string]*mfer.MFFilePath, len(existingFiles)) for _, f := range existingFiles { existingByPath[f.GetPath()] = f } return existingByPath, nil } func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error { log.Debug("freshenManifestOperation()") basePath := ctx.String("base") showProgress := ctx.Bool("progress") // Find manifest file manifestPath, err := mfa.resolveFreshenManifestPath(ctx) if err != nil { return fmt.Errorf("freshen: %w", err) } existingByPath, err := mfa.loadExistingEntries(manifestPath) if err != nil { return err } absBase, err := filepath.Abs(basePath) if err != nil { return fmt.Errorf("freshen: invalid base path: %w", err) } // Phase 1: Scan filesystem scanner, removed, err := mfa.freshenScan(ctx, manifestPath, absBase, existingByPath) if err != nil { return err } // Calculate total bytes to hash totalHashBytes, filesToHash := hashTotals(scanner.entries) // Phase 2: Hash changed and new files if filesToHash > 0 { log.Infof("hashing %d files (%s)...", filesToHash, humanize.IBytes(safeUint64(totalHashBytes))) } hasher := &freshenHasher{ fs: mfa.Fs, absBase: absBase, showProgress: showProgress, totalHashBytes: totalHashBytes, filesToHash: filesToHash, startHash: time.Now(), builder: newFreshenBuilder(ctx), } err = runFreshenHash(ctx, hasher, scanner.entries) if err != nil { return err } if showProgress && filesToHash > 0 { log.ProgressDone() } // Print summary log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed", scanner.unchanged, scanner.changed, scanner.added, removed) // Skip writing if nothing changed if scanner.changed == 0 && scanner.added == 0 && removed == 0 { log.Infof("manifest unchanged, skipping write") return nil } // Write updated manifest atomically (write to temp, then rename) err = writeFreshenedManifest(mfa.Fs, hasher.builder, manifestPath) if err != nil { return err } totalDuration := time.Since(mfa.startupTime) if hasher.hashedBytes > 0 { hashDuration := time.Since(hasher.startHash) hashRate := float64(hasher.hashedBytes) / hashDuration.Seconds() log.Infof("hashed %s in %.1fs (%s/s)", humanize.IBytes(safeUint64(hasher.hashedBytes)), totalDuration.Seconds(), humanize.IBytes(safeRateUint64(hashRate))) } log.Infof("wrote %d files to %s", len(scanner.entries), manifestPath) return nil } // hashFile reads a file and computes its SHA256 multihash. // Progress callback is called with bytes read so far. func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) { h := sha256.New() buf := make([]byte, hashBufSize) var total int64 for { n, err := r.Read(buf) if n > 0 { h.Write(buf[:n]) total += int64(n) if progress != nil { progress(total) } } if err == io.EOF { break } // Returned unwrapped: the caller renders this as // "failed to hash : " and adding a second layer here // would change that message. if err != nil { return nil, total, err } } mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) if err != nil { return nil, total, err } return mh, total, nil } // addFileToBuilder adds a new file entry to the builder func addFileToBuilder( b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte, ) error { return b.AddFileWithHash( mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash) } // addExistingToBuilder adds an existing manifest entry to the builder. // // Entries reach this path only when recordEntry classified them as // unchanged, which requires a recorded mtime, so an absent mtime here is // an error rather than something to paper over with the Unix epoch. func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error { mtime, ok := entryMtime(entry) if !ok { return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath()) } if len(entry.GetHashes()) == 0 { return nil } return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()), mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime), entry.GetHashes()[0].GetMultiHash()) }