Update golangci-lint to v2.12.2 with canonical config (closes #60)
Some checks failed
check / check (push) Has been cancelled
Some checks failed
check / check (push) Has been cancelled
Adopts golangci-lint v2.12.2 and the canonical .golangci.yml (default: all), and fixes all resulting findings across the tree. Two intended behavior changes: absent MFFilePath.Mtime is handled explicitly in freshen, list and export rather than dereferenced (main panicked); gpg positional key IDs now follow an explicit -- end-of-options marker. All twelve reworded user-visible error messages restored to byte-identical parity with main and pinned by tests.
This commit was merged in pull request #59.
This commit is contained in:
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -16,6 +17,19 @@ import (
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
const (
|
||||
// hashBufSize is the read buffer size used when hashing files.
|
||||
hashBufSize = 64 * 1024
|
||||
|
||||
// scanProgressInterval is how many scanned files pass between
|
||||
// progress updates.
|
||||
scanProgressInterval = 100
|
||||
)
|
||||
|
||||
// errEntryMissingMtime indicates a manifest entry that carries no
|
||||
// modification time where one is required to carry it forward unchanged.
|
||||
var errEntryMissingMtime = errors.New("manifest entry has no mtime")
|
||||
|
||||
// FreshenStatus contains progress information for the freshen operation.
|
||||
type FreshenStatus struct {
|
||||
Phase string // "scan" or "hash"
|
||||
@@ -36,195 +50,292 @@ type freshenEntry struct {
|
||||
existing *mfer.MFFilePath // existing manifest entry if unchanged
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
||||
log.Debug("freshenManifestOperation()")
|
||||
// freshenScanner walks the filesystem and compares it against the
|
||||
// entries of an existing manifest.
|
||||
type freshenScanner struct {
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
manifestBase string
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
showProgress bool
|
||||
existingByPath map[string]*mfer.MFFilePath
|
||||
|
||||
basePath := ctx.String("base")
|
||||
showProgress := ctx.Bool("progress")
|
||||
includeDotfiles := ctx.Bool("include-dotfiles")
|
||||
followSymlinks := ctx.Bool("follow-symlinks")
|
||||
entries []*freshenEntry
|
||||
scanCount int64
|
||||
changed int64
|
||||
added int64
|
||||
unchanged int64
|
||||
}
|
||||
|
||||
// Find manifest file
|
||||
var manifestPath string
|
||||
var err error
|
||||
// resolveSymlink resolves a symlink to its target's FileInfo. The
|
||||
// second return value is false when the entry should be skipped.
|
||||
func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) {
|
||||
if !s.followSymlinks {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if ctx.Args().Len() > 0 {
|
||||
arg := ctx.Args().Get(0)
|
||||
info, statErr := mfa.Fs.Stat(arg)
|
||||
if statErr == nil && info.IsDir() {
|
||||
manifestPath, err = findManifest(mfa.Fs, arg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: %w", err)
|
||||
}
|
||||
} else {
|
||||
manifestPath = arg
|
||||
}
|
||||
realPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return nil, false // Skip broken symlinks
|
||||
}
|
||||
|
||||
realInfo, err := s.fs.Stat(realPath)
|
||||
if err != nil || realInfo.IsDir() {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return realInfo, true
|
||||
}
|
||||
|
||||
// recordEntry classifies a scanned file as changed, unchanged, or added
|
||||
// relative to the existing manifest.
|
||||
func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
||||
existing, inManifest := s.existingByPath[relPath]
|
||||
if !inManifest {
|
||||
s.added++
|
||||
|
||||
log.Verbosef("A %s", relPath)
|
||||
s.entries = append(s.entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: true,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Check if changed (size or mtime). An entry with no recorded mtime
|
||||
// cannot be compared, so it counts as changed and gets re-hashed;
|
||||
// silently treating the absent mtime as the Unix epoch would classify
|
||||
// every such entry as changed without saying why.
|
||||
existingMtime, haveMtime := entryMtime(existing)
|
||||
if !haveMtime {
|
||||
log.Debugf("%s: manifest entry has no mtime, treating as changed",
|
||||
relPath)
|
||||
}
|
||||
|
||||
if !haveMtime || existing.GetSize() != info.Size() ||
|
||||
!existingMtime.Equal(info.ModTime()) {
|
||||
s.changed++
|
||||
|
||||
log.Verbosef("M %s", relPath)
|
||||
s.entries = append(s.entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: true,
|
||||
})
|
||||
} else {
|
||||
manifestPath, err = findManifest(mfa.Fs, ".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: %w", err)
|
||||
}
|
||||
s.unchanged++
|
||||
|
||||
s.entries = append(s.entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: false,
|
||||
existing: existing,
|
||||
})
|
||||
}
|
||||
// Mark as seen
|
||||
delete(s.existingByPath, relPath)
|
||||
}
|
||||
|
||||
// walk is the afero.Walk callback for the scan phase.
|
||||
func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
|
||||
log.Infof("loading manifest from %s", manifestPath)
|
||||
|
||||
// Load existing manifest
|
||||
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(s.absBase, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
return fmt.Errorf(
|
||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||
}
|
||||
|
||||
existingFiles := manifest.Files()
|
||||
log.Infof("manifest contains %d files", len(existingFiles))
|
||||
|
||||
// Build map of existing entries by path
|
||||
existingByPath := make(map[string]*mfer.MFFilePath, len(existingFiles))
|
||||
for _, f := range existingFiles {
|
||||
existingByPath[f.Path] = f
|
||||
// Skip the manifest file itself
|
||||
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Phase 1: Scan filesystem
|
||||
log.Infof("scanning filesystem...")
|
||||
startScan := time.Now()
|
||||
|
||||
var entries []*freshenEntry
|
||||
var scanCount int64
|
||||
var removed, changed, added, unchanged int64
|
||||
|
||||
absBase, err := filepath.Abs(basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: invalid base path: %w", err)
|
||||
}
|
||||
|
||||
err = afero.Walk(mfa.Fs, absBase, func(path string, info fs.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(absBase, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: failed to compute relative path for %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Skip the manifest file itself
|
||||
if relPath == filepath.Base(manifestPath) || relPath == "."+filepath.Base(manifestPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle dotfiles
|
||||
if !includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
// Handle dotfiles
|
||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle symlinks
|
||||
if info.Mode()&fs.ModeSymlink != 0 {
|
||||
if !followSymlinks {
|
||||
return nil
|
||||
}
|
||||
realPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return nil // Skip broken symlinks
|
||||
}
|
||||
realInfo, err := mfa.Fs.Stat(realPath)
|
||||
if err != nil || realInfo.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info = realInfo
|
||||
}
|
||||
|
||||
scanCount++
|
||||
|
||||
// Check against existing manifest
|
||||
existing, inManifest := existingByPath[relPath]
|
||||
if inManifest {
|
||||
// Check if changed (size or mtime)
|
||||
existingMtime := time.Unix(existing.Mtime.Seconds, int64(existing.Mtime.Nanos))
|
||||
if existing.Size != info.Size() || !existingMtime.Equal(info.ModTime()) {
|
||||
changed++
|
||||
log.Verbosef("M %s", relPath)
|
||||
entries = append(entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: true,
|
||||
})
|
||||
} else {
|
||||
unchanged++
|
||||
entries = append(entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: false,
|
||||
existing: existing,
|
||||
})
|
||||
}
|
||||
// Mark as seen
|
||||
delete(existingByPath, relPath)
|
||||
} else {
|
||||
added++
|
||||
log.Verbosef("A %s", relPath)
|
||||
entries = append(entries, &freshenEntry{
|
||||
path: relPath,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
needsHash: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Report scan progress
|
||||
if showProgress && scanCount%100 == 0 {
|
||||
log.Progressf("Scanning: %d files found", scanCount)
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if showProgress {
|
||||
log.ProgressDone()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan filesystem: %w", err)
|
||||
// Skip directories
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remaining entries in existingByPath are removed files
|
||||
removed = int64(len(existingByPath))
|
||||
for path := range existingByPath {
|
||||
log.Verbosef("D %s", path)
|
||||
// Handle symlinks
|
||||
if info.Mode()&fs.ModeSymlink != 0 {
|
||||
realInfo, keep := s.resolveSymlink(path)
|
||||
if !keep {
|
||||
return nil
|
||||
}
|
||||
|
||||
info = realInfo
|
||||
}
|
||||
|
||||
scanDuration := time.Since(startScan)
|
||||
log.Infof("scan complete in %s: %d unchanged, %d changed, %d added, %d removed",
|
||||
scanDuration.Round(time.Millisecond), unchanged, changed, added, removed)
|
||||
s.scanCount++
|
||||
|
||||
// Calculate total bytes to hash
|
||||
var totalHashBytes int64
|
||||
var filesToHash int64
|
||||
for _, e := range entries {
|
||||
if e.needsHash {
|
||||
totalHashBytes += e.size
|
||||
filesToHash++
|
||||
// Check against existing manifest
|
||||
s.recordEntry(relPath, info)
|
||||
|
||||
// Report scan progress
|
||||
if s.showProgress && s.scanCount%scanProgressInterval == 0 {
|
||||
log.Progressf("Scanning: %d files found", s.scanCount)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveFreshenManifestPath determines the manifest path from the CLI
|
||||
// arguments, searching directories for a manifest where needed.
|
||||
func (mfa *CLIApp) resolveFreshenManifestPath(ctx *cli.Context) (string, error) {
|
||||
if ctx.Args().Len() == 0 {
|
||||
return findManifest(mfa.Fs, ".")
|
||||
}
|
||||
|
||||
arg := ctx.Args().Get(0)
|
||||
|
||||
info, statErr := mfa.Fs.Stat(arg)
|
||||
if statErr == nil && info.IsDir() {
|
||||
return findManifest(mfa.Fs, arg)
|
||||
}
|
||||
|
||||
return arg, nil
|
||||
}
|
||||
|
||||
// freshenHasher hashes changed and added files and feeds all entries to
|
||||
// a manifest builder.
|
||||
type freshenHasher struct {
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
showProgress bool
|
||||
totalHashBytes int64
|
||||
filesToHash int64
|
||||
startHash time.Time
|
||||
builder *mfer.Builder
|
||||
|
||||
hashedFiles int64
|
||||
hashedBytes int64
|
||||
}
|
||||
|
||||
// reportProgress renders hashing progress for the current byte count.
|
||||
func (h *freshenHasher) reportProgress(n int64) {
|
||||
if !h.showProgress {
|
||||
return
|
||||
}
|
||||
|
||||
currentBytes := h.hashedBytes + n
|
||||
elapsed := time.Since(h.startHash)
|
||||
|
||||
var (
|
||||
rate float64
|
||||
eta time.Duration
|
||||
)
|
||||
|
||||
if elapsed > 0 && currentBytes > 0 {
|
||||
rate = float64(currentBytes) / elapsed.Seconds()
|
||||
|
||||
remaining := h.totalHashBytes - currentBytes
|
||||
if rate > 0 {
|
||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: Hash changed and new files
|
||||
if filesToHash > 0 {
|
||||
log.Infof("hashing %d files (%s)...", filesToHash, humanize.IBytes(uint64(totalHashBytes)))
|
||||
if eta > 0 {
|
||||
log.Progressf("Hashing: %d/%d files, %s/s, ETA %s",
|
||||
h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)),
|
||||
eta.Round(time.Second))
|
||||
} else {
|
||||
log.Progressf("Hashing: %d/%d files, %s/s",
|
||||
h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)))
|
||||
}
|
||||
}
|
||||
|
||||
// processEntry hashes the entry if needed and adds it to the builder.
|
||||
func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
if !e.needsHash {
|
||||
// Use existing entry
|
||||
err := addExistingToBuilder(h.builder, e.existing)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
startHash := time.Now()
|
||||
var hashedFiles int64
|
||||
var hashedBytes int64
|
||||
// Need to read and hash the file
|
||||
absPath := filepath.Join(h.absBase, e.path)
|
||||
|
||||
f, err := h.fs.Open(absPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
hash, bytesRead, err := hashFile(f, h.reportProgress)
|
||||
_ = f.Close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to hash %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
h.hashedBytes += bytesRead
|
||||
h.hashedFiles++
|
||||
|
||||
// Add to builder with computed hash
|
||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeFreshenedManifest writes the manifest atomically (write to a
|
||||
// temp file, then rename over the target).
|
||||
func writeFreshenedManifest(
|
||||
afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||
) error {
|
||||
tmpPath := manifestPath + ".tmp"
|
||||
|
||||
outFile, err := afs.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
err = builder.Build(outFile)
|
||||
_ = outFile.Close()
|
||||
|
||||
if err != nil {
|
||||
_ = afs.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to write manifest: %w", err)
|
||||
}
|
||||
|
||||
// Rename temp to final
|
||||
err = afs.Rename(tmpPath, manifestPath)
|
||||
if err != nil {
|
||||
_ = afs.Remove(tmpPath)
|
||||
|
||||
return fmt.Errorf("failed to rename manifest: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// newFreshenBuilder constructs the manifest builder configured from CLI
|
||||
// flags.
|
||||
func newFreshenBuilder(ctx *cli.Context) *mfer.Builder {
|
||||
builder := mfer.NewBuilder()
|
||||
if ctx.Bool("include-timestamps") {
|
||||
builder.SetIncludeTimestamps(true)
|
||||
@@ -238,6 +349,77 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
||||
log.Infof("signing manifest with GPG key: %s", signKey)
|
||||
}
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
// freshenScan runs the scan phase against the loaded manifest entries
|
||||
// and returns the populated scanner and the count of removed files.
|
||||
func (mfa *CLIApp) freshenScan(
|
||||
ctx *cli.Context, manifestPath, absBase string,
|
||||
existingByPath map[string]*mfer.MFFilePath,
|
||||
) (*freshenScanner, int64, error) {
|
||||
log.Infof("scanning filesystem...")
|
||||
|
||||
startScan := time.Now()
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
scanner := &freshenScanner{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
manifestBase: filepath.Base(manifestPath),
|
||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||
showProgress: showProgress,
|
||||
existingByPath: existingByPath,
|
||||
}
|
||||
|
||||
err := afero.Walk(mfa.Fs, absBase, scanner.walk)
|
||||
|
||||
if showProgress {
|
||||
log.ProgressDone()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to scan filesystem: %w", err)
|
||||
}
|
||||
|
||||
// Remaining entries in existingByPath are removed files
|
||||
removed := int64(len(existingByPath))
|
||||
for path := range existingByPath {
|
||||
log.Verbosef("D %s", path)
|
||||
}
|
||||
|
||||
scanDuration := time.Since(startScan)
|
||||
log.Infof("scan complete in %s: %d unchanged, %d changed, %d added, %d removed",
|
||||
scanDuration.Round(time.Millisecond), scanner.unchanged, scanner.changed,
|
||||
scanner.added, removed)
|
||||
|
||||
return scanner, removed, nil
|
||||
}
|
||||
|
||||
// hashTotals returns the total byte count and file count of entries
|
||||
// that need hashing.
|
||||
func hashTotals(entries []*freshenEntry) (int64, int64) {
|
||||
var (
|
||||
totalHashBytes int64
|
||||
filesToHash int64
|
||||
)
|
||||
|
||||
for _, e := range entries {
|
||||
if e.needsHash {
|
||||
totalHashBytes += e.size
|
||||
filesToHash++
|
||||
}
|
||||
}
|
||||
|
||||
return totalHashBytes, filesToHash
|
||||
}
|
||||
|
||||
// runFreshenHash processes every entry through the hasher, aborting if
|
||||
// the context is canceled.
|
||||
func runFreshenHash(
|
||||
ctx *cli.Context, hasher *freshenHasher, entries []*freshenEntry,
|
||||
) error {
|
||||
for _, e := range entries {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -245,122 +427,154 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
||||
default:
|
||||
}
|
||||
|
||||
if e.needsHash {
|
||||
// Need to read and hash the file
|
||||
absPath := filepath.Join(absBase, e.path)
|
||||
f, err := mfa.Fs.Open(absPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
hash, bytesRead, err := hashFile(f, e.size, func(n int64) {
|
||||
if showProgress {
|
||||
currentBytes := hashedBytes + n
|
||||
elapsed := time.Since(startHash)
|
||||
var rate float64
|
||||
var eta time.Duration
|
||||
if elapsed > 0 && currentBytes > 0 {
|
||||
rate = float64(currentBytes) / elapsed.Seconds()
|
||||
remaining := totalHashBytes - currentBytes
|
||||
if rate > 0 {
|
||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||
}
|
||||
}
|
||||
if eta > 0 {
|
||||
log.Progressf("Hashing: %d/%d files, %s/s, ETA %s",
|
||||
hashedFiles, filesToHash, humanize.IBytes(uint64(rate)), eta.Round(time.Second))
|
||||
} else {
|
||||
log.Progressf("Hashing: %d/%d files, %s/s",
|
||||
hashedFiles, filesToHash, humanize.IBytes(uint64(rate)))
|
||||
}
|
||||
}
|
||||
})
|
||||
_ = f.Close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to hash %s: %w", e.path, err)
|
||||
}
|
||||
|
||||
hashedBytes += bytesRead
|
||||
hashedFiles++
|
||||
|
||||
// Add to builder with computed hash
|
||||
if err := addFileToBuilder(builder, e.path, e.size, e.mtime, hash); err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
} else {
|
||||
// Use existing entry
|
||||
if err := addExistingToBuilder(builder, e.existing); err != nil {
|
||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
||||
}
|
||||
err := hasher.processEntry(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadExistingEntries loads the manifest and indexes its file entries
|
||||
// by path.
|
||||
func (mfa *CLIApp) loadExistingEntries(
|
||||
manifestPath string,
|
||||
) (map[string]*mfer.MFFilePath, error) {
|
||||
log.Infof("loading manifest from %s", manifestPath)
|
||||
|
||||
// Load existing manifest
|
||||
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
||||
}
|
||||
|
||||
existingFiles := manifest.Files()
|
||||
log.Infof("manifest contains %d files", len(existingFiles))
|
||||
|
||||
// Build map of existing entries by path
|
||||
existingByPath := make(map[string]*mfer.MFFilePath, len(existingFiles))
|
||||
for _, f := range existingFiles {
|
||||
existingByPath[f.GetPath()] = f
|
||||
}
|
||||
|
||||
return existingByPath, nil
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
||||
log.Debug("freshenManifestOperation()")
|
||||
|
||||
basePath := ctx.String("base")
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
// Find manifest file
|
||||
manifestPath, err := mfa.resolveFreshenManifestPath(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: %w", err)
|
||||
}
|
||||
|
||||
existingByPath, err := mfa.loadExistingEntries(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
absBase, err := filepath.Abs(basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("freshen: invalid base path: %w", err)
|
||||
}
|
||||
|
||||
// Phase 1: Scan filesystem
|
||||
scanner, removed, err := mfa.freshenScan(ctx, manifestPath, absBase,
|
||||
existingByPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Calculate total bytes to hash
|
||||
totalHashBytes, filesToHash := hashTotals(scanner.entries)
|
||||
|
||||
// Phase 2: Hash changed and new files
|
||||
if filesToHash > 0 {
|
||||
log.Infof("hashing %d files (%s)...", filesToHash,
|
||||
humanize.IBytes(safeUint64(totalHashBytes)))
|
||||
}
|
||||
|
||||
hasher := &freshenHasher{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
showProgress: showProgress,
|
||||
totalHashBytes: totalHashBytes,
|
||||
filesToHash: filesToHash,
|
||||
startHash: time.Now(),
|
||||
builder: newFreshenBuilder(ctx),
|
||||
}
|
||||
|
||||
err = runFreshenHash(ctx, hasher, scanner.entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if showProgress && filesToHash > 0 {
|
||||
log.ProgressDone()
|
||||
}
|
||||
|
||||
// Print summary
|
||||
log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
|
||||
unchanged, changed, added, removed)
|
||||
scanner.unchanged, scanner.changed, scanner.added, removed)
|
||||
|
||||
// Skip writing if nothing changed
|
||||
if changed == 0 && added == 0 && removed == 0 {
|
||||
if scanner.changed == 0 && scanner.added == 0 && removed == 0 {
|
||||
log.Infof("manifest unchanged, skipping write")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write updated manifest atomically (write to temp, then rename)
|
||||
tmpPath := manifestPath + ".tmp"
|
||||
outFile, err := mfa.Fs.Create(tmpPath)
|
||||
err = writeFreshenedManifest(mfa.Fs, hasher.builder, manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
err = builder.Build(outFile)
|
||||
_ = outFile.Close()
|
||||
if err != nil {
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
return fmt.Errorf("failed to write manifest: %w", err)
|
||||
}
|
||||
|
||||
// Rename temp to final
|
||||
if err := mfa.Fs.Rename(tmpPath, manifestPath); err != nil {
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
return fmt.Errorf("failed to rename manifest: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
totalDuration := time.Since(mfa.startupTime)
|
||||
if hashedBytes > 0 {
|
||||
hashDuration := time.Since(startHash)
|
||||
hashRate := float64(hashedBytes) / hashDuration.Seconds()
|
||||
if hasher.hashedBytes > 0 {
|
||||
hashDuration := time.Since(hasher.startHash)
|
||||
hashRate := float64(hasher.hashedBytes) / hashDuration.Seconds()
|
||||
log.Infof("hashed %s in %.1fs (%s/s)",
|
||||
humanize.IBytes(uint64(hashedBytes)), totalDuration.Seconds(), humanize.IBytes(uint64(hashRate)))
|
||||
humanize.IBytes(safeUint64(hasher.hashedBytes)),
|
||||
totalDuration.Seconds(), humanize.IBytes(safeRateUint64(hashRate)))
|
||||
}
|
||||
log.Infof("wrote %d files to %s", len(entries), manifestPath)
|
||||
|
||||
log.Infof("wrote %d files to %s", len(scanner.entries), manifestPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// hashFile reads a file and computes its SHA256 multihash.
|
||||
// Progress callback is called with bytes read so far.
|
||||
func hashFile(r io.Reader, size int64, progress func(int64)) ([]byte, int64, error) {
|
||||
func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
|
||||
h := sha256.New()
|
||||
buf := make([]byte, 64*1024)
|
||||
buf := make([]byte, hashBufSize)
|
||||
|
||||
var total int64
|
||||
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
if n > 0 {
|
||||
h.Write(buf[:n])
|
||||
|
||||
total += int64(n)
|
||||
if progress != nil {
|
||||
progress(total)
|
||||
}
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
// Returned unwrapped: the caller renders this as
|
||||
// "failed to hash <path>: <err>" and adding a second layer here
|
||||
// would change that message.
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
@@ -375,15 +589,29 @@ func hashFile(r io.Reader, size int64, progress func(int64)) ([]byte, int64, err
|
||||
}
|
||||
|
||||
// addFileToBuilder adds a new file entry to the builder
|
||||
func addFileToBuilder(b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte) error {
|
||||
return b.AddFileWithHash(mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash)
|
||||
func addFileToBuilder(
|
||||
b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte,
|
||||
) error {
|
||||
return b.AddFileWithHash(
|
||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash)
|
||||
}
|
||||
|
||||
// addExistingToBuilder adds an existing manifest entry to the builder
|
||||
// addExistingToBuilder adds an existing manifest entry to the builder.
|
||||
//
|
||||
// Entries reach this path only when recordEntry classified them as
|
||||
// unchanged, which requires a recorded mtime, so an absent mtime here is
|
||||
// an error rather than something to paper over with the Unix epoch.
|
||||
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||
mtime := time.Unix(entry.Mtime.Seconds, int64(entry.Mtime.Nanos))
|
||||
if len(entry.Hashes) == 0 {
|
||||
mtime, ok := entryMtime(entry)
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
|
||||
}
|
||||
|
||||
if len(entry.GetHashes()) == 0 {
|
||||
return nil
|
||||
}
|
||||
return b.AddFileWithHash(mfer.RelFilePath(entry.Path), mfer.FileSize(entry.Size), mfer.ModTime(mtime), entry.Hashes[0].MultiHash)
|
||||
|
||||
return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||
entry.GetHashes()[0].GetMultiHash())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user