Update golangci-lint to v2.12.2 with canonical config (closes #60)
All checks were successful
check / check (push) Successful in 35s
All checks were successful
check / check (push) Successful in 35s
- Add canonical .golangci.yml (v2 schema, default: all, project thresholds for lll/funlen/cyclop/dupl) - Bump golangci-lint pins from v2.0.2 to v2.12.2 in Makefile (go install, new /v2 module path) and Dockerfile (tagged+digest Debian image pin) - Fix all lint findings surfaced by the new linter set across cmd/mfer, internal/bork, internal/cli, internal/log, and mfer: static sentinel errors (err113), context-aware HTTP and exec (noctx), guarded integer conversions and stricter permissions (gosec), named constants (mnd, goconst), function decomposition (funlen, cyclop, gocognit, nestif), declaration ordering (funcorder), t.Parallel/t.TempDir/t.Setenv adoption in tests (paralleltest, usetesting), protobuf getters (protogetter), plus formatting and style cleanups (wsl_v5, nlreturn, lll, revive, testifylint, and others) - Serialize CLI runs in tests behind a mutex so parallel tests do not cross-wire the process-global logger's captured output The decompositions are behavior-preserving. In particular: - REPO_POLICIES.md is untouched and stays byte-identical to the authoritative copy in the prompts repo - the mfer.manifest type stays unexported; whether to export it is an open owner design question (README question 13) - directories created by fetch keep mode 0755, because fetched trees are content meant to be readable by other uids - an absent MFFilePath.Mtime is handled explicitly and identically in freshen, list, and export rather than being read as the Unix epoch, which would classify every entry as changed and rewrite the manifest on every freshen - every user-visible error message renders byte-identically to what it did before, with the err113 sentinels wrapped mid-sentence where needed; the rendered strings are now pinned by tests Also fixes an argument-injection defect the lint pass surfaced: key IDs reach gpg as bare positional arguments, so a key ID beginning with "-" was parsed by gpg as an option. All positional arguments now follow an explicit "--" end-of-options marker. The symlink-escape gap in fetch's path handling, which sanitizePath does not and cannot address, is filed separately as #86.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -16,9 +17,78 @@ import (
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
log.Debug("generateManifestOperation()")
|
||||
var (
|
||||
// errPathNotExist indicates an input path that does not exist.
|
||||
errPathNotExist = errors.New("path does not exist")
|
||||
// errOutputExists indicates the output file already exists and
|
||||
// --force was not given. It is wrapped mid-sentence so that the
|
||||
// rendered message stays exactly as mfer has always printed it.
|
||||
errOutputExists = errors.New(
|
||||
"already exists (use --force to overwrite)")
|
||||
)
|
||||
|
||||
// reportEnumProgress renders enumeration progress until the channel
|
||||
// closes.
|
||||
func reportEnumProgress(progress <-chan mfer.EnumerateStatus, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
for status := range progress {
|
||||
log.Progressf("Enumerating: %d files, %s",
|
||||
status.FilesFound,
|
||||
humanize.IBytes(safeUint64(int64(status.BytesFound))))
|
||||
}
|
||||
|
||||
log.ProgressDone()
|
||||
}
|
||||
|
||||
// reportScanProgress renders scan progress until the channel closes.
|
||||
func reportScanProgress(progress <-chan mfer.ScanStatus, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
for status := range progress {
|
||||
if status.ETA > 0 {
|
||||
log.Progressf("Scanning: %d/%d files, %s/s, ETA %s",
|
||||
status.ScannedFiles,
|
||||
status.TotalFiles,
|
||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)),
|
||||
status.ETA.Round(time.Second))
|
||||
} else {
|
||||
log.Progressf("Scanning: %d/%d files, %s/s",
|
||||
status.ScannedFiles,
|
||||
status.TotalFiles,
|
||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)))
|
||||
}
|
||||
}
|
||||
|
||||
log.ProgressDone()
|
||||
}
|
||||
|
||||
// collectInputPaths validates the input path arguments and returns them
|
||||
// as absolute paths.
|
||||
func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
paths := make([]string, 0, args.Len())
|
||||
|
||||
for i := range args.Len() {
|
||||
inputPath := args.Get(i)
|
||||
|
||||
ap, err := filepath.Abs(inputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate: invalid path %q: %w", inputPath, err)
|
||||
}
|
||||
// Validate path exists before adding to list
|
||||
if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
|
||||
return nil, fmt.Errorf("%w: %s", errPathNotExist, inputPath)
|
||||
}
|
||||
|
||||
log.Debugf("enumerating path: %s", ap)
|
||||
paths = append(paths, ap)
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||
opts := &mfer.ScannerOptions{
|
||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||
@@ -29,6 +99,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
// Set seed for deterministic UUID if provided
|
||||
if seed := ctx.String("seed"); seed != "" {
|
||||
opts.Seed = seed
|
||||
|
||||
log.Infof("using deterministic seed for manifest UUID")
|
||||
}
|
||||
|
||||
@@ -40,136 +111,167 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
log.Infof("signing manifest with GPG key: %s", signKey)
|
||||
}
|
||||
|
||||
s := mfer.NewScannerWithOptions(opts)
|
||||
|
||||
// Phase 1: Enumeration - collect paths and stat files
|
||||
args := ctx.Args()
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
// Set up enumeration progress reporting
|
||||
var enumProgress chan mfer.EnumerateStatus
|
||||
var enumWg sync.WaitGroup
|
||||
if showProgress {
|
||||
enumProgress = make(chan mfer.EnumerateStatus, 1)
|
||||
enumWg.Add(1)
|
||||
go func() {
|
||||
defer enumWg.Done()
|
||||
for status := range enumProgress {
|
||||
log.Progressf("Enumerating: %d files, %s",
|
||||
status.FilesFound,
|
||||
humanize.IBytes(uint64(status.BytesFound)))
|
||||
}
|
||||
log.ProgressDone()
|
||||
}()
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// enumerateInputs runs the enumeration phase over the argument paths,
|
||||
// or the current directory when no arguments are given.
|
||||
func (mfa *CLIApp) enumerateInputs(
|
||||
s *mfer.Scanner, args cli.Args, enumProgress chan mfer.EnumerateStatus,
|
||||
) error {
|
||||
if args.Len() == 0 {
|
||||
// Default to current directory
|
||||
if err := s.EnumeratePath(".", enumProgress); err != nil {
|
||||
return fmt.Errorf("generate: failed to enumerate current directory: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Collect and validate all paths first
|
||||
paths := make([]string, 0, args.Len())
|
||||
for i := 0; i < args.Len(); i++ {
|
||||
inputPath := args.Get(i)
|
||||
ap, err := filepath.Abs(inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate: invalid path %q: %w", inputPath, err)
|
||||
}
|
||||
// Validate path exists before adding to list
|
||||
if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
|
||||
return fmt.Errorf("path does not exist: %s", inputPath)
|
||||
}
|
||||
log.Debugf("enumerating path: %s", ap)
|
||||
paths = append(paths, ap)
|
||||
}
|
||||
if err := s.EnumeratePaths(enumProgress, paths...); err != nil {
|
||||
return fmt.Errorf("generate: failed to enumerate paths: %w", err)
|
||||
err := s.EnumeratePath(".", enumProgress)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"generate: failed to enumerate current directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect and validate all paths first
|
||||
paths, err := mfa.collectInputPaths(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.EnumeratePaths(enumProgress, paths...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate: failed to enumerate paths: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupOnSignal installs a handler that removes the temp output file
|
||||
// and exits when the process is interrupted. It returns the signal
|
||||
// channel so the caller can stop and close it when done.
|
||||
func (mfa *CLIApp) cleanupOnSignal(outFile afero.File, tmpPath string) chan os.Signal {
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
|
||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
sig, ok := <-sigChan
|
||||
if !ok || sig == nil {
|
||||
return // Channel closed normally, not a signal
|
||||
}
|
||||
|
||||
_ = outFile.Close()
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
|
||||
os.Exit(1)
|
||||
}()
|
||||
|
||||
return sigChan
|
||||
}
|
||||
|
||||
// runEnumeratePhase enumerates all input paths with optional progress
|
||||
// reporting and logs the totals.
|
||||
func (mfa *CLIApp) runEnumeratePhase(ctx *cli.Context, s *mfer.Scanner) error {
|
||||
// Set up enumeration progress reporting
|
||||
var (
|
||||
enumProgress chan mfer.EnumerateStatus
|
||||
enumWg sync.WaitGroup
|
||||
)
|
||||
|
||||
if ctx.Bool("progress") {
|
||||
enumProgress = make(chan mfer.EnumerateStatus, 1)
|
||||
|
||||
enumWg.Add(1)
|
||||
|
||||
go reportEnumProgress(enumProgress, &enumWg)
|
||||
}
|
||||
|
||||
err := mfa.enumerateInputs(s, ctx.Args(), enumProgress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
enumWg.Wait()
|
||||
|
||||
log.Infof("enumerated %d files, %s total", s.FileCount(), humanize.IBytes(uint64(s.TotalBytes())))
|
||||
log.Infof("enumerated %d files, %s total", s.FileCount(),
|
||||
humanize.IBytes(safeUint64(int64(s.TotalBytes()))))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
log.Debug("generateManifestOperation()")
|
||||
|
||||
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(ctx))
|
||||
|
||||
// Phase 1: Enumeration - collect paths and stat files
|
||||
err := mfa.runEnumeratePhase(ctx, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
// Check if output file exists
|
||||
outputPath := ctx.String("output")
|
||||
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists {
|
||||
if !ctx.Bool("force") {
|
||||
return fmt.Errorf("output file %s already exists (use --force to overwrite)", outputPath)
|
||||
}
|
||||
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !ctx.Bool("force") {
|
||||
return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
tmpPath := outputPath + ".tmp"
|
||||
|
||||
outFile, err := mfa.Fs.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
// Set up signal handler to clean up temp file on Ctrl-C
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
sig, ok := <-sigChan
|
||||
if !ok || sig == nil {
|
||||
return // Channel closed normally, not a signal
|
||||
}
|
||||
_ = outFile.Close()
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
os.Exit(1)
|
||||
}()
|
||||
sigChan := mfa.cleanupOnSignal(outFile, tmpPath)
|
||||
|
||||
// Clean up temp file on any error or interruption
|
||||
success := false
|
||||
|
||||
defer func() {
|
||||
signal.Stop(sigChan)
|
||||
close(sigChan)
|
||||
|
||||
_ = outFile.Close()
|
||||
|
||||
if !success {
|
||||
_ = mfa.Fs.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
// Phase 2: Scan - read file contents and generate manifest
|
||||
var scanProgress chan mfer.ScanStatus
|
||||
var scanWg sync.WaitGroup
|
||||
var (
|
||||
scanProgress chan mfer.ScanStatus
|
||||
scanWg sync.WaitGroup
|
||||
)
|
||||
|
||||
if showProgress {
|
||||
scanProgress = make(chan mfer.ScanStatus, 1)
|
||||
|
||||
scanWg.Add(1)
|
||||
go func() {
|
||||
defer scanWg.Done()
|
||||
for status := range scanProgress {
|
||||
if status.ETA > 0 {
|
||||
log.Progressf("Scanning: %d/%d files, %s/s, ETA %s",
|
||||
status.ScannedFiles,
|
||||
status.TotalFiles,
|
||||
humanize.IBytes(uint64(status.BytesPerSec)),
|
||||
status.ETA.Round(time.Second))
|
||||
} else {
|
||||
log.Progressf("Scanning: %d/%d files, %s/s",
|
||||
status.ScannedFiles,
|
||||
status.TotalFiles,
|
||||
humanize.IBytes(uint64(status.BytesPerSec)))
|
||||
}
|
||||
}
|
||||
log.ProgressDone()
|
||||
}()
|
||||
|
||||
go reportScanProgress(scanProgress, &scanWg)
|
||||
}
|
||||
|
||||
err = s.ToManifest(ctx.Context, outFile, scanProgress)
|
||||
|
||||
scanWg.Wait()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate manifest: %w", err)
|
||||
}
|
||||
|
||||
// Close file before rename to ensure all data is flushed
|
||||
if err := outFile.Close(); err != nil {
|
||||
err = outFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
if err := mfa.Fs.Rename(tmpPath, outputPath); err != nil {
|
||||
err = mfa.Fs.Rename(tmpPath, outputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
@@ -177,7 +279,9 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
|
||||
elapsed := time.Since(mfa.startupTime).Seconds()
|
||||
rate := float64(s.TotalBytes()) / elapsed
|
||||
log.Infof("wrote %d files (%s) to %s in %.1fs (%s/s)", s.FileCount(), humanize.IBytes(uint64(s.TotalBytes())), outputPath, elapsed, humanize.IBytes(uint64(rate)))
|
||||
log.Infof("wrote %d files (%s) to %s in %.1fs (%s/s)", s.FileCount(),
|
||||
humanize.IBytes(safeUint64(int64(s.TotalBytes()))), outputPath, elapsed,
|
||||
humanize.IBytes(safeRateUint64(rate)))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user