Update golangci-lint to v2.12.2 with canonical config (closes #60)
check / check (push) Has been cancelled
check / check (push) Has been cancelled
Adopts golangci-lint v2.12.2 and the canonical .golangci.yml (default: all), and fixes all resulting findings across the tree. Two intended behavior changes: absent MFFilePath.Mtime is handled explicitly in freshen, list and export rather than dereferenced (main panicked); gpg positional key IDs now follow an explicit -- end-of-options marker. All twelve reworded user-visible error messages restored to byte-identical parity with main and pinned by tests.
This commit was merged in pull request #59.
This commit is contained in:
+311
-159
@@ -43,12 +43,20 @@ type ScanStatus struct {
|
||||
|
||||
// ScannerOptions configures scanner behavior.
|
||||
type ScannerOptions struct {
|
||||
IncludeDotfiles bool // Include files and directories starting with a dot (default: exclude)
|
||||
FollowSymLinks bool // Resolve symlinks instead of skipping them
|
||||
IncludeTimestamps bool // Include createdAt timestamp in manifest (default: omit for determinism)
|
||||
Fs afero.Fs // Filesystem to use, defaults to OsFs if nil
|
||||
SigningOptions *SigningOptions // GPG signing options (nil = no signing)
|
||||
Seed string // If set, derive a deterministic UUID from this seed
|
||||
// IncludeDotfiles includes files and directories starting with a dot
|
||||
// (default: exclude).
|
||||
IncludeDotfiles bool
|
||||
// FollowSymLinks resolves symlinks instead of skipping them.
|
||||
FollowSymLinks bool
|
||||
// IncludeTimestamps includes a createdAt timestamp in the manifest
|
||||
// (default: omit for determinism).
|
||||
IncludeTimestamps bool
|
||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
||||
Fs afero.Fs
|
||||
// SigningOptions holds GPG signing options (nil = no signing).
|
||||
SigningOptions *SigningOptions
|
||||
// Seed, if set, derives a deterministic UUID from this seed.
|
||||
Seed string
|
||||
}
|
||||
|
||||
// FileEntry represents a file that has been enumerated.
|
||||
@@ -79,10 +87,12 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
||||
if opts == nil {
|
||||
opts = &ScannerOptions{}
|
||||
}
|
||||
|
||||
fs := opts.Fs
|
||||
if fs == nil {
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
|
||||
return &Scanner{
|
||||
files: make([]*FileEntry, 0),
|
||||
options: opts,
|
||||
@@ -96,47 +106,63 @@ func (s *Scanner) EnumerateFile(filePath string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, err := s.fs.Stat(abs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// For single files, use the filename as the relative path
|
||||
basePath := filepath.Dir(abs)
|
||||
|
||||
return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil)
|
||||
}
|
||||
|
||||
// EnumeratePath walks a directory path and adds all files to the scanner.
|
||||
// If progress is non-nil, status updates are sent as files are discovered.
|
||||
// The progress channel is closed when the method returns.
|
||||
func (s *Scanner) EnumeratePath(inputPath string, progress chan<- EnumerateStatus) error {
|
||||
func (s *Scanner) EnumeratePath(
|
||||
inputPath string,
|
||||
progress chan<- EnumerateStatus,
|
||||
) error {
|
||||
if progress != nil {
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
abs, err := filepath.Abs(inputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
||||
|
||||
return s.enumerateFS(afs, abs, progress)
|
||||
}
|
||||
|
||||
// EnumeratePaths walks multiple directory paths and adds all files to the scanner.
|
||||
// If progress is non-nil, status updates are sent as files are discovered.
|
||||
// The progress channel is closed when the method returns.
|
||||
func (s *Scanner) EnumeratePaths(progress chan<- EnumerateStatus, inputPaths ...string) error {
|
||||
func (s *Scanner) EnumeratePaths(
|
||||
progress chan<- EnumerateStatus,
|
||||
inputPaths ...string,
|
||||
) error {
|
||||
if progress != nil {
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
for _, p := range inputPaths {
|
||||
abs, err := filepath.Abs(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
||||
if err := s.enumerateFS(afs, abs, progress); err != nil {
|
||||
|
||||
err = s.enumerateFS(afs, abs, progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -144,31 +170,231 @@ func (s *Scanner) EnumeratePaths(progress chan<- EnumerateStatus, inputPaths ...
|
||||
// If progress is non-nil, status updates are sent as files are discovered.
|
||||
// The progress channel is closed when the method returns.
|
||||
// basePath is used to compute absolute paths for file reading.
|
||||
func (s *Scanner) EnumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
|
||||
func (s *Scanner) EnumerateFS(
|
||||
afs afero.Fs,
|
||||
basePath string,
|
||||
progress chan<- EnumerateStatus,
|
||||
) error {
|
||||
if progress != nil {
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
return s.enumerateFS(afs, basePath, progress)
|
||||
}
|
||||
|
||||
// enumerateFS is the internal implementation that doesn't close the progress channel.
|
||||
func (s *Scanner) enumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
|
||||
// Files returns a copy of all files added to the scanner.
|
||||
func (s *Scanner) Files() []*FileEntry {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
out := make([]*FileEntry, len(s.files))
|
||||
copy(out, s.files)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// FileCount returns the number of files in the scanner.
|
||||
func (s *Scanner) FileCount() FileCount {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
return FileCount(len(s.files))
|
||||
}
|
||||
|
||||
// TotalBytes returns the total size of all files in the scanner.
|
||||
func (s *Scanner) TotalBytes() FileSize {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
return s.totalBytes
|
||||
}
|
||||
|
||||
// ToManifest reads all file contents, computes hashes, and generates a manifest.
|
||||
// If progress is non-nil, status updates are sent approximately once per second.
|
||||
// The progress channel is closed when the method returns.
|
||||
// The manifest is written to the provided io.Writer.
|
||||
func (s *Scanner) ToManifest(
|
||||
ctx context.Context, w io.Writer, progress chan<- ScanStatus,
|
||||
) error {
|
||||
if progress != nil {
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
files := make([]*FileEntry, len(s.files))
|
||||
copy(files, s.files)
|
||||
totalFiles := FileCount(len(files))
|
||||
|
||||
var totalBytes FileSize
|
||||
for _, f := range files {
|
||||
totalBytes += f.Size
|
||||
}
|
||||
|
||||
s.mu.RUnlock()
|
||||
|
||||
builder := s.configureBuilder()
|
||||
|
||||
var (
|
||||
scannedFiles FileCount
|
||||
scannedBytes FileSize
|
||||
)
|
||||
|
||||
lastProgressTime := time.Now()
|
||||
startTime := time.Now()
|
||||
|
||||
pt := &scanProgressTracker{
|
||||
progress: progress,
|
||||
totalFiles: totalFiles,
|
||||
totalBytes: totalBytes,
|
||||
startTime: startTime,
|
||||
lastProgress: &lastProgressTime,
|
||||
}
|
||||
|
||||
for _, entry := range files {
|
||||
// Check for cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
bytesRead, err := s.scanFile(builder, pt, entry, scannedFiles, scannedBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scannedFiles++
|
||||
scannedBytes += bytesRead
|
||||
}
|
||||
|
||||
// Send final progress (ETA is 0 at completion; remaining bytes are 0,
|
||||
// so computeRateETA yields eta 0 and the same average rate as before)
|
||||
if progress != nil {
|
||||
rate, _ := computeRateETA(time.Since(startTime), scannedBytes, totalBytes)
|
||||
|
||||
sendScanStatus(progress, ScanStatus{
|
||||
TotalFiles: totalFiles,
|
||||
ScannedFiles: scannedFiles,
|
||||
TotalBytes: totalBytes,
|
||||
ScannedBytes: scannedBytes,
|
||||
BytesPerSec: rate,
|
||||
ETA: 0,
|
||||
})
|
||||
}
|
||||
|
||||
// Build and write manifest
|
||||
//nolint:contextcheck // Build's GPG signing exec is not cancellable by design
|
||||
return builder.Build(w)
|
||||
}
|
||||
|
||||
// configureBuilder constructs a manifest builder configured from the
|
||||
// scanner options.
|
||||
func (s *Scanner) configureBuilder() *Builder {
|
||||
builder := NewBuilder()
|
||||
if s.options.IncludeTimestamps {
|
||||
builder.SetIncludeTimestamps(true)
|
||||
}
|
||||
|
||||
if s.options.SigningOptions != nil {
|
||||
builder.SetSigningOptions(s.options.SigningOptions)
|
||||
}
|
||||
|
||||
if s.options.Seed != "" {
|
||||
builder.SetSeed(s.options.Seed)
|
||||
}
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
// scanFile hashes a single file into the builder, forwarding per-file
|
||||
// progress updates, and returns the number of bytes read.
|
||||
func (s *Scanner) scanFile(
|
||||
builder *Builder,
|
||||
pt *scanProgressTracker,
|
||||
entry *FileEntry,
|
||||
scannedFiles FileCount,
|
||||
scannedBytes FileSize,
|
||||
) (FileSize, error) {
|
||||
// Open file
|
||||
f, err := s.fs.Open(string(entry.AbsPath))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Create progress channel for this file
|
||||
var (
|
||||
fileProgress chan FileHashProgress
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
if pt.progress != nil {
|
||||
fileProgress = make(chan FileHashProgress, 1)
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func(base FileSize, done FileCount) {
|
||||
defer wg.Done()
|
||||
|
||||
pt.forward(fileProgress, done, base)
|
||||
}(scannedBytes, scannedFiles)
|
||||
}
|
||||
|
||||
// Add to manifest with progress channel
|
||||
bytesRead, err := builder.AddFile(
|
||||
entry.Path,
|
||||
entry.Size,
|
||||
entry.Mtime,
|
||||
f,
|
||||
fileProgress,
|
||||
)
|
||||
_ = f.Close()
|
||||
|
||||
// Close channel and wait for goroutine to finish
|
||||
if fileProgress != nil {
|
||||
close(fileProgress)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(sizeToUint64(bytesRead)))
|
||||
|
||||
return bytesRead, nil
|
||||
}
|
||||
|
||||
// enumerateFS is the internal implementation that doesn't close the
|
||||
// progress channel.
|
||||
func (s *Scanner) enumerateFS(
|
||||
afs afero.Fs,
|
||||
basePath string,
|
||||
progress chan<- EnumerateStatus,
|
||||
) error {
|
||||
return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !s.options.IncludeDotfiles && IsHiddenPath(p) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.enumerateFileWithInfo(p, basePath, info, progress)
|
||||
})
|
||||
}
|
||||
|
||||
// enumerateFileWithInfo adds a file with pre-existing fs.FileInfo.
|
||||
func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info fs.FileInfo, progress chan<- EnumerateStatus) error {
|
||||
func (s *Scanner) enumerateFileWithInfo(
|
||||
filePath string,
|
||||
basePath string,
|
||||
info fs.FileInfo,
|
||||
progress chan<- EnumerateStatus,
|
||||
) error {
|
||||
if info.IsDir() {
|
||||
// Manifests contain only files, directories are implied
|
||||
return nil
|
||||
@@ -193,11 +419,13 @@ func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info f
|
||||
realPath, err := filepath.EvalSymlinks(absPath)
|
||||
if err != nil {
|
||||
// Skip broken symlinks
|
||||
return nil
|
||||
return nil //nolint:nilerr // broken symlinks are skipped by design
|
||||
}
|
||||
|
||||
realInfo, err := s.fs.Stat(realPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
// Skip symlinks whose target cannot be stat'd
|
||||
return nil //nolint:nilerr // unreadable targets are skipped by design
|
||||
}
|
||||
// Skip if symlink points to a directory
|
||||
if realInfo.IsDir() {
|
||||
@@ -232,160 +460,78 @@ func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info f
|
||||
return nil
|
||||
}
|
||||
|
||||
// Files returns a copy of all files added to the scanner.
|
||||
func (s *Scanner) Files() []*FileEntry {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
out := make([]*FileEntry, len(s.files))
|
||||
copy(out, s.files)
|
||||
return out
|
||||
// scanProgressTracker carries the shared state needed to report rate-limited
|
||||
// scan progress updates.
|
||||
type scanProgressTracker struct {
|
||||
progress chan<- ScanStatus
|
||||
totalFiles FileCount
|
||||
totalBytes FileSize
|
||||
startTime time.Time
|
||||
lastProgress *time.Time
|
||||
}
|
||||
|
||||
// FileCount returns the number of files in the scanner.
|
||||
func (s *Scanner) FileCount() FileCount {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return FileCount(len(s.files))
|
||||
}
|
||||
|
||||
// TotalBytes returns the total size of all files in the scanner.
|
||||
func (s *Scanner) TotalBytes() FileSize {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.totalBytes
|
||||
}
|
||||
|
||||
// ToManifest reads all file contents, computes hashes, and generates a manifest.
|
||||
// If progress is non-nil, status updates are sent approximately once per second.
|
||||
// The progress channel is closed when the method returns.
|
||||
// The manifest is written to the provided io.Writer.
|
||||
func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- ScanStatus) error {
|
||||
if progress != nil {
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
files := make([]*FileEntry, len(s.files))
|
||||
copy(files, s.files)
|
||||
totalFiles := FileCount(len(files))
|
||||
var totalBytes FileSize
|
||||
for _, f := range files {
|
||||
totalBytes += f.Size
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
builder := NewBuilder()
|
||||
if s.options.IncludeTimestamps {
|
||||
builder.SetIncludeTimestamps(true)
|
||||
}
|
||||
if s.options.SigningOptions != nil {
|
||||
builder.SetSigningOptions(s.options.SigningOptions)
|
||||
}
|
||||
if s.options.Seed != "" {
|
||||
builder.SetSeed(s.options.Seed)
|
||||
}
|
||||
|
||||
var scannedFiles FileCount
|
||||
var scannedBytes FileSize
|
||||
lastProgressTime := time.Now()
|
||||
startTime := time.Now()
|
||||
|
||||
for _, entry := range files {
|
||||
// Check for cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// forward relays per-file hash progress to the scan progress channel,
|
||||
// rate-limited to one update per second.
|
||||
func (pt *scanProgressTracker) forward(
|
||||
fileProgress <-chan FileHashProgress,
|
||||
scannedFiles FileCount,
|
||||
baseBytes FileSize,
|
||||
) {
|
||||
for p := range fileProgress {
|
||||
// Send progress at most once per second
|
||||
now := time.Now()
|
||||
if now.Sub(*pt.lastProgress) < time.Second {
|
||||
continue
|
||||
}
|
||||
|
||||
// Open file
|
||||
f, err := s.fs.Open(string(entry.AbsPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
currentBytes := baseBytes + p.BytesRead
|
||||
rate, eta := computeRateETA(now.Sub(pt.startTime), currentBytes, pt.totalBytes)
|
||||
|
||||
// Create progress channel for this file
|
||||
var fileProgress chan FileHashProgress
|
||||
var wg sync.WaitGroup
|
||||
if progress != nil {
|
||||
fileProgress = make(chan FileHashProgress, 1)
|
||||
wg.Add(1)
|
||||
go func(baseScannedBytes FileSize) {
|
||||
defer wg.Done()
|
||||
for p := range fileProgress {
|
||||
// Send progress at most once per second
|
||||
now := time.Now()
|
||||
if now.Sub(lastProgressTime) >= time.Second {
|
||||
elapsed := now.Sub(startTime).Seconds()
|
||||
currentBytes := baseScannedBytes + p.BytesRead
|
||||
var rate float64
|
||||
var eta time.Duration
|
||||
if elapsed > 0 && currentBytes > 0 {
|
||||
rate = float64(currentBytes) / elapsed
|
||||
remainingBytes := totalBytes - currentBytes
|
||||
if rate > 0 {
|
||||
eta = time.Duration(float64(remainingBytes)/rate) * time.Second
|
||||
}
|
||||
}
|
||||
sendScanStatus(progress, ScanStatus{
|
||||
TotalFiles: totalFiles,
|
||||
ScannedFiles: scannedFiles,
|
||||
TotalBytes: totalBytes,
|
||||
ScannedBytes: currentBytes,
|
||||
BytesPerSec: rate,
|
||||
ETA: eta,
|
||||
})
|
||||
lastProgressTime = now
|
||||
}
|
||||
}
|
||||
}(scannedBytes)
|
||||
}
|
||||
|
||||
// Add to manifest with progress channel
|
||||
bytesRead, err := builder.AddFile(
|
||||
entry.Path,
|
||||
entry.Size,
|
||||
entry.Mtime,
|
||||
f,
|
||||
fileProgress,
|
||||
)
|
||||
_ = f.Close()
|
||||
|
||||
// Close channel and wait for goroutine to finish
|
||||
if fileProgress != nil {
|
||||
close(fileProgress)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(uint64(bytesRead)))
|
||||
|
||||
scannedFiles++
|
||||
scannedBytes += bytesRead
|
||||
}
|
||||
|
||||
// Send final progress (ETA is 0 at completion)
|
||||
if progress != nil {
|
||||
elapsed := time.Since(startTime).Seconds()
|
||||
var rate float64
|
||||
if elapsed > 0 {
|
||||
rate = float64(scannedBytes) / elapsed
|
||||
}
|
||||
sendScanStatus(progress, ScanStatus{
|
||||
TotalFiles: totalFiles,
|
||||
sendScanStatus(pt.progress, ScanStatus{
|
||||
TotalFiles: pt.totalFiles,
|
||||
ScannedFiles: scannedFiles,
|
||||
TotalBytes: totalBytes,
|
||||
ScannedBytes: scannedBytes,
|
||||
TotalBytes: pt.totalBytes,
|
||||
ScannedBytes: currentBytes,
|
||||
BytesPerSec: rate,
|
||||
ETA: 0,
|
||||
ETA: eta,
|
||||
})
|
||||
|
||||
*pt.lastProgress = now
|
||||
}
|
||||
}
|
||||
|
||||
// computeRateETA returns the average throughput over elapsed time and the
|
||||
// estimated time to process the remaining bytes at that rate.
|
||||
func computeRateETA(
|
||||
elapsed time.Duration,
|
||||
done FileSize,
|
||||
total FileSize,
|
||||
) (float64, time.Duration) {
|
||||
var (
|
||||
rate float64
|
||||
eta time.Duration
|
||||
)
|
||||
|
||||
if elapsed > 0 && done > 0 {
|
||||
rate = float64(done) / elapsed.Seconds()
|
||||
|
||||
remaining := total - done
|
||||
if rate > 0 {
|
||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// Build and write manifest
|
||||
return builder.Build(w)
|
||||
return rate, eta
|
||||
}
|
||||
|
||||
// sizeToUint64 converts a FileSize to uint64 for display, clamping
|
||||
// negative values to zero so the conversion cannot overflow.
|
||||
func sizeToUint64(v FileSize) uint64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return uint64(v)
|
||||
}
|
||||
|
||||
// IsHiddenPath returns true if the path or any of its parent directories
|
||||
@@ -396,17 +542,21 @@ func IsHiddenPath(p string) bool {
|
||||
if tp == "." || tp == "/" {
|
||||
return false
|
||||
}
|
||||
|
||||
if strings.HasPrefix(tp, ".") {
|
||||
return true
|
||||
}
|
||||
|
||||
for {
|
||||
d, f := path.Split(tp)
|
||||
if strings.HasPrefix(f, ".") {
|
||||
return true
|
||||
}
|
||||
|
||||
if d == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
tp = d[0 : len(d)-1] // trim trailing slash from dir
|
||||
}
|
||||
}
|
||||
@@ -417,6 +567,7 @@ func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) {
|
||||
if ch == nil {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- status:
|
||||
default:
|
||||
@@ -430,6 +581,7 @@ func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) {
|
||||
if ch == nil {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- status:
|
||||
default:
|
||||
|
||||
Reference in New Issue
Block a user