sneak/vaultik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce internal/ui package and rewrite user-facing output

Browse Source 
All user-facing output now goes through a single ui.Writer with a
uniform style:

  》 (white)     for begin / info / notice
  》 (green)     for complete / success
  Warning:      for warnings (orange)
  ERROR:        for errors (red)
    》          (indented) for progress heartbeats

Color is enabled when stdout is a TTY and NO_COLOR is unset.

Standards:
- Complete-sentence messages with fully qualified terms ("backup
  destination store", "local index database", "snapshot source
  files enumeration").
- Every Complete has a matching Begin.
- Natural verb tense conveys state ("Uploading" -> "Uploaded"). The
  words "begin"/"complete" never appear in message bodies; the marker
  color carries that information.
- ETA means clock time, not duration. Progress lines say "estimated
  remaining time (<dur>), finish at <time>" with both labeled.

Adds globals.CommitDate (populated by Makefile/Dockerfile/goreleaser
via ldflags from `git show -s --format=%cI HEAD`) and a startup banner
printed once per invocation.

Strips fx call-chain noise from startup errors so users see the actual
underlying error (e.g. "creating base path: mkdir /Volumes/BACKUPS:
permission denied" instead of three layers of "could not build
arguments for function ...").

README documents the output style and the ui package conventions.
This commit is contained in:
Jeffrey Paul
2026-06-17 04:32:05 +02:00
parent 8de8f8e5cc
commit 00d4b36e35
14 changed files with 533 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/snapshot/module.go
View File

@@ -1,19 +1,18 @@
package snapshot
import (
"io"
"github.com/spf13/afero"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
)
// ScannerParams holds parameters for scanner creation
type ScannerParams struct {
EnableProgress bool
Output io.Writer // Where one-off scanner messages go; nil disables them
UI *ui.Writer // Where user-facing scanner messages go; nil = discard
Fs afero.Fs
Exclude []string // Exclude patterns (combined global + snapshot-specific)
SkipErrors bool // Skip file read errors (log loudly but continue)
@@ -49,7 +48,7 @@ func provideScannerFactory(cfg *config.Config, repos *database.Repositories, sto
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress,
Output: params.Output,
UI: params.UI,
Exclude: excludes,
SkipErrors: params.SkipErrors,
})

196
internal/snapshot/scanner.go
View File

@@ -22,6 +22,7 @@ import (
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/types"
"sneak.berlin/go/vaultik/internal/ui"
)
// FileToProcess holds information about a file that needs processing
@@ -60,8 +61,8 @@ type Scanner struct {
exclude []string // Glob patterns for files/directories to exclude
compiledExclude []compiledPattern // Compiled glob patterns
progress *ProgressReporter
skipErrors bool // Skip file read errors (log loudly but continue)
output io.Writer // User-facing output (os.Stdout or io.Discard in cron mode)
skipErrors bool // Skip file read errors (log loudly but continue)
ui *ui.Writer // User-facing output; never nil (defaults to a discarding writer)
// In-memory cache of known chunk hashes for fast existence checks
knownChunks map[string]struct{}
@@ -92,11 +93,11 @@ type ScannerConfig struct {
Storage storage.Storer
MaxBlobSize int64
CompressionLevel int
AgeRecipients []string // Optional, empty means no encryption
EnableProgress bool // Enable the live progress reporter (ETAs, throughput)
Output io.Writer // Where one-off scanner messages go; nil disables them
Exclude []string // Glob patterns for files/directories to exclude
SkipErrors bool // Skip file read errors (log loudly but continue)
AgeRecipients []string // Optional, empty means no encryption
EnableProgress bool // Enable the live progress reporter (ETAs, throughput)
UI *ui.Writer // Where user-facing scanner messages go; nil = discard
Exclude []string // Glob patterns for files/directories to exclude
SkipErrors bool // Skip file read errors (log loudly but continue)
}
// ScanResult contains the results of a scan operation
@@ -143,9 +144,9 @@ func NewScanner(cfg ScannerConfig) *Scanner {
// Compile exclude patterns
compiledExclude := compileExcludePatterns(cfg.Exclude)
output := cfg.Output
if output == nil {
output = io.Discard
uiw := cfg.UI
if uiw == nil {
uiw = ui.NewWithColor(io.Discard, false)
}
return &Scanner{
@@ -161,7 +162,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
compiledExclude: compiledExclude,
progress: progress,
skipErrors: cfg.SkipErrors,
output: output,
ui: uiw,
pendingChunkHashes: make(map[string]struct{}),
}
}
@@ -212,7 +213,7 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
if len(scanResult.UnchangedFileIDs) > 0 {
_, _ = fmt.Fprintf(s.output, "Associating %s unchanged files with snapshot...\n", formatNumber(len(scanResult.UnchangedFileIDs)))
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
return nil, fmt.Errorf("associating unchanged files: %w", err)
}
@@ -223,13 +224,13 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 2: Process files and create chunks
if len(filesToProcess) > 0 {
_, _ = fmt.Fprintf(s.output, "Processing %s files...\n", formatNumber(len(filesToProcess)))
s.ui.Begin("Processing %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
return nil, fmt.Errorf("process phase failed: %w", err)
}
} else {
_, _ = fmt.Fprintf(s.output, "No files need processing. Creating metadata-only snapshot.\n")
s.ui.Info("Snapshot file processing skipped: no changed files (creating metadata-only snapshot).")
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
}
@@ -242,18 +243,18 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// loadDatabaseState loads known files and chunks from the database into memory for fast lookup
// This avoids per-file and per-chunk database queries during the scan and process phases
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
_, _ = fmt.Fprintln(s.output, "Loading known files from database...")
s.ui.Begin("Loading known files from local index database.")
knownFiles, err := s.loadKnownFiles(ctx, path)
if err != nil {
return nil, fmt.Errorf("loading known files: %w", err)
}
_, _ = fmt.Fprintf(s.output, "Loaded %s known files from database\n", formatNumber(len(knownFiles)))
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
_, _ = fmt.Fprintln(s.output, "Loading known chunks from database...")
s.ui.Begin("Loading known chunks from local index database.")
if err := s.loadKnownChunks(ctx); err != nil {
return nil, fmt.Errorf("loading known chunks: %w", err)
}
_, _ = fmt.Fprintf(s.output, "Loaded %s known chunks from database\n", formatNumber(len(s.knownChunks)))
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
return knownFiles, nil
}
@@ -277,17 +278,17 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
"files_skipped", result.FilesSkipped,
"bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped)))
_, _ = fmt.Fprintf(s.output, "Scan complete: %s examined (%s), %s to process (%s)",
formatNumber(result.FilesScanned),
humanize.Bytes(uint64(totalSizeToProcess+result.BytesSkipped)),
formatNumber(len(filesToProcess)),
humanize.Bytes(uint64(totalSizeToProcess)))
msg := fmt.Sprintf("Enumerated %s snapshot source files (%s total), %s to process (%s)",
s.ui.Count(result.FilesScanned),
s.ui.Size(totalSizeToProcess+result.BytesSkipped),
s.ui.Count(len(filesToProcess)),
s.ui.Size(totalSizeToProcess))
if result.FilesDeleted > 0 {
_, _ = fmt.Fprintf(s.output, ", %s deleted (%s)",
formatNumber(result.FilesDeleted),
humanize.Bytes(uint64(result.BytesDeleted)))
msg += fmt.Sprintf(", %s deleted (%s)",
s.ui.Count(result.FilesDeleted),
s.ui.Size(result.BytesDeleted))
}
_, _ = fmt.Fprintln(s.output)
s.ui.Complete("%s.", msg)
}
// finalizeScanResult populates final blob statistics in the scan result
@@ -628,8 +629,8 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
if s.skipErrors {
log.Error("ERROR: Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
_, _ = fmt.Fprintf(s.output, "ERROR: Failed to access %s: %v (skipping)\n", filePath, err)
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
return nil // Continue scanning
}
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
@@ -775,23 +776,29 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
if rate > 0 && remaining > 0 {
eta = time.Duration(float64(remaining)/rate) * time.Second
}
_, _ = fmt.Fprintf(s.output, "Scan: %s files (~%.0f%%), %s changed/new, %.0f files/sec, %s elapsed",
formatNumber(int(filesScanned)),
pct,
formatNumber(changedCount),
rate,
elapsed.Round(time.Second))
if eta > 0 {
_, _ = fmt.Fprintf(s.output, ", ETA %s", eta.Round(time.Second))
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed %s, enumeration estimated remaining time (%s), finish at %s.",
s.ui.Count(int(filesScanned)),
s.ui.Percent(pct),
s.ui.Count(changedCount),
rate,
s.ui.Duration(elapsed),
s.ui.Duration(eta),
s.ui.Time(time.Now().Add(eta)))
} else {
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed %s.",
s.ui.Count(int(filesScanned)),
s.ui.Percent(pct),
s.ui.Count(changedCount),
rate,
s.ui.Duration(elapsed))
}
_, _ = fmt.Fprintln(s.output)
} else {
// First backup - no estimate available
_, _ = fmt.Fprintf(s.output, "Scan: %s files, %s changed/new, %.0f files/sec, %s elapsed\n",
formatNumber(int(filesScanned)),
formatNumber(changedCount),
s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed %s.",
s.ui.Count(int(filesScanned)),
s.ui.Count(changedCount),
rate,
elapsed.Round(time.Second))
s.ui.Duration(elapsed))
}
}
@@ -957,16 +964,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
elapsed := time.Since(startTime)
rate := float64(end) / elapsed.Seconds()
pct := float64(end) / float64(len(fileIDs)) * 100
_, _ = fmt.Fprintf(s.output, "Associating files: %s/%s (%.1f%%), %.0f files/sec\n",
formatNumber(end), formatNumber(len(fileIDs)), pct, rate)
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
lastStatusTime = time.Now()
}
}
elapsed := time.Since(startTime)
rate := float64(len(fileIDs)) / elapsed.Seconds()
_, _ = fmt.Fprintf(s.output, "Associated %s unchanged files in %s (%.0f files/sec)\n",
formatNumber(len(fileIDs)), elapsed.Round(time.Second), rate)
s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate)
return nil
}
@@ -1034,8 +1041,8 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces
}
// Skip file read errors if --skip-errors is enabled
if s.skipErrors {
log.Error("ERROR: Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
_, _ = fmt.Fprintf(s.output, "ERROR: Failed to process %s: %v (skipping)\n", fileToProcess.Path, err)
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
result.FilesSkipped++
return true, nil
}
@@ -1059,20 +1066,29 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
}
// Format: Progress [5.7k/610k] 6.7 GB/44 GB (15.4%), 106MB/sec, 500 files/sec, running for 1m30s, ETA: 5m49s
_, _ = fmt.Fprintf(s.output, "Progress [%s/%s] %s/%s (%.1f%%), %s/sec, %.0f files/sec, running for %s",
formatCompact(filesProcessed),
formatCompact(totalFiles),
humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalBytes)),
pct,
humanize.Bytes(uint64(byteRate)),
fileRate,
elapsed.Round(time.Second))
if eta > 0 {
_, _ = fmt.Fprintf(s.output, ", ETA: %s", eta.Round(time.Second))
s.ui.Progress("Snapshot file processing: %s/%s files (%s), %s/%s, %s, %.0f files/sec, processing elapsed %s, processing estimated remaining time (%s), finish at %s.",
s.ui.Count(filesProcessed),
s.ui.Count(totalFiles),
s.ui.Percent(pct),
s.ui.Size(bytesProcessed),
s.ui.Size(totalBytes),
s.ui.Speed(byteRate),
fileRate,
s.ui.Duration(elapsed),
s.ui.Duration(eta),
s.ui.Time(time.Now().Add(eta)))
} else {
s.ui.Progress("Snapshot file processing: %s/%s files (%s), %s/%s, %s, %.0f files/sec, processing elapsed %s.",
s.ui.Count(filesProcessed),
s.ui.Count(totalFiles),
s.ui.Percent(pct),
s.ui.Size(bytesProcessed),
s.ui.Size(totalBytes),
s.ui.Speed(byteRate),
fileRate,
s.ui.Duration(elapsed))
}
_, _ = fmt.Fprintln(s.output)
}
// finalizeProcessPhase flushes the packer, writes remaining pending files to the database,
@@ -1164,14 +1180,13 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
if _, err := s.storage.Stat(ctx, blobPath); err == nil {
log.Info("Blob already exists in storage, skipping upload",
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
_, _ = fmt.Fprintf(s.output, "Blob exists: %s (%s, skipped upload)\n",
finishedBlob.Hash[:12]+"...", humanize.Bytes(uint64(finishedBlob.Compressed)))
s.ui.Info("Blob %s (%s) already exists in backup destination store. Skipping upload.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed))
return true, nil
}
_, _ = fmt.Fprintf(s.output, "Uploading blob: %s (%s)\n",
finishedBlob.Hash[:12]+"...",
humanize.Bytes(uint64(finishedBlob.Compressed)))
s.ui.Begin("Uploading blob %s (%s) to backup destination store.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed))
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
@@ -1183,11 +1198,11 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
uploadDuration := time.Since(startTime)
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
_, _ = fmt.Fprintf(s.output, "Blob stored: %s (%s, %s/sec, %s)\n",
finishedBlob.Hash[:12]+"...",
humanize.Bytes(uint64(finishedBlob.Compressed)),
humanize.Bytes(uint64(uploadSpeedBps)),
uploadDuration.Round(time.Millisecond))
s.ui.Complete("Uploaded blob %s (%s) in %s at %s.",
s.ui.Hex(finishedBlob.Hash),
s.ui.Size(finishedBlob.Compressed),
s.ui.Duration(uploadDuration),
s.ui.Speed(uploadSpeedBps))
log.Info("Successfully uploaded blob to storage",
"path", blobPath,
@@ -1236,14 +1251,15 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
if avgSpeed > 0 {
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
}
_, _ = fmt.Fprintf(s.output, " blob upload %s: %s/%s (%.0f%%) at %s/sec, blob upload elapsed %s, blob upload ETA %s\n",
finishedBlob.Hash[:12]+"...",
humanize.Bytes(uint64(uploaded)),
humanize.Bytes(uint64(finishedBlob.Compressed)),
pct,
humanize.Bytes(uint64(avgSpeed)),
totalElapsed.Round(time.Second),
eta.Round(time.Second))
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed %s, blob upload estimated remaining time (%s), finish at %s.",
s.ui.Hex(finishedBlob.Hash),
s.ui.Size(uploaded),
s.ui.Size(finishedBlob.Compressed),
s.ui.Percent(pct),
s.ui.Speed(avgSpeed),
s.ui.Duration(totalElapsed),
s.ui.Duration(eta),
s.ui.Time(now.Add(eta)))
lastStdoutTime = now
}
@@ -1472,7 +1488,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[
}
if result.FilesDeleted > 0 {
_, _ = fmt.Fprintf(s.output, "Found %s deleted files\n", formatNumber(result.FilesDeleted))
s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
}
return nil
@@ -1594,25 +1610,3 @@ func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
return false
}
// formatNumber formats a number with comma separators
func formatNumber(n int) string {
if n < 1000 {
return fmt.Sprintf("%d", n)
}
return humanize.Comma(int64(n))
}
// formatCompact formats a number compactly with k/M suffixes (e.g., 5.7k, 1.2M)
func formatCompact(n int) string {
if n < 1000 {
return fmt.Sprintf("%d", n)
}
if n < 10000 {
return fmt.Sprintf("%.1fk", float64(n)/1000)
}
if n < 1000000 {
return fmt.Sprintf("%.0fk", float64(n)/1000)
}
return fmt.Sprintf("%.1fM", float64(n)/1000000)
}