Major refactoring: UUID-based storage, streaming architecture, and CLI improvements
This commit represents a significant architectural overhaul of vaultik: Database Schema Changes: - Switch files table to use UUID primary keys instead of path-based keys - Add UUID primary keys to blobs table for immediate chunk association - Update all foreign key relationships to use UUIDs - Add comprehensive schema documentation in DATAMODEL.md - Add SQLite busy timeout handling for concurrent operations Streaming and Performance Improvements: - Implement true streaming blob packing without intermediate storage - Add streaming chunk processing to reduce memory usage - Improve progress reporting with real-time metrics - Add upload metrics tracking in new uploads table CLI Refactoring: - Restructure CLI to use subcommands: snapshot create/list/purge/verify - Add store info command for S3 configuration display - Add custom duration parser supporting days/weeks/months/years - Remove old backup.go in favor of enhanced snapshot.go - Add --cron flag for silent operation Configuration Changes: - Remove unused index_prefix configuration option - Add support for snapshot pruning retention policies - Improve configuration validation and error messages Testing Improvements: - Add comprehensive repository tests with edge cases - Add cascade delete debugging tests - Fix concurrent operation tests to use SQLite busy timeout - Remove tolerance for SQLITE_BUSY errors in tests Documentation: - Add MIT LICENSE file - Update README with new command structure - Add comprehensive DATAMODEL.md explaining database schema - Update DESIGN.md with UUID-based architecture Other Changes: - Add test-config.yml for testing - Update Makefile with better test output formatting - Fix various race conditions in concurrent operations - Improve error handling throughout
This commit is contained in:
@@ -15,7 +15,9 @@ import (
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// AppOptions contains common options for creating the fx application
|
||||
// AppOptions contains common options for creating the fx application.
|
||||
// It includes the configuration file path, logging options, and additional
|
||||
// fx modules and invocations that should be included in the application.
|
||||
type AppOptions struct {
|
||||
ConfigPath string
|
||||
LogOptions log.LogOptions
|
||||
@@ -27,13 +29,16 @@ type AppOptions struct {
|
||||
func setupGlobals(lc fx.Lifecycle, g *globals.Globals) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
g.StartTime = time.Now()
|
||||
g.StartTime = time.Now().UTC()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// NewApp creates a new fx application with common modules
|
||||
// NewApp creates a new fx application with common modules.
|
||||
// It sets up the base modules (config, database, logging, globals) and
|
||||
// combines them with any additional modules specified in the options.
|
||||
// The returned fx.App is ready to be started with RunApp.
|
||||
func NewApp(opts AppOptions) *fx.App {
|
||||
baseModules := []fx.Option{
|
||||
fx.Supply(config.ConfigPath(opts.ConfigPath)),
|
||||
@@ -53,7 +58,10 @@ func NewApp(opts AppOptions) *fx.App {
|
||||
return fx.New(allOptions...)
|
||||
}
|
||||
|
||||
// RunApp starts and stops the fx application within the given context
|
||||
// RunApp starts and stops the fx application within the given context.
|
||||
// It handles graceful shutdown on interrupt signals (SIGINT, SIGTERM) and
|
||||
// ensures the application stops cleanly. The function blocks until the
|
||||
// application completes or is interrupted. Returns an error if startup fails.
|
||||
func RunApp(ctx context.Context, app *fx.App) error {
|
||||
// Set up signal handling for graceful shutdown
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
@@ -101,7 +109,9 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
}
|
||||
}
|
||||
|
||||
// RunWithApp is a helper that creates and runs an fx app with the given options
|
||||
// RunWithApp is a helper that creates and runs an fx app with the given options.
|
||||
// It combines NewApp and RunApp into a single convenient function. This is the
|
||||
// preferred way to run CLI commands that need the full application context.
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
app := NewApp(opts)
|
||||
return RunApp(ctx, app)
|
||||
|
||||
@@ -1,287 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/backup"
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/crypto"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// BackupOptions contains options for the backup command
|
||||
type BackupOptions struct {
|
||||
ConfigPath string
|
||||
Daemon bool
|
||||
Cron bool
|
||||
Prune bool
|
||||
}
|
||||
|
||||
// BackupApp contains all dependencies needed for running backups
|
||||
type BackupApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
ScannerFactory backup.ScannerFactory
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Lifecycle fx.Lifecycle
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewBackupCommand creates the backup command
|
||||
func NewBackupCommand() *cobra.Command {
|
||||
opts := &BackupOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "backup",
|
||||
Short: "Perform incremental backup",
|
||||
Long: `Backup configured directories using incremental deduplication and encryption.
|
||||
|
||||
Config is located at /etc/vaultik/config.yml, but can be overridden by specifying
|
||||
a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// If --config not specified, check environment variable
|
||||
if opts.ConfigPath == "" {
|
||||
opts.ConfigPath = os.Getenv("VAULTIK_CONFIG")
|
||||
}
|
||||
// If still not specified, use default
|
||||
if opts.ConfigPath == "" {
|
||||
defaultConfig := "/etc/vaultik/config.yml"
|
||||
if _, err := os.Stat(defaultConfig); err == nil {
|
||||
opts.ConfigPath = defaultConfig
|
||||
} else {
|
||||
return fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultConfig)
|
||||
}
|
||||
}
|
||||
return runBackup(cmd.Context(), opts)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&opts.ConfigPath, "config", "", "Path to config file")
|
||||
cmd.Flags().BoolVar(&opts.Daemon, "daemon", false, "Run in daemon mode with inotify monitoring")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "Delete all previous snapshots and unreferenced blobs after backup")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runBackup(ctx context.Context, opts *BackupOptions) error {
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: opts.ConfigPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Cron: opts.Cron,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
backup.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
scannerFactory backup.ScannerFactory, s3Client *s3.Client, db *database.DB,
|
||||
lc fx.Lifecycle, shutdowner fx.Shutdowner) *BackupApp {
|
||||
return &BackupApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
ScannerFactory: scannerFactory,
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Lifecycle: lc,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *BackupApp, lc fx.Lifecycle) {
|
||||
// Create a cancellable context for the backup
|
||||
backupCtx, backupCancel := context.WithCancel(context.Background())
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the backup in a goroutine
|
||||
go func() {
|
||||
// Run the backup
|
||||
if err := app.runBackup(backupCtx, opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Backup failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when backup completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping backup")
|
||||
// Cancel the backup context
|
||||
backupCancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// runBackup executes the backup operation
|
||||
func (app *BackupApp) runBackup(ctx context.Context, opts *BackupOptions) error {
|
||||
log.Info("Starting backup",
|
||||
"config", opts.ConfigPath,
|
||||
"version", app.Globals.Version,
|
||||
"commit", app.Globals.Commit,
|
||||
"index_path", app.Config.IndexPath,
|
||||
)
|
||||
|
||||
if opts.Daemon {
|
||||
log.Info("Running in daemon mode")
|
||||
// TODO: Implement daemon mode with inotify
|
||||
return fmt.Errorf("daemon mode not yet implemented")
|
||||
}
|
||||
|
||||
// Resolve source directories to absolute paths
|
||||
resolvedDirs := make([]string, 0, len(app.Config.SourceDirs))
|
||||
for _, dir := range app.Config.SourceDirs {
|
||||
absPath, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Resolve symlinks
|
||||
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
||||
if err != nil {
|
||||
// If the path doesn't exist yet, use the absolute path
|
||||
if os.IsNotExist(err) {
|
||||
resolvedPath = absPath
|
||||
} else {
|
||||
return fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
resolvedDirs = append(resolvedDirs, resolvedPath)
|
||||
}
|
||||
|
||||
// Create scanner with progress enabled (unless in cron mode)
|
||||
scanner := app.ScannerFactory(backup.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
})
|
||||
|
||||
// Perform a single backup run
|
||||
log.Notice("Starting backup", "source_dirs", len(resolvedDirs))
|
||||
for i, dir := range resolvedDirs {
|
||||
log.Info("Source directory", "index", i+1, "path", dir)
|
||||
}
|
||||
|
||||
totalFiles := 0
|
||||
totalBytes := int64(0)
|
||||
totalChunks := 0
|
||||
totalBlobs := 0
|
||||
|
||||
// Create a new snapshot at the beginning of backup
|
||||
hostname := app.Config.Hostname
|
||||
if hostname == "" {
|
||||
hostname, _ = os.Hostname()
|
||||
}
|
||||
|
||||
// Create encryptor if age recipients are configured
|
||||
var encryptor backup.Encryptor
|
||||
if len(app.Config.AgeRecipients) > 0 {
|
||||
cryptoEncryptor, err := crypto.NewEncryptor(app.Config.AgeRecipients)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating encryptor: %w", err)
|
||||
}
|
||||
encryptor = cryptoEncryptor
|
||||
}
|
||||
|
||||
snapshotManager := backup.NewSnapshotManager(app.Repositories, app.S3Client, encryptor)
|
||||
snapshotID, err := snapshotManager.CreateSnapshot(ctx, hostname, app.Globals.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating snapshot: %w", err)
|
||||
}
|
||||
log.Info("Created snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
for _, dir := range resolvedDirs {
|
||||
// Check if context is cancelled
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info("Backup cancelled")
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
log.Info("Scanning directory", "path", dir)
|
||||
result, err := scanner.Scan(ctx, dir, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||
}
|
||||
|
||||
totalFiles += result.FilesScanned
|
||||
totalBytes += result.BytesScanned
|
||||
totalChunks += result.ChunksCreated
|
||||
totalBlobs += result.BlobsCreated
|
||||
|
||||
log.Info("Directory scan complete",
|
||||
"path", dir,
|
||||
"files", result.FilesScanned,
|
||||
"files_skipped", result.FilesSkipped,
|
||||
"bytes", result.BytesScanned,
|
||||
"bytes_skipped", result.BytesSkipped,
|
||||
"chunks", result.ChunksCreated,
|
||||
"blobs", result.BlobsCreated,
|
||||
"duration", result.EndTime.Sub(result.StartTime))
|
||||
}
|
||||
|
||||
// Update snapshot statistics
|
||||
stats := backup.BackupStats{
|
||||
FilesScanned: totalFiles,
|
||||
BytesScanned: totalBytes,
|
||||
ChunksCreated: totalChunks,
|
||||
BlobsCreated: totalBlobs,
|
||||
BytesUploaded: totalBytes, // TODO: Track actual uploaded bytes
|
||||
}
|
||||
|
||||
if err := snapshotManager.UpdateSnapshotStats(ctx, snapshotID, stats); err != nil {
|
||||
return fmt.Errorf("updating snapshot stats: %w", err)
|
||||
}
|
||||
|
||||
// Mark snapshot as complete
|
||||
if err := snapshotManager.CompleteSnapshot(ctx, snapshotID); err != nil {
|
||||
return fmt.Errorf("completing snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Export snapshot metadata
|
||||
// Export snapshot metadata without closing the database
|
||||
// The export function should handle its own database connection
|
||||
if err := snapshotManager.ExportSnapshotMetadata(ctx, app.Config.IndexPath, snapshotID); err != nil {
|
||||
return fmt.Errorf("exporting snapshot metadata: %w", err)
|
||||
}
|
||||
|
||||
log.Notice("Backup complete",
|
||||
"snapshot_id", snapshotID,
|
||||
"total_files", totalFiles,
|
||||
"total_bytes", totalBytes,
|
||||
"total_chunks", totalChunks,
|
||||
"total_blobs", totalBlobs)
|
||||
|
||||
if opts.Prune {
|
||||
log.Info("Pruning enabled - will delete old snapshots after backup")
|
||||
// TODO: Implement pruning
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
94
internal/cli/duration.go
Normal file
94
internal/cli/duration.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// parseDuration parses duration strings. Supports standard Go duration format
|
||||
// (e.g., "3h30m", "1h45m30s") as well as extended units:
|
||||
// - d: days (e.g., "30d", "7d")
|
||||
// - w: weeks (e.g., "2w", "4w")
|
||||
// - mo: months (30 days) (e.g., "6mo", "1mo")
|
||||
// - y: years (365 days) (e.g., "1y", "2y")
|
||||
//
|
||||
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
// First try standard Go duration parsing
|
||||
if d, err := time.ParseDuration(s); err == nil {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Extended duration parsing
|
||||
// Check for negative values
|
||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||
return 0, fmt.Errorf("negative durations are not supported")
|
||||
}
|
||||
|
||||
// Pattern matches: number + unit, repeated
|
||||
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
|
||||
matches := re.FindAllStringSubmatch(s, -1)
|
||||
|
||||
if len(matches) == 0 {
|
||||
return 0, fmt.Errorf("invalid duration format: %q", s)
|
||||
}
|
||||
|
||||
var total time.Duration
|
||||
|
||||
for _, match := range matches {
|
||||
valueStr := match[1]
|
||||
unit := strings.ToLower(match[2])
|
||||
|
||||
value, err := strconv.ParseFloat(valueStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
switch unit {
|
||||
// Standard time units
|
||||
case "ns", "nanosecond", "nanoseconds":
|
||||
d = time.Duration(value)
|
||||
case "us", "µs", "microsecond", "microseconds":
|
||||
d = time.Duration(value * float64(time.Microsecond))
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
d = time.Duration(value * float64(time.Millisecond))
|
||||
case "s", "sec", "second", "seconds":
|
||||
d = time.Duration(value * float64(time.Second))
|
||||
case "m", "min", "minute", "minutes":
|
||||
d = time.Duration(value * float64(time.Minute))
|
||||
case "h", "hr", "hour", "hours":
|
||||
d = time.Duration(value * float64(time.Hour))
|
||||
// Extended units
|
||||
case "d", "day", "days":
|
||||
d = time.Duration(value * float64(24*time.Hour))
|
||||
case "w", "week", "weeks":
|
||||
d = time.Duration(value * float64(7*24*time.Hour))
|
||||
case "mo", "month", "months":
|
||||
// Using 30 days as approximation
|
||||
d = time.Duration(value * float64(30*24*time.Hour))
|
||||
case "y", "year", "years":
|
||||
// Using 365 days as approximation
|
||||
d = time.Duration(value * float64(365*24*time.Hour))
|
||||
default:
|
||||
// Try parsing as standard Go duration unit
|
||||
testStr := fmt.Sprintf("1%s", unit)
|
||||
if _, err := time.ParseDuration(testStr); err == nil {
|
||||
// It's a valid Go duration unit, parse the full value
|
||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||
if d, err = time.ParseDuration(fullStr); err != nil {
|
||||
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
||||
}
|
||||
} else {
|
||||
return 0, fmt.Errorf("unknown time unit %q", unit)
|
||||
}
|
||||
}
|
||||
|
||||
total += d
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
263
internal/cli/duration_test.go
Normal file
263
internal/cli/duration_test.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
// Standard Go durations
|
||||
{
|
||||
name: "standard seconds",
|
||||
input: "30s",
|
||||
expected: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard minutes",
|
||||
input: "45m",
|
||||
expected: 45 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard hours",
|
||||
input: "2h",
|
||||
expected: 2 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "standard combined",
|
||||
input: "3h30m",
|
||||
expected: 3*time.Hour + 30*time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard complex",
|
||||
input: "1h45m30s",
|
||||
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard with milliseconds",
|
||||
input: "1s500ms",
|
||||
expected: 1*time.Second + 500*time.Millisecond,
|
||||
},
|
||||
// Extended units - days
|
||||
{
|
||||
name: "single day",
|
||||
input: "1d",
|
||||
expected: 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple days",
|
||||
input: "7d",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "fractional days",
|
||||
input: "1.5d",
|
||||
expected: 36 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days spelled out",
|
||||
input: "3days",
|
||||
expected: 3 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - weeks
|
||||
{
|
||||
name: "single week",
|
||||
input: "1w",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple weeks",
|
||||
input: "4w",
|
||||
expected: 4 * 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "weeks spelled out",
|
||||
input: "2weeks",
|
||||
expected: 2 * 7 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - months
|
||||
{
|
||||
name: "single month",
|
||||
input: "1mo",
|
||||
expected: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple months",
|
||||
input: "6mo",
|
||||
expected: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "months spelled out",
|
||||
input: "3months",
|
||||
expected: 3 * 30 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - years
|
||||
{
|
||||
name: "single year",
|
||||
input: "1y",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple years",
|
||||
input: "2y",
|
||||
expected: 2 * 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years spelled out",
|
||||
input: "1year",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
// Combined extended units
|
||||
{
|
||||
name: "weeks and days",
|
||||
input: "2w3d",
|
||||
expected: 2*7*24*time.Hour + 3*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years and months",
|
||||
input: "1y6mo",
|
||||
expected: 365*24*time.Hour + 6*30*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days and hours",
|
||||
input: "1d12h",
|
||||
expected: 24*time.Hour + 12*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "complex combination",
|
||||
input: "1y2mo3w4d5h6m7s",
|
||||
expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second,
|
||||
},
|
||||
{
|
||||
name: "with spaces",
|
||||
input: "1d 12h 30m",
|
||||
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
|
||||
},
|
||||
// Edge cases
|
||||
{
|
||||
name: "zero duration",
|
||||
input: "0s",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "large duration",
|
||||
input: "10y",
|
||||
expected: 10 * 365 * 24 * time.Hour,
|
||||
},
|
||||
// Error cases
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
input: "abc",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
input: "5x",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid number",
|
||||
input: "xyzd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative not supported",
|
||||
input: "-5d",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "expected error for input %q", tt.input)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err, "unexpected error for input %q", tt.input)
|
||||
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationSpecialCases(t *testing.T) {
|
||||
// Test that standard Go durations work exactly as expected
|
||||
standardDurations := []string{
|
||||
"300ms",
|
||||
"1.5h",
|
||||
"2h45m",
|
||||
"72h",
|
||||
"1us",
|
||||
"1µs",
|
||||
"1ns",
|
||||
}
|
||||
|
||||
for _, d := range standardDurations {
|
||||
expected, err := time.ParseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
|
||||
got, err := parseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationRealWorldExamples(t *testing.T) {
|
||||
// Test real-world snapshot purge scenarios
|
||||
tests := []struct {
|
||||
description string
|
||||
input string
|
||||
olderThan time.Duration
|
||||
}{
|
||||
{
|
||||
description: "keep snapshots from last 30 days",
|
||||
input: "30d",
|
||||
olderThan: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 6 months",
|
||||
input: "6mo",
|
||||
olderThan: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last year",
|
||||
input: "1y",
|
||||
olderThan: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last week and a half",
|
||||
input: "1w3d",
|
||||
olderThan: 10 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 90 days",
|
||||
input: "90d",
|
||||
olderThan: 90 * 24 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.description, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.olderThan, got)
|
||||
|
||||
// Verify the duration makes sense for snapshot purging
|
||||
assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// CLIEntry is the main entry point for the CLI application
|
||||
// CLIEntry is the main entry point for the CLI application.
|
||||
// It creates the root command, executes it, and exits with status 1
|
||||
// if an error occurs. This function should be called from main().
|
||||
func CLIEntry() {
|
||||
rootCmd := NewRootCommand()
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestCLIEntry(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all subcommands are registered
|
||||
expectedCommands := []string{"backup", "restore", "prune", "verify", "fetch"}
|
||||
expectedCommands := []string{"snapshot", "store", "restore", "prune", "verify", "fetch"}
|
||||
for _, expected := range expectedCommands {
|
||||
found := false
|
||||
for _, cmd := range cmd.Commands() {
|
||||
@@ -32,19 +32,24 @@ func TestCLIEntry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Verify backup command has proper flags
|
||||
backupCmd, _, err := cmd.Find([]string{"backup"})
|
||||
// Verify snapshot command has subcommands
|
||||
snapshotCmd, _, err := cmd.Find([]string{"snapshot"})
|
||||
if err != nil {
|
||||
t.Errorf("Failed to find backup command: %v", err)
|
||||
t.Errorf("Failed to find snapshot command: %v", err)
|
||||
} else {
|
||||
if backupCmd.Flag("config") == nil {
|
||||
t.Error("Backup command missing --config flag")
|
||||
}
|
||||
if backupCmd.Flag("daemon") == nil {
|
||||
t.Error("Backup command missing --daemon flag")
|
||||
}
|
||||
if backupCmd.Flag("cron") == nil {
|
||||
t.Error("Backup command missing --cron flag")
|
||||
// Check snapshot subcommands
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify"}
|
||||
for _, expected := range expectedSubCommands {
|
||||
found := false
|
||||
for _, subcmd := range snapshotCmd.Commands() {
|
||||
if subcmd.Use == expected || subcmd.Name() == expected {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,25 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// RootFlags holds global flags
|
||||
// RootFlags holds global flags that apply to all commands.
|
||||
// These flags are defined on the root command and inherited by all subcommands.
|
||||
type RootFlags struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
ConfigPath string
|
||||
Verbose bool
|
||||
Debug bool
|
||||
}
|
||||
|
||||
var rootFlags RootFlags
|
||||
|
||||
// NewRootCommand creates the root cobra command
|
||||
// NewRootCommand creates the root cobra command for the vaultik CLI.
|
||||
// It sets up the command structure, global flags, and adds all subcommands.
|
||||
// This is the main entry point for the CLI command hierarchy.
|
||||
func NewRootCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "vaultik",
|
||||
@@ -24,23 +31,49 @@ on the source system.`,
|
||||
}
|
||||
|
||||
// Add global flags
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or /etc/vaultik/config.yml)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(
|
||||
NewBackupCommand(),
|
||||
NewRestoreCommand(),
|
||||
NewPruneCommand(),
|
||||
NewVerifyCommand(),
|
||||
NewFetchCommand(),
|
||||
SnapshotCmd(),
|
||||
NewStoreCommand(),
|
||||
NewSnapshotCommand(),
|
||||
)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// GetRootFlags returns the global flags
|
||||
// GetRootFlags returns the global flags that were parsed from the command line.
|
||||
// This allows subcommands to access global flag values like verbosity and config path.
|
||||
func GetRootFlags() RootFlags {
|
||||
return rootFlags
|
||||
}
|
||||
|
||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||
// It checks in order: 1) --config flag, 2) VAULTIK_CONFIG environment variable,
|
||||
// 3) default location /etc/vaultik/config.yml. Returns an error if no valid
|
||||
// config file can be found through any of these methods.
|
||||
func ResolveConfigPath() (string, error) {
|
||||
// First check global flag
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath, nil
|
||||
}
|
||||
|
||||
// Then check environment variable
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath, nil
|
||||
}
|
||||
|
||||
// Finally check default location
|
||||
defaultPath := "/etc/vaultik/config.yml"
|
||||
if _, err := os.Stat(defaultPath); err == nil {
|
||||
return defaultPath, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultPath)
|
||||
}
|
||||
|
||||
@@ -1,90 +1,892 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/backup"
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/crypto"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
func SnapshotCmd() *cobra.Command {
|
||||
// SnapshotCreateOptions contains options for the snapshot create command
|
||||
type SnapshotCreateOptions struct {
|
||||
Daemon bool
|
||||
Cron bool
|
||||
Prune bool
|
||||
}
|
||||
|
||||
// SnapshotCreateApp contains all dependencies needed for creating snapshots
|
||||
type SnapshotCreateApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
ScannerFactory backup.ScannerFactory
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Lifecycle fx.Lifecycle
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// SnapshotApp contains dependencies for snapshot commands
|
||||
type SnapshotApp struct {
|
||||
*SnapshotCreateApp // Reuse snapshot creation functionality
|
||||
S3Client *s3.Client
|
||||
}
|
||||
|
||||
// SnapshotInfo represents snapshot information for listing
|
||||
type SnapshotInfo struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
}
|
||||
|
||||
// NewSnapshotCommand creates the snapshot command and subcommands
|
||||
func NewSnapshotCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "snapshot",
|
||||
Short: "Manage snapshots",
|
||||
Long: "Commands for listing, removing, and querying snapshots",
|
||||
Short: "Snapshot management commands",
|
||||
Long: "Commands for creating, listing, and managing snapshots",
|
||||
}
|
||||
|
||||
cmd.AddCommand(snapshotListCmd())
|
||||
cmd.AddCommand(snapshotRmCmd())
|
||||
cmd.AddCommand(snapshotLatestCmd())
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newSnapshotCreateCommand())
|
||||
cmd.AddCommand(newSnapshotListCommand())
|
||||
cmd.AddCommand(newSnapshotPurgeCommand())
|
||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func snapshotListCmd() *cobra.Command {
|
||||
var (
|
||||
bucket string
|
||||
prefix string
|
||||
limit int
|
||||
// newSnapshotCreateCommand creates the 'snapshot create' subcommand
|
||||
func newSnapshotCreateCommand() *cobra.Command {
|
||||
opts := &SnapshotCreateOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "create",
|
||||
Short: "Create a new snapshot",
|
||||
Long: `Creates a new snapshot of the configured directories.
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the backup functionality from cli package
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Cron: opts.Cron,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
backup.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
scannerFactory backup.ScannerFactory, s3Client *s3.Client, db *database.DB,
|
||||
lc fx.Lifecycle, shutdowner fx.Shutdowner) *SnapshotCreateApp {
|
||||
return &SnapshotCreateApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
ScannerFactory: scannerFactory,
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Lifecycle: lc,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *SnapshotCreateApp, lc fx.Lifecycle) {
|
||||
// Create a cancellable context for the snapshot
|
||||
snapshotCtx, snapshotCancel := context.WithCancel(context.Background())
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the snapshot creation in a goroutine
|
||||
go func() {
|
||||
// Run the snapshot creation
|
||||
if err := app.runSnapshot(snapshotCtx, opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when snapshot completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping snapshot creation")
|
||||
// Cancel the snapshot context
|
||||
snapshotCancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Daemon, "daemon", false, "Run in daemon mode with inotify monitoring")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "Delete all previous snapshots and unreferenced blobs after backup")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// runSnapshot executes the snapshot creation operation
|
||||
func (app *SnapshotCreateApp) runSnapshot(ctx context.Context, opts *SnapshotCreateOptions) error {
|
||||
snapshotStartTime := time.Now()
|
||||
|
||||
log.Info("Starting snapshot creation",
|
||||
"version", app.Globals.Version,
|
||||
"commit", app.Globals.Commit,
|
||||
"index_path", app.Config.IndexPath,
|
||||
)
|
||||
|
||||
// Clean up incomplete snapshots FIRST, before any scanning
|
||||
// This is critical for data safety - see CleanupIncompleteSnapshots for details
|
||||
hostname := app.Config.Hostname
|
||||
if hostname == "" {
|
||||
hostname, _ = os.Hostname()
|
||||
}
|
||||
|
||||
// Create encryptor if needed for snapshot manager
|
||||
var encryptor backup.Encryptor
|
||||
if len(app.Config.AgeRecipients) > 0 {
|
||||
cryptoEncryptor, err := crypto.NewEncryptor(app.Config.AgeRecipients)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating encryptor: %w", err)
|
||||
}
|
||||
encryptor = cryptoEncryptor
|
||||
}
|
||||
|
||||
snapshotManager := backup.NewSnapshotManager(app.Repositories, app.S3Client, encryptor)
|
||||
// CRITICAL: This MUST succeed. If we fail to clean up incomplete snapshots,
|
||||
// the deduplication logic will think files from the incomplete snapshot were
|
||||
// already backed up and skip them, resulting in data loss.
|
||||
if err := snapshotManager.CleanupIncompleteSnapshots(ctx, hostname); err != nil {
|
||||
return fmt.Errorf("cleanup incomplete snapshots: %w", err)
|
||||
}
|
||||
|
||||
if opts.Daemon {
|
||||
log.Info("Running in daemon mode")
|
||||
// TODO: Implement daemon mode with inotify
|
||||
return fmt.Errorf("daemon mode not yet implemented")
|
||||
}
|
||||
|
||||
// Resolve source directories to absolute paths
|
||||
resolvedDirs := make([]string, 0, len(app.Config.SourceDirs))
|
||||
for _, dir := range app.Config.SourceDirs {
|
||||
absPath, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Resolve symlinks
|
||||
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
||||
if err != nil {
|
||||
// If the path doesn't exist yet, use the absolute path
|
||||
if os.IsNotExist(err) {
|
||||
resolvedPath = absPath
|
||||
} else {
|
||||
return fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
resolvedDirs = append(resolvedDirs, resolvedPath)
|
||||
}
|
||||
|
||||
// Create scanner with progress enabled (unless in cron mode)
|
||||
scanner := app.ScannerFactory(backup.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
})
|
||||
|
||||
// Perform a single snapshot run
|
||||
log.Notice("Starting snapshot", "source_dirs", len(resolvedDirs))
|
||||
for i, dir := range resolvedDirs {
|
||||
log.Info("Source directory", "index", i+1, "path", dir)
|
||||
}
|
||||
|
||||
// Statistics tracking
|
||||
totalFiles := 0
|
||||
totalBytes := int64(0)
|
||||
totalChunks := 0
|
||||
totalBlobs := 0
|
||||
totalBytesSkipped := int64(0)
|
||||
totalFilesSkipped := 0
|
||||
totalBytesUploaded := int64(0)
|
||||
totalBlobsUploaded := 0
|
||||
uploadDuration := time.Duration(0)
|
||||
|
||||
// Create a new snapshot at the beginning
|
||||
// (hostname, encryptor, and snapshotManager already created above for cleanup)
|
||||
snapshotID, err := snapshotManager.CreateSnapshot(ctx, hostname, app.Globals.Version, app.Globals.Commit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating snapshot: %w", err)
|
||||
}
|
||||
log.Info("Created snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
for _, dir := range resolvedDirs {
|
||||
// Check if context is cancelled
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info("Snapshot creation cancelled")
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
log.Info("Scanning directory", "path", dir)
|
||||
result, err := scanner.Scan(ctx, dir, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||
}
|
||||
|
||||
totalFiles += result.FilesScanned
|
||||
totalBytes += result.BytesScanned
|
||||
totalChunks += result.ChunksCreated
|
||||
totalBlobs += result.BlobsCreated
|
||||
totalFilesSkipped += result.FilesSkipped
|
||||
totalBytesSkipped += result.BytesSkipped
|
||||
|
||||
log.Info("Directory scan complete",
|
||||
"path", dir,
|
||||
"files", result.FilesScanned,
|
||||
"files_skipped", result.FilesSkipped,
|
||||
"bytes", result.BytesScanned,
|
||||
"bytes_skipped", result.BytesSkipped,
|
||||
"chunks", result.ChunksCreated,
|
||||
"blobs", result.BlobsCreated,
|
||||
"duration", result.EndTime.Sub(result.StartTime))
|
||||
}
|
||||
|
||||
// Get upload statistics from scanner progress if available
|
||||
if s := scanner.GetProgress(); s != nil {
|
||||
stats := s.GetStats()
|
||||
totalBytesUploaded = stats.BytesUploaded.Load()
|
||||
totalBlobsUploaded = int(stats.BlobsUploaded.Load())
|
||||
uploadDuration = time.Duration(stats.UploadDurationMs.Load()) * time.Millisecond
|
||||
}
|
||||
|
||||
// Update snapshot statistics with extended fields
|
||||
extStats := backup.ExtendedBackupStats{
|
||||
BackupStats: backup.BackupStats{
|
||||
FilesScanned: totalFiles,
|
||||
BytesScanned: totalBytes,
|
||||
ChunksCreated: totalChunks,
|
||||
BlobsCreated: totalBlobs,
|
||||
BytesUploaded: totalBytesUploaded,
|
||||
},
|
||||
BlobUncompressedSize: 0, // Will be set from database query below
|
||||
CompressionLevel: app.Config.CompressionLevel,
|
||||
UploadDurationMs: uploadDuration.Milliseconds(),
|
||||
}
|
||||
|
||||
if err := snapshotManager.UpdateSnapshotStatsExtended(ctx, snapshotID, extStats); err != nil {
|
||||
return fmt.Errorf("updating snapshot stats: %w", err)
|
||||
}
|
||||
|
||||
// Mark snapshot as complete
|
||||
if err := snapshotManager.CompleteSnapshot(ctx, snapshotID); err != nil {
|
||||
return fmt.Errorf("completing snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Export snapshot metadata
|
||||
// Export snapshot metadata without closing the database
|
||||
// The export function should handle its own database connection
|
||||
if err := snapshotManager.ExportSnapshotMetadata(ctx, app.Config.IndexPath, snapshotID); err != nil {
|
||||
return fmt.Errorf("exporting snapshot metadata: %w", err)
|
||||
}
|
||||
|
||||
// Calculate final statistics
|
||||
snapshotDuration := time.Since(snapshotStartTime)
|
||||
totalFilesChanged := totalFiles - totalFilesSkipped
|
||||
totalBytesChanged := totalBytes
|
||||
totalBytesAll := totalBytes + totalBytesSkipped
|
||||
|
||||
// Calculate upload speed
|
||||
var avgUploadSpeed string
|
||||
if totalBytesUploaded > 0 && uploadDuration > 0 {
|
||||
bytesPerSec := float64(totalBytesUploaded) / uploadDuration.Seconds()
|
||||
bitsPerSec := bytesPerSec * 8
|
||||
if bitsPerSec >= 1e9 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9)
|
||||
} else if bitsPerSec >= 1e6 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6)
|
||||
} else if bitsPerSec >= 1e3 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3)
|
||||
} else {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f bit/s", bitsPerSec)
|
||||
}
|
||||
} else {
|
||||
avgUploadSpeed = "N/A"
|
||||
}
|
||||
|
||||
// Get total blob sizes from database
|
||||
totalBlobSizeCompressed := int64(0)
|
||||
totalBlobSizeUncompressed := int64(0)
|
||||
if blobHashes, err := app.Repositories.Snapshots.GetBlobHashes(ctx, snapshotID); err == nil {
|
||||
for _, hash := range blobHashes {
|
||||
if blob, err := app.Repositories.Blobs.GetByHash(ctx, hash); err == nil && blob != nil {
|
||||
totalBlobSizeCompressed += blob.CompressedSize
|
||||
totalBlobSizeUncompressed += blob.UncompressedSize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate compression ratio
|
||||
var compressionRatio float64
|
||||
if totalBlobSizeUncompressed > 0 {
|
||||
compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed)
|
||||
} else {
|
||||
compressionRatio = 1.0
|
||||
}
|
||||
|
||||
// Print comprehensive summary
|
||||
log.Notice("=== Snapshot Summary ===")
|
||||
log.Info("Snapshot ID", "id", snapshotID)
|
||||
log.Info("Source files",
|
||||
"total_count", formatNumber(totalFiles),
|
||||
"total_size", humanize.Bytes(uint64(totalBytesAll)))
|
||||
log.Info("Changed files",
|
||||
"count", formatNumber(totalFilesChanged),
|
||||
"size", humanize.Bytes(uint64(totalBytesChanged)))
|
||||
log.Info("Unchanged files",
|
||||
"count", formatNumber(totalFilesSkipped),
|
||||
"size", humanize.Bytes(uint64(totalBytesSkipped)))
|
||||
log.Info("Blob storage",
|
||||
"total_uncompressed", humanize.Bytes(uint64(totalBlobSizeUncompressed)),
|
||||
"total_compressed", humanize.Bytes(uint64(totalBlobSizeCompressed)),
|
||||
"compression_ratio", fmt.Sprintf("%.2fx", compressionRatio),
|
||||
"compression_level", app.Config.CompressionLevel)
|
||||
log.Info("Upload activity",
|
||||
"bytes_uploaded", humanize.Bytes(uint64(totalBytesUploaded)),
|
||||
"blobs_uploaded", totalBlobsUploaded,
|
||||
"upload_time", formatDuration(uploadDuration),
|
||||
"avg_speed", avgUploadSpeed)
|
||||
log.Info("Total time", "duration", formatDuration(snapshotDuration))
|
||||
log.Notice("==========================")
|
||||
|
||||
if opts.Prune {
|
||||
log.Info("Pruning enabled - will delete old snapshots after snapshot")
|
||||
// TODO: Implement pruning
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// newSnapshotListCommand creates the 'snapshot list' subcommand
|
||||
func newSnapshotListCommand() *cobra.Command {
|
||||
var jsonOutput bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List snapshots",
|
||||
Long: "List all snapshots in the bucket, sorted by timestamp",
|
||||
Short: "List all snapshots",
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
panic("unimplemented")
|
||||
return runSnapshotCommand(cmd.Context(), func(app *SnapshotApp) error {
|
||||
return app.List(cmd.Context(), jsonOutput)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix")
|
||||
cmd.Flags().IntVar(&limit, "limit", 10, "Maximum number of snapshots to list")
|
||||
_ = cmd.MarkFlagRequired("bucket")
|
||||
cmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func snapshotRmCmd() *cobra.Command {
|
||||
var (
|
||||
bucket string
|
||||
prefix string
|
||||
snapshot string
|
||||
)
|
||||
// newSnapshotPurgeCommand creates the 'snapshot purge' subcommand
|
||||
func newSnapshotPurgeCommand() *cobra.Command {
|
||||
var keepLatest bool
|
||||
var olderThan string
|
||||
var force bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "rm",
|
||||
Short: "Remove a snapshot",
|
||||
Long: "Remove a snapshot and optionally its associated blobs",
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: "Removes snapshots based on age or count criteria",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
panic("unimplemented")
|
||||
// Validate flags
|
||||
if !keepLatest && olderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
if keepLatest && olderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
}
|
||||
|
||||
return runSnapshotCommand(cmd.Context(), func(app *SnapshotApp) error {
|
||||
return app.Purge(cmd.Context(), keepLatest, olderThan, force)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix")
|
||||
cmd.Flags().StringVar(&snapshot, "snapshot", "", "Snapshot ID to remove")
|
||||
_ = cmd.MarkFlagRequired("bucket")
|
||||
_ = cmd.MarkFlagRequired("snapshot")
|
||||
cmd.Flags().BoolVar(&keepLatest, "keep-latest", false, "Keep only the latest snapshot")
|
||||
cmd.Flags().StringVar(&olderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&force, "force", false, "Skip confirmation prompt")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func snapshotLatestCmd() *cobra.Command {
|
||||
var (
|
||||
bucket string
|
||||
prefix string
|
||||
)
|
||||
// newSnapshotVerifyCommand creates the 'snapshot verify' subcommand
|
||||
func newSnapshotVerifyCommand() *cobra.Command {
|
||||
var deep bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "latest",
|
||||
Short: "Get the latest snapshot ID",
|
||||
Long: "Display the ID of the most recent snapshot",
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: "Verifies that all blobs referenced in a snapshot exist",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
panic("unimplemented")
|
||||
return runSnapshotCommand(cmd.Context(), func(app *SnapshotApp) error {
|
||||
return app.Verify(cmd.Context(), args[0], deep)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix")
|
||||
_ = cmd.MarkFlagRequired("bucket")
|
||||
cmd.Flags().BoolVar(&deep, "deep", false, "Download and verify blob hashes")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// List lists all snapshots
|
||||
func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error {
|
||||
snapshots, err := app.getSnapshots(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
sort.Slice(snapshots, func(i, j int) bool {
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
|
||||
if jsonOutput {
|
||||
// JSON output
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(snapshots)
|
||||
}
|
||||
|
||||
// Table output
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
|
||||
if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, snap := range snapshots {
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\n",
|
||||
snap.ID,
|
||||
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
||||
formatBytes(snap.CompressedSize)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
// Purge removes old snapshots based on criteria
|
||||
func (app *SnapshotApp) Purge(ctx context.Context, keepLatest bool, olderThan string, force bool) error {
|
||||
snapshots, err := app.getSnapshots(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
sort.Slice(snapshots, func(i, j int) bool {
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
|
||||
var toDelete []SnapshotInfo
|
||||
|
||||
if keepLatest {
|
||||
// Keep only the most recent snapshot
|
||||
if len(snapshots) > 1 {
|
||||
toDelete = snapshots[1:]
|
||||
}
|
||||
} else if olderThan != "" {
|
||||
// Parse duration
|
||||
duration, err := parseDuration(olderThan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration: %w", err)
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-duration)
|
||||
for _, snap := range snapshots {
|
||||
if snap.Timestamp.Before(cutoff) {
|
||||
toDelete = append(toDelete, snap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
fmt.Println("No snapshots to delete")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Show what will be deleted
|
||||
fmt.Printf("The following snapshots will be deleted:\n\n")
|
||||
for _, snap := range toDelete {
|
||||
fmt.Printf(" %s (%s, %s)\n",
|
||||
snap.ID,
|
||||
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
||||
formatBytes(snap.CompressedSize))
|
||||
}
|
||||
|
||||
// Confirm unless --force is used
|
||||
if !force {
|
||||
fmt.Printf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete))
|
||||
var confirm string
|
||||
if _, err := fmt.Scanln(&confirm); err != nil {
|
||||
// Treat EOF or error as "no"
|
||||
fmt.Println("Cancelled")
|
||||
return nil
|
||||
}
|
||||
if strings.ToLower(confirm) != "y" {
|
||||
fmt.Println("Cancelled")
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete))
|
||||
}
|
||||
|
||||
// Delete snapshots
|
||||
for _, snap := range toDelete {
|
||||
log.Info("Deleting snapshot", "id", snap.ID)
|
||||
if err := app.deleteSnapshot(ctx, snap.ID); err != nil {
|
||||
return fmt.Errorf("deleting snapshot %s: %w", snap.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Deleted %d snapshot(s)\n", len(toDelete))
|
||||
|
||||
// TODO: Run blob pruning to clean up unreferenced blobs
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify checks snapshot integrity
|
||||
func (app *SnapshotApp) Verify(ctx context.Context, snapshotID string, deep bool) error {
|
||||
fmt.Printf("Verifying snapshot %s...\n", snapshotID)
|
||||
|
||||
// Download and parse manifest
|
||||
manifest, err := app.downloadManifest(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading manifest: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Manifest contains %d blobs\n", len(manifest))
|
||||
|
||||
// Check each blob exists
|
||||
missing := 0
|
||||
verified := 0
|
||||
|
||||
for _, blobHash := range manifest {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||
|
||||
if deep {
|
||||
// Download and verify hash
|
||||
// TODO: Implement deep verification
|
||||
fmt.Printf("Deep verification not yet implemented\n")
|
||||
return nil
|
||||
} else {
|
||||
// Just check existence
|
||||
_, err := app.S3Client.StatObject(ctx, blobPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" Missing: %s\n", blobHash)
|
||||
missing++
|
||||
} else {
|
||||
verified++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nVerification complete:\n")
|
||||
fmt.Printf(" Verified: %d\n", verified)
|
||||
fmt.Printf(" Missing: %d\n", missing)
|
||||
|
||||
if missing > 0 {
|
||||
return fmt.Errorf("%d blobs are missing", missing)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getSnapshots retrieves all snapshots from S3
|
||||
func (app *SnapshotApp) getSnapshots(ctx context.Context) ([]SnapshotInfo, error) {
|
||||
var snapshots []SnapshotInfo
|
||||
|
||||
// List all objects under metadata/
|
||||
objectCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true)
|
||||
|
||||
// Track unique snapshots
|
||||
snapshotMap := make(map[string]*SnapshotInfo)
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing objects: %w", object.Err)
|
||||
}
|
||||
|
||||
// Extract snapshot ID from paths like metadata/2024-01-15-143052-hostname/manifest.json.zst
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) < 3 || parts[0] != "metadata" {
|
||||
continue
|
||||
}
|
||||
|
||||
snapshotID := parts[1]
|
||||
if snapshotID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Initialize snapshot info if not seen
|
||||
if _, exists := snapshotMap[snapshotID]; !exists {
|
||||
timestamp, err := parseSnapshotTimestamp(snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
snapshotMap[snapshotID] = &SnapshotInfo{
|
||||
ID: snapshotID,
|
||||
Timestamp: timestamp,
|
||||
CompressedSize: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For each snapshot, download manifest and calculate total blob size
|
||||
for _, snap := range snapshotMap {
|
||||
manifest, err := app.downloadManifest(ctx, snap.ID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to download manifest", "id", snap.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Calculate total size of referenced blobs
|
||||
for _, blobHash := range manifest {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||
info, err := app.S3Client.StatObject(ctx, blobPath)
|
||||
if err != nil {
|
||||
log.Warn("Failed to stat blob", "blob", blobHash, "error", err)
|
||||
continue
|
||||
}
|
||||
snap.CompressedSize += info.Size
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, *snap)
|
||||
}
|
||||
|
||||
return snapshots, nil
|
||||
}
|
||||
|
||||
// downloadManifest downloads and parses a snapshot manifest
|
||||
func (app *SnapshotApp) downloadManifest(ctx context.Context, snapshotID string) ([]string, error) {
|
||||
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
|
||||
reader, err := app.S3Client.GetObject(ctx, manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Decompress
|
||||
zr, err := zstd.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
// Decode JSON
|
||||
var manifest []string
|
||||
if err := json.NewDecoder(zr).Decode(&manifest); err != nil {
|
||||
return nil, fmt.Errorf("decoding manifest: %w", err)
|
||||
}
|
||||
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// deleteSnapshot removes a snapshot and its metadata
|
||||
func (app *SnapshotApp) deleteSnapshot(ctx context.Context, snapshotID string) error {
|
||||
// List all objects under metadata/{snapshotID}/
|
||||
prefix := fmt.Sprintf("metadata/%s/", snapshotID)
|
||||
objectCh := app.S3Client.ListObjectsStream(ctx, prefix, true)
|
||||
|
||||
var objectsToDelete []string
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing objects: %w", object.Err)
|
||||
}
|
||||
objectsToDelete = append(objectsToDelete, object.Key)
|
||||
}
|
||||
|
||||
// Delete all objects
|
||||
for _, key := range objectsToDelete {
|
||||
if err := app.S3Client.RemoveObject(ctx, key); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseSnapshotTimestamp extracts timestamp from snapshot ID
|
||||
// Format: hostname-20240115-143052Z
|
||||
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
||||
// Find the last hyphen to separate hostname from timestamp
|
||||
lastHyphen := strings.LastIndex(snapshotID, "-")
|
||||
if lastHyphen == -1 {
|
||||
return time.Time{}, fmt.Errorf("invalid snapshot ID format")
|
||||
}
|
||||
|
||||
// Extract timestamp part (everything after hostname)
|
||||
timestampPart := snapshotID[lastHyphen+1:]
|
||||
|
||||
// The timestamp format is YYYYMMDD-HHMMSSZ
|
||||
// We need to find where the date ends and time begins
|
||||
if len(timestampPart) < 8 {
|
||||
return time.Time{}, fmt.Errorf("invalid snapshot ID format: timestamp too short")
|
||||
}
|
||||
|
||||
// Find where the hostname ends by looking for pattern YYYYMMDD
|
||||
hostnameEnd := strings.LastIndex(snapshotID[:lastHyphen], "-")
|
||||
if hostnameEnd == -1 {
|
||||
return time.Time{}, fmt.Errorf("invalid snapshot ID format: missing date separator")
|
||||
}
|
||||
|
||||
// Get the full timestamp including date from before the last hyphen
|
||||
fullTimestamp := snapshotID[hostnameEnd+1:]
|
||||
|
||||
// Parse the timestamp with Z suffix
|
||||
return time.Parse("20060102-150405Z", fullTimestamp)
|
||||
}
|
||||
|
||||
// parseDuration is now in duration.go
|
||||
|
||||
// runSnapshotCommand creates the FX app and runs the given function
|
||||
func runSnapshotCommand(ctx context.Context, fn func(*SnapshotApp) error) error {
|
||||
var result error
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
s3.Module,
|
||||
fx.Provide(func(
|
||||
g *globals.Globals,
|
||||
cfg *config.Config,
|
||||
db *database.DB,
|
||||
repos *database.Repositories,
|
||||
s3Client *s3.Client,
|
||||
lc fx.Lifecycle,
|
||||
shutdowner fx.Shutdowner,
|
||||
) *SnapshotApp {
|
||||
snapshotCreateApp := &SnapshotCreateApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
ScannerFactory: nil, // Not needed for snapshot commands
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Lifecycle: lc,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
return &SnapshotApp{
|
||||
SnapshotCreateApp: snapshotCreateApp,
|
||||
S3Client: s3Client,
|
||||
}
|
||||
}),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *SnapshotApp, shutdowner fx.Shutdowner) {
|
||||
result = fn(app)
|
||||
// Shutdown after command completes
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||
if err := shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// formatNumber formats a number with comma separators
|
||||
func formatNumber(n int) string {
|
||||
if n < 1000 {
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
return humanize.Comma(int64(n))
|
||||
}
|
||||
|
||||
// formatDuration formats a duration in a human-readable way
|
||||
func formatDuration(d time.Duration) string {
|
||||
if d < time.Second {
|
||||
return fmt.Sprintf("%dms", d.Milliseconds())
|
||||
}
|
||||
if d < time.Minute {
|
||||
return fmt.Sprintf("%.1fs", d.Seconds())
|
||||
}
|
||||
if d < time.Hour {
|
||||
mins := int(d.Minutes())
|
||||
secs := int(d.Seconds()) % 60
|
||||
if secs > 0 {
|
||||
return fmt.Sprintf("%dm%ds", mins, secs)
|
||||
}
|
||||
return fmt.Sprintf("%dm", mins)
|
||||
}
|
||||
hours := int(d.Hours())
|
||||
mins := int(d.Minutes()) % 60
|
||||
if mins > 0 {
|
||||
return fmt.Sprintf("%dh%dm", hours, mins)
|
||||
}
|
||||
return fmt.Sprintf("%dh", hours)
|
||||
}
|
||||
|
||||
159
internal/cli/store.go
Normal file
159
internal/cli/store.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// StoreApp contains dependencies for store commands
|
||||
type StoreApp struct {
|
||||
S3Client *s3.Client
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewStoreCommand creates the store command and subcommands
|
||||
func NewStoreCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "store",
|
||||
Short: "Storage information commands",
|
||||
Long: "Commands for viewing information about the S3 storage backend",
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newStoreInfoCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newStoreInfoCommand creates the 'store info' subcommand
|
||||
func newStoreInfoCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "info",
|
||||
Short: "Display storage information",
|
||||
Long: "Shows S3 bucket configuration and storage statistics including snapshots and blobs",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
||||
return app.Info(cmd.Context())
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Info displays storage information
|
||||
func (app *StoreApp) Info(ctx context.Context) error {
|
||||
// Get bucket info
|
||||
bucketName := app.S3Client.BucketName()
|
||||
endpoint := app.S3Client.Endpoint()
|
||||
|
||||
fmt.Printf("Storage Information\n")
|
||||
fmt.Printf("==================\n\n")
|
||||
fmt.Printf("S3 Configuration:\n")
|
||||
fmt.Printf(" Endpoint: %s\n", endpoint)
|
||||
fmt.Printf(" Bucket: %s\n\n", bucketName)
|
||||
|
||||
// Count snapshots by listing metadata/ prefix
|
||||
snapshotCount := 0
|
||||
snapshotCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true)
|
||||
snapshotDirs := make(map[string]bool)
|
||||
|
||||
for object := range snapshotCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing snapshots: %w", object.Err)
|
||||
}
|
||||
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
snapshotDirs[parts[1]] = true
|
||||
}
|
||||
}
|
||||
snapshotCount = len(snapshotDirs)
|
||||
|
||||
// Count blobs and calculate total size by listing blobs/ prefix
|
||||
blobCount := 0
|
||||
var totalSize int64
|
||||
|
||||
blobCh := app.S3Client.ListObjectsStream(ctx, "blobs/", false)
|
||||
for object := range blobCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", object.Err)
|
||||
}
|
||||
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
||||
blobCount++
|
||||
totalSize += object.Size
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Storage Statistics:\n")
|
||||
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
||||
fmt.Printf(" Blobs: %d\n", blobCount)
|
||||
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatBytes formats bytes into human-readable format
|
||||
func formatBytes(bytes int64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// runWithApp creates the FX app and runs the given function
|
||||
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
||||
var result error
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
s3.Module,
|
||||
fx.Provide(func(s3Client *s3.Client, shutdowner fx.Shutdowner) *StoreApp {
|
||||
return &StoreApp{
|
||||
S3Client: s3Client,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
}),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
||||
result = fn(app)
|
||||
// Shutdown after command completes
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||
if err := shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user