Refactor: Move Vaultik struct and methods to internal/vaultik package
- Created new internal/vaultik package with unified Vaultik struct - Moved all command methods (snapshot, info, prune, verify) from CLI to vaultik package - Implemented single constructor that handles crypto capabilities automatically - Added CanDecrypt() method to check if decryption is available - Updated all CLI commands to use the new vaultik.Vaultik struct - Removed old fragmented App structs and WithCrypto wrapper - Fixed context management - Vaultik now owns its context lifecycle - Cleaned up package imports and dependencies This creates a cleaner separation between CLI/Cobra code and business logic, with all vaultik operations now centralized in the internal/vaultik package.
This commit is contained in:
@@ -9,9 +9,13 @@ import (
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/crypto"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
@@ -48,6 +52,10 @@ func NewApp(opts AppOptions) *fx.App {
|
||||
config.Module,
|
||||
database.Module,
|
||||
log.Module,
|
||||
s3.Module,
|
||||
snapshot.Module,
|
||||
crypto.Module, // This will provide crypto only if age_secret_key is configured
|
||||
fx.Provide(vaultik.New),
|
||||
fx.Invoke(setupGlobals),
|
||||
fx.NopLogger,
|
||||
}
|
||||
|
||||
@@ -3,20 +3,29 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// FetchOptions contains options for the fetch command
|
||||
type FetchOptions struct {
|
||||
Bucket string
|
||||
Prefix string
|
||||
SnapshotID string
|
||||
FilePath string
|
||||
Target string
|
||||
}
|
||||
|
||||
// FetchApp contains all dependencies needed for fetch
|
||||
type FetchApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewFetchCommand creates the fetch command
|
||||
@@ -24,65 +33,107 @@ func NewFetchCommand() *cobra.Command {
|
||||
opts := &FetchOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "fetch",
|
||||
Use: "fetch <snapshot-id> <file-path> <target-path>",
|
||||
Short: "Extract single file from backup",
|
||||
Long: `Download and decrypt a single file from a backup snapshot`,
|
||||
Args: cobra.NoArgs,
|
||||
Long: `Download and decrypt a single file from a backup snapshot.
|
||||
|
||||
This command extracts a specific file from the snapshot and saves it to the target path.
|
||||
The age_secret_key must be configured in the config file for decryption.`,
|
||||
Args: cobra.ExactArgs(3),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate required flags
|
||||
if opts.Bucket == "" {
|
||||
return fmt.Errorf("--bucket is required")
|
||||
snapshotID := args[0]
|
||||
filePath := args[1]
|
||||
targetPath := args[2]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Prefix == "" {
|
||||
return fmt.Errorf("--prefix is required")
|
||||
}
|
||||
if opts.SnapshotID == "" {
|
||||
return fmt.Errorf("--snapshot is required")
|
||||
}
|
||||
if opts.FilePath == "" {
|
||||
return fmt.Errorf("--file is required")
|
||||
}
|
||||
if opts.Target == "" {
|
||||
return fmt.Errorf("--target is required")
|
||||
}
|
||||
return runFetch(cmd.Context(), opts)
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
snapshot.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *FetchApp {
|
||||
return &FetchApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *FetchApp, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the fetch operation in a goroutine
|
||||
go func() {
|
||||
// Run the fetch operation
|
||||
if err := app.runFetch(ctx, snapshotID, filePath, targetPath, opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Fetch operation failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when fetch completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping fetch operation")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
|
||||
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID")
|
||||
cmd.Flags().StringVar(&opts.FilePath, "file", "", "Path of file to extract from backup")
|
||||
cmd.Flags().StringVar(&opts.Target, "target", "", "Target path for extracted file")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runFetch(ctx context.Context, opts *FetchOptions) error {
|
||||
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
|
||||
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
|
||||
// runFetch executes the fetch operation
|
||||
func (app *FetchApp) runFetch(ctx context.Context, snapshotID, filePath, targetPath string, opts *FetchOptions) error {
|
||||
// Check for age_secret_key
|
||||
if app.Config.AgeSecretKey == "" {
|
||||
return fmt.Errorf("age_secret_key missing from config - required for fetch")
|
||||
}
|
||||
|
||||
app := fx.New(
|
||||
fx.Supply(opts),
|
||||
fx.Provide(globals.New),
|
||||
// Additional modules will be added here
|
||||
fx.Invoke(func(g *globals.Globals) error {
|
||||
// TODO: Implement fetch logic
|
||||
fmt.Printf("Fetching %s from snapshot %s to %s\n", opts.FilePath, opts.SnapshotID, opts.Target)
|
||||
return nil
|
||||
}),
|
||||
fx.NopLogger,
|
||||
log.Info("Starting fetch operation",
|
||||
"snapshot_id", snapshotID,
|
||||
"file_path", filePath,
|
||||
"target_path", targetPath,
|
||||
"bucket", app.Config.S3.Bucket,
|
||||
"prefix", app.Config.S3.Prefix,
|
||||
)
|
||||
|
||||
if err := app.Start(ctx); err != nil {
|
||||
return fmt.Errorf("failed to start fetch: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := app.Stop(ctx); err != nil {
|
||||
fmt.Printf("error stopping app: %v\n", err)
|
||||
}
|
||||
}()
|
||||
// TODO: Implement fetch logic
|
||||
// 1. Download and decrypt database from S3
|
||||
// 2. Find the file metadata and chunk list
|
||||
// 3. Download and decrypt only the necessary blobs
|
||||
// 4. Reconstruct the file from chunks
|
||||
// 5. Write file to target path with proper metadata
|
||||
|
||||
fmt.Printf("Fetching %s from snapshot %s to %s\n", filePath, snapshotID, targetPath)
|
||||
fmt.Println("TODO: Implement fetch logic")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
70
internal/cli/info.go
Normal file
70
internal/cli/info.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// NewInfoCommand creates the info command
|
||||
func NewInfoCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "info",
|
||||
Short: "Display system and configuration information",
|
||||
Long: `Shows information about the current vaultik configuration, including:
|
||||
- System details (OS, architecture, version)
|
||||
- Storage configuration (S3 bucket, endpoint)
|
||||
- Backup settings (source directories, compression)
|
||||
- Encryption configuration (recipients)
|
||||
- Local database statistics`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ShowInfo(); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to show info", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -2,51 +2,28 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/dustin/go-humanize"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// PruneOptions contains options for the prune command
|
||||
type PruneOptions struct {
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// PruneApp contains all dependencies needed for pruning
|
||||
type PruneApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewPruneCommand creates the prune command
|
||||
func NewPruneCommand() *cobra.Command {
|
||||
opts := &PruneOptions{}
|
||||
opts := &vaultik.PruneOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Remove unreferenced blobs",
|
||||
Long: `Delete blobs that are no longer referenced by any snapshot.
|
||||
Long: `Removes blobs that are not referenced by any snapshot.
|
||||
|
||||
This command will:
|
||||
1. Download the manifest from the last successful snapshot
|
||||
2. List all blobs in S3
|
||||
3. Delete any blobs not referenced in the manifest
|
||||
This command scans all snapshots and their manifests to build a list of
|
||||
referenced blobs, then removes any blobs in storage that are not in this list.
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Use this command after deleting snapshots with 'vaultik purge' to reclaim
|
||||
storage space.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
@@ -63,38 +40,23 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
snapshot.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *PruneApp {
|
||||
return &PruneApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *PruneApp, lc fx.Lifecycle) {
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the prune operation in a goroutine
|
||||
go func() {
|
||||
// Run the prune operation
|
||||
if err := app.runPrune(ctx, opts); err != nil {
|
||||
if err := v.PruneBlobs(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when prune completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
@@ -102,6 +64,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping prune operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -111,186 +74,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be deleted without actually deleting")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// runPrune executes the prune operation
|
||||
func (app *PruneApp) runPrune(ctx context.Context, opts *PruneOptions) error {
|
||||
log.Info("Starting prune operation",
|
||||
"bucket", app.Config.S3.Bucket,
|
||||
"prefix", app.Config.S3.Prefix,
|
||||
"dry_run", opts.DryRun,
|
||||
)
|
||||
|
||||
// Step 1: Get the latest complete snapshot from the database
|
||||
log.Info("Getting latest snapshot from database")
|
||||
snapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing snapshots: %w", err)
|
||||
}
|
||||
|
||||
if len(snapshots) == 0 {
|
||||
return fmt.Errorf("no snapshots found in database")
|
||||
}
|
||||
|
||||
latestSnapshot := snapshots[0]
|
||||
if latestSnapshot.CompletedAt == nil {
|
||||
return fmt.Errorf("latest snapshot %s is incomplete", latestSnapshot.ID)
|
||||
}
|
||||
|
||||
log.Info("Found latest snapshot",
|
||||
"id", latestSnapshot.ID,
|
||||
"completed_at", latestSnapshot.CompletedAt.Format("2006-01-02 15:04:05"))
|
||||
|
||||
// Step 2: Find and download the manifest from the last successful snapshot in S3
|
||||
log.Info("Finding last successful snapshot in S3")
|
||||
metadataPrefix := "metadata/"
|
||||
|
||||
// List all snapshots in S3
|
||||
var s3Snapshots []string
|
||||
objectCh := app.S3Client.ListObjectsStream(ctx, metadataPrefix, false)
|
||||
for obj := range objectCh {
|
||||
if obj.Err != nil {
|
||||
return fmt.Errorf("listing metadata objects: %w", obj.Err)
|
||||
}
|
||||
// Extract snapshot ID from path like "metadata/hostname-20240115-143052Z/manifest.json.zst"
|
||||
parts := strings.Split(obj.Key, "/")
|
||||
if len(parts) >= 2 && strings.HasSuffix(obj.Key, "/manifest.json.zst") {
|
||||
s3Snapshots = append(s3Snapshots, parts[1])
|
||||
}
|
||||
}
|
||||
|
||||
if len(s3Snapshots) == 0 {
|
||||
return fmt.Errorf("no snapshot manifests found in S3")
|
||||
}
|
||||
|
||||
// Find the most recent snapshot (they're named with timestamps)
|
||||
var lastS3Snapshot string
|
||||
for _, snap := range s3Snapshots {
|
||||
if lastS3Snapshot == "" || snap > lastS3Snapshot {
|
||||
lastS3Snapshot = snap
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Found last S3 snapshot", "id", lastS3Snapshot)
|
||||
|
||||
// Step 3: Verify the last S3 snapshot matches the latest DB snapshot
|
||||
if lastS3Snapshot != latestSnapshot.ID {
|
||||
return fmt.Errorf("latest snapshot in database (%s) does not match last successful snapshot in S3 (%s)",
|
||||
latestSnapshot.ID, lastS3Snapshot)
|
||||
}
|
||||
|
||||
// Step 4: Download and parse the manifest
|
||||
log.Info("Downloading manifest", "snapshot_id", lastS3Snapshot)
|
||||
manifest, err := app.downloadManifest(ctx, lastS3Snapshot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading manifest: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Manifest loaded", "blob_count", len(manifest.Blobs))
|
||||
|
||||
// Step 5: Build set of referenced blobs
|
||||
referencedBlobs := make(map[string]bool)
|
||||
for _, blob := range manifest.Blobs {
|
||||
referencedBlobs[blob.Hash] = true
|
||||
}
|
||||
|
||||
// Step 6: List all blobs in S3
|
||||
log.Info("Listing all blobs in S3")
|
||||
blobPrefix := "blobs/"
|
||||
var totalBlobs int
|
||||
var unreferencedBlobs []s3.ObjectInfo
|
||||
var unreferencedSize int64
|
||||
|
||||
objectCh = app.S3Client.ListObjectsStream(ctx, blobPrefix, true)
|
||||
for obj := range objectCh {
|
||||
if obj.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", obj.Err)
|
||||
}
|
||||
|
||||
totalBlobs++
|
||||
|
||||
// Extract blob hash from path like "blobs/ca/fe/cafebabe..."
|
||||
parts := strings.Split(obj.Key, "/")
|
||||
if len(parts) == 4 {
|
||||
blobHash := parts[3]
|
||||
if !referencedBlobs[blobHash] {
|
||||
unreferencedBlobs = append(unreferencedBlobs, obj)
|
||||
unreferencedSize += obj.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Blob scan complete",
|
||||
"total_blobs", totalBlobs,
|
||||
"referenced_blobs", len(referencedBlobs),
|
||||
"unreferenced_blobs", len(unreferencedBlobs),
|
||||
"unreferenced_size", humanize.Bytes(uint64(unreferencedSize)))
|
||||
|
||||
// Step 7: Delete or report unreferenced blobs
|
||||
if opts.DryRun {
|
||||
fmt.Printf("\nDry run mode - would delete %d unreferenced blobs\n", len(unreferencedBlobs))
|
||||
fmt.Printf("Total size of blobs to delete: %s\n", humanize.Bytes(uint64(unreferencedSize)))
|
||||
|
||||
if len(unreferencedBlobs) > 0 {
|
||||
log.Debug("Unreferenced blobs found", "count", len(unreferencedBlobs))
|
||||
for _, obj := range unreferencedBlobs {
|
||||
log.Debug("Would delete blob", "key", obj.Key, "size", humanize.Bytes(uint64(obj.Size)))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if len(unreferencedBlobs) == 0 {
|
||||
fmt.Println("No unreferenced blobs to delete")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("\nDeleting %d unreferenced blobs (%s)...\n",
|
||||
len(unreferencedBlobs), humanize.Bytes(uint64(unreferencedSize)))
|
||||
|
||||
deletedCount := 0
|
||||
deletedSize := int64(0)
|
||||
|
||||
for _, obj := range unreferencedBlobs {
|
||||
if err := app.S3Client.RemoveObject(ctx, obj.Key); err != nil {
|
||||
log.Error("Failed to delete blob", "key", obj.Key, "error", err)
|
||||
continue
|
||||
}
|
||||
deletedCount++
|
||||
deletedSize += obj.Size
|
||||
|
||||
// Show progress every 100 blobs
|
||||
if deletedCount%100 == 0 {
|
||||
fmt.Printf(" Deleted %d/%d blobs (%s)...\n",
|
||||
deletedCount, len(unreferencedBlobs),
|
||||
humanize.Bytes(uint64(deletedSize)))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nDeleted %d blobs (%s)\n", deletedCount, humanize.Bytes(uint64(deletedSize)))
|
||||
}
|
||||
|
||||
log.Info("Prune operation completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadManifest downloads and decompresses a snapshot manifest
|
||||
func (app *PruneApp) downloadManifest(ctx context.Context, snapshotID string) (*snapshot.Manifest, error) {
|
||||
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
|
||||
// Download the compressed manifest
|
||||
reader, err := app.S3Client.GetObject(ctx, manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("downloading manifest: %w", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Decode manifest
|
||||
manifest, err := snapshot.DecodeManifest(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding manifest: %w", err)
|
||||
}
|
||||
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
99
internal/cli/purge.go
Normal file
99
internal/cli/purge.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// PurgeOptions contains options for the purge command
|
||||
type PurgeOptions struct {
|
||||
KeepLatest bool
|
||||
OlderThan string
|
||||
Force bool
|
||||
}
|
||||
|
||||
// NewPurgeCommand creates the purge command
|
||||
func NewPurgeCommand() *cobra.Command {
|
||||
opts := &PurgeOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: `Removes snapshots based on age or count criteria.
|
||||
|
||||
This command allows you to:
|
||||
- Keep only the latest snapshot (--keep-latest)
|
||||
- Remove snapshots older than a specific duration (--older-than)
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate flags
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the purge operation in a goroutine
|
||||
go func() {
|
||||
// Run the purge operation
|
||||
if err := v.PurgeSnapshots(opts.KeepLatest, opts.OlderThan, opts.Force); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Purge operation failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when purge completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping purge operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -3,19 +3,30 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// RestoreOptions contains options for the restore command
|
||||
type RestoreOptions struct {
|
||||
Bucket string
|
||||
Prefix string
|
||||
SnapshotID string
|
||||
TargetDir string
|
||||
TargetDir string
|
||||
}
|
||||
|
||||
// RestoreApp contains all dependencies needed for restore
|
||||
type RestoreApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewRestoreCommand creates the restore command
|
||||
@@ -23,61 +34,104 @@ func NewRestoreCommand() *cobra.Command {
|
||||
opts := &RestoreOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "restore",
|
||||
Use: "restore <snapshot-id> <target-dir>",
|
||||
Short: "Restore files from backup",
|
||||
Long: `Download and decrypt files from a backup snapshot`,
|
||||
Args: cobra.NoArgs,
|
||||
Long: `Download and decrypt files from a backup snapshot.
|
||||
|
||||
This command will restore all files from the specified snapshot to the target directory.
|
||||
The age_secret_key must be configured in the config file for decryption.`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate required flags
|
||||
if opts.Bucket == "" {
|
||||
return fmt.Errorf("--bucket is required")
|
||||
snapshotID := args[0]
|
||||
opts.TargetDir = args[1]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Prefix == "" {
|
||||
return fmt.Errorf("--prefix is required")
|
||||
}
|
||||
if opts.SnapshotID == "" {
|
||||
return fmt.Errorf("--snapshot is required")
|
||||
}
|
||||
if opts.TargetDir == "" {
|
||||
return fmt.Errorf("--target is required")
|
||||
}
|
||||
return runRestore(cmd.Context(), opts)
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
snapshot.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *RestoreApp {
|
||||
return &RestoreApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Repositories: repos,
|
||||
S3Client: s3Client,
|
||||
DB: db,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the restore operation in a goroutine
|
||||
go func() {
|
||||
// Run the restore operation
|
||||
if err := app.runRestore(ctx, snapshotID, opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when restore completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping restore operation")
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
|
||||
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID to restore")
|
||||
cmd.Flags().StringVar(&opts.TargetDir, "target", "", "Target directory for restore")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runRestore(ctx context.Context, opts *RestoreOptions) error {
|
||||
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
|
||||
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
|
||||
// runRestore executes the restore operation
|
||||
func (app *RestoreApp) runRestore(ctx context.Context, snapshotID string, opts *RestoreOptions) error {
|
||||
// Check for age_secret_key
|
||||
if app.Config.AgeSecretKey == "" {
|
||||
return fmt.Errorf("age_secret_key missing from config - required for restore")
|
||||
}
|
||||
|
||||
app := fx.New(
|
||||
fx.Supply(opts),
|
||||
fx.Provide(globals.New),
|
||||
// Additional modules will be added here
|
||||
fx.Invoke(func(g *globals.Globals) error {
|
||||
// TODO: Implement restore logic
|
||||
fmt.Printf("Restoring snapshot %s to %s\n", opts.SnapshotID, opts.TargetDir)
|
||||
return nil
|
||||
}),
|
||||
fx.NopLogger,
|
||||
log.Info("Starting restore operation",
|
||||
"snapshot_id", snapshotID,
|
||||
"target_dir", opts.TargetDir,
|
||||
"bucket", app.Config.S3.Bucket,
|
||||
"prefix", app.Config.S3.Prefix,
|
||||
)
|
||||
|
||||
if err := app.Start(ctx); err != nil {
|
||||
return fmt.Errorf("failed to start restore: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := app.Stop(ctx); err != nil {
|
||||
fmt.Printf("error stopping app: %v\n", err)
|
||||
}
|
||||
}()
|
||||
// TODO: Implement restore logic
|
||||
// 1. Download and decrypt database from S3
|
||||
// 2. Download and decrypt blobs
|
||||
// 3. Reconstruct files from chunks
|
||||
// 4. Write files to target directory with proper metadata
|
||||
|
||||
fmt.Printf("Restoring snapshot %s to %s\n", snapshotID, opts.TargetDir)
|
||||
fmt.Println("TODO: Implement restore logic")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ on the source system.`,
|
||||
NewFetchCommand(),
|
||||
NewStoreCommand(),
|
||||
NewSnapshotCommand(),
|
||||
NewInfoCommand(),
|
||||
)
|
||||
|
||||
return cmd
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
10
internal/cli/vaultik_snapshot_types.go
Normal file
10
internal/cli/vaultik_snapshot_types.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package cli
|
||||
|
||||
import "time"
|
||||
|
||||
// SnapshotInfo represents snapshot information for listing
|
||||
type SnapshotInfo struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
}
|
||||
@@ -2,85 +2,93 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// VerifyOptions contains options for the verify command
|
||||
type VerifyOptions struct {
|
||||
Bucket string
|
||||
Prefix string
|
||||
SnapshotID string
|
||||
Quick bool
|
||||
}
|
||||
|
||||
// NewVerifyCommand creates the verify command
|
||||
func NewVerifyCommand() *cobra.Command {
|
||||
opts := &VerifyOptions{}
|
||||
opts := &vaultik.VerifyOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify",
|
||||
Short: "Verify backup integrity",
|
||||
Long: `Check that all referenced blobs exist and verify metadata integrity`,
|
||||
Args: cobra.NoArgs,
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: `Verifies that all blobs referenced in a snapshot exist and optionally verifies their contents.
|
||||
|
||||
Shallow verification (default):
|
||||
- Downloads and decompresses manifest
|
||||
- Checks existence of all blobs in S3
|
||||
- Reports missing blobs
|
||||
|
||||
Deep verification (--deep):
|
||||
- Downloads and decrypts database
|
||||
- Verifies blob lists match between manifest and database
|
||||
- Downloads, decrypts, and decompresses each blob
|
||||
- Verifies SHA256 hash of each chunk matches database
|
||||
- Ensures chunks are ordered correctly
|
||||
|
||||
The command will fail immediately on any verification error and exit with non-zero status.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate required flags
|
||||
if opts.Bucket == "" {
|
||||
return fmt.Errorf("--bucket is required")
|
||||
snapshotID := args[0]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Prefix == "" {
|
||||
return fmt.Errorf("--prefix is required")
|
||||
}
|
||||
return runVerify(cmd.Context(), opts)
|
||||
|
||||
// Use the app framework for all verification
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Run the verify operation directly
|
||||
go func() {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshot(snapshotID, false)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Verification failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping verify operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
|
||||
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
|
||||
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID to verify (optional, defaults to latest)")
|
||||
cmd.Flags().BoolVar(&opts.Quick, "quick", false, "Perform quick verification by checking blob existence and S3 content hashes without downloading")
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Perform deep verification by downloading and verifying all blob contents")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runVerify(ctx context.Context, opts *VerifyOptions) error {
|
||||
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
|
||||
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
|
||||
}
|
||||
|
||||
app := fx.New(
|
||||
fx.Supply(opts),
|
||||
fx.Provide(globals.New),
|
||||
// Additional modules will be added here
|
||||
fx.Invoke(func(g *globals.Globals) error {
|
||||
// TODO: Implement verify logic
|
||||
if opts.SnapshotID == "" {
|
||||
fmt.Printf("Verifying latest snapshot in bucket %s with prefix %s\n", opts.Bucket, opts.Prefix)
|
||||
} else {
|
||||
fmt.Printf("Verifying snapshot %s in bucket %s with prefix %s\n", opts.SnapshotID, opts.Bucket, opts.Prefix)
|
||||
}
|
||||
if opts.Quick {
|
||||
fmt.Println("Performing quick verification")
|
||||
} else {
|
||||
fmt.Println("Performing deep verification")
|
||||
}
|
||||
return nil
|
||||
}),
|
||||
fx.NopLogger,
|
||||
)
|
||||
|
||||
if err := app.Start(ctx); err != nil {
|
||||
return fmt.Errorf("failed to start verify: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := app.Stop(ctx); err != nil {
|
||||
fmt.Printf("error stopping app: %v\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user