sneak/vaultik
1
0
Fork 0
Code Issues 7 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Refactor: Move Vaultik struct and methods to internal/vaultik package

Browse Source 
- Created new internal/vaultik package with unified Vaultik struct
- Moved all command methods (snapshot, info, prune, verify) from CLI to vaultik package
- Implemented single constructor that handles crypto capabilities automatically
- Added CanDecrypt() method to check if decryption is available
- Updated all CLI commands to use the new vaultik.Vaultik struct
- Removed old fragmented App structs and WithCrypto wrapper
- Fixed context management - Vaultik now owns its context lifecycle
- Cleaned up package imports and dependencies

This creates a cleaner separation between CLI/Cobra code and business logic,
with all vaultik operations now centralized in the internal/vaultik package.
This commit is contained in:
Jeffrey Paul
2025-07-26 14:47:26 +02:00
parent 5c70405a85
commit e29a995120
22 changed files with 1494 additions and 1320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/blob/packer.go
View File

@@ -420,7 +420,6 @@ func (p *Packer) finalizeCurrentBlob() error {
// Call blob handler if set
if p.blobHandler != nil {
log.Debug("Invoking blob handler callback", "blob_hash", blobHash[:8]+"...")
// Reset file position for handler
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile()

8
internal/cli/app.go
View File

@@ -9,9 +9,13 @@ import (
"time"
"git.eeqj.de/sneak/vaultik/internal/config"
"git.eeqj.de/sneak/vaultik/internal/crypto"
"git.eeqj.de/sneak/vaultik/internal/database"
"git.eeqj.de/sneak/vaultik/internal/globals"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/s3"
"git.eeqj.de/sneak/vaultik/internal/snapshot"
"git.eeqj.de/sneak/vaultik/internal/vaultik"
"go.uber.org/fx"
)
@@ -48,6 +52,10 @@ func NewApp(opts AppOptions) *fx.App {
config.Module,
database.Module,
log.Module,
s3.Module,
snapshot.Module,
crypto.Module, // This will provide crypto only if age_secret_key is configured
fx.Provide(vaultik.New),
fx.Invoke(setupGlobals),
fx.NopLogger,
}

155
internal/cli/fetch.go
View File

@@ -3,20 +3,29 @@ package cli
import (
"context"
"fmt"
"os"
"git.eeqj.de/sneak/vaultik/internal/config"
"git.eeqj.de/sneak/vaultik/internal/database"
"git.eeqj.de/sneak/vaultik/internal/globals"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/s3"
"git.eeqj.de/sneak/vaultik/internal/snapshot"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// FetchOptions contains options for the fetch command
type FetchOptions struct {
Bucket string
Prefix string
SnapshotID string
FilePath string
Target string
}
// FetchApp contains all dependencies needed for fetch
type FetchApp struct {
Globals *globals.Globals
Config *config.Config
Repositories *database.Repositories
S3Client *s3.Client
DB *database.DB
Shutdowner fx.Shutdowner
}
// NewFetchCommand creates the fetch command
@@ -24,65 +33,107 @@ func NewFetchCommand() *cobra.Command {
opts := &FetchOptions{}
cmd := &cobra.Command{
Use: "fetch",
Use: "fetch <snapshot-id> <file-path> <target-path>",
Short: "Extract single file from backup",
Long: `Download and decrypt a single file from a backup snapshot`,
Args: cobra.NoArgs,
Long: `Download and decrypt a single file from a backup snapshot.
This command extracts a specific file from the snapshot and saves it to the target path.
The age_secret_key must be configured in the config file for decryption.`,
Args: cobra.ExactArgs(3),
RunE: func(cmd *cobra.Command, args []string) error {
// Validate required flags
if opts.Bucket == "" {
return fmt.Errorf("--bucket is required")
snapshotID := args[0]
filePath := args[1]
targetPath := args[2]
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
if opts.Prefix == "" {
return fmt.Errorf("--prefix is required")
}
if opts.SnapshotID == "" {
return fmt.Errorf("--snapshot is required")
}
if opts.FilePath == "" {
return fmt.Errorf("--file is required")
}
if opts.Target == "" {
return fmt.Errorf("--target is required")
}
return runFetch(cmd.Context(), opts)
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{
snapshot.Module,
s3.Module,
fx.Provide(fx.Annotate(
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *FetchApp {
return &FetchApp{
Globals: g,
Config: cfg,
Repositories: repos,
S3Client: s3Client,
DB: db,
Shutdowner: shutdowner,
}
},
)),
},
Invokes: []fx.Option{
fx.Invoke(func(app *FetchApp, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
// Start the fetch operation in a goroutine
go func() {
// Run the fetch operation
if err := app.runFetch(ctx, snapshotID, filePath, targetPath, opts); err != nil {
if err != context.Canceled {
log.Error("Fetch operation failed", "error", err)
}
}
// Shutdown the app when fetch completes
if err := app.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping fetch operation")
return nil
},
})
}),
},
})
},
}
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID")
cmd.Flags().StringVar(&opts.FilePath, "file", "", "Path of file to extract from backup")
cmd.Flags().StringVar(&opts.Target, "target", "", "Target path for extracted file")
return cmd
}
func runFetch(ctx context.Context, opts *FetchOptions) error {
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
// runFetch executes the fetch operation
func (app *FetchApp) runFetch(ctx context.Context, snapshotID, filePath, targetPath string, opts *FetchOptions) error {
// Check for age_secret_key
if app.Config.AgeSecretKey == "" {
return fmt.Errorf("age_secret_key missing from config - required for fetch")
}
app := fx.New(
fx.Supply(opts),
fx.Provide(globals.New),
// Additional modules will be added here
fx.Invoke(func(g *globals.Globals) error {
// TODO: Implement fetch logic
fmt.Printf("Fetching %s from snapshot %s to %s\n", opts.FilePath, opts.SnapshotID, opts.Target)
return nil
}),
fx.NopLogger,
log.Info("Starting fetch operation",
"snapshot_id", snapshotID,
"file_path", filePath,
"target_path", targetPath,
"bucket", app.Config.S3.Bucket,
"prefix", app.Config.S3.Prefix,
)
if err := app.Start(ctx); err != nil {
return fmt.Errorf("failed to start fetch: %w", err)
}
defer func() {
if err := app.Stop(ctx); err != nil {
fmt.Printf("error stopping app: %v\n", err)
}
}()
// TODO: Implement fetch logic
// 1. Download and decrypt database from S3
// 2. Find the file metadata and chunk list
// 3. Download and decrypt only the necessary blobs
// 4. Reconstruct the file from chunks
// 5. Write file to target path with proper metadata
fmt.Printf("Fetching %s from snapshot %s to %s\n", filePath, snapshotID, targetPath)
fmt.Println("TODO: Implement fetch logic")
return nil
}

70
internal/cli/info.go Normal file
View File

@@ -0,0 +1,70 @@
package cli
import (
"context"
"os"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/vaultik"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// NewInfoCommand creates the info command
func NewInfoCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "info",
Short: "Display system and configuration information",
Long: `Shows information about the current vaultik configuration, including:
- System details (OS, architecture, version)
- Storage configuration (S3 bucket, endpoint)
- Backup settings (source directories, compression)
- Encryption configuration (recipients)
- Local database statistics`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
// Use the app framework
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.ShowInfo(); err != nil {
if err != context.Canceled {
log.Error("Failed to show info", "error", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
}),
},
})
},
}
return cmd
}

246
internal/cli/prune.go
View File

@@ -2,51 +2,28 @@ package cli
import (
"context"
"fmt"
"strings"
"os"
"git.eeqj.de/sneak/vaultik/internal/config"
"git.eeqj.de/sneak/vaultik/internal/database"
"git.eeqj.de/sneak/vaultik/internal/globals"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/s3"
"git.eeqj.de/sneak/vaultik/internal/snapshot"
"github.com/dustin/go-humanize"
"git.eeqj.de/sneak/vaultik/internal/vaultik"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// PruneOptions contains options for the prune command
type PruneOptions struct {
DryRun bool
}
// PruneApp contains all dependencies needed for pruning
type PruneApp struct {
Globals *globals.Globals
Config *config.Config
Repositories *database.Repositories
S3Client *s3.Client
DB *database.DB
Shutdowner fx.Shutdowner
}
// NewPruneCommand creates the prune command
func NewPruneCommand() *cobra.Command {
opts := &PruneOptions{}
opts := &vaultik.PruneOptions{}
cmd := &cobra.Command{
Use: "prune",
Short: "Remove unreferenced blobs",
Long: `Delete blobs that are no longer referenced by any snapshot.
Long: `Removes blobs that are not referenced by any snapshot.
This command will:
1. Download the manifest from the last successful snapshot
2. List all blobs in S3
3. Delete any blobs not referenced in the manifest
This command scans all snapshots and their manifests to build a list of
referenced blobs, then removes any blobs in storage that are not in this list.
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
Use this command after deleting snapshots with 'vaultik purge' to reclaim
storage space.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// Use unified config resolution
@@ -63,38 +40,23 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{
snapshot.Module,
s3.Module,
fx.Provide(fx.Annotate(
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *PruneApp {
return &PruneApp{
Globals: g,
Config: cfg,
Repositories: repos,
S3Client: s3Client,
DB: db,
Shutdowner: shutdowner,
}
},
)),
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(app *PruneApp, lc fx.Lifecycle) {
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
// Start the prune operation in a goroutine
go func() {
// Run the prune operation
if err := app.runPrune(ctx, opts); err != nil {
if err := v.PruneBlobs(opts); err != nil {
if err != context.Canceled {
log.Error("Prune operation failed", "error", err)
os.Exit(1)
}
}
// Shutdown the app when prune completes
if err := app.Shutdowner.Shutdown(); err != nil {
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
@@ -102,6 +64,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping prune operation")
v.Cancel()
return nil
},
})
@@ -111,186 +74,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
},
}
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be deleted without actually deleting")
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
return cmd
}
// runPrune executes the prune operation
func (app *PruneApp) runPrune(ctx context.Context, opts *PruneOptions) error {
log.Info("Starting prune operation",
"bucket", app.Config.S3.Bucket,
"prefix", app.Config.S3.Prefix,
"dry_run", opts.DryRun,
)
// Step 1: Get the latest complete snapshot from the database
log.Info("Getting latest snapshot from database")
snapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 1)
if err != nil {
return fmt.Errorf("listing snapshots: %w", err)
}
if len(snapshots) == 0 {
return fmt.Errorf("no snapshots found in database")
}
latestSnapshot := snapshots[0]
if latestSnapshot.CompletedAt == nil {
return fmt.Errorf("latest snapshot %s is incomplete", latestSnapshot.ID)
}
log.Info("Found latest snapshot",
"id", latestSnapshot.ID,
"completed_at", latestSnapshot.CompletedAt.Format("2006-01-02 15:04:05"))
// Step 2: Find and download the manifest from the last successful snapshot in S3
log.Info("Finding last successful snapshot in S3")
metadataPrefix := "metadata/"
// List all snapshots in S3
var s3Snapshots []string
objectCh := app.S3Client.ListObjectsStream(ctx, metadataPrefix, false)
for obj := range objectCh {
if obj.Err != nil {
return fmt.Errorf("listing metadata objects: %w", obj.Err)
}
// Extract snapshot ID from path like "metadata/hostname-20240115-143052Z/manifest.json.zst"
parts := strings.Split(obj.Key, "/")
if len(parts) >= 2 && strings.HasSuffix(obj.Key, "/manifest.json.zst") {
s3Snapshots = append(s3Snapshots, parts[1])
}
}
if len(s3Snapshots) == 0 {
return fmt.Errorf("no snapshot manifests found in S3")
}
// Find the most recent snapshot (they're named with timestamps)
var lastS3Snapshot string
for _, snap := range s3Snapshots {
if lastS3Snapshot == "" || snap > lastS3Snapshot {
lastS3Snapshot = snap
}
}
log.Info("Found last S3 snapshot", "id", lastS3Snapshot)
// Step 3: Verify the last S3 snapshot matches the latest DB snapshot
if lastS3Snapshot != latestSnapshot.ID {
return fmt.Errorf("latest snapshot in database (%s) does not match last successful snapshot in S3 (%s)",
latestSnapshot.ID, lastS3Snapshot)
}
// Step 4: Download and parse the manifest
log.Info("Downloading manifest", "snapshot_id", lastS3Snapshot)
manifest, err := app.downloadManifest(ctx, lastS3Snapshot)
if err != nil {
return fmt.Errorf("downloading manifest: %w", err)
}
log.Info("Manifest loaded", "blob_count", len(manifest.Blobs))
// Step 5: Build set of referenced blobs
referencedBlobs := make(map[string]bool)
for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = true
}
// Step 6: List all blobs in S3
log.Info("Listing all blobs in S3")
blobPrefix := "blobs/"
var totalBlobs int
var unreferencedBlobs []s3.ObjectInfo
var unreferencedSize int64
objectCh = app.S3Client.ListObjectsStream(ctx, blobPrefix, true)
for obj := range objectCh {
if obj.Err != nil {
return fmt.Errorf("listing blobs: %w", obj.Err)
}
totalBlobs++
// Extract blob hash from path like "blobs/ca/fe/cafebabe..."
parts := strings.Split(obj.Key, "/")
if len(parts) == 4 {
blobHash := parts[3]
if !referencedBlobs[blobHash] {
unreferencedBlobs = append(unreferencedBlobs, obj)
unreferencedSize += obj.Size
}
}
}
log.Info("Blob scan complete",
"total_blobs", totalBlobs,
"referenced_blobs", len(referencedBlobs),
"unreferenced_blobs", len(unreferencedBlobs),
"unreferenced_size", humanize.Bytes(uint64(unreferencedSize)))
// Step 7: Delete or report unreferenced blobs
if opts.DryRun {
fmt.Printf("\nDry run mode - would delete %d unreferenced blobs\n", len(unreferencedBlobs))
fmt.Printf("Total size of blobs to delete: %s\n", humanize.Bytes(uint64(unreferencedSize)))
if len(unreferencedBlobs) > 0 {
log.Debug("Unreferenced blobs found", "count", len(unreferencedBlobs))
for _, obj := range unreferencedBlobs {
log.Debug("Would delete blob", "key", obj.Key, "size", humanize.Bytes(uint64(obj.Size)))
}
}
} else {
if len(unreferencedBlobs) == 0 {
fmt.Println("No unreferenced blobs to delete")
return nil
}
fmt.Printf("\nDeleting %d unreferenced blobs (%s)...\n",
len(unreferencedBlobs), humanize.Bytes(uint64(unreferencedSize)))
deletedCount := 0
deletedSize := int64(0)
for _, obj := range unreferencedBlobs {
if err := app.S3Client.RemoveObject(ctx, obj.Key); err != nil {
log.Error("Failed to delete blob", "key", obj.Key, "error", err)
continue
}
deletedCount++
deletedSize += obj.Size
// Show progress every 100 blobs
if deletedCount%100 == 0 {
fmt.Printf(" Deleted %d/%d blobs (%s)...\n",
deletedCount, len(unreferencedBlobs),
humanize.Bytes(uint64(deletedSize)))
}
}
fmt.Printf("\nDeleted %d blobs (%s)\n", deletedCount, humanize.Bytes(uint64(deletedSize)))
}
log.Info("Prune operation completed successfully")
return nil
}
// downloadManifest downloads and decompresses a snapshot manifest
func (app *PruneApp) downloadManifest(ctx context.Context, snapshotID string) (*snapshot.Manifest, error) {
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
// Download the compressed manifest
reader, err := app.S3Client.GetObject(ctx, manifestPath)
if err != nil {
return nil, fmt.Errorf("downloading manifest: %w", err)
}
defer func() { _ = reader.Close() }()
// Decode manifest
manifest, err := snapshot.DecodeManifest(reader)
if err != nil {
return nil, fmt.Errorf("decoding manifest: %w", err)
}
return manifest, nil
}

99
internal/cli/purge.go Normal file
View File

@@ -0,0 +1,99 @@
package cli
import (
"context"
"fmt"
"os"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/vaultik"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// PurgeOptions contains options for the purge command
type PurgeOptions struct {
KeepLatest bool
OlderThan string
Force bool
}
// NewPurgeCommand creates the purge command
func NewPurgeCommand() *cobra.Command {
opts := &PurgeOptions{}
cmd := &cobra.Command{
Use: "purge",
Short: "Purge old snapshots",
Long: `Removes snapshots based on age or count criteria.
This command allows you to:
- Keep only the latest snapshot (--keep-latest)
- Remove snapshots older than a specific duration (--older-than)
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// Validate flags
if !opts.KeepLatest && opts.OlderThan == "" {
return fmt.Errorf("must specify either --keep-latest or --older-than")
}
if opts.KeepLatest && opts.OlderThan != "" {
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
}
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
// Start the purge operation in a goroutine
go func() {
// Run the purge operation
if err := v.PurgeSnapshots(opts.KeepLatest, opts.OlderThan, opts.Force); err != nil {
if err != context.Canceled {
log.Error("Purge operation failed", "error", err)
os.Exit(1)
}
}
// Shutdown the app when purge completes
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping purge operation")
v.Cancel()
return nil
},
})
}),
},
})
},
}
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot")
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)")
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts")
return cmd
}

148
internal/cli/restore.go
View File

@@ -3,19 +3,30 @@ package cli
import (
"context"
"fmt"
"os"
"git.eeqj.de/sneak/vaultik/internal/config"
"git.eeqj.de/sneak/vaultik/internal/database"
"git.eeqj.de/sneak/vaultik/internal/globals"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/s3"
"git.eeqj.de/sneak/vaultik/internal/snapshot"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// RestoreOptions contains options for the restore command
type RestoreOptions struct {
Bucket string
Prefix string
SnapshotID string
TargetDir string
TargetDir string
}
// RestoreApp contains all dependencies needed for restore
type RestoreApp struct {
Globals *globals.Globals
Config *config.Config
Repositories *database.Repositories
S3Client *s3.Client
DB *database.DB
Shutdowner fx.Shutdowner
}
// NewRestoreCommand creates the restore command
@@ -23,61 +34,104 @@ func NewRestoreCommand() *cobra.Command {
opts := &RestoreOptions{}
cmd := &cobra.Command{
Use: "restore",
Use: "restore <snapshot-id> <target-dir>",
Short: "Restore files from backup",
Long: `Download and decrypt files from a backup snapshot`,
Args: cobra.NoArgs,
Long: `Download and decrypt files from a backup snapshot.
This command will restore all files from the specified snapshot to the target directory.
The age_secret_key must be configured in the config file for decryption.`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
// Validate required flags
if opts.Bucket == "" {
return fmt.Errorf("--bucket is required")
snapshotID := args[0]
opts.TargetDir = args[1]
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
if opts.Prefix == "" {
return fmt.Errorf("--prefix is required")
}
if opts.SnapshotID == "" {
return fmt.Errorf("--snapshot is required")
}
if opts.TargetDir == "" {
return fmt.Errorf("--target is required")
}
return runRestore(cmd.Context(), opts)
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{
snapshot.Module,
s3.Module,
fx.Provide(fx.Annotate(
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
s3Client *s3.Client, db *database.DB, shutdowner fx.Shutdowner) *RestoreApp {
return &RestoreApp{
Globals: g,
Config: cfg,
Repositories: repos,
S3Client: s3Client,
DB: db,
Shutdowner: shutdowner,
}
},
)),
},
Invokes: []fx.Option{
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
// Start the restore operation in a goroutine
go func() {
// Run the restore operation
if err := app.runRestore(ctx, snapshotID, opts); err != nil {
if err != context.Canceled {
log.Error("Restore operation failed", "error", err)
}
}
// Shutdown the app when restore completes
if err := app.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping restore operation")
return nil
},
})
}),
},
})
},
}
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID to restore")
cmd.Flags().StringVar(&opts.TargetDir, "target", "", "Target directory for restore")
return cmd
}
func runRestore(ctx context.Context, opts *RestoreOptions) error {
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
// runRestore executes the restore operation
func (app *RestoreApp) runRestore(ctx context.Context, snapshotID string, opts *RestoreOptions) error {
// Check for age_secret_key
if app.Config.AgeSecretKey == "" {
return fmt.Errorf("age_secret_key missing from config - required for restore")
}
app := fx.New(
fx.Supply(opts),
fx.Provide(globals.New),
// Additional modules will be added here
fx.Invoke(func(g *globals.Globals) error {
// TODO: Implement restore logic
fmt.Printf("Restoring snapshot %s to %s\n", opts.SnapshotID, opts.TargetDir)
return nil
}),
fx.NopLogger,
log.Info("Starting restore operation",
"snapshot_id", snapshotID,
"target_dir", opts.TargetDir,
"bucket", app.Config.S3.Bucket,
"prefix", app.Config.S3.Prefix,
)
if err := app.Start(ctx); err != nil {
return fmt.Errorf("failed to start restore: %w", err)
}
defer func() {
if err := app.Stop(ctx); err != nil {
fmt.Printf("error stopping app: %v\n", err)
}
}()
// TODO: Implement restore logic
// 1. Download and decrypt database from S3
// 2. Download and decrypt blobs
// 3. Reconstruct files from chunks
// 4. Write files to target directory with proper metadata
fmt.Printf("Restoring snapshot %s to %s\n", snapshotID, opts.TargetDir)
fmt.Println("TODO: Implement restore logic")
return nil
}

1
internal/cli/root.go
View File

@@ -43,6 +43,7 @@ on the source system.`,
NewFetchCommand(),
NewStoreCommand(),
NewSnapshotCommand(),
NewInfoCommand(),
)
return cmd

941
internal/cli/snapshot.go
View File

File diff suppressed because it is too large Load Diff

10
internal/cli/vaultik_snapshot_types.go Normal file
View File

@@ -0,0 +1,10 @@
package cli
import "time"
// SnapshotInfo represents snapshot information for listing
type SnapshotInfo struct {
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`
CompressedSize int64 `json:"compressed_size"`
}

136
internal/cli/verify.go
View File

@@ -2,85 +2,93 @@ package cli
import (
"context"
"fmt"
"os"
"git.eeqj.de/sneak/vaultik/internal/globals"
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/vaultik"
"github.com/spf13/cobra"
"go.uber.org/fx"
)
// VerifyOptions contains options for the verify command
type VerifyOptions struct {
Bucket string
Prefix string
SnapshotID string
Quick bool
}
// NewVerifyCommand creates the verify command
func NewVerifyCommand() *cobra.Command {
opts := &VerifyOptions{}
opts := &vaultik.VerifyOptions{}
cmd := &cobra.Command{
Use: "verify",
Short: "Verify backup integrity",
Long: `Check that all referenced blobs exist and verify metadata integrity`,
Args: cobra.NoArgs,
Use: "verify <snapshot-id>",
Short: "Verify snapshot integrity",
Long: `Verifies that all blobs referenced in a snapshot exist and optionally verifies their contents.
Shallow verification (default):
- Downloads and decompresses manifest
- Checks existence of all blobs in S3
- Reports missing blobs
Deep verification (--deep):
- Downloads and decrypts database
- Verifies blob lists match between manifest and database
- Downloads, decrypts, and decompresses each blob
- Verifies SHA256 hash of each chunk matches database
- Ensures chunks are ordered correctly
The command will fail immediately on any verification error and exit with non-zero status.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
// Validate required flags
if opts.Bucket == "" {
return fmt.Errorf("--bucket is required")
snapshotID := args[0]
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
if opts.Prefix == "" {
return fmt.Errorf("--prefix is required")
}
return runVerify(cmd.Context(), opts)
// Use the app framework for all verification
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
// Run the verify operation directly
go func() {
var err error
if opts.Deep {
err = v.RunDeepVerify(snapshotID, opts)
} else {
err = v.VerifySnapshot(snapshotID, false)
}
if err != nil {
if err != context.Canceled {
log.Error("Verification failed", "error", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping verify operation")
v.Cancel()
return nil
},
})
}),
},
})
},
}
cmd.Flags().StringVar(&opts.Bucket, "bucket", "", "S3 bucket name")
cmd.Flags().StringVar(&opts.Prefix, "prefix", "", "S3 prefix")
cmd.Flags().StringVar(&opts.SnapshotID, "snapshot", "", "Snapshot ID to verify (optional, defaults to latest)")
cmd.Flags().BoolVar(&opts.Quick, "quick", false, "Perform quick verification by checking blob existence and S3 content hashes without downloading")
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Perform deep verification by downloading and verifying all blob contents")
return cmd
}
func runVerify(ctx context.Context, opts *VerifyOptions) error {
if os.Getenv("VAULTIK_PRIVATE_KEY") == "" {
return fmt.Errorf("VAULTIK_PRIVATE_KEY environment variable must be set")
}
app := fx.New(
fx.Supply(opts),
fx.Provide(globals.New),
// Additional modules will be added here
fx.Invoke(func(g *globals.Globals) error {
// TODO: Implement verify logic
if opts.SnapshotID == "" {
fmt.Printf("Verifying latest snapshot in bucket %s with prefix %s\n", opts.Bucket, opts.Prefix)
} else {
fmt.Printf("Verifying snapshot %s in bucket %s with prefix %s\n", opts.SnapshotID, opts.Bucket, opts.Prefix)
}
if opts.Quick {
fmt.Println("Performing quick verification")
} else {
fmt.Println("Performing deep verification")
}
return nil
}),
fx.NopLogger,
)
if err := app.Start(ctx); err != nil {
return fmt.Errorf("failed to start verify: %w", err)
}
defer func() {
if err := app.Stop(ctx); err != nil {
fmt.Printf("error stopping app: %v\n", err)
}
}()
return nil
}

18
internal/config/config.go
View File

@@ -5,6 +5,7 @@ import (
"os"
"time"
"git.eeqj.de/sneak/smartconfig"
"go.uber.org/fx"
"gopkg.in/yaml.v3"
)
@@ -15,6 +16,7 @@ import (
// Configuration is typically loaded from a YAML file.
type Config struct {
AgeRecipients []string `yaml:"age_recipients"`
AgeSecretKey string `yaml:"age_secret_key"`
BackupInterval time.Duration `yaml:"backup_interval"`
BlobSizeLimit Size `yaml:"blob_size_limit"`
ChunkSize Size `yaml:"chunk_size"`
@@ -65,13 +67,14 @@ func New(path ConfigPath) (*Config, error) {
// Load reads and parses the configuration file from the specified path.
// It applies default values for optional fields, performs environment variable
// substitution for certain fields (like IndexPath), and validates the configuration.
// substitution using smartconfig, and validates the configuration.
// The configuration file should be in YAML format. Returns an error if the file
// cannot be read, parsed, or if validation fails.
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
// Load config using smartconfig for interpolation
sc, err := smartconfig.NewFromConfigPath(path)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
return nil, fmt.Errorf("failed to load config file: %w", err)
}
cfg := &Config{
@@ -85,7 +88,14 @@ func Load(path string) (*Config, error) {
CompressionLevel: 3,
}
if err := yaml.Unmarshal(data, cfg); err != nil {
// Convert smartconfig data to YAML then unmarshal
configData := sc.Data()
yamlBytes, err := yaml.Marshal(configData)
if err != nil {
return nil, fmt.Errorf("failed to marshal config data: %w", err)
}
if err := yaml.Unmarshal(yamlBytes, cfg); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}

9
internal/config/size.go
View File

@@ -51,3 +51,12 @@ func (s Size) Int64() int64 {
func (s Size) String() string {
return humanize.Bytes(uint64(s))
}
// ParseSize parses a size string into a Size value
func ParseSize(s string) (Size, error) {
bytes, err := humanize.ParseBytes(s)
if err != nil {
return 0, fmt.Errorf("invalid size format: %w", err)
}
return Size(bytes), nil
}

64
internal/crypto/encryption.go
View File

@@ -7,6 +7,7 @@ import (
"sync"
"filippo.io/age"
"go.uber.org/fx"
)
// Encryptor provides thread-safe encryption using the age encryption library.
@@ -143,3 +144,66 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
return nil
}
// Decryptor provides thread-safe decryption using the age encryption library.
// It uses a private key to decrypt data that was encrypted for the corresponding
// public key.
type Decryptor struct {
identity age.Identity
mu sync.RWMutex
}
// NewDecryptor creates a new decryptor with the given age private key.
// The private key should be a valid age X25519 identity string.
// Returns an error if the private key is invalid.
func NewDecryptor(privateKey string) (*Decryptor, error) {
identity, err := age.ParseX25519Identity(privateKey)
if err != nil {
return nil, fmt.Errorf("parsing age identity: %w", err)
}
return &Decryptor{
identity: identity,
}, nil
}
// Decrypt decrypts data using age decryption.
// This method is suitable for small to medium amounts of data that fit in memory.
// For large data streams, use DecryptStream instead.
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(bytes.NewReader(data), identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
decrypted, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading decrypted data: %w", err)
}
return decrypted, nil
}
// DecryptStream returns a reader that decrypts data from the provided reader.
// This method is suitable for decrypting large files or streams as it processes
// data in a streaming fashion without loading everything into memory.
// The caller should close the input reader when done.
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(src, identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
return r, nil
}
// Module exports the crypto module for fx dependency injection.
var Module = fx.Module("crypto")

6
internal/database/chunks.go
View File

@@ -139,7 +139,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
return chunks, rows.Err()
}
// DeleteOrphaned deletes chunks that are not referenced by any file
// DeleteOrphaned deletes chunks that are not referenced by any file or blob
func (r *ChunkRepository) DeleteOrphaned(ctx context.Context) error {
query := `
DELETE FROM chunks
@@ -147,6 +147,10 @@ func (r *ChunkRepository) DeleteOrphaned(ctx context.Context) error {
SELECT 1 FROM file_chunks
WHERE file_chunks.chunk_hash = chunks.chunk_hash
)
AND NOT EXISTS (
SELECT 1 FROM blob_chunks
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
)
`
result, err := r.db.ExecWithLog(ctx, query)

30
internal/database/snapshots.go
View File

@@ -337,6 +337,24 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
return blobs, rows.Err()
}
// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) {
query := `
SELECT COALESCE(SUM(b.compressed_size), 0)
FROM snapshot_blobs sb
JOIN blobs b ON sb.blob_hash = b.blob_hash
WHERE sb.snapshot_id = ?
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying total compressed size: %w", err)
}
return totalSize, nil
}
// GetIncompleteSnapshots returns all snapshots that haven't been completed
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
query := `
@@ -474,3 +492,15 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID
return nil
}
// DeleteSnapshotUploads removes all uploads entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error {
query := `DELETE FROM uploads WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads: %w", err)
}
return nil
}

73
internal/snapshot/progress.go
View File

@@ -22,6 +22,9 @@ const (
// DetailInterval defines how often multi-line detailed status reports are printed.
// These reports include comprehensive statistics about files, chunks, blobs, and uploads.
DetailInterval = 60 * time.Second
// UploadProgressInterval defines how often upload progress messages are logged.
UploadProgressInterval = 15 * time.Second
)
// ProgressStats holds atomic counters for progress tracking
@@ -52,9 +55,10 @@ type ProgressStats struct {
// UploadInfo tracks current upload progress
type UploadInfo struct {
BlobHash string
Size int64
StartTime time.Time
BlobHash string
Size int64
StartTime time.Time
LastLogTime time.Time
}
// ProgressReporter handles periodic progress reporting
@@ -330,6 +334,11 @@ func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload to S3",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
@@ -377,36 +386,34 @@ func (pr *ProgressReporter) UpdateChunkingActivity() {
func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) {
// Update the current upload info with progress
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
// Format speed in bits/second
bitsPerSec := instantSpeed * 8
var speedStr string
if bitsPerSec >= 1e9 {
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9)
} else if bitsPerSec >= 1e6 {
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6)
} else if bitsPerSec >= 1e3 {
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3)
} else {
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * 8
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * 100
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:8]+"...",
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"total", humanize.Bytes(uint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
percent := float64(bytesUploaded) / float64(totalSize) * 100
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:8]+"...",
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"total", humanize.Bytes(uint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
}
}

76
internal/snapshot/scanner.go
View File

@@ -69,8 +69,10 @@ type ScannerConfig struct {
type ScanResult struct {
FilesScanned int
FilesSkipped int
FilesDeleted int
BytesScanned int64
BytesSkipped int64
BytesDeleted int64
ChunksCreated int
BlobsCreated int
StartTime time.Time
@@ -138,6 +140,11 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
defer s.progress.Stop()
}
// Phase 0: Check for deleted files from previous snapshots
if err := s.detectDeletedFiles(ctx, path, result); err != nil {
return nil, fmt.Errorf("detecting deleted files: %w", err)
}
// Phase 1: Scan directory and collect files to process
log.Info("Phase 1/3: Scanning directory structure")
filesToProcess, err := s.scanPhase(ctx, path, result)
@@ -163,28 +170,29 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
"files_skipped", result.FilesSkipped,
"bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped)))
// Print detailed scan summary
fmt.Printf("\n=== Scan Summary ===\n")
fmt.Printf("Total files examined: %d\n", result.FilesScanned)
fmt.Printf("Files with content changes: %d\n", len(filesToProcess))
fmt.Printf("Files with unchanged content: %d\n", result.FilesSkipped)
fmt.Printf("Total size of changed files: %s\n", humanize.Bytes(uint64(totalSizeToProcess)))
fmt.Printf("Total size of unchanged files: %s\n", humanize.Bytes(uint64(result.BytesSkipped)))
if len(filesToProcess) > 0 {
fmt.Printf("\nStarting snapshot of %d changed files...\n\n", len(filesToProcess))
} else {
fmt.Printf("\nNo file contents have changed.\n")
fmt.Printf("Creating metadata-only snapshot to capture current state...\n\n")
// Print scan summary
fmt.Printf("Scan complete: %s examined (%s), %s to process (%s)",
formatNumber(result.FilesScanned),
humanize.Bytes(uint64(totalSizeToProcess+result.BytesSkipped)),
formatNumber(len(filesToProcess)),
humanize.Bytes(uint64(totalSizeToProcess)))
if result.FilesDeleted > 0 {
fmt.Printf(", %s deleted (%s)",
formatNumber(result.FilesDeleted),
humanize.Bytes(uint64(result.BytesDeleted)))
}
fmt.Println()
// Phase 2: Process files and create chunks
if len(filesToProcess) > 0 {
fmt.Printf("Processing %s files...\n", formatNumber(len(filesToProcess)))
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
return nil, fmt.Errorf("process phase failed: %w", err)
}
} else {
log.Info("Phase 2/3: Skipping (no file contents changed, metadata-only snapshot)")
fmt.Printf("No files need processing. Creating metadata-only snapshot.\n")
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
}
// Get final stats from packer
@@ -266,10 +274,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
changedCount := len(filesToProcess)
mu.Unlock()
fmt.Printf("Scan progress: %d files examined, %s total size, %d files changed\n",
filesScanned,
humanize.Bytes(uint64(bytesScanned)),
changedCount)
fmt.Printf("Scan progress: %s files examined, %s changed\n",
formatNumber(int(filesScanned)),
formatNumber(changedCount))
lastStatusTime = time.Now()
}
@@ -320,8 +327,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
eta = elapsed / time.Duration(filesProcessed) * time.Duration(remaining)
}
fmt.Printf("Snapshot progress: %d/%d files processed, %d chunks created, %d blobs uploaded",
filesProcessed, totalFiles, result.ChunksCreated, result.BlobsCreated)
fmt.Printf("Progress: %s/%s files", formatNumber(filesProcessed), formatNumber(totalFiles))
if remaining > 0 && eta > 0 {
fmt.Printf(", ETA: %s", eta.Round(time.Second))
}
@@ -558,8 +564,6 @@ func (s *Scanner) associateExistingChunks(ctx context.Context, path string) erro
// handleBlobReady is called by the packer when a blob is finalized
func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
log.Debug("Invoking blob upload handler", "blob_hash", blobWithReader.Hash[:8]+"...")
startTime := time.Now().UTC()
finishedBlob := blobWithReader.FinishedBlob
@@ -854,3 +858,33 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
func (s *Scanner) GetProgress() *ProgressReporter {
return s.progress
}
// detectDeletedFiles finds files that existed in previous snapshots but no longer exist
func (s *Scanner) detectDeletedFiles(ctx context.Context, path string, result *ScanResult) error {
// Get all files with this path prefix from the database
files, err := s.repos.Files.ListByPrefix(ctx, path)
if err != nil {
return fmt.Errorf("listing files by prefix: %w", err)
}
for _, file := range files {
// Check if the file still exists on disk
_, err := s.fs.Stat(file.Path)
if os.IsNotExist(err) {
// File has been deleted
result.FilesDeleted++
result.BytesDeleted += file.Size
log.Debug("Detected deleted file", "path", file.Path, "size", file.Size)
}
}
return nil
}
// formatNumber formats a number with comma separators
func formatNumber(n int) string {
if n < 1000 {
return fmt.Sprintf("%d", n)
}
return humanize.Comma(int64(n))
}

60
internal/snapshot/snapshot.go
View File

@@ -214,7 +214,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
log.Debug("Database copy complete", "size", getFileSize(tempDBPath))
// Step 2: Clean the temp database to only contain current snapshot data
log.Debug("Cleaning temporary database to contain only current snapshot data", "snapshot_id", snapshotID, "db_path", tempDBPath)
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
if err != nil {
return fmt.Errorf("cleaning snapshot database: %w", err)
@@ -231,29 +231,27 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
// Step 3: Dump the cleaned database to SQL
dumpPath := filepath.Join(tempDir, "snapshot.sql")
log.Debug("Dumping database to SQL", "source", tempDBPath, "destination", dumpPath)
if err := sm.dumpDatabase(tempDBPath, dumpPath); err != nil {
return fmt.Errorf("dumping database: %w", err)
}
log.Debug("SQL dump complete", "size", getFileSize(dumpPath))
log.Debug("SQL dump complete", "size", humanize.Bytes(uint64(getFileSize(dumpPath))))
// Step 4: Compress and encrypt the SQL dump
compressedPath := filepath.Join(tempDir, "snapshot.sql.zst.age")
log.Debug("Compressing and encrypting SQL dump", "source", dumpPath, "destination", compressedPath)
if err := sm.compressDump(dumpPath, compressedPath); err != nil {
return fmt.Errorf("compressing dump: %w", err)
}
log.Debug("Compression complete", "original_size", getFileSize(dumpPath), "compressed_size", getFileSize(compressedPath))
log.Debug("Compression complete",
"original_size", humanize.Bytes(uint64(getFileSize(dumpPath))),
"compressed_size", humanize.Bytes(uint64(getFileSize(compressedPath))))
// Step 5: Read compressed and encrypted data for upload
log.Debug("Reading compressed and encrypted data for upload", "path", compressedPath)
finalData, err := os.ReadFile(compressedPath)
if err != nil {
return fmt.Errorf("reading compressed dump: %w", err)
}
// Step 6: Generate blob manifest (before closing temp DB)
log.Debug("Generating blob manifest from temporary database", "db_path", tempDBPath)
blobManifest, err := sm.generateBlobManifest(ctx, tempDBPath, snapshotID)
if err != nil {
return fmt.Errorf("generating blob manifest: %w", err)
@@ -263,7 +261,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
// Upload database backup (compressed and encrypted)
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID)
log.Debug("Uploading snapshot database to S3", "key", dbKey, "size", len(finalData))
dbUploadStart := time.Now()
if err := sm.s3Client.PutObject(ctx, dbKey, bytes.NewReader(finalData)); err != nil {
return fmt.Errorf("uploading snapshot database: %w", err)
@@ -278,7 +275,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
// Upload blob manifest (compressed only, not encrypted)
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
log.Debug("Uploading blob manifest to S3", "key", manifestKey, "size", len(blobManifest))
manifestUploadStart := time.Now()
if err := sm.s3Client.PutObject(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil {
return fmt.Errorf("uploading blob manifest: %w", err)
@@ -411,7 +407,6 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
stats.CompressedSize = compressedSize.Int64
stats.UncompressedSize = uncompressedSize.Int64
log.Debug("[Temp DB Cleanup] Database cleanup complete", "stats", stats)
return stats, nil
}
@@ -425,7 +420,7 @@ func (sm *SnapshotManager) dumpDatabase(dbPath, dumpPath string) error {
return fmt.Errorf("running sqlite3 dump: %w", err)
}
log.Debug("SQL dump generated", "size", len(output))
log.Debug("SQL dump generated", "size", humanize.Bytes(uint64(len(output))))
if err := os.WriteFile(dumpPath, output, 0644); err != nil {
return fmt.Errorf("writing dump file: %w", err)
}
@@ -435,43 +430,43 @@ func (sm *SnapshotManager) dumpDatabase(dbPath, dumpPath string) error {
// compressDump compresses the SQL dump using zstd
func (sm *SnapshotManager) compressDump(inputPath, outputPath string) error {
log.Debug("Opening SQL dump for compression", "path", inputPath)
input, err := os.Open(inputPath)
if err != nil {
return fmt.Errorf("opening input file: %w", err)
}
defer func() {
log.Debug("Closing input file", "path", inputPath)
if err := input.Close(); err != nil {
log.Debug("Failed to close input file", "path", inputPath, "error", err)
}
}()
log.Debug("Creating output file for compressed and encrypted data", "path", outputPath)
output, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("creating output file: %w", err)
}
defer func() {
log.Debug("Closing output file", "path", outputPath)
if err := output.Close(); err != nil {
log.Debug("Failed to close output file", "path", outputPath, "error", err)
}
}()
// Use blobgen for compression and encryption
log.Debug("Creating compressor/encryptor", "level", sm.config.CompressionLevel)
log.Debug("Compressing and encrypting data")
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
if err != nil {
return fmt.Errorf("creating blobgen writer: %w", err)
}
// Track if writer has been closed to avoid double-close
writerClosed := false
defer func() {
if err := writer.Close(); err != nil {
log.Debug("Failed to close writer", "error", err)
if !writerClosed {
if err := writer.Close(); err != nil {
log.Debug("Failed to close writer", "error", err)
}
}
}()
log.Debug("Compressing and encrypting data")
if _, err := io.Copy(writer, input); err != nil {
return fmt.Errorf("compressing data: %w", err)
}
@@ -480,6 +475,7 @@ func (sm *SnapshotManager) compressDump(inputPath, outputPath string) error {
if err := writer.Close(); err != nil {
return fmt.Errorf("closing writer: %w", err)
}
writerClosed = true
log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256()))
@@ -524,7 +520,6 @@ func copyFile(src, dst string) error {
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
log.Debug("Generating blob manifest", "db_path", dbPath, "snapshot_id", snapshotID)
// Open the cleaned database using the database package
db, err := database.New(ctx, dbPath)
@@ -573,7 +568,6 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
}
// Encode manifest
log.Debug("Encoding manifest")
compressedData, err := EncodeManifest(manifest, sm.config.CompressionLevel)
if err != nil {
return nil, fmt.Errorf("encoding manifest: %w", err)
@@ -731,6 +725,17 @@ func (sm *SnapshotManager) cleanupOrphanedData(ctx context.Context) error {
// deleteOtherSnapshots deletes all snapshots except the current one
func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", "keeping", currentSnapshotID)
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
}
uploadsDeleted, _ := uploadResult.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
// Now we can safely delete the snapshots
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
if err != nil {
@@ -842,16 +847,21 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context
return nil
}
// deleteOrphanedChunks deletes chunks not referenced by any file
// deleteOrphanedChunks deletes chunks not referenced by any file or blob
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
database.LogSQL("Execute", `DELETE FROM chunks WHERE NOT EXISTS (SELECT 1 FROM file_chunks WHERE file_chunks.chunk_hash = chunks.chunk_hash)`)
result, err := tx.ExecContext(ctx, `
query := `
DELETE FROM chunks
WHERE NOT EXISTS (
SELECT 1 FROM file_chunks
WHERE file_chunks.chunk_hash = chunks.chunk_hash
)`)
)
AND NOT EXISTS (
SELECT 1 FROM blob_chunks
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}