diff --git a/README.md b/README.md index 2c5b1a9..d77ed85 100644 --- a/README.md +++ b/README.md @@ -99,15 +99,13 @@ vaultik [--config ] snapshot create [snapshot-names...] [--cron] [--prune] vaultik [--config ] snapshot list [--json] vaultik [--config ] snapshot verify [--deep] [--json] vaultik [--config ] snapshot purge [--keep-latest | --older-than ] [--snapshot ...] [--force] -vaultik [--config ] snapshot remove [--dry-run] [--force] [--local-only] [--json] -vaultik [--config ] snapshot cleanup +vaultik [--config ] snapshot remove [--dry-run] [--force] [--local-only] [--json] vaultik [--config ] snapshot restore [paths...] [--verify] vaultik [--config ] prune [--force] [--json] vaultik [--config ] info vaultik [--config ] remote info [--json] vaultik [--config ] remote nuke --force -vaultik [--config ] store info -vaultik [--config ] database purge [--force] +vaultik [--config ] database delete [--force] vaultik completion vaultik version ``` @@ -198,7 +196,7 @@ latest globally). * `--snapshot `: Restrict to specific snapshot names (repeat for multiple) * `--force`: Skip confirmation prompt -**`snapshot remove`**: Remove a snapshot. By default this removes the +**`snapshot remove`**: Remove one snapshot. By default this removes the snapshot from the local index and strips the snapshot's metadata from the backup destination store. Blobs are NOT touched — deleting blobs requires reading every remaining remote manifest (the destination store @@ -208,17 +206,13 @@ prune` invocation to run as a follow-up. Local row cleanup (files, chunks, blobs the snapshot was the last referrer for) runs automatically. If the destination store is unreachable, the local-DB removal still completes and a warning is emitted; rerun `vaultik prune` -once the store is reachable to finish remote cleanup. +once the store is reachable to finish remote cleanup. To wipe everything +on the destination in one go, use `vaultik remote nuke --force`. * `--local-only`: Skip remote cleanup; only touch the local index -* `--all`: Remove all snapshots (requires `--force`) * `--dry-run`: Show what would be deleted without deleting * `--force`: Skip confirmation prompt * `--json`: Output result as JSON -**`snapshot cleanup`**: Remove stale local snapshot records that have no -corresponding metadata in remote storage. These are typically left behind -by incomplete or interrupted backups. Does not touch remote storage. - **`snapshot restore`**: Restore files from a backup snapshot. * Requires `VAULTIK_AGE_SECRET_KEY` environment variable * Optional path arguments to restore specific files/directories (default: all) @@ -226,31 +220,38 @@ by incomplete or interrupted backups. Does not touch remote storage. symlinks, and empty directories * `--verify`: After restoring, verify every file's chunk hashes match -**`prune`**: Tidy up everything that isn't needed. Removes orphaned local -database rows (files, chunks, blobs no longer referenced by any completed -snapshot) AND deletes unreferenced blobs from remote storage. `snapshot -create --prune`, `snapshot remove`, and `snapshot purge` run the same -cleanup automatically; this is the manual entry point for the same work. +**`prune`**: Tidy up everything that isn't needed. Runs three passes: +(1) reconcile the local index against the destination store — any +local snapshot whose remote metadata is missing is dropped from the +local index; (2) delete orphaned local rows (files, chunks, blobs no +longer referenced by any completed snapshot); (3) list every remote +manifest on the destination store to compute the still-referenced blob +set and delete any blob not in that set. Step (3) reads all remote +manifests — network cost scales with the number of snapshots. `snapshot +create --prune` runs the same cleanup automatically; this is the +manual entry point for the same work. * `--force`: Skip confirmation prompt * `--json`: Output stats as JSON **`info`**: Display system configuration, storage settings, encryption recipients, and local database statistics. -**`remote info`**: Show detailed remote storage information including per-snapshot -metadata sizes, blob counts, and orphaned blob detection. +**`remote info`**: Show storage backend type and location plus detailed +remote storage inventory: per-snapshot metadata sizes, blob counts, and +orphaned blob detection. * `--json`: Output as JSON **`remote nuke`**: Delete every snapshot's metadata and every blob from the backup destination store, leaving the bucket prefix empty. Destructive and -irreversible. +irreversible. This is the single supported way to wipe the entire +destination store. * `--force`: Required to confirm destruction. -**`store info`**: Display storage backend type and statistics. - -**`database purge`**: Delete the local SQLite state database entirely. Remote -storage is unaffected; the next backup will do a full scan and re-deduplicate -against existing remote blobs. +**`database delete`**: Delete the local SQLite state database file +entirely. Remote storage is unaffected; the next backup will do a full +scan and re-deduplicate against existing remote blobs, and the local +index will re-bind to the currently configured storage destination. +Use this after changing `storage_url` to a different destination. * `--force`: Skip confirmation prompt --- @@ -387,7 +388,7 @@ Key fields: * **Device nodes, named pipes, and sockets are silently skipped.** Only regular files, directories, and symlinks are backed up. * **No database migrations.** If the local SQLite schema changes between - versions, delete the local database (`vaultik database purge`) and run + versions, delete the local database (`vaultik database delete`) and run a full backup. Remote storage is unaffected. * **Files that change during backup may be inconsistent.** There is no filesystem snapshot or freeze. If a file is modified between the scan @@ -459,7 +460,7 @@ priority. first-class operation in this README. Worth a dedicated section once it's settled. * **Schema migrations.** Currently nonexistent — pre-1.0 schema - changes are handled by `vaultik database purge` plus a full + changes are handled by `vaultik database delete` plus a full re-scan. Post-1.0 we'll need a migration story to keep existing index databases usable across upgrades. * **Storage backend coverage tests.** S3, file://, and rclone:// diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index 3ff1c3f..01a6dec 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -7,7 +7,7 @@ Vaultik uses a local SQLite database to track file metadata, chunk mappings, and **Important Notes:** - **No Migration Support (pre-1.0)**: Vaultik does not support database schema migrations. The local index is treated as disposable — if the schema changes, - delete the local SQLite database (`vaultik database purge`) and run a full + delete the local SQLite database (`vaultik database delete`) and run a full backup. The remote storage is unaffected; the new index will re-deduplicate against existing remote blobs. - **Version Compatibility**: In rare cases, you may need to use the same version diff --git a/internal/cli/database.go b/internal/cli/database.go index cc2c1a0..cf83da0 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -18,28 +18,33 @@ func NewDatabaseCommand() *cobra.Command { } cmd.AddCommand( - newDatabasePurgeCommand(), + newDatabaseDeleteCommand(), ) return cmd } -// newDatabasePurgeCommand creates the database purge command -func newDatabasePurgeCommand() *cobra.Command { +// newDatabaseDeleteCommand creates the database delete command. +// (Renamed from "purge"; the operation removes the SQLite file +// entirely, which is a delete, not a purge of content.) +func newDatabaseDeleteCommand() *cobra.Command { var force bool cmd := &cobra.Command{ - Use: "purge", - Short: "Delete the local state database", + Use: "delete", + Short: "Delete the local state database file", Long: `Completely removes the local SQLite state database. This will erase all local tracking of: - File metadata and change detection state - Chunk and blob mappings - Local snapshot records +- The storage-binding record -The remote storage is NOT affected. After purging, the next backup will -perform a full scan and re-deduplicate against existing remote blobs. +The remote storage is NOT affected. After deletion, the next backup +will perform a full scan and re-deduplicate against existing remote +blobs, and the local index will re-bind to the currently configured +storage destination on that run. Use --force to skip the confirmation prompt.`, Args: cobra.NoArgs, @@ -88,10 +93,10 @@ Use --force to skip the confirmation prompt.`, rootFlags := GetRootFlags() if !rootFlags.Quiet { - fmt.Printf("Database purged: %s\n", dbPath) + fmt.Printf("Database deleted: %s\n", dbPath) } - log.Info("Local state database purged", "path", dbPath) + log.Info("Local state database deleted", "path", dbPath) return nil }, } diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index c6ed0b8..cc7e623 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -18,7 +18,7 @@ func TestCLIEntry(t *testing.T) { } // Verify all subcommands are registered - expectedCommands := []string{"config", "snapshot", "store", "prune", "info", "version", "remote", "database"} + expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} for _, expected := range expectedCommands { found := false for _, cmd := range cmd.Commands() { @@ -38,7 +38,7 @@ func TestCLIEntry(t *testing.T) { t.Errorf("Failed to find snapshot command: %v", err) } else { // Check snapshot subcommands - expectedSubCommands := []string{"create", "list", "purge", "verify", "cleanup", "restore"} + expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} for _, expected := range expectedSubCommands { found := false for _, subcmd := range snapshotCmd.Commands() { diff --git a/internal/cli/root.go b/internal/cli/root.go index ecaee21..5c29b14 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -53,7 +53,6 @@ on the source system.`, cmd.AddCommand( NewConfigCommand(), NewPruneCommand(), - NewStoreCommand(), NewSnapshotCommand(), NewInfoCommand(), NewVersionCommand(), diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 7c733ad..ef01eaa 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -25,7 +25,6 @@ func NewSnapshotCommand() *cobra.Command { cmd.AddCommand(newSnapshotPurgeCommand()) cmd.AddCommand(newSnapshotVerifyCommand()) cmd.AddCommand(newSnapshotRemoveCommand()) - cmd.AddCommand(newSnapshotCleanupCommand()) cmd.AddCommand(newSnapshotRestoreCommand()) return cmd @@ -322,7 +321,7 @@ func newSnapshotRemoveCommand() *cobra.Command { opts := &vaultik.RemoveOptions{} cmd := &cobra.Command{ - Use: "remove [snapshot-id]", + Use: "remove ", Aliases: []string{"rm"}, Short: "Remove a snapshot from local index and remote metadata", Long: `Removes a snapshot. @@ -341,20 +340,13 @@ If the remote is unreachable, the local-database removal still completes and a warning is emitted; rerun 'vaultik prune' once the destination store is reachable to finish remote cleanup. -Use --all --force to remove all snapshots.`, +To wipe the entire destination store and start over, use 'vaultik remote +nuke --force' — it is the single supported entry point for that.`, Args: func(cmd *cobra.Command, args []string) error { - all, _ := cmd.Flags().GetBool("all") - if all { - if len(args) > 0 { - _ = cmd.Help() - return fmt.Errorf("--all cannot be used with a snapshot ID") - } - return nil - } if len(args) != 1 { _ = cmd.Help() if len(args) == 0 { - return fmt.Errorf("snapshot ID required (or use --all --force)") + return fmt.Errorf("snapshot ID required") } return fmt.Errorf("expected 1 argument, got %d", len(args)) } @@ -381,12 +373,7 @@ Use --all --force to remove all snapshots.`, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - var err error - if opts.All { - _, err = v.RemoveAllSnapshots(opts) - } else { - _, err = v.RemoveSnapshot(args[0], opts) - } + _, err := v.RemoveSnapshot(args[0], opts) if err != nil { if err != context.Canceled { if !opts.JSON { @@ -417,65 +404,6 @@ Use --all --force to remove all snapshots.`, cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing") cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON") cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, "Skip remote cleanup; only touch the local index") - cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)") - - return cmd -} - -// newSnapshotCleanupCommand creates the 'snapshot cleanup' subcommand -func newSnapshotCleanupCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "cleanup", - Short: "Remove stale local snapshot records not found in remote storage", - Long: `Removes local database records for snapshots whose metadata no longer -exists in remote storage. These are typically left behind by incomplete -or interrupted backups. - -This command does not delete anything from remote storage.`, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - if err := v.CleanupLocalSnapshots(); err != nil { - if err != context.Canceled { - log.Error("Cleanup failed", "error", err) - ReportError("Cleanup failed: %v", err) - os.Exit(1) - } - } - if err := v.Shutdowner.Shutdown(); err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - return nil - }, - }) - }), - }, - }) - }, - } return cmd } diff --git a/internal/cli/store.go b/internal/cli/store.go deleted file mode 100644 index 9c5c7a5..0000000 --- a/internal/cli/store.go +++ /dev/null @@ -1,158 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/spf13/cobra" - "go.uber.org/fx" - "sneak.berlin/go/vaultik/internal/log" - "sneak.berlin/go/vaultik/internal/storage" -) - -// StoreApp contains dependencies for store commands -type StoreApp struct { - Storage storage.Storer - Shutdowner fx.Shutdowner -} - -// NewStoreCommand creates the store command and subcommands -func NewStoreCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "store", - Short: "Storage information commands", - Long: "Commands for viewing information about the storage backend", - } - - // Add subcommands - cmd.AddCommand(newStoreInfoCommand()) - - return cmd -} - -// newStoreInfoCommand creates the 'store info' subcommand -func newStoreInfoCommand() *cobra.Command { - return &cobra.Command{ - Use: "info", - Short: "Display storage information", - Long: "Shows storage configuration and statistics including snapshots and blobs", - RunE: func(cmd *cobra.Command, args []string) error { - return runWithApp(cmd.Context(), func(app *StoreApp) error { - return app.Info(cmd.Context()) - }) - }, - } -} - -// Info displays storage information -func (app *StoreApp) Info(ctx context.Context) error { - // Get storage info - storageInfo := app.Storage.Info() - - fmt.Printf("Storage Information\n") - fmt.Printf("==================\n\n") - fmt.Printf("Storage Configuration:\n") - fmt.Printf(" Type: %s\n", storageInfo.Type) - fmt.Printf(" Location: %s\n\n", storageInfo.Location) - - // Count snapshots by listing metadata/ prefix - snapshotCount := 0 - snapshotCh := app.Storage.ListStream(ctx, "metadata/") - snapshotDirs := make(map[string]bool) - - for object := range snapshotCh { - if object.Err != nil { - return fmt.Errorf("listing snapshots: %w", object.Err) - } - // Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/ - parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - snapshotDirs[parts[1]] = true - } - } - snapshotCount = len(snapshotDirs) - - // Count blobs and calculate total size by listing blobs/ prefix - blobCount := 0 - var totalSize int64 - - blobCh := app.Storage.ListStream(ctx, "blobs/") - for object := range blobCh { - if object.Err != nil { - return fmt.Errorf("listing blobs: %w", object.Err) - } - if !strings.HasSuffix(object.Key, "/") { // Skip directories - blobCount++ - totalSize += object.Size - } - } - - fmt.Printf("Storage Statistics:\n") - fmt.Printf(" Snapshots: %d\n", snapshotCount) - fmt.Printf(" Blobs: %d\n", blobCount) - fmt.Printf(" Total Size: %s\n", formatBytes(totalSize)) - - return nil -} - -// formatBytes formats bytes into human-readable format -func formatBytes(bytes int64) string { - const unit = 1024 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} - -// runWithApp creates the FX app and runs the given function -func runWithApp(ctx context.Context, fn func(*StoreApp) error) error { - var result error - rootFlags := GetRootFlags() - - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - err = RunWithApp(ctx, AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{ - fx.Provide(func(storer storage.Storer, shutdowner fx.Shutdowner) *StoreApp { - return &StoreApp{ - Storage: storer, - Shutdowner: shutdowner, - } - }), - }, - Invokes: []fx.Option{ - fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) { - result = fn(app) - // Shutdown after command completes - go func() { - time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown - if err := shutdowner.Shutdown(); err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - }), - }, - }) - - if err != nil { - return err - } - return result -} diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 1a510ff..8b3cfac 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -58,6 +58,14 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { if err := v.EnsureStorageBinding(); err != nil { return err } + // First reconcile local snapshot records against remote metadata: + // any local snapshot whose manifest is missing from the destination + // store is treated as gone. This used to be the separate 'snapshot + // cleanup' command and is now folded in so a single 'vaultik prune' + // gets the local index fully back in sync with the destination. + if err := v.CleanupLocalSnapshots(); err != nil { + return fmt.Errorf("reconciling local snapshots with remote: %w", err) + } if _, err := v.PruneDatabase(); err != nil { return fmt.Errorf("pruning local database: %w", err) } diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index b61b722..5dd03ec 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -291,7 +291,7 @@ func TestRemoveAllSnapshots_RequiresForce(t *testing.T) { tv := vaultik.NewForTesting(store) - opts := &vaultik.RemoveOptions{All: true} // No Force + opts := &vaultik.RemoveOptions{} // No Force _, err := tv.RemoveAllSnapshots(opts) assert.Error(t, err) @@ -310,7 +310,7 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { tv := vaultik.NewForTesting(store) - opts := &vaultik.RemoveOptions{All: true, Force: true} + opts := &vaultik.RemoveOptions{Force: true} result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) @@ -342,7 +342,7 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) { // Default (no LocalOnly) enumerates the orphan remote keys, which // matches what NewForTesting has — local DB is empty, so the two // addManifest calls land as orphan remote keys. - opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true} + opts := &vaultik.RemoveOptions{Force: true, DryRun: true} result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) @@ -362,7 +362,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) { tv := vaultik.NewForTesting(store) - opts := &vaultik.RemoveOptions{All: true, Force: true} + opts := &vaultik.RemoveOptions{Force: true} result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index c17e080..ba5c017 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -987,7 +987,6 @@ type RemoveOptions struct { DryRun bool JSON bool LocalOnly bool // Skip remote cleanup; only touch the local index - All bool // Remove all snapshots (requires Force) } // RemoveResult contains the result of a snapshot removal diff --git a/internal/vaultik/storage_bind.go b/internal/vaultik/storage_bind.go index f414578..8c1083d 100644 --- a/internal/vaultik/storage_bind.go +++ b/internal/vaultik/storage_bind.go @@ -86,6 +86,6 @@ func buildBindingMismatchMessage(stored, configured string) string { "\n" + "To proceed, either:\n" + " - revert storage_url in your config to the bound destination, or\n" + - " - run 'vaultik database purge' to discard the local index and rebuild it\n" + + " - run 'vaultik database delete' to discard the local index and rebuild it\n" + " from a fresh full backup against the new destination" } diff --git a/internal/vaultik/storage_bind_test.go b/internal/vaultik/storage_bind_test.go index bd3b26e..087031b 100644 --- a/internal/vaultik/storage_bind_test.go +++ b/internal/vaultik/storage_bind_test.go @@ -68,5 +68,5 @@ func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "file:///mnt/backups/old") assert.Contains(t, err.Error(), "file:///mnt/backups/new") - assert.Contains(t, err.Error(), "vaultik database purge") + assert.Contains(t, err.Error(), "vaultik database delete") }