Add deterministic deduplication, rclone backend, and database purge command
- Implement deterministic blob hashing using double SHA256 of uncompressed plaintext data, enabling deduplication even after local DB is cleared - Add Stat() check before blob upload to skip existing blobs in storage - Add rclone storage backend for additional remote storage options - Add 'vaultik database purge' command to erase local state DB - Add 'vaultik remote check' command to verify remote connectivity - Show configured snapshots in 'vaultik snapshot list' output - Skip macOS resource fork files (._*) when listing remote snapshots - Use multi-threaded zstd compression (CPUs - 2 threads) - Add writer tests for double hashing behavior
This commit is contained in:
@@ -111,10 +111,11 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
var jsonOutput bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List all snapshots",
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
Args: cobra.NoArgs,
|
||||
Use: "list",
|
||||
Aliases: []string{"ls"},
|
||||
Short: "List all snapshots",
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
@@ -242,7 +243,16 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: "Verifies that all blobs referenced in a snapshot exist",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
@@ -266,7 +276,13 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
@@ -302,17 +318,37 @@ func newSnapshotRemoveCommand() *cobra.Command {
|
||||
opts := &vaultik.RemoveOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "remove <snapshot-id>",
|
||||
Use: "remove [snapshot-id]",
|
||||
Aliases: []string{"rm"},
|
||||
Short: "Remove a snapshot and its orphaned blobs",
|
||||
Long: `Removes a snapshot and any blobs that are no longer referenced by other snapshots.
|
||||
Short: "Remove a snapshot from the local database",
|
||||
Long: `Removes a snapshot from the local database.
|
||||
|
||||
This command downloads manifests from all other snapshots to determine which blobs
|
||||
are still in use, then deletes any blobs that would become orphaned.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
By default, only removes from the local database. Use --remote to also remove
|
||||
the snapshot metadata from remote storage.
|
||||
|
||||
Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs
|
||||
after removing snapshots.
|
||||
|
||||
Use --all --force to remove all snapshots.`,
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
all, _ := cmd.Flags().GetBool("all")
|
||||
if all {
|
||||
if len(args) > 0 {
|
||||
_ = cmd.Help()
|
||||
return fmt.Errorf("--all cannot be used with a snapshot ID")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required (or use --all --force)")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -333,7 +369,13 @@ are still in use, then deletes any blobs that would become orphaned.`,
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if _, err := v.RemoveSnapshot(snapshotID, opts); err != nil {
|
||||
var err error
|
||||
if opts.All {
|
||||
_, err = v.RemoveAllSnapshots(opts)
|
||||
} else {
|
||||
_, err = v.RemoveSnapshot(args[0], opts)
|
||||
}
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Failed to remove snapshot", "error", err)
|
||||
@@ -359,8 +401,10 @@ are still in use, then deletes any blobs that would become orphaned.`,
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be deleted without deleting")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output deletion stats as JSON")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
|
||||
cmd.Flags().BoolVar(&opts.Remote, "remote", false, "Also remove snapshot metadata from remote storage")
|
||||
cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user