Add deterministic deduplication, rclone backend, and database purge command

- Implement deterministic blob hashing using double SHA256 of uncompressed
  plaintext data, enabling deduplication even after local DB is cleared
- Add Stat() check before blob upload to skip existing blobs in storage
- Add rclone storage backend for additional remote storage options
- Add 'vaultik database purge' command to erase local state DB
- Add 'vaultik remote check' command to verify remote connectivity
- Show configured snapshots in 'vaultik snapshot list' output
- Skip macOS resource fork files (._*) when listing remote snapshots
- Use multi-threaded zstd compression (CPUs - 2 threads)
- Add writer tests for double hashing behavior
This commit is contained in:
2026-01-28 15:50:17 -08:00
parent bdaaadf990
commit 470bf648c4
26 changed files with 2966 additions and 777 deletions

View File

@@ -111,10 +111,11 @@ func newSnapshotListCommand() *cobra.Command {
var jsonOutput bool
cmd := &cobra.Command{
Use: "list",
Short: "List all snapshots",
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
Args: cobra.NoArgs,
Use: "list",
Aliases: []string{"ls"},
Short: "List all snapshots",
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
@@ -242,7 +243,16 @@ func newSnapshotVerifyCommand() *cobra.Command {
Use: "verify <snapshot-id>",
Short: "Verify snapshot integrity",
Long: "Verifies that all blobs referenced in a snapshot exist",
Args: cobra.ExactArgs(1),
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
snapshotID := args[0]
@@ -266,7 +276,13 @@ func newSnapshotVerifyCommand() *cobra.Command {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil {
var err error
if opts.Deep {
err = v.RunDeepVerify(snapshotID, opts)
} else {
err = v.VerifySnapshotWithOptions(snapshotID, opts)
}
if err != nil {
if err != context.Canceled {
if !opts.JSON {
log.Error("Verification failed", "error", err)
@@ -302,17 +318,37 @@ func newSnapshotRemoveCommand() *cobra.Command {
opts := &vaultik.RemoveOptions{}
cmd := &cobra.Command{
Use: "remove <snapshot-id>",
Use: "remove [snapshot-id]",
Aliases: []string{"rm"},
Short: "Remove a snapshot and its orphaned blobs",
Long: `Removes a snapshot and any blobs that are no longer referenced by other snapshots.
Short: "Remove a snapshot from the local database",
Long: `Removes a snapshot from the local database.
This command downloads manifests from all other snapshots to determine which blobs
are still in use, then deletes any blobs that would become orphaned.`,
Args: cobra.ExactArgs(1),
By default, only removes from the local database. Use --remote to also remove
the snapshot metadata from remote storage.
Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs
after removing snapshots.
Use --all --force to remove all snapshots.`,
Args: func(cmd *cobra.Command, args []string) error {
all, _ := cmd.Flags().GetBool("all")
if all {
if len(args) > 0 {
_ = cmd.Help()
return fmt.Errorf("--all cannot be used with a snapshot ID")
}
return nil
}
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required (or use --all --force)")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
snapshotID := args[0]
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
@@ -333,7 +369,13 @@ are still in use, then deletes any blobs that would become orphaned.`,
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if _, err := v.RemoveSnapshot(snapshotID, opts); err != nil {
var err error
if opts.All {
_, err = v.RemoveAllSnapshots(opts)
} else {
_, err = v.RemoveSnapshot(args[0], opts)
}
if err != nil {
if err != context.Canceled {
if !opts.JSON {
log.Error("Failed to remove snapshot", "error", err)
@@ -359,8 +401,10 @@ are still in use, then deletes any blobs that would become orphaned.`,
}
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be deleted without deleting")
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output deletion stats as JSON")
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
cmd.Flags().BoolVar(&opts.Remote, "remote", false, "Also remove snapshot metadata from remote storage")
cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)")
return cmd
}