Add deterministic deduplication, rclone backend, and database purge command

- Implement deterministic blob hashing using double SHA256 of uncompressed
  plaintext data, enabling deduplication even after local DB is cleared
- Add Stat() check before blob upload to skip existing blobs in storage
- Add rclone storage backend for additional remote storage options
- Add 'vaultik database purge' command to erase local state DB
- Add 'vaultik remote check' command to verify remote connectivity
- Show configured snapshots in 'vaultik snapshot list' output
- Skip macOS resource fork files (._*) when listing remote snapshots
- Use multi-threaded zstd compression (CPUs - 2 threads)
- Add writer tests for double hashing behavior

internal/vaultik/restore.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"time"
 
@@ -173,7 +172,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 
 // downloadSnapshotDB downloads and decrypts the snapshot metadata database
 func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (*database.DB, error) {
-	// Download encrypted database from S3
+	// Download encrypted database from storage
 	dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID)
 
 	reader, err := v.Storage.Get(v.ctx, dbKey)
@@ -196,43 +195,30 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
 	}
 	defer func() { _ = blobReader.Close() }()
 
-	// Read the SQL dump
-	sqlDump, err := io.ReadAll(blobReader)
+	// Read the binary SQLite database
+	dbData, err := io.ReadAll(blobReader)
 	if err != nil {
 		return nil, fmt.Errorf("decrypting and decompressing: %w", err)
 	}
-	log.Debug("Decrypted database SQL dump", "size", humanize.Bytes(uint64(len(sqlDump))))
+	log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData))))
 
-	// Create a temporary database file
+	// Create a temporary database file and write the binary SQLite data directly
 	tempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.db")
 	if err != nil {
 		return nil, fmt.Errorf("creating temp file: %w", err)
 	}
 	tempPath := tempFile.Name()
+
+	// Write the binary SQLite database directly
+	if _, err := tempFile.Write(dbData); err != nil {
+		_ = tempFile.Close()
+		_ = v.Fs.Remove(tempPath)
+		return nil, fmt.Errorf("writing database file: %w", err)
+	}
 	if err := tempFile.Close(); err != nil {
+		_ = v.Fs.Remove(tempPath)
 		return nil, fmt.Errorf("closing temp file: %w", err)
 	}
-
-	// Write SQL to a temp file for sqlite3 to read
-	sqlTempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.sql")
-	if err != nil {
-		return nil, fmt.Errorf("creating SQL temp file: %w", err)
-	}
-	sqlTempPath := sqlTempFile.Name()
-	if _, err := sqlTempFile.Write(sqlDump); err != nil {
-		_ = sqlTempFile.Close()
-		return nil, fmt.Errorf("writing SQL dump: %w", err)
-	}
-	if err := sqlTempFile.Close(); err != nil {
-		return nil, fmt.Errorf("closing SQL temp file: %w", err)
-	}
-	defer func() { _ = v.Fs.Remove(sqlTempPath) }()
-
-	// Execute the SQL dump to create the database
-	cmd := exec.Command("sqlite3", tempPath, ".read "+sqlTempPath)
-	if output, err := cmd.CombinedOutput(); err != nil {
-		return nil, fmt.Errorf("executing SQL dump: %w\nOutput: %s", err, output)
-	}
 	log.Debug("Created restore database", "path", tempPath)
 
 	// Open the database
@@ -433,13 +419,13 @@ func (v *Vaultik) restoreRegularFile(
 		blobHashStr := blob.Hash.String()
 		blobData, ok := blobCache[blobHashStr]
 		if !ok {
-			blobData, err = v.downloadBlob(ctx, blobHashStr, identity)
+			blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity)
 			if err != nil {
 				return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err)
 			}
 			blobCache[blobHashStr] = blobData
 			result.BlobsDownloaded++
-			result.BytesDownloaded += int64(len(blobData))
+			result.BytesDownloaded += blob.CompressedSize
 		}
 
 		// Extract chunk from blob
@@ -488,41 +474,12 @@ func (v *Vaultik) restoreRegularFile(
 }
 
 // downloadBlob downloads and decrypts a blob
-func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, identity age.Identity) ([]byte, error) {
-	// Construct blob path with sharding
-	blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
-
-	reader, err := v.Storage.Get(ctx, blobPath)
+func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) ([]byte, error) {
+	result, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity)
 	if err != nil {
-		return nil, fmt.Errorf("downloading blob: %w", err)
+		return nil, err
 	}
-	defer func() { _ = reader.Close() }()
-
-	// Read encrypted data
-	encryptedData, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("reading blob data: %w", err)
-	}
-
-	// Decrypt and decompress
-	blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
-	if err != nil {
-		return nil, fmt.Errorf("creating decryption reader: %w", err)
-	}
-	defer func() { _ = blobReader.Close() }()
-
-	data, err := io.ReadAll(blobReader)
-	if err != nil {
-		return nil, fmt.Errorf("decrypting blob: %w", err)
-	}
-
-	log.Debug("Downloaded and decrypted blob",
-		"hash", blobHash[:16],
-		"encrypted_size", humanize.Bytes(uint64(len(encryptedData))),
-		"decrypted_size", humanize.Bytes(uint64(len(data))),
-	)
-
-	return data, nil
+	return result.Data, nil
 }
 
 // verifyRestoredFiles verifies that all restored files match their expected chunk hashes