Add deterministic deduplication, rclone backend, and database purge command
- Implement deterministic blob hashing using double SHA256 of uncompressed plaintext data, enabling deduplication even after local DB is cleared - Add Stat() check before blob upload to skip existing blobs in storage - Add rclone storage backend for additional remote storage options - Add 'vaultik database purge' command to erase local state DB - Add 'vaultik remote check' command to verify remote connectivity - Show configured snapshots in 'vaultik snapshot list' output - Skip macOS resource fork files (._*) when listing remote snapshots - Use multi-threaded zstd compression (CPUs - 2 threads) - Add writer tests for double hashing behavior
This commit is contained in:
@@ -1,10 +1,14 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
@@ -108,3 +112,237 @@ func (v *Vaultik) ShowInfo() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotMetadataInfo contains information about a single snapshot's metadata
|
||||
type SnapshotMetadataInfo struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
ManifestSize int64 `json:"manifest_size"`
|
||||
DatabaseSize int64 `json:"database_size"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
BlobCount int `json:"blob_count"`
|
||||
BlobsSize int64 `json:"blobs_size"`
|
||||
}
|
||||
|
||||
// RemoteInfoResult contains all remote storage information
|
||||
type RemoteInfoResult struct {
|
||||
// Storage info
|
||||
StorageType string `json:"storage_type"`
|
||||
StorageLocation string `json:"storage_location"`
|
||||
|
||||
// Snapshot metadata
|
||||
Snapshots []SnapshotMetadataInfo `json:"snapshots"`
|
||||
TotalMetadataSize int64 `json:"total_metadata_size"`
|
||||
TotalMetadataCount int `json:"total_metadata_count"`
|
||||
|
||||
// All blobs on remote
|
||||
TotalBlobCount int `json:"total_blob_count"`
|
||||
TotalBlobSize int64 `json:"total_blob_size"`
|
||||
|
||||
// Referenced blobs (from manifests)
|
||||
ReferencedBlobCount int `json:"referenced_blob_count"`
|
||||
ReferencedBlobSize int64 `json:"referenced_blob_size"`
|
||||
|
||||
// Orphaned blobs
|
||||
OrphanedBlobCount int `json:"orphaned_blob_count"`
|
||||
OrphanedBlobSize int64 `json:"orphaned_blob_size"`
|
||||
}
|
||||
|
||||
// RemoteInfo displays information about remote storage
|
||||
func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
result := &RemoteInfoResult{}
|
||||
|
||||
// Get storage info
|
||||
storageInfo := v.Storage.Info()
|
||||
result.StorageType = storageInfo.Type
|
||||
result.StorageLocation = storageInfo.Location
|
||||
|
||||
if !jsonOutput {
|
||||
fmt.Printf("=== Remote Storage ===\n")
|
||||
fmt.Printf("Type: %s\n", storageInfo.Type)
|
||||
fmt.Printf("Location: %s\n", storageInfo.Location)
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// List all snapshot metadata
|
||||
if !jsonOutput {
|
||||
fmt.Printf("Scanning snapshot metadata...\n")
|
||||
}
|
||||
|
||||
snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
|
||||
|
||||
// Collect metadata files
|
||||
metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
for obj := range metadataCh {
|
||||
if obj.Err != nil {
|
||||
return fmt.Errorf("listing metadata: %w", obj.Err)
|
||||
}
|
||||
|
||||
// Parse key: metadata/<snapshot-id>/<filename>
|
||||
parts := strings.Split(obj.Key, "/")
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
}
|
||||
snapshotID := parts[1]
|
||||
|
||||
if _, exists := snapshotMetadata[snapshotID]; !exists {
|
||||
snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{
|
||||
SnapshotID: snapshotID,
|
||||
}
|
||||
}
|
||||
|
||||
info := snapshotMetadata[snapshotID]
|
||||
filename := parts[2]
|
||||
|
||||
if strings.HasPrefix(filename, "manifest") {
|
||||
info.ManifestSize = obj.Size
|
||||
} else if strings.HasPrefix(filename, "db") {
|
||||
info.DatabaseSize = obj.Size
|
||||
}
|
||||
info.TotalSize = info.ManifestSize + info.DatabaseSize
|
||||
}
|
||||
|
||||
// Sort snapshots by ID for consistent output
|
||||
var snapshotIDs []string
|
||||
for id := range snapshotMetadata {
|
||||
snapshotIDs = append(snapshotIDs, id)
|
||||
}
|
||||
sort.Strings(snapshotIDs)
|
||||
|
||||
// Download and parse all manifests to get referenced blobs
|
||||
if !jsonOutput {
|
||||
fmt.Printf("Downloading %d manifest(s)...\n", len(snapshotIDs))
|
||||
}
|
||||
|
||||
referencedBlobs := make(map[string]int64) // hash -> compressed size
|
||||
|
||||
for _, snapshotID := range snapshotIDs {
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
reader, err := v.Storage.Get(v.ctx, manifestKey)
|
||||
if err != nil {
|
||||
log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
manifest, err := snapshot.DecodeManifest(reader)
|
||||
_ = reader.Close()
|
||||
if err != nil {
|
||||
log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Record blob info from manifest
|
||||
info := snapshotMetadata[snapshotID]
|
||||
info.BlobCount = manifest.BlobCount
|
||||
|
||||
var blobsSize int64
|
||||
for _, blob := range manifest.Blobs {
|
||||
referencedBlobs[blob.Hash] = blob.CompressedSize
|
||||
blobsSize += blob.CompressedSize
|
||||
}
|
||||
info.BlobsSize = blobsSize
|
||||
}
|
||||
|
||||
// Build result snapshots
|
||||
var totalMetadataSize int64
|
||||
for _, id := range snapshotIDs {
|
||||
info := snapshotMetadata[id]
|
||||
result.Snapshots = append(result.Snapshots, *info)
|
||||
totalMetadataSize += info.TotalSize
|
||||
}
|
||||
result.TotalMetadataSize = totalMetadataSize
|
||||
result.TotalMetadataCount = len(snapshotIDs)
|
||||
|
||||
// Calculate referenced blob stats
|
||||
for _, size := range referencedBlobs {
|
||||
result.ReferencedBlobCount++
|
||||
result.ReferencedBlobSize += size
|
||||
}
|
||||
|
||||
// List all blobs on remote
|
||||
if !jsonOutput {
|
||||
fmt.Printf("Scanning blobs...\n")
|
||||
}
|
||||
|
||||
allBlobs := make(map[string]int64) // hash -> size from storage
|
||||
|
||||
blobCh := v.Storage.ListStream(v.ctx, "blobs/")
|
||||
for obj := range blobCh {
|
||||
if obj.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", obj.Err)
|
||||
}
|
||||
|
||||
// Extract hash from key: blobs/xx/yy/hash
|
||||
parts := strings.Split(obj.Key, "/")
|
||||
if len(parts) < 4 {
|
||||
continue
|
||||
}
|
||||
hash := parts[3]
|
||||
allBlobs[hash] = obj.Size
|
||||
result.TotalBlobCount++
|
||||
result.TotalBlobSize += obj.Size
|
||||
}
|
||||
|
||||
// Calculate orphaned blobs
|
||||
for hash, size := range allBlobs {
|
||||
if _, referenced := referencedBlobs[hash]; !referenced {
|
||||
result.OrphanedBlobCount++
|
||||
result.OrphanedBlobSize += size
|
||||
}
|
||||
}
|
||||
|
||||
// Output results
|
||||
if jsonOutput {
|
||||
enc := json.NewEncoder(v.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(result)
|
||||
}
|
||||
|
||||
// Human-readable output
|
||||
fmt.Printf("\n=== Snapshot Metadata ===\n")
|
||||
if len(result.Snapshots) == 0 {
|
||||
fmt.Printf("No snapshots found\n")
|
||||
} else {
|
||||
fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
|
||||
fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
for _, info := range result.Snapshots {
|
||||
fmt.Printf("%-45s %12s %12s %12s %10s %12s\n",
|
||||
truncateString(info.SnapshotID, 45),
|
||||
humanize.Bytes(uint64(info.ManifestSize)),
|
||||
humanize.Bytes(uint64(info.DatabaseSize)),
|
||||
humanize.Bytes(uint64(info.TotalSize)),
|
||||
humanize.Comma(int64(info.BlobCount)),
|
||||
humanize.Bytes(uint64(info.BlobsSize)),
|
||||
)
|
||||
}
|
||||
fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
fmt.Printf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
|
||||
}
|
||||
|
||||
fmt.Printf("\n=== Blob Storage ===\n")
|
||||
fmt.Printf("Total blobs on remote: %s (%s)\n",
|
||||
humanize.Comma(int64(result.TotalBlobCount)),
|
||||
humanize.Bytes(uint64(result.TotalBlobSize)))
|
||||
fmt.Printf("Referenced by snapshots: %s (%s)\n",
|
||||
humanize.Comma(int64(result.ReferencedBlobCount)),
|
||||
humanize.Bytes(uint64(result.ReferencedBlobSize)))
|
||||
fmt.Printf("Orphaned (unreferenced): %s (%s)\n",
|
||||
humanize.Comma(int64(result.OrphanedBlobCount)),
|
||||
humanize.Bytes(uint64(result.OrphanedBlobSize)))
|
||||
|
||||
if result.OrphanedBlobCount > 0 {
|
||||
fmt.Printf("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// truncateString truncates a string to maxLen, adding "..." if truncated
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
if maxLen <= 3 {
|
||||
return s[:maxLen]
|
||||
}
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user