Add exclude patterns, snapshot prune, and other improvements
- Implement exclude patterns with anchored pattern support: - Patterns starting with / only match from root of source dir - Unanchored patterns match anywhere in path - Support for glob patterns (*.log, .*, **/*.pack) - Directory patterns skip entire subtrees - Add gobwas/glob dependency for pattern matching - Add 16 comprehensive tests for exclude functionality - Add snapshot prune command to clean orphaned data: - Removes incomplete snapshots from database - Cleans orphaned files, chunks, and blobs - Runs automatically at backup start for consistency - Add snapshot remove command for deleting snapshots - Add VAULTIK_AGE_SECRET_KEY environment variable support - Fix duplicate fx module provider in restore command - Change snapshot ID format to hostname_YYYY-MM-DDTHH:MM:SSZ
This commit is contained in:
@@ -43,8 +43,12 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
// CRITICAL: This MUST succeed. If we fail to clean up incomplete snapshots,
|
||||
// the deduplication logic will think files from the incomplete snapshot were
|
||||
// already backed up and skip them, resulting in data loss.
|
||||
if err := v.SnapshotManager.CleanupIncompleteSnapshots(v.ctx, hostname); err != nil {
|
||||
return fmt.Errorf("cleanup incomplete snapshots: %w", err)
|
||||
//
|
||||
// Prune the database before starting: delete incomplete snapshots and orphaned data.
|
||||
// This ensures the database is consistent before we start a new snapshot.
|
||||
// Since we use locking, only one vaultik instance accesses the DB at a time.
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
return fmt.Errorf("prune database: %w", err)
|
||||
}
|
||||
|
||||
if opts.Daemon {
|
||||
@@ -633,21 +637,23 @@ func (v *Vaultik) deleteSnapshot(snapshotID string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Then, delete from local database
|
||||
// Delete related records first to avoid foreign key constraints
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot files", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
// Then, delete from local database (if we have a local database)
|
||||
if v.Repositories != nil {
|
||||
// Delete related records first to avoid foreign key constraints
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot files", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshotID, "error", err)
|
||||
}
|
||||
|
||||
// Now delete the snapshot itself
|
||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil {
|
||||
return fmt.Errorf("deleting snapshot from database: %w", err)
|
||||
// Now delete the snapshot itself
|
||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil {
|
||||
return fmt.Errorf("deleting snapshot from database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -699,3 +705,277 @@ func (v *Vaultik) syncWithRemote() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveOptions contains options for the snapshot remove command
|
||||
type RemoveOptions struct {
|
||||
Force bool
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// RemoveResult contains the result of a snapshot removal
|
||||
type RemoveResult struct {
|
||||
SnapshotID string
|
||||
BlobsDeleted int
|
||||
BytesFreed int64
|
||||
BlobsFailed int
|
||||
}
|
||||
|
||||
// RemoveSnapshot removes a snapshot and any blobs that become orphaned
|
||||
func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*RemoveResult, error) {
|
||||
log.Info("Starting snapshot removal", "snapshot_id", snapshotID)
|
||||
|
||||
result := &RemoveResult{
|
||||
SnapshotID: snapshotID,
|
||||
}
|
||||
|
||||
// Step 1: List all snapshots in storage
|
||||
log.Info("Listing remote snapshots")
|
||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
|
||||
var allSnapshotIDs []string
|
||||
targetExists := false
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing remote snapshots: %w", object.Err)
|
||||
}
|
||||
|
||||
// Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
|
||||
sid := parts[1]
|
||||
// Only add unique snapshot IDs
|
||||
found := false
|
||||
for _, id := range allSnapshotIDs {
|
||||
if id == sid {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
allSnapshotIDs = append(allSnapshotIDs, sid)
|
||||
if sid == snapshotID {
|
||||
targetExists = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !targetExists {
|
||||
return nil, fmt.Errorf("snapshot not found: %s", snapshotID)
|
||||
}
|
||||
|
||||
log.Info("Found snapshots", "total", len(allSnapshotIDs))
|
||||
|
||||
// Step 2: Download target snapshot's manifest
|
||||
log.Info("Downloading target manifest", "snapshot_id", snapshotID)
|
||||
targetManifest, err := v.downloadManifest(snapshotID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("downloading target manifest: %w", err)
|
||||
}
|
||||
|
||||
// Build set of target blob hashes with sizes
|
||||
targetBlobs := make(map[string]int64) // hash -> size
|
||||
for _, blob := range targetManifest.Blobs {
|
||||
targetBlobs[blob.Hash] = blob.CompressedSize
|
||||
}
|
||||
log.Info("Target snapshot has blobs", "count", len(targetBlobs))
|
||||
|
||||
// Step 3: Download manifests from all OTHER snapshots to build "in-use" set
|
||||
inUseBlobs := make(map[string]bool)
|
||||
otherCount := 0
|
||||
|
||||
for _, sid := range allSnapshotIDs {
|
||||
if sid == snapshotID {
|
||||
continue // Skip target snapshot
|
||||
}
|
||||
|
||||
log.Debug("Processing manifest", "snapshot_id", sid)
|
||||
manifest, err := v.downloadManifest(sid)
|
||||
if err != nil {
|
||||
log.Error("Failed to download manifest", "snapshot_id", sid, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, blob := range manifest.Blobs {
|
||||
inUseBlobs[blob.Hash] = true
|
||||
}
|
||||
otherCount++
|
||||
}
|
||||
|
||||
log.Info("Processed other manifests", "count", otherCount, "in_use_blobs", len(inUseBlobs))
|
||||
|
||||
// Step 4: Find orphaned blobs (in target but not in use by others)
|
||||
var orphanedBlobs []string
|
||||
var totalSize int64
|
||||
for hash, size := range targetBlobs {
|
||||
if !inUseBlobs[hash] {
|
||||
orphanedBlobs = append(orphanedBlobs, hash)
|
||||
totalSize += size
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Found orphaned blobs",
|
||||
"count", len(orphanedBlobs),
|
||||
"total_size", humanize.Bytes(uint64(totalSize)),
|
||||
)
|
||||
|
||||
// Show summary
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nSnapshot: %s\n", snapshotID)
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Blobs in snapshot: %d\n", len(targetBlobs))
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Orphaned blobs to delete: %d (%s)\n", len(orphanedBlobs), humanize.Bytes(uint64(totalSize)))
|
||||
|
||||
if opts.DryRun {
|
||||
_, _ = fmt.Fprintln(v.Stdout, "\n[Dry run - no changes made]")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Confirm unless --force is used
|
||||
if !opts.Force {
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nDelete snapshot and %d orphaned blob(s)? [y/N] ", len(orphanedBlobs))
|
||||
var confirm string
|
||||
if _, err := fmt.Fscanln(v.Stdin, &confirm); err != nil {
|
||||
_, _ = fmt.Fprintln(v.Stdout, "Cancelled")
|
||||
return result, nil
|
||||
}
|
||||
if strings.ToLower(confirm) != "y" {
|
||||
_, _ = fmt.Fprintln(v.Stdout, "Cancelled")
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Delete orphaned blobs
|
||||
if len(orphanedBlobs) > 0 {
|
||||
log.Info("Deleting orphaned blobs")
|
||||
for i, hash := range orphanedBlobs {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash)
|
||||
|
||||
if err := v.Storage.Delete(v.ctx, blobPath); err != nil {
|
||||
log.Error("Failed to delete blob", "hash", hash, "error", err)
|
||||
result.BlobsFailed++
|
||||
continue
|
||||
}
|
||||
|
||||
result.BlobsDeleted++
|
||||
result.BytesFreed += targetBlobs[hash]
|
||||
|
||||
// Progress update every 100 blobs
|
||||
if (i+1)%100 == 0 || i == len(orphanedBlobs)-1 {
|
||||
log.Info("Deletion progress",
|
||||
"deleted", i+1,
|
||||
"total", len(orphanedBlobs),
|
||||
"percent", fmt.Sprintf("%.1f%%", float64(i+1)/float64(len(orphanedBlobs))*100),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 6: Delete snapshot metadata
|
||||
log.Info("Deleting snapshot metadata")
|
||||
if err := v.deleteSnapshot(snapshotID); err != nil {
|
||||
return result, fmt.Errorf("deleting snapshot metadata: %w", err)
|
||||
}
|
||||
|
||||
// Print summary
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nRemoved snapshot %s\n", snapshotID)
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Blobs deleted: %d\n", result.BlobsDeleted)
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Storage freed: %s\n", humanize.Bytes(uint64(result.BytesFreed)))
|
||||
if result.BlobsFailed > 0 {
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Blobs failed: %d\n", result.BlobsFailed)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PruneResult contains statistics about the prune operation
|
||||
type PruneResult struct {
|
||||
SnapshotsDeleted int64
|
||||
FilesDeleted int64
|
||||
ChunksDeleted int64
|
||||
BlobsDeleted int64
|
||||
}
|
||||
|
||||
// PruneDatabase removes incomplete snapshots and orphaned files, chunks,
|
||||
// and blobs from the local database. This ensures database consistency
|
||||
// before starting a new backup or on-demand via the prune command.
|
||||
func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
|
||||
log.Info("Pruning database: removing incomplete snapshots and orphaned data")
|
||||
|
||||
result := &PruneResult{}
|
||||
|
||||
// First, delete any incomplete snapshots
|
||||
incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting incomplete snapshots: %w", err)
|
||||
}
|
||||
|
||||
for _, snapshot := range incompleteSnapshots {
|
||||
log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID)
|
||||
// Delete related records first
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshot.ID); err != nil {
|
||||
log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshot.ID); err != nil {
|
||||
log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshot.ID); err != nil {
|
||||
log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshot.ID); err != nil {
|
||||
log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err)
|
||||
} else {
|
||||
result.SnapshotsDeleted++
|
||||
}
|
||||
}
|
||||
|
||||
// Get counts before cleanup for reporting
|
||||
fileCountBefore, _ := v.getTableCount("files")
|
||||
chunkCountBefore, _ := v.getTableCount("chunks")
|
||||
blobCountBefore, _ := v.getTableCount("blobs")
|
||||
|
||||
// Run the cleanup
|
||||
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
|
||||
return nil, fmt.Errorf("cleanup orphaned data: %w", err)
|
||||
}
|
||||
|
||||
// Get counts after cleanup
|
||||
fileCountAfter, _ := v.getTableCount("files")
|
||||
chunkCountAfter, _ := v.getTableCount("chunks")
|
||||
blobCountAfter, _ := v.getTableCount("blobs")
|
||||
|
||||
result.FilesDeleted = fileCountBefore - fileCountAfter
|
||||
result.ChunksDeleted = chunkCountBefore - chunkCountAfter
|
||||
result.BlobsDeleted = blobCountBefore - blobCountAfter
|
||||
|
||||
log.Info("Prune complete",
|
||||
"incomplete_snapshots", result.SnapshotsDeleted,
|
||||
"orphaned_files", result.FilesDeleted,
|
||||
"orphaned_chunks", result.ChunksDeleted,
|
||||
"orphaned_blobs", result.BlobsDeleted,
|
||||
)
|
||||
|
||||
// Print summary
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Prune complete:\n")
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Incomplete snapshots removed: %d\n", result.SnapshotsDeleted)
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Orphaned files removed: %d\n", result.FilesDeleted)
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Orphaned chunks removed: %d\n", result.ChunksDeleted)
|
||||
_, _ = fmt.Fprintf(v.Stdout, " Orphaned blobs removed: %d\n", result.BlobsDeleted)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// getTableCount returns the count of rows in a table
|
||||
func (v *Vaultik) getTableCount(tableName string) (int64, error) {
|
||||
if v.DB == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var count int64
|
||||
query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)
|
||||
err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user