package vaultik import ( "encoding/json" "fmt" "os" "path/filepath" "regexp" "sort" "strings" "text/tabwriter" "time" "git.eeqj.de/sneak/vaultik/internal/database" "git.eeqj.de/sneak/vaultik/internal/log" "git.eeqj.de/sneak/vaultik/internal/snapshot" "git.eeqj.de/sneak/vaultik/internal/types" "github.com/dustin/go-humanize" ) // SnapshotCreateOptions contains options for the snapshot create command type SnapshotCreateOptions struct { Daemon bool Cron bool Prune bool SkipErrors bool // Skip file read errors (log them loudly but continue) Snapshots []string // Optional list of snapshot names to process (empty = all) } // CreateSnapshot executes the snapshot creation operation func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { overallStartTime := time.Now() log.Info("Starting snapshot creation", "version", v.Globals.Version, "commit", v.Globals.Commit, "index_path", v.Config.IndexPath, ) // Clean up incomplete snapshots FIRST, before any scanning // This is critical for data safety - see CleanupIncompleteSnapshots for details hostname := v.Config.Hostname if hostname == "" { hostname, _ = os.Hostname() } // CRITICAL: This MUST succeed. If we fail to clean up incomplete snapshots, // the deduplication logic will think files from the incomplete snapshot were // already backed up and skip them, resulting in data loss. // // Prune the database before starting: delete incomplete snapshots and orphaned data. // This ensures the database is consistent before we start a new snapshot. // Since we use locking, only one vaultik instance accesses the DB at a time. if _, err := v.PruneDatabase(); err != nil { return fmt.Errorf("prune database: %w", err) } if opts.Daemon { log.Info("Running in daemon mode") // TODO: Implement daemon mode with inotify return fmt.Errorf("daemon mode not yet implemented") } // Determine which snapshots to process snapshotNames := opts.Snapshots if len(snapshotNames) == 0 { snapshotNames = v.Config.SnapshotNames() } else { // Validate requested snapshot names exist for _, name := range snapshotNames { if _, ok := v.Config.Snapshots[name]; !ok { return fmt.Errorf("snapshot %q not found in config", name) } } } if len(snapshotNames) == 0 { return fmt.Errorf("no snapshots configured") } // Process each named snapshot for snapIdx, snapName := range snapshotNames { if err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)); err != nil { return err } } // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { v.printfStdout("\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second)) } // Prune old snapshots and unreferenced blobs if --prune was specified if opts.Prune { log.Info("Pruning enabled - deleting old snapshots and unreferenced blobs") v.printlnStdout("\nPruning old snapshots (keeping latest)...") if err := v.PurgeSnapshots(true, "", true); err != nil { return fmt.Errorf("prune: purging old snapshots: %w", err) } v.printlnStdout("Pruning unreferenced blobs...") if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { return fmt.Errorf("prune: removing unreferenced blobs: %w", err) } log.Info("Pruning complete") } return nil } // snapshotStats tracks aggregate statistics across directory scans type snapshotStats struct { totalFiles int totalBytes int64 totalChunks int totalBlobs int totalBytesSkipped int64 totalFilesSkipped int totalFilesDeleted int totalBytesDeleted int64 totalBytesUploaded int64 totalBlobsUploaded int uploadDuration time.Duration } // createNamedSnapshot creates a single named snapshot func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, snapName string, idx, total int) error { snapshotStartTime := time.Now() if total > 1 { v.printfStdout("\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName) } resolvedDirs, err := v.resolveSnapshotPaths(snapName) if err != nil { return err } scanner := v.ScannerFactory(snapshot.ScannerParams{ EnableProgress: !opts.Cron, Fs: v.Fs, Exclude: v.Config.GetExcludes(snapName), SkipErrors: opts.SkipErrors, }) snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) if err != nil { return fmt.Errorf("creating snapshot: %w", err) } log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) v.printfStdout("Beginning snapshot: %s\n", snapshotID) stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) if err != nil { return err } v.collectUploadStats(scanner, stats) if err := v.finalizeSnapshotMetadata(snapshotID, stats); err != nil { return err } log.Info("Snapshot complete", "snapshot_id", snapshotID, "name", snapName, "files", stats.totalFiles, "blobs_uploaded", stats.totalBlobsUploaded, "bytes_uploaded", stats.totalBytesUploaded, "duration", time.Since(snapshotStartTime)) v.printSnapshotSummary(snapshotID, snapshotStartTime, stats) return nil } // resolveSnapshotPaths resolves source directories to absolute paths with symlink resolution func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { snapConfig := v.Config.Snapshots[snapName] resolvedDirs := make([]string, 0, len(snapConfig.Paths)) for _, dir := range snapConfig.Paths { absPath, err := filepath.Abs(dir) if err != nil { return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err) } resolvedPath, err := filepath.EvalSymlinks(absPath) if err != nil { if os.IsNotExist(err) { resolvedPath = absPath } else { return nil, fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err) } } resolvedDirs = append(resolvedDirs, resolvedPath) } return resolvedDirs, nil } // scanAllDirectories runs the scanner on each resolved directory and accumulates stats func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string) (*snapshotStats, error) { stats := &snapshotStats{} for i, dir := range resolvedDirs { select { case <-v.ctx.Done(): log.Info("Snapshot creation cancelled") return nil, v.ctx.Err() default: } log.Info("Scanning directory", "path", dir) v.printfStdout("Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { return nil, fmt.Errorf("failed to scan %s: %w", dir, err) } stats.totalFiles += result.FilesScanned stats.totalBytes += result.BytesScanned stats.totalChunks += result.ChunksCreated stats.totalBlobs += result.BlobsCreated stats.totalFilesSkipped += result.FilesSkipped stats.totalBytesSkipped += result.BytesSkipped stats.totalFilesDeleted += result.FilesDeleted stats.totalBytesDeleted += result.BytesDeleted log.Info("Directory scan complete", "path", dir, "files", result.FilesScanned, "files_skipped", result.FilesSkipped, "bytes", result.BytesScanned, "bytes_skipped", result.BytesSkipped, "chunks", result.ChunksCreated, "blobs", result.BlobsCreated, "duration", result.EndTime.Sub(result.StartTime)) } return stats, nil } // collectUploadStats gathers upload statistics from the scanner's progress reporter func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) { if s := scanner.GetProgress(); s != nil { progressStats := s.GetStats() stats.totalBytesUploaded = progressStats.BytesUploaded.Load() stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load()) stats.uploadDuration = time.Duration(progressStats.UploadDurationMs.Load()) * time.Millisecond } } // finalizeSnapshotMetadata updates stats, marks complete, and exports metadata func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotStats) error { extStats := snapshot.ExtendedBackupStats{ BackupStats: snapshot.BackupStats{ FilesScanned: stats.totalFiles, BytesScanned: stats.totalBytes, ChunksCreated: stats.totalChunks, BlobsCreated: stats.totalBlobs, BytesUploaded: stats.totalBytesUploaded, }, BlobUncompressedSize: 0, CompressionLevel: v.Config.CompressionLevel, UploadDurationMs: stats.uploadDuration.Milliseconds(), } if err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats); err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } if err := v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID); err != nil { return fmt.Errorf("completing snapshot: %w", err) } if err := v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID); err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } return nil } // formatUploadSpeed formats bytes uploaded and duration into a human-readable speed string func formatUploadSpeed(bytesUploaded int64, duration time.Duration) string { if bytesUploaded <= 0 || duration <= 0 { return "N/A" } bytesPerSec := float64(bytesUploaded) / duration.Seconds() bitsPerSec := bytesPerSec * 8 switch { case bitsPerSec >= 1e9: return fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9) case bitsPerSec >= 1e6: return fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6) case bitsPerSec >= 1e3: return fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3) default: return fmt.Sprintf("%.0f bit/s", bitsPerSec) } } // printSnapshotSummary prints the comprehensive snapshot completion summary func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, stats *snapshotStats) { snapshotDuration := time.Since(startTime) totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped totalBytesAll := stats.totalBytes + stats.totalBytesSkipped // Get total blob sizes from database totalBlobSizeCompressed, totalBlobSizeUncompressed := v.getSnapshotBlobSizes(snapshotID) var compressionRatio float64 if totalBlobSizeUncompressed > 0 { compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed) } else { compressionRatio = 1.0 } v.printfStdout("=== Snapshot Complete ===\n") v.printfStdout("ID: %s\n", snapshotID) v.printfStdout("Files: %s examined, %s to process, %s unchanged", formatNumber(stats.totalFiles), formatNumber(totalFilesChanged), formatNumber(stats.totalFilesSkipped)) if stats.totalFilesDeleted > 0 { v.printfStdout(", %s deleted", formatNumber(stats.totalFilesDeleted)) } v.printlnStdout() v.printfStdout("Data: %s total (%s to process)", humanize.Bytes(uint64(totalBytesAll)), humanize.Bytes(uint64(stats.totalBytes))) if stats.totalBytesDeleted > 0 { v.printfStdout(", %s deleted", humanize.Bytes(uint64(stats.totalBytesDeleted))) } v.printlnStdout() if stats.totalBlobsUploaded > 0 { v.printfStdout("Storage: %s compressed from %s (%.2fx)\n", humanize.Bytes(uint64(totalBlobSizeCompressed)), humanize.Bytes(uint64(totalBlobSizeUncompressed)), compressionRatio) v.printfStdout("Upload: %d blobs, %s in %s (%s)\n", stats.totalBlobsUploaded, humanize.Bytes(uint64(stats.totalBytesUploaded)), formatDuration(stats.uploadDuration), formatUploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration)) } // getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, uncompressed int64) { blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID) if err != nil { return 0, 0 } for _, hash := range blobHashes { if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { compressed += blob.CompressedSize uncompressed += blob.UncompressedSize } } return compressed, uncompressed } // ListSnapshots lists all snapshots func (v *Vaultik) ListSnapshots(jsonOutput bool) error { log.Info("Listing snapshots") remoteSnapshots, err := v.listRemoteSnapshotIDs() if err != nil { return err } localSnapshotMap, err := v.reconcileLocalWithRemote(remoteSnapshots) if err != nil { return err } snapshots, err := v.buildSnapshotInfoList(remoteSnapshots, localSnapshotMap) if err != nil { return err } // Sort by timestamp (newest first) sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) if jsonOutput { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(snapshots) } return v.printSnapshotTable(snapshots) } // listRemoteSnapshotIDs returns a set of snapshot IDs found in remote storage func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) { remoteSnapshots := make(map[string]bool) objectCh := v.Storage.ListStream(v.ctx, "metadata/") for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) } parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { if strings.HasPrefix(parts[1], ".") { continue } remoteSnapshots[parts[1]] = true } } return remoteSnapshots, nil } // reconcileLocalWithRemote removes local snapshots not in remote and returns the surviving local map func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) { localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { return nil, fmt.Errorf("listing local snapshots: %w", err) } localSnapshotMap := make(map[string]*database.Snapshot) for _, s := range localSnapshots { localSnapshotMap[s.ID.String()] = s } for _, snap := range localSnapshots { snapshotIDStr := snap.ID.String() if !remoteSnapshots[snapshotIDStr] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID) if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err) } else { log.Info("Deleted local snapshot not found in remote", "snapshot_id", snap.ID) delete(localSnapshotMap, snapshotIDStr) } } } return localSnapshotMap, nil } // buildSnapshotInfoList constructs SnapshotInfo entries from remote IDs and local data func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSnapshotMap map[string]*database.Snapshot) ([]SnapshotInfo, error) { snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots)) for snapshotID := range remoteSnapshots { if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil { totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, snapshotID) if err != nil { log.Warn("Failed to get total compressed size", "id", snapshotID, "error", err) totalSize = localSnap.BlobSize } snapshots = append(snapshots, SnapshotInfo{ ID: localSnap.ID, Timestamp: localSnap.StartedAt, CompressedSize: totalSize, }) } else { timestamp, err := parseSnapshotTimestamp(snapshotID) if err != nil { log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err) continue } totalSize, err := v.getManifestSize(snapshotID) if err != nil { return nil, fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err) } snapshots = append(snapshots, SnapshotInfo{ ID: types.SnapshotID(snapshotID), Timestamp: timestamp, CompressedSize: totalSize, }) } } return snapshots, nil } // printSnapshotTable renders the snapshot list as a formatted table func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { return err } if _, err := fmt.Fprintln(w, "NAME\tPATHS"); err != nil { return err } if _, err := fmt.Fprintln(w, "────\t─────"); err != nil { return err } for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] paths := strings.Join(snap.Paths, ", ") if _, err := fmt.Fprintf(w, "%s\t%s\n", name, paths); err != nil { return err } } if _, err := fmt.Fprintln(w); err != nil { return err } if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil { return err } if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE"); err != nil { return err } if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────"); err != nil { return err } for _, snap := range snapshots { if _, err := fmt.Fprintf(w, "%s\t%s\t%s\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)); err != nil { return err } } return w.Flush() } // PurgeSnapshots removes old snapshots based on criteria func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) error { // Sync with remote first if err := v.syncWithRemote(); err != nil { return fmt.Errorf("syncing with remote: %w", err) } // Get snapshots from local database dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { return fmt.Errorf("listing snapshots: %w", err) } // Convert to SnapshotInfo format, only including completed snapshots snapshots := make([]SnapshotInfo, 0, len(dbSnapshots)) for _, s := range dbSnapshots { if s.CompletedAt != nil { snapshots = append(snapshots, SnapshotInfo{ ID: s.ID, Timestamp: s.StartedAt, CompressedSize: s.BlobSize, }) } } // Sort by timestamp (newest first) sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) toDelete, err := v.collectSnapshotsToPurge(snapshots, keepLatest, olderThan) if err != nil { return err } if len(toDelete) == 0 { v.printlnStdout("No snapshots to delete") return nil } return v.confirmAndExecutePurge(toDelete, force) } // collectSnapshotsToPurge determines which snapshots to delete based on retention criteria func (v *Vaultik) collectSnapshotsToPurge(snapshots []SnapshotInfo, keepLatest bool, olderThan string) ([]SnapshotInfo, error) { if keepLatest { // Keep only the most recent snapshot if len(snapshots) > 1 { return snapshots[1:], nil } return nil, nil } if olderThan != "" { // Parse duration duration, err := parseDuration(olderThan) if err != nil { return nil, fmt.Errorf("invalid duration: %w", err) } cutoff := time.Now().UTC().Add(-duration) var toDelete []SnapshotInfo for _, snap := range snapshots { if snap.Timestamp.Before(cutoff) { toDelete = append(toDelete, snap) } } return toDelete, nil } return nil, nil } // confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force bool) error { // Show what will be deleted v.printfStdout("The following snapshots will be deleted:\n\n") for _, snap := range toDelete { v.printfStdout(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)) } // Confirm unless --force is used if !force { v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string if _, err := v.scanStdin(&confirm); err != nil { // Treat EOF or error as "no" v.printlnStdout("Cancelled") return nil } if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") return nil } } else { v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) } // Delete snapshots (both local and remote) for _, snap := range toDelete { snapshotID := snap.ID.String() log.Info("Deleting snapshot", "id", snapshotID) if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) } if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err) } } v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) // Note: Run 'vaultik prune' separately to clean up unreferenced blobs v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.") return nil } // VerifySnapshot checks snapshot integrity func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { opts := &VerifyOptions{Deep: deep} if deep { return v.RunDeepVerify(snapshotID, opts) } return v.VerifySnapshotWithOptions(snapshotID, opts) } // VerifySnapshotWithOptions checks snapshot integrity with full options func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptions) error { result := &VerifyResult{ SnapshotID: snapshotID, Mode: "shallow", } if opts.Deep { result.Mode = "deep" } v.printVerifyHeader(snapshotID, opts) // Download and parse manifest manifest, err := v.downloadManifest(snapshotID) if err != nil { if opts.JSON { result.Status = "failed" result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) return v.outputVerifyJSON(result) } return fmt.Errorf("downloading manifest: %w", err) } result.BlobCount = manifest.BlobCount result.TotalSize = manifest.TotalCompressedSize if !opts.JSON { v.printfStdout("Snapshot information:\n") v.printfStdout(" Blob count: %d\n", manifest.BlobCount) v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) if manifest.Timestamp != "" { if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) } } v.printlnStdout() // Check each blob exists v.printfStdout("Checking blob existence...\n") } result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) return v.formatVerifyResult(result, manifest, opts) } // printVerifyHeader prints the snapshot ID and parsed timestamp for verification output func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { // Parse snapshot ID to extract timestamp parts := strings.Split(snapshotID, "-") var snapshotTime time.Time if len(parts) >= 3 { // Format: hostname-YYYYMMDD-HHMMSSZ dateStr := parts[len(parts)-2] timeStr := parts[len(parts)-1] if len(dateStr) == 8 && len(timeStr) == 7 && strings.HasSuffix(timeStr, "Z") { timeStr = timeStr[:6] // Remove Z timestamp, err := time.Parse("20060102150405", dateStr+timeStr) if err == nil { snapshotTime = timestamp } } } if !opts.JSON { v.printfStdout("Verifying snapshot %s\n", snapshotID) if !snapshotTime.IsZero() { v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) } v.printlnStdout() } } // verifyManifestBlobsExist checks that each blob in the manifest exists in storage func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *VerifyOptions) (verified, missing int, missingSize int64) { for _, blob := range manifest.Blobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) // Just check existence (deep verification is handled by RunDeepVerify) _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) } missing++ missingSize += blob.CompressedSize } else { verified++ } } return verified, missing, missingSize } // formatVerifyResult outputs the final verification results as JSON or human-readable text func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions) error { if opts.JSON { if result.Missing > 0 { result.Status = "failed" result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing) } else { result.Status = "ok" } return v.outputVerifyJSON(result) } v.printfStdout("\nVerification complete:\n") v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified, humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) if result.Missing > 0 { v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) } else { v.printfStdout(" Missing: 0 blobs\n") } v.printfStdout(" Status: ") if result.Missing > 0 { v.printfStdout("FAILED - %d blobs are missing\n", result.Missing) return fmt.Errorf("%d blobs are missing", result.Missing) } v.printfStdout("OK - All blobs verified\n") return nil } // outputVerifyJSON outputs the verification result as JSON func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") if err := encoder.Encode(result); err != nil { return fmt.Errorf("encoding JSON: %w", err) } if result.Status == "failed" { return fmt.Errorf("verification failed: %s", result.ErrorMessage) } return nil } // Helper methods that were previously on SnapshotApp func (v *Vaultik) getManifestSize(snapshotID string) (int64, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) reader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { return 0, fmt.Errorf("downloading manifest: %w", err) } defer func() { _ = reader.Close() }() manifest, err := snapshot.DecodeManifest(reader) if err != nil { return 0, fmt.Errorf("decoding manifest: %w", err) } return manifest.TotalCompressedSize, nil } func (v *Vaultik) downloadManifest(snapshotID string) (*snapshot.Manifest, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) reader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { return nil, err } defer func() { _ = reader.Close() }() manifest, err := snapshot.DecodeManifest(reader) if err != nil { return nil, fmt.Errorf("decoding manifest: %w", err) } return manifest, nil } func (v *Vaultik) syncWithRemote() error { log.Info("Syncing with remote snapshots") // Get all remote snapshot IDs remoteSnapshots := make(map[string]bool) objectCh := v.Storage.ListStream(v.ctx, "metadata/") for object := range objectCh { if object.Err != nil { return fmt.Errorf("listing remote snapshots: %w", object.Err) } // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } remoteSnapshots[parts[1]] = true } } log.Debug("Found remote snapshots", "count", len(remoteSnapshots)) // Get all local snapshots (use a high limit to get all) localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } // Remove local snapshots that don't exist remotely removedCount := 0 for _, snapshot := range localSnapshots { snapshotIDStr := snapshot.ID.String() if !remoteSnapshots[snapshotIDStr] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) } else { removedCount++ } } } if removedCount > 0 { log.Info("Removed local snapshots not found in remote", "count", removedCount) } return nil } // RemoveOptions contains options for the snapshot remove command type RemoveOptions struct { Force bool DryRun bool JSON bool Remote bool // Also remove metadata from remote storage All bool // Remove all snapshots (requires Force) } // RemoveResult contains the result of a snapshot removal type RemoveResult struct { SnapshotID string `json:"snapshot_id,omitempty"` SnapshotsRemoved []string `json:"snapshots_removed,omitempty"` RemoteRemoved bool `json:"remote_removed,omitempty"` DryRun bool `json:"dry_run,omitempty"` } // RemoveSnapshot removes a snapshot from the local database and optionally from remote storage // Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs. func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*RemoveResult, error) { result := &RemoveResult{ SnapshotID: snapshotID, } if opts.DryRun { result.DryRun = true if !opts.JSON { v.printfStdout("Would remove snapshot: %s\n", snapshotID) if opts.Remote { v.printlnStdout("Would also remove from remote storage") } v.printlnStdout("[Dry run - no changes made]") } if opts.JSON { return result, v.outputRemoveJSON(result) } return result, nil } // Confirm unless --force is used (skip in JSON mode - require --force) if !opts.Force && !opts.JSON { if opts.Remote { v.printfStdout("Remove snapshot '%s' from local database and remote storage? [y/N] ", snapshotID) } else { v.printfStdout("Remove snapshot '%s' from local database? [y/N] ", snapshotID) } var confirm string if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") return result, nil } if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") return result, nil } } log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) // Remove from local database if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { return result, fmt.Errorf("removing from local database: %w", err) } // If --remote, also remove from remote storage if opts.Remote { log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { return result, fmt.Errorf("removing from remote storage: %w", err) } result.RemoteRemoved = true } // Output result if opts.JSON { return result, v.outputRemoveJSON(result) } // Print summary v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) if opts.Remote { v.printlnStdout("Removed snapshot metadata from remote storage") v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") } return result, nil } // RemoveAllSnapshots removes all snapshots from local database and optionally from remote func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) { snapshotIDs, err := v.listAllRemoteSnapshotIDs() if err != nil { return nil, err } if len(snapshotIDs) == 0 { if !opts.JSON { v.printlnStdout("No snapshots found") } return &RemoveResult{}, nil } if opts.DryRun { return v.handleRemoveAllDryRun(snapshotIDs, opts) } return v.executeRemoveAll(snapshotIDs, opts) } // listAllRemoteSnapshotIDs collects all unique snapshot IDs from remote storage func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { log.Info("Listing all snapshots") objectCh := v.Storage.ListStream(v.ctx, "metadata/") var snapshotIDs []string for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) } parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { sid := parts[1] found := false for _, id := range snapshotIDs { if id == sid { found = true break } } if !found { snapshotIDs = append(snapshotIDs, sid) } } } } return snapshotIDs, nil } // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots func (v *Vaultik) handleRemoveAllDryRun(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { result := &RemoveResult{ DryRun: true, SnapshotsRemoved: snapshotIDs, } if !opts.JSON { v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) for _, id := range snapshotIDs { v.printfStdout(" %s\n", id) } if opts.Remote { v.printlnStdout("Would also remove from remote storage") } v.printlnStdout("[Dry run - no changes made]") } if opts.JSON { return result, v.outputRemoveJSON(result) } return result, nil } // executeRemoveAll removes all snapshots from local database and optionally from remote storage func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { // --all requires --force if !opts.Force { return nil, fmt.Errorf("--all requires --force") } log.Info("Removing all snapshots", "count", len(snapshotIDs)) result := &RemoveResult{} for _, snapshotID := range snapshotIDs { log.Info("Removing snapshot", "snapshot_id", snapshotID) if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) continue } if opts.Remote { if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { log.Error("Failed to remove from remote", "snapshot_id", snapshotID, "error", err) continue } } result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) } if opts.Remote { result.RemoteRemoved = true } if opts.JSON { return result, v.outputRemoveJSON(result) } v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) if opts.Remote { v.printlnStdout("Removed snapshot metadata from remote storage") v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") } return result, nil } // deleteSnapshotFromLocalDB removes a snapshot from the local database only func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { if v.Repositories == nil { return nil // No local database } // Delete related records first to avoid foreign key constraints if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil { return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil { return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil { return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil { return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err) } return nil } // deleteSnapshotFromRemote removes snapshot metadata files from remote storage func (v *Vaultik) deleteSnapshotFromRemote(snapshotID string) error { prefix := fmt.Sprintf("metadata/%s/", snapshotID) objectCh := v.Storage.ListStream(v.ctx, prefix) var objectsToDelete []string for object := range objectCh { if object.Err != nil { return fmt.Errorf("listing objects: %w", object.Err) } objectsToDelete = append(objectsToDelete, object.Key) } for _, key := range objectsToDelete { if err := v.Storage.Delete(v.ctx, key); err != nil { return fmt.Errorf("removing %s: %w", key, err) } log.Debug("Deleted remote object", "key", key) } return nil } // outputRemoveJSON outputs the removal result as JSON func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(result) } // PruneResult contains statistics about the prune operation type PruneResult struct { SnapshotsDeleted int64 FilesDeleted int64 ChunksDeleted int64 BlobsDeleted int64 } // PruneDatabase removes incomplete snapshots and orphaned files, chunks, // and blobs from the local database. This ensures database consistency // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { log.Info("Pruning local database: removing incomplete snapshots and orphaned data") result := &PruneResult{} // First, delete any incomplete snapshots incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx) if err != nil { return nil, fmt.Errorf("getting incomplete snapshots: %w", err) } for _, snapshot := range incompleteSnapshots { snapshotIDStr := snapshot.ID.String() log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID) // Delete related records first if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr); err != nil { log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) } if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr); err != nil { log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) } if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr); err != nil { log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) } if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err) } else { result.SnapshotsDeleted++ } } // Get counts before cleanup for reporting fileCountBefore, _ := v.getTableCount("files") chunkCountBefore, _ := v.getTableCount("chunks") blobCountBefore, _ := v.getTableCount("blobs") // Run the cleanup if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { return nil, fmt.Errorf("cleanup orphaned data: %w", err) } // Get counts after cleanup fileCountAfter, _ := v.getTableCount("files") chunkCountAfter, _ := v.getTableCount("chunks") blobCountAfter, _ := v.getTableCount("blobs") result.FilesDeleted = fileCountBefore - fileCountAfter result.ChunksDeleted = chunkCountBefore - chunkCountAfter result.BlobsDeleted = blobCountBefore - blobCountAfter log.Info("Local database prune complete", "incomplete_snapshots", result.SnapshotsDeleted, "orphaned_files", result.FilesDeleted, "orphaned_chunks", result.ChunksDeleted, "orphaned_blobs", result.BlobsDeleted, ) // Print summary v.printfStdout("Local database prune complete:\n") v.printfStdout(" Incomplete snapshots removed: %d\n", result.SnapshotsDeleted) v.printfStdout(" Orphaned files removed: %d\n", result.FilesDeleted) v.printfStdout(" Orphaned chunks removed: %d\n", result.ChunksDeleted) v.printfStdout(" Orphaned blobs removed: %d\n", result.BlobsDeleted) return result, nil } // validTableNameRe matches table names containing only lowercase alphanumeric characters and underscores. var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`) // getTableCount returns the count of rows in a table. // The tableName is sanitized to only allow [a-z0-9_] characters to prevent SQL injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } if !validTableNameRe.MatchString(tableName) { return 0, fmt.Errorf("invalid table name: %q", tableName) } var count int64 query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) if err != nil { return 0, err } return count, nil }