package vaultik import ( "encoding/json" "errors" "fmt" "os" "path/filepath" "regexp" "slices" "sort" "strconv" "strings" "time" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/types" ) // Sentinel errors for snapshot management. var ( errSnapshotNotInConfig = errors.New("snapshot not found in config") errNoSnapshotsInConfig = errors.New("no snapshots configured") errSnapshotVerifyFailed = errors.New("verification failed") errProblemsInStorage = errors.New("problems found in storage") errRemoveAllNeedsForce = errors.New("--all requires --force") errSnapshotNotFound = errors.New("snapshot not found") errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed") errInvalidTableName = errors.New("invalid table name") errNoAgeRecipients = errors.New( "creating a snapshot needs at least one public key in " + "age_recipients (generate a keypair with: age-keygen)") errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune") ) // ErrPostBackupPruneFailed wraps a failure of the post-backup prune, // which runs only after every snapshot was created and uploaded, so the // caller can report it apart from a failed backup. var ErrPostBackupPruneFailed = errors.New("post-backup prune failed") // ErrFilesUnreadable is returned when --skip-errors skipped files or // directories it could not read or list. The snapshot is complete and // kept, so the caller can report it apart from a failed backup. var ErrFilesUnreadable = errors.New("file(s) could not be read") // ErrSourcePathEmpty is returned when --prune skipped the post-backup // prune because the scan found nothing in a configured path of a // snapshot, as in the mount point of a volume that is not mounted. The // snapshots are complete and kept, so the caller can report it apart // from a failed backup. A run that also skipped unreadable files // returns ErrFilesUnreadable instead. var ErrSourcePathEmpty = errors.New("source path(s) were empty") // ErrVolumeNotMounted is returned when --prune skipped the post-backup // prune because a directory that an earlier snapshot of the same name // found as a mount point was not found as one in this run: the volume // mounted there is not mounted, the run found no entry in it, or the // directory is no longer backed up. The snapshots are complete and // kept, so the caller can report it apart from a failed backup. A run // that also skipped unreadable files or found a configured path empty // returns ErrFilesUnreadable or ErrSourcePathEmpty instead. var ErrVolumeNotMounted = errors.New( "mount point(s) an earlier snapshot recorded were not found") // listRecentLimit caps how many snapshot rows are fetched from the // local index for listing and purge operations. const listRecentLimit = 10000 // SnapshotCreateOptions contains options for the snapshot create command type SnapshotCreateOptions struct { Cron bool Prune bool // KeepNewerThan is used with --prune: keep snapshots newer than this // duration (e.g. "4w"); default: keep only latest. KeepNewerThan string SkipErrors bool // Skip file read errors (log them loudly but continue) Snapshots []string // Optional list of snapshot names to process (empty = all) } // Validate fails when KeepNewerThan is set without Prune or is not a // valid duration. Only the post-backup prune reads KeepNewerThan, after // the backup, so the caller checks it before starting one. func (opts *SnapshotCreateOptions) Validate() error { if opts.KeepNewerThan == "" { return nil } if !opts.Prune { return errKeepNewerThanNeedsPrune } _, err := parseDuration(opts.KeepNewerThan) if err != nil { return fmt.Errorf("--keep-newer-than: %w", err) } return nil } // CreateSnapshot executes the snapshot creation operation func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // config.Load accepts an empty list, since listing, verifying and // restoring need no public key. if len(v.Config.AgeRecipients) == 0 { return errNoAgeRecipients } overallStartTime := time.Now() log.Info("Starting snapshot creation", "version", v.Globals.Version, "commit", v.Globals.Commit, "index_path", v.Config.IndexPath, ) err := v.EnsureStorageBinding() if err != nil { return err } // Before PruneDatabase: it drops incomplete snapshots, which could // leave the local index without one and so skip the marker check. err = v.markDestination() if err != nil { return err } // Clean up incomplete snapshots FIRST, before any scanning. // This is critical for data safety; PruneDatabase below does it. hostname := v.Config.Hostname if hostname == "" { hostname, _ = os.Hostname() } // CRITICAL: This MUST succeed. If we fail to clean up incomplete snapshots, // the deduplication logic will think files from the incomplete snapshot were // already backed up and skip them, resulting in data loss. // // Prune the database before starting: delete incomplete snapshots and orphaned data. // This ensures the database is consistent before we start a new snapshot. // Since we use locking, only one vaultik instance accesses the DB at a time. // A snapshot whose metadata export was interrupted is left incomplete by // finalizeSnapshotMetadata, so it is among the incomplete snapshots dropped // here (https://git.eeqj.de/sneak/vaultik/issues/177). _, err = v.PruneDatabase() if err != nil { return fmt.Errorf("prune database: %w", err) } snapshotNames, err := v.snapshotNamesToCreate(opts.Snapshots) if err != nil { return err } filesUnreadable, emptyPaths, unmountedPaths, err := v.createNamedSnapshots( opts, hostname, snapshotNames) if err != nil { return err } // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { v.UI.Completef("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) } switch { case opts.Prune && (filesUnreadable > 0 || len(emptyPaths) > 0 || len(unmountedPaths) > 0): return v.skipPostBackupPrune( filesUnreadable, emptyPaths, unmountedPaths) case opts.Prune: err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) if err != nil { return fmt.Errorf("%w: %w", ErrPostBackupPruneFailed, err) } } return v.finishCreateSnapshot(filesUnreadable) } // createNamedSnapshots creates each named snapshot in turn. It returns // the number of entries --skip-errors skipped because it could not read // or list them, each configured path in which the scan found nothing, // and, under --prune, each directory where an earlier snapshot found a // volume mounted and this run did not, the paths named with their // snapshot. func (v *Vaultik) createNamedSnapshots( opts *SnapshotCreateOptions, hostname string, snapshotNames []string, ) (int, []string, []string, error) { filesUnreadable := 0 var emptyPaths, unmountedPaths []string for snapIdx, snapName := range snapshotNames { stats, err := v.createNamedSnapshot( opts, hostname, snapName, snapIdx+1, len(snapshotNames)) if err != nil { return 0, nil, nil, err } filesUnreadable += stats.totalFilesUnreadable for _, dir := range stats.emptyPaths { emptyPaths = append(emptyPaths, dir+" of snapshot "+snapName) } for _, dir := range stats.unmountedPaths { unmountedPaths = append(unmountedPaths, dir+" of snapshot "+snapName) } } return filesUnreadable, emptyPaths, unmountedPaths, nil } // skipPostBackupPrune says why the post-backup prune does not run, naming // every cause, and returns the error the run ends with. An older snapshot // may be the only one that still holds a file this run could not read, // or what an empty path or a volume that is not mounted held. The // previous snapshot is no guide: after one such run it lacks them too. func (v *Vaultik) skipPostBackupPrune( filesUnreadable int, emptyPaths, unmountedPaths []string, ) error { log.Warn("Skipping post-backup prune", "files_unreadable", filesUnreadable, "empty_paths", emptyPaths, "unmounted_paths", unmountedPaths) if len(emptyPaths) > 0 { v.UI.Warningf("Skipped the post-backup prune because nothing was "+ "found in %s; older snapshots are kept.", strings.Join(emptyPaths, ", ")) } if len(unmountedPaths) > 0 { v.UI.Warningf("Skipped the post-backup prune because this backup "+ "did not find a mount point at %s, where an earlier snapshot "+ "found one: the volume is not mounted, this backup found no "+ "entry in it, or the directory is no longer backed up. Older "+ "snapshots are kept. Mount the volume again, or, if that is on "+ "purpose, remove the older snapshots by hand with "+ "'vaultik snapshot purge'.", strings.Join(unmountedPaths, ", ")) } if filesUnreadable > 0 { v.UI.Warningf("Skipped the post-backup prune because %d file(s) "+ "could not be read; older snapshots are kept.", filesUnreadable) return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable) } if len(emptyPaths) > 0 { return fmt.Errorf("%d %w", len(emptyPaths), ErrSourcePathEmpty) } return fmt.Errorf("%d %w", len(unmountedPaths), ErrVolumeNotMounted) } // finishCreateSnapshot ends a run whose snapshots are all complete and // kept. Like finishRestore, it returns an error when --skip-errors // skipped anything it could not read or list; otherwise it prints the // closing line. A file removed during the run is counted as failed in // the summary but is not an error, with or without --skip-errors. func (v *Vaultik) finishCreateSnapshot(filesUnreadable int) error { if filesUnreadable > 0 { return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable) } // Terminus must obey the --cron invariant: silent on total // success only. UI.Complete is dropped in cron/quiet mode (that's // the success path), but if any warnings fired during the run we // emit the summary via UI.Warning so cron actually delivers // something for the user to look at. if v.UI.WarningCount() > 0 { v.UI.Warningf("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) } else { v.UI.Completef("Finished successfully.") } return nil } // snapshotNamesToCreate returns the requested snapshot names, each checked // against the config, or every configured name when none was requested. func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) { snapshotNames := requested if len(snapshotNames) == 0 { snapshotNames = v.Config.SnapshotNames() } else { // Validate requested snapshot names exist for _, name := range snapshotNames { if _, ok := v.Config.Snapshots[name]; !ok { return nil, fmt.Errorf("%w: %q", errSnapshotNotInConfig, name) } } } if len(snapshotNames) == 0 { return nil, errNoSnapshotsInConfig } return snapshotNames, nil } // runPostBackupPrune drops older snapshots of the given names and removes // from remote storage the orphan blobs and the directories under // metadata/ with no manifest, as PruneBlobs does. If keepNewerThan is set // (e.g. "4w"), snapshots newer than that duration are kept. Otherwise only // the latest snapshot of each name is kept. func (v *Vaultik) runPostBackupPrune( snapshotNames []string, keepNewerThan string, ) error { log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) v.UI.Beginf("Running post-backup prune.") purgeOpts := &SnapshotPurgeOptions{ Force: true, Names: snapshotNames, Quiet: true, } if keepNewerThan != "" { purgeOpts.OlderThan = keepNewerThan } else { purgeOpts.KeepLatest = true } err := v.PurgeSnapshotsWithOptions(purgeOpts) if err != nil { return fmt.Errorf("purging old snapshots: %w", err) } err = v.PruneBlobs(&PruneOptions{Force: true}) if err != nil { return fmt.Errorf("pruning the destination store: %w", err) } return nil } // snapshotStats tracks aggregate statistics across directory scans type snapshotStats struct { totalFiles int totalBytes int64 totalChunks int newChunkSize int64 totalBlobs int totalBytesSkipped int64 totalFilesSkipped int totalFilesFailed int totalFilesDeleted int totalBytesDeleted int64 totalBytesUploaded int64 totalBlobsUploaded int uploadDuration time.Duration // The part of totalFilesFailed that --skip-errors skipped because it // could not be read or listed. totalFilesUnreadable int // The configured paths in which the scan found nothing. emptyPaths []string // Under --prune, the directories where an earlier snapshot of the // same name found a volume mounted and this one did not. unmountedPaths []string // The sizes of all blobs the snapshot references, set by // finalizeSnapshotMetadata once snapshot_blobs is populated. blobSize int64 blobUncompressedSize int64 } // createNamedSnapshot creates a single named snapshot and returns the // statistics of its scan. func (v *Vaultik) createNamedSnapshot( opts *SnapshotCreateOptions, hostname, snapName string, idx, total int, ) (*snapshotStats, error) { snapshotStartTime := time.Now() if total > 1 { v.UI.Infof("Snapshot %d/%d: %s.", idx, total, snapName) } resolvedDirs, err := v.resolveSnapshotPaths(snapName) if err != nil { return nil, err } previousSnapshotID, err := v.previousSnapshotID(snapName) if err != nil { return nil, err } scanner := v.ScannerFactory(snapshot.ScannerParams{ EnableProgress: !opts.Cron, UI: v.UI, Fs: v.Fs, Exclude: v.Config.GetExcludes(snapName), SkipErrors: opts.SkipErrors, PreviousSnapshotID: previousSnapshotID, }) snapshotID, err := v.SnapshotManager.CreateSnapshotWithName( v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) if err != nil { return nil, fmt.Errorf("creating snapshot: %w", err) } log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) v.UI.Beginf("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) if err != nil { return nil, err } err = v.finalizeSnapshotMetadata(snapshotID, stats) if err != nil { return nil, err } log.Info("Snapshot complete", "snapshot_id", snapshotID, "name", snapName, "files", stats.totalFiles, "blobs_uploaded", stats.totalBlobsUploaded, "bytes_uploaded", stats.totalBytesUploaded, "duration", time.Since(snapshotStartTime)) v.printSnapshotSummary(snapshotID, snapshotStartTime, stats) if opts.Prune { stats.unmountedPaths, err = v.findUnmountedPaths(snapName, snapshotID) if err != nil { return nil, err } } return stats, nil } // findUnmountedPaths returns each directory that an earlier snapshot // named snapName in the local index recorded as a mount point and // snapshot snapshotID did not, as when the volume mounted there is not // mounted. func (v *Vaultik) findUnmountedPaths( snapName, snapshotID string, ) ([]string, error) { found, err := v.Repositories.Snapshots.GetMountPoints(v.ctx, snapshotID) if err != nil { return nil, err } snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return nil, fmt.Errorf("listing snapshots: %w", err) } var unmounted []string for _, s := range snapshots { if s.ID.String() == snapshotID || parseSnapshotName(s.ID.String(), s.Hostname.String()) != snapName { continue } recorded, err := v.Repositories.Snapshots.GetMountPoints( v.ctx, s.ID.String()) if err != nil { return nil, err } for _, dir := range recorded { if !slices.Contains(found, dir) && !slices.Contains(unmounted, dir) { unmounted = append(unmounted, dir) } } } return unmounted, nil } // previousSnapshotID returns the ID of the most recent completed snapshot // named snapName in the local index, or "" when there is none. func (v *Vaultik) previousSnapshotID(snapName string) (string, error) { snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return "", fmt.Errorf("listing snapshots: %w", err) } for _, s := range snapshots { if s.CompletedAt != nil && parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName { return s.ID.String(), nil } } return "", nil } // resolveSnapshotPaths resolves source directories to absolute paths // with symlink resolution. A path that is a symlink whose target does // not exist is an error. func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { snapConfig := v.Config.Snapshots[snapName] resolvedDirs := make([]string, 0, len(snapConfig.Paths)) for _, dir := range snapConfig.Paths { absPath, err := filepath.Abs(dir) if err != nil { return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err) } resolvedPath, err := filepath.EvalSymlinks(absPath) if err != nil { // A path that does not exist is left for the scan to report. // One that exists is a symlink whose target is gone, as one // into a volume that is not mounted: the scan would record // only the link, and --prune would then drop the older // snapshots that hold the files. _, lstatErr := os.Lstat(absPath) if !os.IsNotExist(err) || lstatErr == nil { return nil, fmt.Errorf( "failed to resolve symlinks for %s: %w", absPath, err) } resolvedPath = absPath } resolvedDirs = append(resolvedDirs, resolvedPath) } return resolvedDirs, nil } // scanAllDirectories runs the scanner on each resolved directory and // accumulates stats. func (v *Vaultik) scanAllDirectories( scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string, ) (*snapshotStats, error) { if progress := scanner.GetProgress(); progress != nil { progress.Start() defer progress.Stop() } stats := &snapshotStats{} for i, dir := range resolvedDirs { select { case <-v.ctx.Done(): log.Info("Snapshot creation cancelled") return nil, v.ctx.Err() default: } log.Info("Scanning directory", "path", dir) v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { return nil, fmt.Errorf("failed to scan %s: %w", dir, err) } stats.totalFiles += result.FilesScanned stats.totalBytes += result.BytesScanned stats.totalChunks += result.ChunksCreated stats.newChunkSize += result.NewChunkSize stats.totalBlobs += result.BlobsCreated stats.totalFilesSkipped += result.FilesSkipped stats.totalFilesFailed += result.FilesFailed stats.totalFilesUnreadable += result.FilesUnreadable stats.totalBytesSkipped += result.BytesSkipped stats.totalFilesDeleted += result.FilesDeleted stats.totalBytesDeleted += result.BytesDeleted stats.totalBlobsUploaded += result.BlobsUploaded stats.totalBytesUploaded += result.BytesUploaded stats.uploadDuration += result.UploadDuration if result.EntriesFound == 0 { stats.emptyPaths = append(stats.emptyPaths, dir) } log.Info("Directory scan complete", "path", dir, "files", result.FilesScanned, "files_skipped", result.FilesSkipped, "files_failed", result.FilesFailed, "bytes", result.BytesScanned, "bytes_skipped", result.BytesSkipped, "chunks", result.ChunksCreated, "blobs", result.BlobsCreated, "duration", result.EndTime.Sub(result.StartTime)) } return stats, nil } // finalizeSnapshotMetadata updates stats, exports metadata, and only then // marks the snapshot complete. Recording completion last is deliberate: an // export interrupted by a crash leaves the snapshot incomplete rather than // looking complete with no manifest or database at the destination. The next // run's PruneDatabase drops the incomplete snapshot and re-backs-up its data. // See https://git.eeqj.de/sneak/vaultik/issues/177. func (v *Vaultik) finalizeSnapshotMetadata( snapshotID string, stats *snapshotStats, ) error { // snapshot_blobs must be populated before the blob sizes below, which // total the snapshot's blobs, and before the export, which builds the // manifest and the trimmed metadata database from it. err := v.SnapshotManager.PopulateSnapshotBlobs(v.ctx, snapshotID) if err != nil { return fmt.Errorf("populating snapshot blobs: %w", err) } stats.blobSize, stats.blobUncompressedSize, err = v.Repositories.Snapshots.GetSnapshotBlobSizes(v.ctx, snapshotID) if err != nil { return fmt.Errorf("getting snapshot blob sizes: %w", err) } // file_count and total_size leave out the files that could not be // stored; stats.totalBytes already does. extStats := snapshot.ExtendedBackupStats{ BackupStats: snapshot.BackupStats{ FilesScanned: stats.totalFiles - stats.totalFilesFailed, TotalSize: stats.totalBytes + stats.totalBytesSkipped, ChunksCreated: stats.totalChunks, BlobsCreated: stats.totalBlobs, BytesUploaded: stats.totalBytesUploaded, }, BlobSize: stats.blobSize, BlobUncompressedSize: stats.blobUncompressedSize, CompressionLevel: v.Config.CompressionLevel, UploadDurationMs: stats.uploadDuration.Milliseconds(), NewChunkSize: stats.newChunkSize, } err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) if err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } err = v.SnapshotManager.ExportSnapshotMetadata( v.ctx, v.Config.IndexPath, snapshotID) if err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } // Record completion last, so an interrupted export never leaves a // snapshot marked complete without its metadata at the destination. err = v.SnapshotManager.MarkSnapshotComplete(v.ctx, snapshotID) if err != nil { return fmt.Errorf("marking snapshot complete: %w", err) } return nil } // uploadSpeed returns the average network upload rate as a colorized // bits/sec string, or "N/A" when there's no usable data. func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) string { if bytesUploaded <= 0 || duration <= 0 { return v.UI.Speed(0) } return v.UI.Speed(float64(bytesUploaded) / duration.Seconds()) } // printSnapshotSummary prints the comprehensive snapshot completion summary func (v *Vaultik) printSnapshotSummary( snapshotID string, startTime time.Time, stats *snapshotStats, ) { snapshotDuration := time.Since(startTime) totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped - stats.totalFilesFailed totalBytesAll := stats.totalBytes + stats.totalBytesSkipped var compressionRatio float64 if stats.blobUncompressedSize > 0 { compressionRatio = float64(stats.blobSize) / float64(stats.blobUncompressedSize) } else { compressionRatio = 1.0 } v.UI.Completef("Created snapshot %s.", v.UI.Snapshot(snapshotID)) filesMsg := fmt.Sprintf("Files: %s examined, %s backed up, %s unchanged", v.UI.Count(stats.totalFiles), v.UI.Count(totalFilesChanged), v.UI.Count(stats.totalFilesSkipped)) if stats.totalFilesFailed > 0 { filesMsg += fmt.Sprintf(", %s failed", v.UI.Count(stats.totalFilesFailed)) } if stats.totalFilesDeleted > 0 { filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted)) } v.UI.Detailf("%s.", filesMsg) dataMsg := fmt.Sprintf("Data: %s total (%s backed up)", v.UI.Size(totalBytesAll), v.UI.Size(stats.totalBytes)) if stats.totalBytesDeleted > 0 { dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted)) } v.UI.Detailf("%s.", dataMsg) if stats.totalBlobsUploaded > 0 { v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", v.UI.Size(stats.blobSize), v.UI.Size(stats.blobUncompressedSize), compressionRatio) v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", stats.totalBlobsUploaded, v.UI.Size(stats.totalBytesUploaded), v.UI.Duration(stats.uploadDuration), v.uploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) } // SnapshotPurgeOptions contains options for the snapshot purge command. type SnapshotPurgeOptions struct { KeepLatest bool // Keep only the most recent snapshot per name // OlderThan drops snapshots older than this duration (e.g. "30d", // "6m", "1y"). OlderThan string Force bool // Skip confirmation prompt // Names restricts the operation to snapshots with one of these // names when non-empty. Names []string Quiet bool // Suppress informational output (used by --prune flag) } // PurgeSnapshotsWithOptions removes old snapshots based on criteria. // Retention is per-snapshot-name: KeepLatest keeps the latest of EACH configured // snapshot name, not the latest globally. This prevents `home` and `system` // snapshots from cannibalizing each other. func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { err := v.EnsureStorageBinding() if err != nil { return err } // Sync with remote first err = v.syncWithRemote() if err != nil { return fmt.Errorf("syncing with remote: %w", err) } // Get snapshots from local database dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing snapshots: %w", err) } // Build name filter set if --snapshot was specified. nameFilter := make(map[string]struct{}, len(opts.Names)) for _, n := range opts.Names { nameFilter[n] = struct{}{} } // Collect completed snapshots and their names, applying the name filter. snapshots := make([]SnapshotInfo, 0, len(dbSnapshots)) names := make(map[types.SnapshotID]string, len(dbSnapshots)) for _, s := range dbSnapshots { if s.CompletedAt == nil { continue } name := parseSnapshotName(s.ID.String(), s.Hostname.String()) if len(nameFilter) > 0 { if _, ok := nameFilter[name]; !ok { continue } } names[s.ID] = name snapshots = append(snapshots, SnapshotInfo{ ID: s.ID, Timestamp: s.StartedAt, CompressedSize: s.BlobSize, }) } // Sort by timestamp (newest first) sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) toDelete, err := selectSnapshotsToPurge(snapshots, names, opts) if err != nil { return err } if len(toDelete) == 0 { if !opts.Quiet { v.UI.Infof("No snapshots to delete.") } return nil } return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet) } // selectSnapshotsToPurge applies the purge retention criteria to the // newest-first sorted snapshot list and returns the deletion // candidates. names maps each snapshot's ID to its snapshot name. func selectSnapshotsToPurge( snapshots []SnapshotInfo, names map[types.SnapshotID]string, opts *SnapshotPurgeOptions, ) ([]SnapshotInfo, error) { var toDelete []SnapshotInfo switch { case opts.KeepLatest: // Keep the latest snapshot per snapshot name. Snapshots are sorted // newest-first, so the first occurrence of each name is kept. seen := make(map[string]bool) for _, snap := range snapshots { name := names[snap.ID] if seen[name] { toDelete = append(toDelete, snap) continue } seen[name] = true } case opts.OlderThan != "": duration, err := parseDuration(opts.OlderThan) if err != nil { return nil, err } cutoff := time.Now().UTC().Add(-duration) for _, snap := range snapshots { if snap.Timestamp.Before(cutoff) { toDelete = append(toDelete, snap) } } } return toDelete, nil } // confirmAndExecutePurge shows deletion candidates, confirms with the // user, and deletes snapshots. func (v *Vaultik) confirmAndExecutePurge( toDelete []SnapshotInfo, force, quiet bool, ) error { // Confirm unless --force is used. The list, the prompt and its answer // bypass the UI: they are an interactive exchange the operator must // see, so --quiet does not silence them. if !force { v.stdoutf("The following snapshots will be deleted:\n\n") for _, snap := range toDelete { v.stdoutf(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format(time.DateTime), ubytes(snap.CompressedSize)) } v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string _, err := v.scanStdin(&confirm) if err != nil { v.printlnStdout("Cancelled") return nil //nolint:nilerr // treat EOF or read error as "no" } if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") return nil } } else if !quiet { v.UI.Infof("The following snapshots will be deleted:") for _, snap := range toDelete { v.UI.Detailf("%s (%s, %s)", v.UI.Snapshot(snap.ID.String()), snap.Timestamp.Format(time.DateTime), ubytes(snap.CompressedSize)) } v.UI.Infof("Deleting %d snapshot(s) (--force specified).", len(toDelete)) } // Delete snapshots (both local and remote) for _, snap := range toDelete { snapshotID := snap.ID.String() log.Info("Deleting snapshot", "id", snapshotID) // Stop before the remote delete, so a snapshot still in the local // index keeps its metadata on the destination store. err := v.deleteSnapshotFromLocalDB(snapshotID) if err != nil { return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err) } _, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err) } } // Tidy up local DB orphans now so users don't have to run a // separate command after a purge. Guarded against nil for tests // that don't wire up a SnapshotManager. if v.SnapshotManager != nil { err := v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { log.Warn("Failed to clean up orphaned local data after purge", "error", err) } } if !quiet { v.UI.Infof("Deleted %d snapshot(s).", len(toDelete)) v.UI.Noticef("Run '%s' to clean up unreferenced remote blobs.", pruneCommandHint) } return nil } // VerifySnapshot checks snapshot integrity func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { opts := &VerifyOptions{Deep: deep} if deep { return v.RunDeepVerify(snapshotID, opts) } return v.VerifySnapshotWithOptions(snapshotID, opts) } // VerifySnapshotWithOptions checks snapshot integrity with full options. // Deep verification is delegated to RunDeepVerify so this function only // implements the shallow (existence-only) path. func (v *Vaultik) VerifySnapshotWithOptions( snapshotID string, opts *VerifyOptions, ) error { if opts.Deep { return v.RunDeepVerify(snapshotID, opts) } result := &VerifyResult{ SnapshotID: snapshotID, Mode: "shallow", } v.printVerifyHeader(snapshotID, opts) // Resolve the identifier to the snapshot's remote key. A human ID is // hashed; a remote key (or its abbreviation, as printed for a // remote-only snapshot) is used as-is, so a host with no local index // can verify a snapshot it can only see on the store. The key is kept // so we can also check for the snapshot's encrypted database below. remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID) if err != nil { if opts.JSON { result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err) return v.outputVerifyJSON(result) } return fmt.Errorf("resolving snapshot identifier: %w", err) } manifest, err := v.downloadManifestByKey(remoteKey) if err != nil { if opts.JSON { result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) return v.outputVerifyJSON(result) } return fmt.Errorf("downloading manifest: %w", err) } result.BlobCount = manifest.BlobCount result.TotalSize = manifest.TotalCompressedSize if !opts.JSON && !v.UI.Quiet() { v.stdoutf("Snapshot information:\n") v.stdoutf(" Blob count: %d\n", manifest.BlobCount) v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize)) if manifest.Timestamp != "" { t, terr := time.Parse(time.RFC3339, manifest.Timestamp) if terr == nil { v.stdoutf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) } } v.printlnStdout() // Check each blob is present with the size the manifest records. v.UI.Beginf("Checking blob presence and sizes...") } v.checkSnapshotDatabase(remoteKey, result) result.Verified, result.Missing, result.Mismatched, result.MissingSize, result.Unchecked, err = v.verifyManifestBlobs(manifest, opts) if err != nil { if opts.JSON { result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("verifying manifest blobs: %v", err) return v.outputVerifyJSON(result) } return fmt.Errorf("verifying manifest blobs: %w", err) } return v.formatVerifyResult(result, opts) } // printVerifyHeader prints the snapshot ID and parsed timestamp for // verification output. Snapshot ID format: hostname[_name]_ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { var snapshotTime time.Time t, err := parseSnapshotTimestamp(snapshotID) if err == nil { snapshotTime = t } if !opts.JSON && !v.UI.Quiet() { v.stdoutf("Verifying snapshot %s\n", snapshotID) if !snapshotTime.IsZero() { v.stdoutf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) } v.printlnStdout() } } // checkSnapshotDatabase records in result whether the snapshot's // encrypted database is missing or could not be checked. A snapshot is // only restorable if its database is present alongside the blobs. // Shallow verify checks that the object exists; it does not decrypt it // (that is deep verify's job). func (v *Vaultik) checkSnapshotDatabase(remoteKey string, result *VerifyResult) { dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) _, err := v.Storage.Stat(v.ctx, dbPath) switch { case errors.Is(err, storage.ErrNotFound): result.DatabaseMissing = true case err != nil: result.DatabaseCheckError = err.Error() } } // verifyManifestBlobs checks that each blob in the manifest is present in // storage with the size the manifest records, returning the counts of // blobs that were present with the right size, absent, and present but the // wrong size, the total bytes of the absent blobs, and the blobs it could // not check because storage returned an error other than not-found. It // does not read blob contents; deep verification (RunDeepVerify) does // that. The size comparison matches the deep path (see // verifyBlobExistenceFromDB). func (v *Vaultik) verifyManifestBlobs( manifest *snapshot.Manifest, opts *VerifyOptions, ) (int, int, int, int64, []UncheckedBlob, error) { var ( verified, missing, mismatched int missingSize int64 unchecked []UncheckedBlob ) for _, blob := range manifest.Blobs { // The manifest is unauthenticated, so its blob hashes are checked // before being spliced into a storage path. if !isBlobHash(blob.Hash) { return 0, 0, 0, 0, nil, fmt.Errorf("%w: %s", errInvalidBlobHash, shortHash(blob.Hash)) } blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) stat, err := v.Storage.Stat(v.ctx, blobPath) switch { case errors.Is(err, storage.ErrNotFound): if !opts.JSON && !v.UI.Quiet() { v.stdoutf(" Missing: %s (%s)\n", blob.Hash, ubytes(blob.CompressedSize)) } missing++ missingSize += blob.CompressedSize case err != nil: if !opts.JSON && !v.UI.Quiet() { v.stdoutf(" Could not check: %s (%v)\n", blob.Hash, err) } unchecked = append(unchecked, UncheckedBlob{Hash: blob.Hash, Error: err.Error()}) case stat.Size != blob.CompressedSize: if !opts.JSON && !v.UI.Quiet() { v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n", blob.Hash, ubytes(stat.Size), ubytes(blob.CompressedSize)) } mismatched++ default: verified++ } } return verified, missing, mismatched, missingSize, unchecked, nil } // formatVerifyResult outputs the final verification results as JSON or // human-readable text. func (v *Vaultik) formatVerifyResult( result *VerifyResult, opts *VerifyOptions, ) error { failure := shallowVerifyFailure(result) if opts.JSON { if failure != "" { result.Status = verifyStatusFailed result.ErrorMessage = failure } else { result.Status = "ok" } return v.outputVerifyJSON(result) } // --quiet drops the report but not the errors storage returned, which // are logged on stderr. A failure is still returned either way, and // the cli layer prints it on stderr. if v.UI.Quiet() { for _, blob := range result.Unchecked { log.Warn("Could not check blob", "blob", blob.Hash, "error", blob.Error) } if result.DatabaseCheckError != "" { log.Warn("Could not check the encrypted database", "snapshot", result.SnapshotID, "error", result.DatabaseCheckError) } } else { v.printVerifySummary(result, failure) } // Not errSnapshotVerifyFailed: the cli layer prints this after its own // "Verification failed:". if failure != "" { return fmt.Errorf("%w: %s", errProblemsInStorage, failure) } return nil } // printVerifySummary prints the counts and the status line that end the // human-readable shallow verify report. failure is empty when it passed. func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) { v.stdoutf("\nVerification complete:\n") v.stdoutf(" Present with listed size: %d blobs\n", result.Verified) if result.Missing > 0 { v.stdoutf(" Missing: %d blobs (%s)\n", result.Missing, ubytes(result.MissingSize)) } if result.Mismatched > 0 { v.stdoutf(" Wrong size: %d blobs\n", result.Mismatched) } if len(result.Unchecked) > 0 { v.stdoutf(" Could not check: %d blobs\n", len(result.Unchecked)) } if result.DatabaseMissing { v.stdoutf(" Encrypted database: missing\n") } if result.DatabaseCheckError != "" { v.stdoutf(" Encrypted database: could not be checked (%s)\n", result.DatabaseCheckError) } v.stdoutf(" Status: ") if failure != "" { v.stdoutf("FAILED - %s\n", failure) return } // Report only what was actually checked: presence and size, not contents. v.stdoutf("OK - all %d blobs listed in the manifest are present with the "+ "listed size; contents not checked (use --deep)\n", result.Verified) } // shallowVerifyFailure returns a human-readable description of everything // that failed shallow verification, or the empty string if it passed. func shallowVerifyFailure(result *VerifyResult) string { var parts []string if result.Missing > 0 { parts = append(parts, fmt.Sprintf("%d blobs are missing", result.Missing)) } if result.Mismatched > 0 { parts = append(parts, fmt.Sprintf("%d blobs have the wrong size", result.Mismatched)) } if len(result.Unchecked) > 0 { parts = append(parts, fmt.Sprintf("%d blobs could not be checked", len(result.Unchecked))) } if result.DatabaseMissing { parts = append(parts, "the encrypted database is missing") } if result.DatabaseCheckError != "" { parts = append(parts, "the encrypted database could not be checked") } return strings.Join(parts, "; ") } // outputVerifyJSON outputs the verification result as JSON func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") err := encoder.Encode(result) if err != nil { return fmt.Errorf("encoding JSON: %w", err) } if result.Status == verifyStatusFailed { return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, result.ErrorMessage) } return nil } // CleanupLocalSnapshots removes local snapshot records that have no // corresponding metadata in remote storage. These are typically left // behind by incomplete or interrupted backups. Each local snapshot's // human ID is hashed via RemoteSnapshotKey and compared against the // remote listing. // // It takes the whole *PruneOptions, symmetric with PruneBlobs, because // it is the other half of one command: Prune runs this phase and then // that one. Only JSON is read here. Under --json every write below is // suppressed, because stdout carries the PruneBlobsResult document and // nothing else — prose ahead of it is what made `vaultik prune --json | // jq` fail (issue #108). The narration is duplicated as log records, // which go to stderr and so cannot corrupt the document. func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error { err := v.EnsureStorageBinding() if err != nil { return err } remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { return err } remoteSet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteSet[k] = true } localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } var removed int for _, snap := range localSnapshots { id := snap.ID.String() if !remoteSet[snapshot.RemoteSnapshotKey(id)] { log.Info("Removing stale local snapshot record", "snapshot_id", id) if !opts.JSON { v.UI.Infof("Removing stale local record: %s.", v.UI.Snapshot(id)) } err = v.deleteSnapshotFromLocalDB(id) if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) continue } removed++ } } log.Info("Reconciled local snapshot records against remote metadata", "removed", removed, "examined", len(localSnapshots)) if opts.JSON { return nil } if removed == 0 { v.UI.Infof("No stale local snapshots found.") } else { v.UI.Infof("Removed %d stale local snapshot record(s).", removed) } return nil } // Helper methods that were previously on SnapshotApp // downloadManifestByKey fetches the manifest at // metadata//manifest.json.zst. The remoteKey is the double- // SHA256 derivation produced by snapshot.RemoteSnapshotKey, not the // human snapshot ID. Callers that have a human ID must hash first. // // This is the only place vaultik reads a manifest off the destination // store, deliberately: the manifest is currently stored compressed but // unencrypted, which is what lets `snapshot list` enumerate the // destination on a host holding no private key. Whether to encrypt it // is open (issue #81), and routing every read through here means that // decision has exactly one call site to change. Keep it that way — do // not open metadata//manifest.json.zst directly elsewhere. func (v *Vaultik) downloadManifestByKey(remoteKey string) (*snapshot.Manifest, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) reader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { return nil, err } defer func() { _ = reader.Close() }() manifest, err := snapshot.DecodeManifest(reader) if err != nil { return nil, fmt.Errorf("decoding manifest: %w", err) } return manifest, nil } func (v *Vaultik) syncWithRemote() error { log.Info("Syncing with remote snapshots") // Remote metadata lives under metadata//, where the // directory name is snapshot.RemoteSnapshotKey(id), not the human // snapshot ID. Compare each local row's hashed key against that set // so a row still backed by remote metadata is kept. Comparing human // IDs against the hashed directory names matches nothing and deletes // every local snapshot record (issue #160). remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { return err } remoteKeySet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteKeySet[k] = true } log.Debug("Found remote snapshots", "count", len(remoteKeySet)) // Get all local snapshots (use a high limit to get all) localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } // Remove local snapshots whose metadata is absent from the remote. removedCount := 0 for _, snap := range localSnapshots { snapshotIDStr := snap.ID.String() if !remoteKeySet[snapshot.RemoteSnapshotKey(snapshotIDStr)] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID) err = v.deleteSnapshotFromLocalDB(snapshotIDStr) if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err) } else { removedCount++ } } } if removedCount > 0 { log.Info("Removed local snapshots not found in remote", "count", removedCount) } return nil } // RemoveOptions contains options for the snapshot remove command type RemoveOptions struct { Force bool DryRun bool JSON bool LocalOnly bool // Skip remote cleanup; only touch the local index } // RemoveResult contains the result of a snapshot removal // //nolint:tagliatelle // snake_case is the established JSON output format type RemoveResult struct { SnapshotID string `json:"snapshot_id,omitempty"` SnapshotsRemoved []string `json:"snapshots_removed,omitempty"` RemoteRemoved bool `json:"remote_removed,omitempty"` DryRun bool `json:"dry_run,omitempty"` } // pruneCommandHint is the exact command suggested after a remove // completes so the user knows how to clean up the blobs that the // just-removed snapshot left behind on the destination store. const pruneCommandHint = "vaultik prune" // snapshotRemoveCommandHint is the command suggested, with the // snapshot's ID, when a remove could not reach the destination store: // running it again removes the snapshot's metadata there, which // `vaultik prune` never does while the manifest is there. const snapshotRemoveCommandHint = "vaultik snapshot remove" // RemoveSnapshot removes a snapshot from the local index database and, // unless LocalOnly is set, also strips the snapshot's metadata from the // destination store. Blobs are NOT touched: removing a snapshot's // blobs requires reading every remaining remote manifest (the remote // may hold snapshots this host doesn't know about), which is what // `vaultik prune` is for — when metadata was removed from the // destination store, the text output names the prune invocation to run // as a follow-up. When removing the metadata fails, a snapshot in the // local index is still removed from it, with a warning; a snapshot // only on the destination store is an error. func (v *Vaultik) RemoveSnapshot( snapshotID string, opts *RemoveOptions, ) (*RemoveResult, error) { result := &RemoveResult{ SnapshotID: snapshotID, } err := v.EnsureStorageBinding() if err != nil { return result, err } inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts) if err != nil { return result, err } if opts.DryRun { return v.removeSnapshotDryRun(result, snapshotID, opts) } if !v.confirmRemoveSnapshot(snapshotID, opts) { return result, nil } // Checked before the local record is deleted: it may be the last // snapshot in the local index, and the check applies only while the // local index holds one. var markerErr error if !opts.LocalOnly { markerErr = v.checkDestinationMarker() } if inLocalIndex { log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) err = v.deleteSnapshotFromLocalDB(snapshotID) if err != nil { return result, fmt.Errorf("removing from local database: %w", err) } } result.RemoteRemoved, err = v.removeSnapshotRemote( snapshotID, inLocalIndex, markerErr, opts) if err != nil { return result, err } if v.SnapshotManager != nil { err = v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { log.Warn("Failed to clean up orphaned local data after removal", "error", err) } } if opts.JSON { return result, v.outputRemoveJSON(result) } if inLocalIndex { v.UI.Infof("Removed snapshot '%s' from local database.", v.UI.Snapshot(snapshotID)) } if result.RemoteRemoved { v.UI.Infof("Removed snapshot metadata from remote storage.") v.UI.Noticef("The removed snapshot's blobs remain on the remote. "+ "Run '%s' to delete any blobs no longer referenced by any "+ "remaining remote snapshot.", pruneCommandHint) } return result, nil } // findSnapshotToRemove reports whether the local index has a row for // snapshotID, and returns errSnapshotNotFound when there is nothing to // remove: no row and, unless LocalOnly is set, no metadata on the // destination store either. The destination store is consulted only // when the local index has no row, so a snapshot in the local index can // still be removed from it while the destination store is unreachable, // and only after checkDestinationMarker passes, so an unplugged volume's // empty mount point is not taken for a store without the snapshot. func (v *Vaultik) findSnapshotToRemove( snapshotID string, opts *RemoveOptions, ) (bool, error) { if v.Repositories != nil { snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID) if err != nil { return false, fmt.Errorf("looking up snapshot in local index: %w", err) } if snap != nil { return true, nil } } if opts.LocalOnly { return false, fmt.Errorf("%w in the local index: %s", errSnapshotNotFound, snapshotID) } err := v.checkDestinationMarker() if err != nil { return false, err } objects, err := v.remoteSnapshotObjects(snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { return false, fmt.Errorf("looking up snapshot on destination store: %w", err) } if len(objects) == 0 { return false, fmt.Errorf( "%w in the local index or on the destination store: %s", errSnapshotNotFound, snapshotID) } return false, nil } // removeSnapshotDryRun reports what RemoveSnapshot would do without // making changes. func (v *Vaultik) removeSnapshotDryRun( result *RemoveResult, snapshotID string, opts *RemoveOptions, ) (*RemoveResult, error) { result.DryRun = true if opts.JSON { return result, v.outputRemoveJSON(result) } v.UI.Infof("Would remove snapshot: %s.", v.UI.Snapshot(snapshotID)) if !opts.LocalOnly { v.UI.Infof("Would also remove snapshot metadata from " + "remote storage (blobs untouched).") } v.UI.Noticef("Dry run: no changes made.") return result, nil } // confirmRemoveSnapshot prompts for confirmation before removing a // snapshot, unless --force or --json is set. Returns false (treating // EOF/read errors as "no") unless the user answers "y". The prompt and // its answer bypass the UI: they are an interactive exchange the // operator must see, so --quiet does not silence them. func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool { if opts.Force || opts.JSON { return true } if opts.LocalOnly { v.stdoutf("Remove snapshot '%s' from local database "+ "(remote untouched)? [y/N] ", snapshotID) } else { v.stdoutf("Remove snapshot '%s' from local database AND its metadata "+ "from remote storage? [y/N] ", snapshotID) } var confirm string _, err := v.scanStdin(&confirm) if err != nil { v.printlnStdout("Cancelled") return false } if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") return false } return true } // removeSnapshotRemote strips the snapshot's metadata from the // destination store. It returns true only when the delete finished // without an error and removed at least one object; under LocalOnly it // does nothing. markerErr is the result of checkDestinationMarker; when // it is set, nothing is deleted and it is the failure. When the snapshot // is only on the destination store, a failure is returned as an error, // because that removal is all the command does. When the snapshot was in // the local index, its removal there has already happened, so a failure // is only a warning telling the user to run `vaultik snapshot remove` // for the snapshot again once the destination store is reachable // (`vaultik prune` never removes snapshot metadata that has its // manifest). func (v *Vaultik) removeSnapshotRemote( snapshotID string, inLocalIndex bool, markerErr error, opts *RemoveOptions, ) (bool, error) { if opts.LocalOnly { return false, nil } log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) var deleted int err := markerErr if err == nil { deleted, err = v.deleteRemoteSnapshotByKey( snapshot.RemoteSnapshotKey(snapshotID)) } if err != nil && !inLocalIndex { return false, fmt.Errorf( "removing snapshot metadata from destination store: %w", err) } if err != nil { log.Warn("Could not remove snapshot metadata from remote storage; "+ "run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+ "again once the remote is reachable", "snapshot_id", snapshotID, "error", err) // The UI writes to stdout, which under --json holds only the // document; the log record above is the warning on stderr. if v.UI != nil && !opts.JSON { v.UI.Warningf("Could not remove snapshot metadata from remote: "+ "%v. Run '%s %s' again once the remote is reachable.", err, snapshotRemoveCommandHint, snapshotID) } return false, nil } return deleted > 0, nil } // RemoveAllSnapshots removes every snapshot known to the local // database from the local index, and (with --remote) every snapshot // metadata directory in remote storage. Both sides are processed so a // "remove --all" leaves nothing behind, even when the local DB and // remote storage have diverged. Unless LocalOnly is set, it fails // before it removes anything when the destination store cannot be // listed or is not present by checkDestinationMarker. func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) { err := v.EnsureStorageBinding() if err != nil { return nil, err } localSnaps, err := v.localSnapshotIDs() if err != nil { return nil, fmt.Errorf("listing local snapshots: %w", err) } // remoteKeys is the set of metadata// subdirectories on the // destination store. Only for a local-only remove is a failure // downgraded to a warning, so a permission-denied or unreachable // remote can't block it. remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys() if remoteErr != nil { if !opts.LocalOnly { return nil, remoteErr } log.Warn("Could not list remote snapshots", "error", remoteErr) v.UI.Warningf("Could not list remote snapshots: %v.", remoteErr) } // Anything visible on the remote that doesn't correspond to a // known local human ID is treated as an orphan key — handled only // when --remote is in effect. knownLocalKeys := make(map[string]string, len(localSnaps)) for _, id := range localSnaps { knownLocalKeys[snapshot.RemoteSnapshotKey(id)] = id } var orphanRemoteKeys []string for _, key := range remoteKeys { if _, known := knownLocalKeys[key]; !known { orphanRemoteKeys = append(orphanRemoteKeys, key) } } if len(localSnaps) == 0 && len(orphanRemoteKeys) == 0 { if !opts.JSON { v.UI.Infof("No snapshots found.") } return &RemoveResult{}, nil } if opts.DryRun { return v.handleRemoveAllDryRun(localSnaps, orphanRemoteKeys, opts) } return v.executeRemoveAll(localSnaps, orphanRemoteKeys, opts) } // localSnapshotIDs returns every snapshot ID present in the local // index database, sorted for deterministic iteration. Empty slice if // the database has no Repositories (e.g. tests). func (v *Vaultik) localSnapshotIDs() ([]string, error) { if v.Repositories == nil { return nil, nil } snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return nil, err } ids := make([]string, 0, len(snaps)) for _, s := range snaps { ids = append(ids, s.ID.String()) } sort.Strings(ids) return ids, nil } // listAllRemoteSnapshotKeys collects the hashed remote keys // (subdirectories under metadata/) currently present in the // destination store. Returns (nil, err) when the store cannot be // listed, or is not present by checkDestinationMarker. func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { log.Info("Listing all remote snapshots") objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) var keys []string for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) } parts := strings.Split(object.Key, "/") if len(parts) < minSnapshotIDParts || parts[0] != metadataDirName || parts[1] == "" { continue } // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } if !strings.HasSuffix(object.Key, "/") && !strings.Contains(object.Key, "/manifest.json.zst") { continue } key := parts[1] // A remote snapshot key is a SHA-256 hash: 64 lowercase hex // characters. The listing comes from the untrusted destination, // so accept a key only in that form. if !isBlobHash(key) { log.Warn("Skipping non-conforming key under metadata/", "key", object.Key) continue } if !seen[key] { seen[key] = true keys = append(keys, key) } } err := v.checkDestinationMarker() if err != nil { return nil, err } return keys, nil } // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots func (v *Vaultik) handleRemoveAllDryRun( localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, ) (*RemoveResult, error) { result := &RemoveResult{DryRun: true} result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) if !opts.LocalOnly { result.SnapshotsRemoved = append( result.SnapshotsRemoved, orphanRemoteKeys...) } if !opts.JSON { v.UI.Infof("Would remove %d local snapshot(s):", len(localSnaps)) for _, id := range localSnaps { v.UI.Detailf("%s", v.UI.Snapshot(id)) } if !opts.LocalOnly { if len(orphanRemoteKeys) > 0 { v.UI.Infof("Would also remove %d orphan remote snapshot key(s):", len(orphanRemoteKeys)) for _, key := range orphanRemoteKeys { v.UI.Detailf("%s", key) } } else { v.UI.Infof("Would also remove snapshot metadata from " + "remote storage (blobs untouched).") } } v.UI.Noticef("Dry run: no changes made.") } if opts.JSON { return result, v.outputRemoveJSON(result) } return result, nil } // executeRemoveAll deletes every local snapshot and, unless LocalOnly // is set, every corresponding remote metadata directory plus any // orphan remote keys. Blobs are not touched. It returns an error when // any snapshot could not be removed. func (v *Vaultik) executeRemoveAll( localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, ) (*RemoveResult, error) { if !opts.Force { return nil, errRemoveAllNeedsForce } log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} failures := v.removeAllLocalSnapshots(localSnaps, opts, result) if !opts.LocalOnly { failures += v.removeAllOrphanRemotes(orphanRemoteKeys, result) result.RemoteRemoved = failures == 0 } if v.SnapshotManager != nil { err := v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err) } } if failures > 0 { return result, fmt.Errorf("%d %w", failures, errSnapshotRemoveFailed) } if opts.JSON { return result, v.outputRemoveJSON(result) } v.UI.Infof("Removed %d snapshot(s).", len(result.SnapshotsRemoved)) if !opts.LocalOnly && result.RemoteRemoved { v.UI.Infof("Removed snapshot metadata from remote storage.") } return result, nil } // deleteIncompleteSnapshot deletes one incomplete snapshot and its // association rows, reporting whether the snapshot row itself was // deleted. func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool { log.Info("Deleting incomplete snapshot", "snapshot_id", id) // Delete related records first err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) if err != nil { log.Error("Failed to delete snapshot files", "snapshot_id", id, "error", err) } err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) if err != nil { log.Error("Failed to delete snapshot blobs", "snapshot_id", id, "error", err) } err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) if err != nil { log.Error("Failed to delete snapshot uploads", "snapshot_id", id, "error", err) } err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) if err != nil { log.Error("Failed to delete snapshot", "snapshot_id", id, "error", err) return false } return true } // removeAllLocalSnapshots removes each known local snapshot's remote // metadata, unless --local-only, and then its local record, returning // the number of snapshots it could not remove. The local record goes // only once the metadata is deleted: dropped first, it would leave that // metadata on the destination store as a remote-only snapshot whose name // is lost. func (v *Vaultik) removeAllLocalSnapshots( localSnaps []string, opts *RemoveOptions, result *RemoveResult, ) int { failures := 0 for _, snapshotID := range localSnaps { log.Info("Removing snapshot", "snapshot_id", snapshotID) if !opts.LocalOnly { _, err := v.deleteRemoteSnapshotByKey( snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { log.Warn("Failed to remove snapshot metadata from remote", "snapshot_id", snapshotID, "error", err) failures++ continue } } err := v.deleteSnapshotFromLocalDB(snapshotID) if err != nil { log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) failures++ continue } result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) } return failures } // removeAllOrphanRemotes deletes remote snapshot metadata that has no // local counterpart, returning the number of deletion failures. func (v *Vaultik) removeAllOrphanRemotes( orphanRemoteKeys []string, result *RemoveResult, ) int { remoteErrors := 0 for _, key := range orphanRemoteKeys { log.Info("Removing orphan remote snapshot", "remote_key", key) _, err := v.deleteRemoteSnapshotByKey(key) if err != nil { log.Warn("Failed to remove orphan from remote", "remote_key", key, "error", err) remoteErrors++ continue } result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) } return remoteErrors } // deleteSnapshotFromLocalDB removes a snapshot from the local database only func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { if v.Repositories == nil { return nil // No local database } // Delete related records first to avoid foreign key constraints err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID) if err != nil { return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err) } err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID) if err != nil { return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err) } err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID) if err != nil { return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err) } err = v.Repositories.Snapshots.Delete(v.ctx, snapshotID) if err != nil { return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err) } return nil } // deleteRemoteSnapshotByKey removes everything under // metadata// on the destination store and returns how many // objects it deleted. The argument is a remote key (double-SHA256 // derivation), not a human snapshot ID; callers that have a human ID // must hash via snapshot.RemoteSnapshotKey first. func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) (int, error) { objectsToDelete, err := v.remoteSnapshotObjects(remoteKey) if err != nil { return 0, err } for i, key := range objectsToDelete { err = v.Storage.Delete(v.ctx, key) if err != nil { return i, fmt.Errorf("removing %s: %w", key, err) } log.Debug("Deleted remote object", "key", key) } return len(objectsToDelete), nil } // remoteSnapshotObjects lists the keys of every object under // metadata// on the destination store. func (v *Vaultik) remoteSnapshotObjects(remoteKey string) ([]string, error) { prefix := fmt.Sprintf("metadata/%s/", remoteKey) var keys []string for object := range v.Storage.ListStream(v.ctx, prefix) { if object.Err != nil { return nil, fmt.Errorf("listing objects: %w", object.Err) } keys = append(keys, object.Key) } return keys, nil } // outputRemoveJSON outputs the removal result as JSON func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(result) } // PruneResult contains statistics about the prune operation. // SnapshotsDeleted counts snapshots actually deleted. FilesDeleted, // ChunksDeleted, and BlobsDeleted are derived from before/after row // counts of the local index; each is nil when a count could not be read, // so an unreadable count is reported as unknown rather than silently // as 0. type PruneResult struct { SnapshotsDeleted int64 FilesDeleted *int64 ChunksDeleted *int64 BlobsDeleted *int64 } // PruneDatabase removes incomplete snapshots and orphaned files, chunks, // and blobs from the local database. This ensures database consistency // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { log.Info("Pruning local database: " + "removing incomplete snapshots and orphaned data") v.UI.Beginf("Pruning local index database " + "(removing incomplete snapshots and orphaned data).") result := &PruneResult{} // Snapshot counts before deletion of incompletes. snapshotCountBefore := v.tableCountForReport("snapshots") // First, delete any incomplete snapshots incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx) if err != nil { return nil, fmt.Errorf("getting incomplete snapshots: %w", err) } for _, snap := range incompleteSnapshots { if v.deleteIncompleteSnapshot(snap.ID.String(), snap.ID) { result.SnapshotsDeleted++ } } // Get counts before cleanup for reporting fileCountBefore := v.tableCountForReport("files") chunkCountBefore := v.tableCountForReport("chunks") blobCountBefore := v.tableCountForReport("blobs") // Run the cleanup err = v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { return nil, fmt.Errorf("cleanup orphaned data: %w", err) } // Get counts after cleanup fileCountAfter := v.tableCountForReport("files") chunkCountAfter := v.tableCountForReport("chunks") blobCountAfter := v.tableCountForReport("blobs") result.FilesDeleted = countDiff(fileCountBefore, fileCountAfter) result.ChunksDeleted = countDiff(chunkCountBefore, chunkCountAfter) result.BlobsDeleted = countDiff(blobCountBefore, blobCountAfter) log.Info("Local database prune complete", "incomplete_snapshots", result.SnapshotsDeleted, "orphaned_files", countText(result.FilesDeleted), "orphaned_chunks", countText(result.ChunksDeleted), "orphaned_blobs", countText(result.BlobsDeleted), ) // Snapshots remaining after removing the incomplete ones; unknown if // the pre-prune snapshot count could not be read. snapshotsRemain := countDiff(snapshotCountBefore, &result.SnapshotsDeleted) v.UI.Completef("Pruned local index database.") v.UI.Detailf("Incomplete snapshots: %s removed "+ "(total snapshots left in the local index: %s).", countText(&result.SnapshotsDeleted), countText(snapshotsRemain)) v.UI.Detailf("Orphaned files: %s removed "+ "(total files left in the local index: %s).", countText(result.FilesDeleted), countText(fileCountAfter)) v.UI.Detailf("Orphaned chunks: %s removed "+ "(total chunks left in the local index: %s).", countText(result.ChunksDeleted), countText(chunkCountAfter)) v.UI.Detailf("Orphaned blobs: %s removed "+ "(total blobs left in the local index: %s).", countText(result.BlobsDeleted), countText(blobCountAfter)) return result, nil } // unknownText is what a count or size reads as when it could not be // determined, distinct from "0", which is a real zero. const unknownText = "unknown" // tableCountForReport returns the row count of a table for the prune // summary, or nil if the count could not be read. A read failure is // logged at warn — visible even under --json, which routes warnings to // stderr — and then rendered as unknown rather than silently becoming 0, // so a broken query is a visible failure instead of a plausible wrong // number. func (v *Vaultik) tableCountForReport(tableName string) *int64 { count, err := v.getTableCount(tableName) if err != nil { log.Warn("could not read table row count for prune summary", "table", tableName, "error", err) return nil } return &count } // countDiff returns before-after, or nil if either count is unknown so // that an unreadable count does not collapse into a plausible delta. func countDiff(before, after *int64) *int64 { if before == nil || after == nil { return nil } diff := *before - *after return &diff } // countText renders a count that may be unknown: nil (the read failed) // becomes "unknown", never "0", so a reader can tell an empty table from // one that could not be queried. func countText(count *int64) string { if count == nil { return unknownText } return strconv.FormatInt(*count, 10) } // validTableNameRe matches table names containing only lowercase // alphanumeric characters and underscores. var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`) // getTableCount returns the count of rows in a table. The tableName is // sanitized to only allow [a-z0-9_] characters to prevent SQL // injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } if !validTableNameRe.MatchString(tableName) { return 0, fmt.Errorf("%w: %q", errInvalidTableName, tableName) } var count int64 query := "SELECT COUNT(*) FROM " + tableName err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) if err != nil { return 0, err } return count, nil }