Files
vaultik/internal/vaultik/snapshot.go
T
clawbot 4418dc3e97
check / check (push) Successful in 26m13s
Print remote info and verify status lines through internal/ui (closes #410)
The status and progress lines of remote info and snapshot verify (shallow and deep) were written with stdoutf, so they had no marker and no color, unlike every other command's. They now go through Beginf, Completef and Progressf, as the README's output style section says. The reports stay plain, and --quiet and --json still drop these lines.

Model: opus-5-5
2026-11-10 22:12:46 +01:00

2287 lines
69 KiB
Go

package vaultik
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"time"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/types"
)
// Sentinel errors for snapshot management.
var (
errSnapshotNotInConfig = errors.New("snapshot not found in config")
errNoSnapshotsInConfig = errors.New("no snapshots configured")
errSnapshotVerifyFailed = errors.New("verification failed")
errProblemsInStorage = errors.New("problems found in storage")
errRemoveAllNeedsForce = errors.New("--all requires --force")
errSnapshotNotFound = errors.New("snapshot not found")
errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed")
errInvalidTableName = errors.New("invalid table name")
errNoAgeRecipients = errors.New(
"creating a snapshot needs at least one public key in " +
"age_recipients (generate a keypair with: age-keygen)")
errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune")
)
// ErrPostBackupPruneFailed wraps a failure of the post-backup prune,
// which runs only after every snapshot was created and uploaded, so the
// caller can report it apart from a failed backup.
var ErrPostBackupPruneFailed = errors.New("post-backup prune failed")
// ErrFilesUnreadable is returned when --skip-errors skipped files or
// directories it could not read or list. The snapshot is complete and
// kept, so the caller can report it apart from a failed backup.
var ErrFilesUnreadable = errors.New("file(s) could not be read")
// ErrSourcePathEmpty is returned when --prune skipped the post-backup
// prune because the scan found nothing in a configured path of a
// snapshot, as in the mount point of a volume that is not mounted. The
// snapshots are complete and kept, so the caller can report it apart
// from a failed backup. A run that also skipped unreadable files
// returns ErrFilesUnreadable instead.
var ErrSourcePathEmpty = errors.New("source path(s) were empty")
// ErrVolumeNotMounted is returned when --prune skipped the post-backup
// prune because a directory that an earlier snapshot of the same name
// found as a mount point was not found as one in this run: the volume
// mounted there is not mounted, the run found no entry in it, or the
// directory is no longer backed up. The snapshots are complete and
// kept, so the caller can report it apart from a failed backup. A run
// that also skipped unreadable files or found a configured path empty
// returns ErrFilesUnreadable or ErrSourcePathEmpty instead.
var ErrVolumeNotMounted = errors.New(
"mount point(s) an earlier snapshot recorded were not found")
// listRecentLimit caps how many snapshot rows are fetched from the
// local index for listing and purge operations.
const listRecentLimit = 10000
// SnapshotCreateOptions contains options for the snapshot create command
type SnapshotCreateOptions struct {
Cron bool
Prune bool
// KeepNewerThan is used with --prune: keep snapshots newer than this
// duration (e.g. "4w"); default: keep only latest.
KeepNewerThan string
SkipErrors bool // Skip file read errors (log them loudly but continue)
Snapshots []string // Optional list of snapshot names to process (empty = all)
}
// Validate fails when KeepNewerThan is set without Prune or is not a
// valid duration. Only the post-backup prune reads KeepNewerThan, after
// the backup, so the caller checks it before starting one.
func (opts *SnapshotCreateOptions) Validate() error {
if opts.KeepNewerThan == "" {
return nil
}
if !opts.Prune {
return errKeepNewerThanNeedsPrune
}
_, err := parseDuration(opts.KeepNewerThan)
if err != nil {
return fmt.Errorf("--keep-newer-than: %w", err)
}
return nil
}
// CreateSnapshot executes the snapshot creation operation
func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
// config.Load accepts an empty list, since listing, verifying and
// restoring need no public key.
if len(v.Config.AgeRecipients) == 0 {
return errNoAgeRecipients
}
overallStartTime := time.Now()
log.Info("Starting snapshot creation",
"version", v.Globals.Version,
"commit", v.Globals.Commit,
"index_path", v.Config.IndexPath,
)
err := v.EnsureStorageBinding()
if err != nil {
return err
}
// Before PruneDatabase: it drops incomplete snapshots, which could
// leave the local index without one and so skip the marker check.
err = v.markDestination()
if err != nil {
return err
}
// Clean up incomplete snapshots FIRST, before any scanning.
// This is critical for data safety; PruneDatabase below does it.
hostname := v.Config.Hostname
if hostname == "" {
hostname, _ = os.Hostname()
}
// CRITICAL: This MUST succeed. If we fail to clean up incomplete snapshots,
// the deduplication logic will think files from the incomplete snapshot were
// already backed up and skip them, resulting in data loss.
//
// Prune the database before starting: delete incomplete snapshots and orphaned data.
// This ensures the database is consistent before we start a new snapshot.
// Since we use locking, only one vaultik instance accesses the DB at a time.
// A snapshot whose metadata export was interrupted is left incomplete by
// finalizeSnapshotMetadata, so it is among the incomplete snapshots dropped
// here (https://git.eeqj.de/sneak/vaultik/issues/177).
_, err = v.PruneDatabase()
if err != nil {
return fmt.Errorf("prune database: %w", err)
}
snapshotNames, err := v.snapshotNamesToCreate(opts.Snapshots)
if err != nil {
return err
}
filesUnreadable, emptyPaths, unmountedPaths, err := v.createNamedSnapshots(
opts, hostname, snapshotNames)
if err != nil {
return err
}
// Print overall summary if multiple snapshots
if len(snapshotNames) > 1 {
v.UI.Completef("All %d snapshots completed in %s.",
len(snapshotNames), v.UI.Duration(time.Since(overallStartTime)))
}
switch {
case opts.Prune && (filesUnreadable > 0 || len(emptyPaths) > 0 ||
len(unmountedPaths) > 0):
return v.skipPostBackupPrune(
filesUnreadable, emptyPaths, unmountedPaths)
case opts.Prune:
err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan)
if err != nil {
return fmt.Errorf("%w: %w", ErrPostBackupPruneFailed, err)
}
}
return v.finishCreateSnapshot(filesUnreadable)
}
// createNamedSnapshots creates each named snapshot in turn. It returns
// the number of entries --skip-errors skipped because it could not read
// or list them, each configured path in which the scan found nothing,
// and, under --prune, each directory where an earlier snapshot found a
// volume mounted and this run did not, the paths named with their
// snapshot.
func (v *Vaultik) createNamedSnapshots(
opts *SnapshotCreateOptions, hostname string, snapshotNames []string,
) (int, []string, []string, error) {
filesUnreadable := 0
var emptyPaths, unmountedPaths []string
for snapIdx, snapName := range snapshotNames {
stats, err := v.createNamedSnapshot(
opts, hostname, snapName, snapIdx+1, len(snapshotNames))
if err != nil {
return 0, nil, nil, err
}
filesUnreadable += stats.totalFilesUnreadable
for _, dir := range stats.emptyPaths {
emptyPaths = append(emptyPaths, dir+" of snapshot "+snapName)
}
for _, dir := range stats.unmountedPaths {
unmountedPaths = append(unmountedPaths,
dir+" of snapshot "+snapName)
}
}
return filesUnreadable, emptyPaths, unmountedPaths, nil
}
// skipPostBackupPrune says why the post-backup prune does not run, naming
// every cause, and returns the error the run ends with. An older snapshot
// may be the only one that still holds a file this run could not read,
// or what an empty path or a volume that is not mounted held. The
// previous snapshot is no guide: after one such run it lacks them too.
func (v *Vaultik) skipPostBackupPrune(
filesUnreadable int, emptyPaths, unmountedPaths []string,
) error {
log.Warn("Skipping post-backup prune",
"files_unreadable", filesUnreadable, "empty_paths", emptyPaths,
"unmounted_paths", unmountedPaths)
if len(emptyPaths) > 0 {
v.UI.Warningf("Skipped the post-backup prune because nothing was "+
"found in %s; older snapshots are kept.",
strings.Join(emptyPaths, ", "))
}
if len(unmountedPaths) > 0 {
v.UI.Warningf("Skipped the post-backup prune because this backup "+
"did not find a mount point at %s, where an earlier snapshot "+
"found one: the volume is not mounted, this backup found no "+
"entry in it, or the directory is no longer backed up. Older "+
"snapshots are kept. Mount the volume again, or, if that is on "+
"purpose, remove the older snapshots by hand with "+
"'vaultik snapshot purge'.",
strings.Join(unmountedPaths, ", "))
}
if filesUnreadable > 0 {
v.UI.Warningf("Skipped the post-backup prune because %d file(s) "+
"could not be read; older snapshots are kept.", filesUnreadable)
return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable)
}
if len(emptyPaths) > 0 {
return fmt.Errorf("%d %w", len(emptyPaths), ErrSourcePathEmpty)
}
return fmt.Errorf("%d %w", len(unmountedPaths), ErrVolumeNotMounted)
}
// finishCreateSnapshot ends a run whose snapshots are all complete and
// kept. Like finishRestore, it returns an error when --skip-errors
// skipped anything it could not read or list; otherwise it prints the
// closing line. A file removed during the run is counted as failed in
// the summary but is not an error, with or without --skip-errors.
func (v *Vaultik) finishCreateSnapshot(filesUnreadable int) error {
if filesUnreadable > 0 {
return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable)
}
// Terminus must obey the --cron invariant: silent on total
// success only. UI.Complete is dropped in cron/quiet mode (that's
// the success path), but if any warnings fired during the run we
// emit the summary via UI.Warning so cron actually delivers
// something for the user to look at.
if v.UI.WarningCount() > 0 {
v.UI.Warningf("Finished with %d warning(s) — review the output above.",
v.UI.WarningCount())
} else {
v.UI.Completef("Finished successfully.")
}
return nil
}
// snapshotNamesToCreate returns the requested snapshot names, each checked
// against the config, or every configured name when none was requested.
func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) {
snapshotNames := requested
if len(snapshotNames) == 0 {
snapshotNames = v.Config.SnapshotNames()
} else {
// Validate requested snapshot names exist
for _, name := range snapshotNames {
if _, ok := v.Config.Snapshots[name]; !ok {
return nil, fmt.Errorf("%w: %q", errSnapshotNotInConfig, name)
}
}
}
if len(snapshotNames) == 0 {
return nil, errNoSnapshotsInConfig
}
return snapshotNames, nil
}
// runPostBackupPrune drops older snapshots of the given names and removes
// from remote storage the orphan blobs and the directories under
// metadata/ with no manifest, as PruneBlobs does. If keepNewerThan is set
// (e.g. "4w"), snapshots newer than that duration are kept. Otherwise only
// the latest snapshot of each name is kept.
func (v *Vaultik) runPostBackupPrune(
snapshotNames []string, keepNewerThan string,
) error {
log.Info("Running post-backup prune",
"snapshots", snapshotNames, "keep_newer_than", keepNewerThan)
v.UI.Beginf("Running post-backup prune.")
purgeOpts := &SnapshotPurgeOptions{
Force: true,
Names: snapshotNames,
Quiet: true,
}
if keepNewerThan != "" {
purgeOpts.OlderThan = keepNewerThan
} else {
purgeOpts.KeepLatest = true
}
err := v.PurgeSnapshotsWithOptions(purgeOpts)
if err != nil {
return fmt.Errorf("purging old snapshots: %w", err)
}
err = v.PruneBlobs(&PruneOptions{Force: true})
if err != nil {
return fmt.Errorf("pruning the destination store: %w", err)
}
return nil
}
// snapshotStats tracks aggregate statistics across directory scans
type snapshotStats struct {
totalFiles int
totalBytes int64
totalChunks int
newChunkSize int64
totalBlobs int
totalBytesSkipped int64
totalFilesSkipped int
totalFilesFailed int
totalFilesDeleted int
totalBytesDeleted int64
totalBytesUploaded int64
totalBlobsUploaded int
uploadDuration time.Duration
// The part of totalFilesFailed that --skip-errors skipped because it
// could not be read or listed.
totalFilesUnreadable int
// The configured paths in which the scan found nothing.
emptyPaths []string
// Under --prune, the directories where an earlier snapshot of the
// same name found a volume mounted and this one did not.
unmountedPaths []string
// The sizes of all blobs the snapshot references, set by
// finalizeSnapshotMetadata once snapshot_blobs is populated.
blobSize int64
blobUncompressedSize int64
}
// createNamedSnapshot creates a single named snapshot and returns the
// statistics of its scan.
func (v *Vaultik) createNamedSnapshot(
opts *SnapshotCreateOptions, hostname, snapName string, idx, total int,
) (*snapshotStats, error) {
snapshotStartTime := time.Now()
if total > 1 {
v.UI.Infof("Snapshot %d/%d: %s.", idx, total, snapName)
}
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
if err != nil {
return nil, err
}
previousSnapshotID, err := v.previousSnapshotID(snapName)
if err != nil {
return nil, err
}
scanner := v.ScannerFactory(snapshot.ScannerParams{
EnableProgress: !opts.Cron,
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
PreviousSnapshotID: previousSnapshotID,
})
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit)
if err != nil {
return nil, fmt.Errorf("creating snapshot: %w", err)
}
log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
v.UI.Beginf("Creating snapshot %s.", v.UI.Snapshot(snapshotID))
stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID)
if err != nil {
return nil, err
}
err = v.finalizeSnapshotMetadata(snapshotID, stats)
if err != nil {
return nil, err
}
log.Info("Snapshot complete",
"snapshot_id", snapshotID,
"name", snapName,
"files", stats.totalFiles,
"blobs_uploaded", stats.totalBlobsUploaded,
"bytes_uploaded", stats.totalBytesUploaded,
"duration", time.Since(snapshotStartTime))
v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
if opts.Prune {
stats.unmountedPaths, err = v.findUnmountedPaths(snapName, snapshotID)
if err != nil {
return nil, err
}
}
return stats, nil
}
// findUnmountedPaths returns each directory that an earlier snapshot
// named snapName in the local index recorded as a mount point and
// snapshot snapshotID did not, as when the volume mounted there is not
// mounted.
func (v *Vaultik) findUnmountedPaths(
snapName, snapshotID string,
) ([]string, error) {
found, err := v.Repositories.Snapshots.GetMountPoints(v.ctx, snapshotID)
if err != nil {
return nil, err
}
snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return nil, fmt.Errorf("listing snapshots: %w", err)
}
var unmounted []string
for _, s := range snapshots {
if s.ID.String() == snapshotID ||
parseSnapshotName(s.ID.String(), s.Hostname.String()) != snapName {
continue
}
recorded, err := v.Repositories.Snapshots.GetMountPoints(
v.ctx, s.ID.String())
if err != nil {
return nil, err
}
for _, dir := range recorded {
if !slices.Contains(found, dir) && !slices.Contains(unmounted, dir) {
unmounted = append(unmounted, dir)
}
}
}
return unmounted, nil
}
// previousSnapshotID returns the ID of the most recent completed snapshot
// named snapName in the local index, or "" when there is none.
func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return "", fmt.Errorf("listing snapshots: %w", err)
}
for _, s := range snapshots {
if s.CompletedAt != nil &&
parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
return s.ID.String(), nil
}
}
return "", nil
}
// resolveSnapshotPaths resolves source directories to absolute paths
// with symlink resolution. A path that is a symlink whose target does
// not exist is an error.
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
snapConfig := v.Config.Snapshots[snapName]
resolvedDirs := make([]string, 0, len(snapConfig.Paths))
for _, dir := range snapConfig.Paths {
absPath, err := filepath.Abs(dir)
if err != nil {
return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err)
}
resolvedPath, err := filepath.EvalSymlinks(absPath)
if err != nil {
// A path that does not exist is left for the scan to report.
// One that exists is a symlink whose target is gone, as one
// into a volume that is not mounted: the scan would record
// only the link, and --prune would then drop the older
// snapshots that hold the files.
_, lstatErr := os.Lstat(absPath)
if !os.IsNotExist(err) || lstatErr == nil {
return nil, fmt.Errorf(
"failed to resolve symlinks for %s: %w", absPath, err)
}
resolvedPath = absPath
}
resolvedDirs = append(resolvedDirs, resolvedPath)
}
return resolvedDirs, nil
}
// scanAllDirectories runs the scanner on each resolved directory and
// accumulates stats.
func (v *Vaultik) scanAllDirectories(
scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string,
) (*snapshotStats, error) {
if progress := scanner.GetProgress(); progress != nil {
progress.Start()
defer progress.Stop()
}
stats := &snapshotStats{}
for i, dir := range resolvedDirs {
select {
case <-v.ctx.Done():
log.Info("Snapshot creation cancelled")
return nil, v.ctx.Err()
default:
}
log.Info("Scanning directory", "path", dir)
v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).",
v.UI.Path(dir), i+1, len(resolvedDirs))
result, err := scanner.Scan(v.ctx, dir, snapshotID)
if err != nil {
return nil, fmt.Errorf("failed to scan %s: %w", dir, err)
}
stats.totalFiles += result.FilesScanned
stats.totalBytes += result.BytesScanned
stats.totalChunks += result.ChunksCreated
stats.newChunkSize += result.NewChunkSize
stats.totalBlobs += result.BlobsCreated
stats.totalFilesSkipped += result.FilesSkipped
stats.totalFilesFailed += result.FilesFailed
stats.totalFilesUnreadable += result.FilesUnreadable
stats.totalBytesSkipped += result.BytesSkipped
stats.totalFilesDeleted += result.FilesDeleted
stats.totalBytesDeleted += result.BytesDeleted
stats.totalBlobsUploaded += result.BlobsUploaded
stats.totalBytesUploaded += result.BytesUploaded
stats.uploadDuration += result.UploadDuration
if result.EntriesFound == 0 {
stats.emptyPaths = append(stats.emptyPaths, dir)
}
log.Info("Directory scan complete",
"path", dir,
"files", result.FilesScanned,
"files_skipped", result.FilesSkipped,
"files_failed", result.FilesFailed,
"bytes", result.BytesScanned,
"bytes_skipped", result.BytesSkipped,
"chunks", result.ChunksCreated,
"blobs", result.BlobsCreated,
"duration", result.EndTime.Sub(result.StartTime))
}
return stats, nil
}
// finalizeSnapshotMetadata updates stats, exports metadata, and only then
// marks the snapshot complete. Recording completion last is deliberate: an
// export interrupted by a crash leaves the snapshot incomplete rather than
// looking complete with no manifest or database at the destination. The next
// run's PruneDatabase drops the incomplete snapshot and re-backs-up its data.
// See https://git.eeqj.de/sneak/vaultik/issues/177.
func (v *Vaultik) finalizeSnapshotMetadata(
snapshotID string, stats *snapshotStats,
) error {
// snapshot_blobs must be populated before the blob sizes below, which
// total the snapshot's blobs, and before the export, which builds the
// manifest and the trimmed metadata database from it.
err := v.SnapshotManager.PopulateSnapshotBlobs(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("populating snapshot blobs: %w", err)
}
stats.blobSize, stats.blobUncompressedSize, err =
v.Repositories.Snapshots.GetSnapshotBlobSizes(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("getting snapshot blob sizes: %w", err)
}
// file_count and total_size leave out the files that could not be
// stored; stats.totalBytes already does.
extStats := snapshot.ExtendedBackupStats{
BackupStats: snapshot.BackupStats{
FilesScanned: stats.totalFiles - stats.totalFilesFailed,
TotalSize: stats.totalBytes + stats.totalBytesSkipped,
ChunksCreated: stats.totalChunks,
BlobsCreated: stats.totalBlobs,
BytesUploaded: stats.totalBytesUploaded,
},
BlobSize: stats.blobSize,
BlobUncompressedSize: stats.blobUncompressedSize,
CompressionLevel: v.Config.CompressionLevel,
UploadDurationMs: stats.uploadDuration.Milliseconds(),
NewChunkSize: stats.newChunkSize,
}
err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats)
if err != nil {
return fmt.Errorf("updating snapshot stats: %w", err)
}
err = v.SnapshotManager.ExportSnapshotMetadata(
v.ctx, v.Config.IndexPath, snapshotID)
if err != nil {
return fmt.Errorf("exporting snapshot metadata: %w", err)
}
// Record completion last, so an interrupted export never leaves a
// snapshot marked complete without its metadata at the destination.
err = v.SnapshotManager.MarkSnapshotComplete(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("marking snapshot complete: %w", err)
}
return nil
}
// uploadSpeed returns the average network upload rate as a colorized
// bits/sec string, or "N/A" when there's no usable data.
func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) string {
if bytesUploaded <= 0 || duration <= 0 {
return v.UI.Speed(0)
}
return v.UI.Speed(float64(bytesUploaded) / duration.Seconds())
}
// printSnapshotSummary prints the comprehensive snapshot completion summary
func (v *Vaultik) printSnapshotSummary(
snapshotID string, startTime time.Time, stats *snapshotStats,
) {
snapshotDuration := time.Since(startTime)
totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped -
stats.totalFilesFailed
totalBytesAll := stats.totalBytes + stats.totalBytesSkipped
var compressionRatio float64
if stats.blobUncompressedSize > 0 {
compressionRatio = float64(stats.blobSize) /
float64(stats.blobUncompressedSize)
} else {
compressionRatio = 1.0
}
v.UI.Completef("Created snapshot %s.", v.UI.Snapshot(snapshotID))
filesMsg := fmt.Sprintf("Files: %s examined, %s backed up, %s unchanged",
v.UI.Count(stats.totalFiles),
v.UI.Count(totalFilesChanged),
v.UI.Count(stats.totalFilesSkipped))
if stats.totalFilesFailed > 0 {
filesMsg += fmt.Sprintf(", %s failed", v.UI.Count(stats.totalFilesFailed))
}
if stats.totalFilesDeleted > 0 {
filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted))
}
v.UI.Detailf("%s.", filesMsg)
dataMsg := fmt.Sprintf("Data: %s total (%s backed up)",
v.UI.Size(totalBytesAll),
v.UI.Size(stats.totalBytes))
if stats.totalBytesDeleted > 0 {
dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted))
}
v.UI.Detailf("%s.", dataMsg)
if stats.totalBlobsUploaded > 0 {
v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).",
v.UI.Size(stats.blobSize),
v.UI.Size(stats.blobUncompressedSize),
compressionRatio)
v.UI.Detailf("Upload: %d blobs, %s in %s (%s).",
stats.totalBlobsUploaded,
v.UI.Size(stats.totalBytesUploaded),
v.UI.Duration(stats.uploadDuration),
v.uploadSpeed(stats.totalBytesUploaded, stats.uploadDuration))
}
v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration))
}
// SnapshotPurgeOptions contains options for the snapshot purge command.
type SnapshotPurgeOptions struct {
KeepLatest bool // Keep only the most recent snapshot per name
// OlderThan drops snapshots older than this duration (e.g. "30d",
// "6m", "1y").
OlderThan string
Force bool // Skip confirmation prompt
// Names restricts the operation to snapshots with one of these
// names when non-empty.
Names []string
Quiet bool // Suppress informational output (used by --prune flag)
}
// PurgeSnapshotsWithOptions removes old snapshots based on criteria.
// Retention is per-snapshot-name: KeepLatest keeps the latest of EACH configured
// snapshot name, not the latest globally. This prevents `home` and `system`
// snapshots from cannibalizing each other.
func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
err := v.EnsureStorageBinding()
if err != nil {
return err
}
// Sync with remote first
err = v.syncWithRemote()
if err != nil {
return fmt.Errorf("syncing with remote: %w", err)
}
// Get snapshots from local database
dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return fmt.Errorf("listing snapshots: %w", err)
}
// Build name filter set if --snapshot was specified.
nameFilter := make(map[string]struct{}, len(opts.Names))
for _, n := range opts.Names {
nameFilter[n] = struct{}{}
}
// Collect completed snapshots and their names, applying the name filter.
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
names := make(map[types.SnapshotID]string, len(dbSnapshots))
for _, s := range dbSnapshots {
if s.CompletedAt == nil {
continue
}
name := parseSnapshotName(s.ID.String(), s.Hostname.String())
if len(nameFilter) > 0 {
if _, ok := nameFilter[name]; !ok {
continue
}
}
names[s.ID] = name
snapshots = append(snapshots, SnapshotInfo{
ID: s.ID,
Timestamp: s.StartedAt,
CompressedSize: s.BlobSize,
})
}
// Sort by timestamp (newest first)
sort.Slice(snapshots, func(i, j int) bool {
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
})
toDelete, err := selectSnapshotsToPurge(snapshots, names, opts)
if err != nil {
return err
}
if len(toDelete) == 0 {
if !opts.Quiet {
v.UI.Infof("No snapshots to delete.")
}
return nil
}
return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet)
}
// selectSnapshotsToPurge applies the purge retention criteria to the
// newest-first sorted snapshot list and returns the deletion
// candidates. names maps each snapshot's ID to its snapshot name.
func selectSnapshotsToPurge(
snapshots []SnapshotInfo,
names map[types.SnapshotID]string,
opts *SnapshotPurgeOptions,
) ([]SnapshotInfo, error) {
var toDelete []SnapshotInfo
switch {
case opts.KeepLatest:
// Keep the latest snapshot per snapshot name. Snapshots are sorted
// newest-first, so the first occurrence of each name is kept.
seen := make(map[string]bool)
for _, snap := range snapshots {
name := names[snap.ID]
if seen[name] {
toDelete = append(toDelete, snap)
continue
}
seen[name] = true
}
case opts.OlderThan != "":
duration, err := parseDuration(opts.OlderThan)
if err != nil {
return nil, err
}
cutoff := time.Now().UTC().Add(-duration)
for _, snap := range snapshots {
if snap.Timestamp.Before(cutoff) {
toDelete = append(toDelete, snap)
}
}
}
return toDelete, nil
}
// confirmAndExecutePurge shows deletion candidates, confirms with the
// user, and deletes snapshots.
func (v *Vaultik) confirmAndExecutePurge(
toDelete []SnapshotInfo, force, quiet bool,
) error {
// Confirm unless --force is used. The list, the prompt and its answer
// bypass the UI: they are an interactive exchange the operator must
// see, so --quiet does not silence them.
if !force {
v.stdoutf("The following snapshots will be deleted:\n\n")
for _, snap := range toDelete {
v.stdoutf(" %s (%s, %s)\n",
snap.ID,
snap.Timestamp.Format(time.DateTime),
ubytes(snap.CompressedSize))
}
v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete))
var confirm string
_, err := v.scanStdin(&confirm)
if err != nil {
v.printlnStdout("Cancelled")
return nil //nolint:nilerr // treat EOF or read error as "no"
}
if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return nil
}
} else if !quiet {
v.UI.Infof("The following snapshots will be deleted:")
for _, snap := range toDelete {
v.UI.Detailf("%s (%s, %s)",
v.UI.Snapshot(snap.ID.String()),
snap.Timestamp.Format(time.DateTime),
ubytes(snap.CompressedSize))
}
v.UI.Infof("Deleting %d snapshot(s) (--force specified).", len(toDelete))
}
// Delete snapshots (both local and remote)
for _, snap := range toDelete {
snapshotID := snap.ID.String()
log.Info("Deleting snapshot", "id", snapshotID)
// Stop before the remote delete, so a snapshot still in the local
// index keeps its metadata on the destination store.
err := v.deleteSnapshotFromLocalDB(snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
}
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
if err != nil {
return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err)
}
}
// Tidy up local DB orphans now so users don't have to run a
// separate command after a purge. Guarded against nil for tests
// that don't wire up a SnapshotManager.
if v.SnapshotManager != nil {
err := v.SnapshotManager.CleanupOrphanedData(v.ctx)
if err != nil {
log.Warn("Failed to clean up orphaned local data after purge",
"error", err)
}
}
if !quiet {
v.UI.Infof("Deleted %d snapshot(s).", len(toDelete))
v.UI.Noticef("Run '%s' to clean up unreferenced remote blobs.",
pruneCommandHint)
}
return nil
}
// VerifySnapshot checks snapshot integrity
func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error {
opts := &VerifyOptions{Deep: deep}
if deep {
return v.RunDeepVerify(snapshotID, opts)
}
return v.VerifySnapshotWithOptions(snapshotID, opts)
}
// VerifySnapshotWithOptions checks snapshot integrity with full options.
// Deep verification is delegated to RunDeepVerify so this function only
// implements the shallow (existence-only) path.
func (v *Vaultik) VerifySnapshotWithOptions(
snapshotID string, opts *VerifyOptions,
) error {
if opts.Deep {
return v.RunDeepVerify(snapshotID, opts)
}
result := &VerifyResult{
SnapshotID: snapshotID,
Mode: "shallow",
}
v.printVerifyHeader(snapshotID, opts)
// Resolve the identifier to the snapshot's remote key. A human ID is
// hashed; a remote key (or its abbreviation, as printed for a
// remote-only snapshot) is used as-is, so a host with no local index
// can verify a snapshot it can only see on the store. The key is kept
// so we can also check for the snapshot's encrypted database below.
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
if err != nil {
if opts.JSON {
result.Status = verifyStatusFailed
result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err)
return v.outputVerifyJSON(result)
}
return fmt.Errorf("resolving snapshot identifier: %w", err)
}
manifest, err := v.downloadManifestByKey(remoteKey)
if err != nil {
if opts.JSON {
result.Status = verifyStatusFailed
result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err)
return v.outputVerifyJSON(result)
}
return fmt.Errorf("downloading manifest: %w", err)
}
result.BlobCount = manifest.BlobCount
result.TotalSize = manifest.TotalCompressedSize
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Snapshot information:\n")
v.stdoutf(" Blob count: %d\n", manifest.BlobCount)
v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize))
if manifest.Timestamp != "" {
t, terr := time.Parse(time.RFC3339, manifest.Timestamp)
if terr == nil {
v.stdoutf(" Created: %s\n",
t.Format("2006-01-02 15:04:05 MST"))
}
}
v.printlnStdout()
// Check each blob is present with the size the manifest records.
v.UI.Beginf("Checking blob presence and sizes...")
}
v.checkSnapshotDatabase(remoteKey, result)
result.Verified, result.Missing, result.Mismatched, result.MissingSize,
result.Unchecked, err = v.verifyManifestBlobs(manifest, opts)
if err != nil {
if opts.JSON {
result.Status = verifyStatusFailed
result.ErrorMessage = fmt.Sprintf("verifying manifest blobs: %v", err)
return v.outputVerifyJSON(result)
}
return fmt.Errorf("verifying manifest blobs: %w", err)
}
return v.formatVerifyResult(result, opts)
}
// printVerifyHeader prints the snapshot ID and parsed timestamp for
// verification output. Snapshot ID format: hostname[_name]_<RFC3339>
func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
var snapshotTime time.Time
t, err := parseSnapshotTimestamp(snapshotID)
if err == nil {
snapshotTime = t
}
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Verifying snapshot %s\n", snapshotID)
if !snapshotTime.IsZero() {
v.stdoutf("Snapshot time: %s\n",
snapshotTime.Format("2006-01-02 15:04:05 MST"))
}
v.printlnStdout()
}
}
// checkSnapshotDatabase records in result whether the snapshot's
// encrypted database is missing or could not be checked. A snapshot is
// only restorable if its database is present alongside the blobs.
// Shallow verify checks that the object exists; it does not decrypt it
// (that is deep verify's job).
func (v *Vaultik) checkSnapshotDatabase(remoteKey string, result *VerifyResult) {
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
_, err := v.Storage.Stat(v.ctx, dbPath)
switch {
case errors.Is(err, storage.ErrNotFound):
result.DatabaseMissing = true
case err != nil:
result.DatabaseCheckError = err.Error()
}
}
// verifyManifestBlobs checks that each blob in the manifest is present in
// storage with the size the manifest records, returning the counts of
// blobs that were present with the right size, absent, and present but the
// wrong size, the total bytes of the absent blobs, and the blobs it could
// not check because storage returned an error other than not-found. It
// does not read blob contents; deep verification (RunDeepVerify) does
// that. The size comparison matches the deep path (see
// verifyBlobExistenceFromDB).
func (v *Vaultik) verifyManifestBlobs(
manifest *snapshot.Manifest, opts *VerifyOptions,
) (int, int, int, int64, []UncheckedBlob, error) {
var (
verified, missing, mismatched int
missingSize int64
unchecked []UncheckedBlob
)
for _, blob := range manifest.Blobs {
// The manifest is unauthenticated, so its blob hashes are checked
// before being spliced into a storage path.
if !isBlobHash(blob.Hash) {
return 0, 0, 0, 0, nil, fmt.Errorf("%w: %s",
errInvalidBlobHash, shortHash(blob.Hash))
}
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
blob.Hash[:2], blob.Hash[2:4], blob.Hash)
stat, err := v.Storage.Stat(v.ctx, blobPath)
switch {
case errors.Is(err, storage.ErrNotFound):
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Missing: %s (%s)\n",
blob.Hash, ubytes(blob.CompressedSize))
}
missing++
missingSize += blob.CompressedSize
case err != nil:
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Could not check: %s (%v)\n", blob.Hash, err)
}
unchecked = append(unchecked,
UncheckedBlob{Hash: blob.Hash, Error: err.Error()})
case stat.Size != blob.CompressedSize:
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n",
blob.Hash, ubytes(stat.Size), ubytes(blob.CompressedSize))
}
mismatched++
default:
verified++
}
}
return verified, missing, mismatched, missingSize, unchecked, nil
}
// formatVerifyResult outputs the final verification results as JSON or
// human-readable text.
func (v *Vaultik) formatVerifyResult(
result *VerifyResult, opts *VerifyOptions,
) error {
failure := shallowVerifyFailure(result)
if opts.JSON {
if failure != "" {
result.Status = verifyStatusFailed
result.ErrorMessage = failure
} else {
result.Status = "ok"
}
return v.outputVerifyJSON(result)
}
// --quiet drops the report but not the errors storage returned, which
// are logged on stderr. A failure is still returned either way, and
// the cli layer prints it on stderr.
if v.UI.Quiet() {
for _, blob := range result.Unchecked {
log.Warn("Could not check blob", "blob", blob.Hash, "error", blob.Error)
}
if result.DatabaseCheckError != "" {
log.Warn("Could not check the encrypted database",
"snapshot", result.SnapshotID, "error", result.DatabaseCheckError)
}
} else {
v.printVerifySummary(result, failure)
}
// Not errSnapshotVerifyFailed: the cli layer prints this after its own
// "Verification failed:".
if failure != "" {
return fmt.Errorf("%w: %s", errProblemsInStorage, failure)
}
return nil
}
// printVerifySummary prints the counts and the status line that end the
// human-readable shallow verify report. failure is empty when it passed.
func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) {
v.stdoutf("\nVerification complete:\n")
v.stdoutf(" Present with listed size: %d blobs\n", result.Verified)
if result.Missing > 0 {
v.stdoutf(" Missing: %d blobs (%s)\n",
result.Missing, ubytes(result.MissingSize))
}
if result.Mismatched > 0 {
v.stdoutf(" Wrong size: %d blobs\n", result.Mismatched)
}
if len(result.Unchecked) > 0 {
v.stdoutf(" Could not check: %d blobs\n", len(result.Unchecked))
}
if result.DatabaseMissing {
v.stdoutf(" Encrypted database: missing\n")
}
if result.DatabaseCheckError != "" {
v.stdoutf(" Encrypted database: could not be checked (%s)\n",
result.DatabaseCheckError)
}
v.stdoutf(" Status: ")
if failure != "" {
v.stdoutf("FAILED - %s\n", failure)
return
}
// Report only what was actually checked: presence and size, not contents.
v.stdoutf("OK - all %d blobs listed in the manifest are present with the "+
"listed size; contents not checked (use --deep)\n", result.Verified)
}
// shallowVerifyFailure returns a human-readable description of everything
// that failed shallow verification, or the empty string if it passed.
func shallowVerifyFailure(result *VerifyResult) string {
var parts []string
if result.Missing > 0 {
parts = append(parts, fmt.Sprintf("%d blobs are missing", result.Missing))
}
if result.Mismatched > 0 {
parts = append(parts,
fmt.Sprintf("%d blobs have the wrong size", result.Mismatched))
}
if len(result.Unchecked) > 0 {
parts = append(parts, fmt.Sprintf("%d blobs could not be checked",
len(result.Unchecked)))
}
if result.DatabaseMissing {
parts = append(parts, "the encrypted database is missing")
}
if result.DatabaseCheckError != "" {
parts = append(parts, "the encrypted database could not be checked")
}
return strings.Join(parts, "; ")
}
// outputVerifyJSON outputs the verification result as JSON
func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
encoder := json.NewEncoder(v.Stdout)
encoder.SetIndent("", " ")
err := encoder.Encode(result)
if err != nil {
return fmt.Errorf("encoding JSON: %w", err)
}
if result.Status == verifyStatusFailed {
return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, result.ErrorMessage)
}
return nil
}
// CleanupLocalSnapshots removes local snapshot records that have no
// corresponding metadata in remote storage. These are typically left
// behind by incomplete or interrupted backups. Each local snapshot's
// human ID is hashed via RemoteSnapshotKey and compared against the
// remote listing.
//
// It takes the whole *PruneOptions, symmetric with PruneBlobs, because
// it is the other half of one command: Prune runs this phase and then
// that one. Only JSON is read here. Under --json every write below is
// suppressed, because stdout carries the PruneBlobsResult document and
// nothing else — prose ahead of it is what made `vaultik prune --json |
// jq` fail (issue #108). The narration is duplicated as log records,
// which go to stderr and so cannot corrupt the document.
func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
err := v.EnsureStorageBinding()
if err != nil {
return err
}
remoteKeys, err := v.listAllRemoteSnapshotKeys()
if err != nil {
return err
}
remoteSet := make(map[string]bool, len(remoteKeys))
for _, k := range remoteKeys {
remoteSet[k] = true
}
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return fmt.Errorf("listing local snapshots: %w", err)
}
var removed int
for _, snap := range localSnapshots {
id := snap.ID.String()
if !remoteSet[snapshot.RemoteSnapshotKey(id)] {
log.Info("Removing stale local snapshot record", "snapshot_id", id)
if !opts.JSON {
v.UI.Infof("Removing stale local record: %s.", v.UI.Snapshot(id))
}
err = v.deleteSnapshotFromLocalDB(id)
if err != nil {
log.Error("Failed to delete local snapshot",
"snapshot_id", id, "error", err)
continue
}
removed++
}
}
log.Info("Reconciled local snapshot records against remote metadata",
"removed", removed, "examined", len(localSnapshots))
if opts.JSON {
return nil
}
if removed == 0 {
v.UI.Infof("No stale local snapshots found.")
} else {
v.UI.Infof("Removed %d stale local snapshot record(s).", removed)
}
return nil
}
// Helper methods that were previously on SnapshotApp
// downloadManifestByKey fetches the manifest at
// metadata/<remoteKey>/manifest.json.zst. The remoteKey is the double-
// SHA256 derivation produced by snapshot.RemoteSnapshotKey, not the
// human snapshot ID. Callers that have a human ID must hash first.
//
// This is the only place vaultik reads a manifest off the destination
// store, deliberately: the manifest is currently stored compressed but
// unencrypted, which is what lets `snapshot list` enumerate the
// destination on a host holding no private key. Whether to encrypt it
// is open (issue #81), and routing every read through here means that
// decision has exactly one call site to change. Keep it that way — do
// not open metadata/<key>/manifest.json.zst directly elsewhere.
func (v *Vaultik) downloadManifestByKey(remoteKey string) (*snapshot.Manifest, error) {
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
reader, err := v.Storage.Get(v.ctx, manifestPath)
if err != nil {
return nil, err
}
defer func() { _ = reader.Close() }()
manifest, err := snapshot.DecodeManifest(reader)
if err != nil {
return nil, fmt.Errorf("decoding manifest: %w", err)
}
return manifest, nil
}
func (v *Vaultik) syncWithRemote() error {
log.Info("Syncing with remote snapshots")
// Remote metadata lives under metadata/<remote-key>/, where the
// directory name is snapshot.RemoteSnapshotKey(id), not the human
// snapshot ID. Compare each local row's hashed key against that set
// so a row still backed by remote metadata is kept. Comparing human
// IDs against the hashed directory names matches nothing and deletes
// every local snapshot record (issue #160).
remoteKeys, err := v.listAllRemoteSnapshotKeys()
if err != nil {
return err
}
remoteKeySet := make(map[string]bool, len(remoteKeys))
for _, k := range remoteKeys {
remoteKeySet[k] = true
}
log.Debug("Found remote snapshots", "count", len(remoteKeySet))
// Get all local snapshots (use a high limit to get all)
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return fmt.Errorf("listing local snapshots: %w", err)
}
// Remove local snapshots whose metadata is absent from the remote.
removedCount := 0
for _, snap := range localSnapshots {
snapshotIDStr := snap.ID.String()
if !remoteKeySet[snapshot.RemoteSnapshotKey(snapshotIDStr)] {
log.Info("Removing local snapshot not found in remote",
"snapshot_id", snap.ID)
err = v.deleteSnapshotFromLocalDB(snapshotIDStr)
if err != nil {
log.Error("Failed to delete local snapshot",
"snapshot_id", snap.ID, "error", err)
} else {
removedCount++
}
}
}
if removedCount > 0 {
log.Info("Removed local snapshots not found in remote", "count", removedCount)
}
return nil
}
// RemoveOptions contains options for the snapshot remove command
type RemoveOptions struct {
Force bool
DryRun bool
JSON bool
LocalOnly bool // Skip remote cleanup; only touch the local index
}
// RemoveResult contains the result of a snapshot removal
//
//nolint:tagliatelle // snake_case is the established JSON output format
type RemoveResult struct {
SnapshotID string `json:"snapshot_id,omitempty"`
SnapshotsRemoved []string `json:"snapshots_removed,omitempty"`
RemoteRemoved bool `json:"remote_removed,omitempty"`
DryRun bool `json:"dry_run,omitempty"`
}
// pruneCommandHint is the exact command suggested after a remove
// completes so the user knows how to clean up the blobs that the
// just-removed snapshot left behind on the destination store.
const pruneCommandHint = "vaultik prune"
// snapshotRemoveCommandHint is the command suggested, with the
// snapshot's ID, when a remove could not reach the destination store:
// running it again removes the snapshot's metadata there, which
// `vaultik prune` never does while the manifest is there.
const snapshotRemoveCommandHint = "vaultik snapshot remove"
// RemoveSnapshot removes a snapshot from the local index database and,
// unless LocalOnly is set, also strips the snapshot's metadata from the
// destination store. Blobs are NOT touched: removing a snapshot's
// blobs requires reading every remaining remote manifest (the remote
// may hold snapshots this host doesn't know about), which is what
// `vaultik prune` is for — when metadata was removed from the
// destination store, the text output names the prune invocation to run
// as a follow-up. When removing the metadata fails, a snapshot in the
// local index is still removed from it, with a warning; a snapshot
// only on the destination store is an error.
func (v *Vaultik) RemoveSnapshot(
snapshotID string, opts *RemoveOptions,
) (*RemoveResult, error) {
result := &RemoveResult{
SnapshotID: snapshotID,
}
err := v.EnsureStorageBinding()
if err != nil {
return result, err
}
inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts)
if err != nil {
return result, err
}
if opts.DryRun {
return v.removeSnapshotDryRun(result, snapshotID, opts)
}
if !v.confirmRemoveSnapshot(snapshotID, opts) {
return result, nil
}
// Checked before the local record is deleted: it may be the last
// snapshot in the local index, and the check applies only while the
// local index holds one.
var markerErr error
if !opts.LocalOnly {
markerErr = v.checkDestinationMarker()
}
if inLocalIndex {
log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)
err = v.deleteSnapshotFromLocalDB(snapshotID)
if err != nil {
return result, fmt.Errorf("removing from local database: %w", err)
}
}
result.RemoteRemoved, err = v.removeSnapshotRemote(
snapshotID, inLocalIndex, markerErr, opts)
if err != nil {
return result, err
}
if v.SnapshotManager != nil {
err = v.SnapshotManager.CleanupOrphanedData(v.ctx)
if err != nil {
log.Warn("Failed to clean up orphaned local data after removal",
"error", err)
}
}
if opts.JSON {
return result, v.outputRemoveJSON(result)
}
if inLocalIndex {
v.UI.Infof("Removed snapshot '%s' from local database.",
v.UI.Snapshot(snapshotID))
}
if result.RemoteRemoved {
v.UI.Infof("Removed snapshot metadata from remote storage.")
v.UI.Noticef("The removed snapshot's blobs remain on the remote. "+
"Run '%s' to delete any blobs no longer referenced by any "+
"remaining remote snapshot.", pruneCommandHint)
}
return result, nil
}
// findSnapshotToRemove reports whether the local index has a row for
// snapshotID, and returns errSnapshotNotFound when there is nothing to
// remove: no row and, unless LocalOnly is set, no metadata on the
// destination store either. The destination store is consulted only
// when the local index has no row, so a snapshot in the local index can
// still be removed from it while the destination store is unreachable,
// and only after checkDestinationMarker passes, so an unplugged volume's
// empty mount point is not taken for a store without the snapshot.
func (v *Vaultik) findSnapshotToRemove(
snapshotID string, opts *RemoveOptions,
) (bool, error) {
if v.Repositories != nil {
snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID)
if err != nil {
return false, fmt.Errorf("looking up snapshot in local index: %w", err)
}
if snap != nil {
return true, nil
}
}
if opts.LocalOnly {
return false, fmt.Errorf("%w in the local index: %s",
errSnapshotNotFound, snapshotID)
}
err := v.checkDestinationMarker()
if err != nil {
return false, err
}
objects, err := v.remoteSnapshotObjects(snapshot.RemoteSnapshotKey(snapshotID))
if err != nil {
return false, fmt.Errorf("looking up snapshot on destination store: %w", err)
}
if len(objects) == 0 {
return false, fmt.Errorf(
"%w in the local index or on the destination store: %s",
errSnapshotNotFound, snapshotID)
}
return false, nil
}
// removeSnapshotDryRun reports what RemoveSnapshot would do without
// making changes.
func (v *Vaultik) removeSnapshotDryRun(
result *RemoveResult, snapshotID string, opts *RemoveOptions,
) (*RemoveResult, error) {
result.DryRun = true
if opts.JSON {
return result, v.outputRemoveJSON(result)
}
v.UI.Infof("Would remove snapshot: %s.", v.UI.Snapshot(snapshotID))
if !opts.LocalOnly {
v.UI.Infof("Would also remove snapshot metadata from " +
"remote storage (blobs untouched).")
}
v.UI.Noticef("Dry run: no changes made.")
return result, nil
}
// confirmRemoveSnapshot prompts for confirmation before removing a
// snapshot, unless --force or --json is set. Returns false (treating
// EOF/read errors as "no") unless the user answers "y". The prompt and
// its answer bypass the UI: they are an interactive exchange the
// operator must see, so --quiet does not silence them.
func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool {
if opts.Force || opts.JSON {
return true
}
if opts.LocalOnly {
v.stdoutf("Remove snapshot '%s' from local database "+
"(remote untouched)? [y/N] ", snapshotID)
} else {
v.stdoutf("Remove snapshot '%s' from local database AND its metadata "+
"from remote storage? [y/N] ", snapshotID)
}
var confirm string
_, err := v.scanStdin(&confirm)
if err != nil {
v.printlnStdout("Cancelled")
return false
}
if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return false
}
return true
}
// removeSnapshotRemote strips the snapshot's metadata from the
// destination store. It returns true only when the delete finished
// without an error and removed at least one object; under LocalOnly it
// does nothing. markerErr is the result of checkDestinationMarker; when
// it is set, nothing is deleted and it is the failure. When the snapshot
// is only on the destination store, a failure is returned as an error,
// because that removal is all the command does. When the snapshot was in
// the local index, its removal there has already happened, so a failure
// is only a warning telling the user to run `vaultik snapshot remove`
// for the snapshot again once the destination store is reachable
// (`vaultik prune` never removes snapshot metadata that has its
// manifest).
func (v *Vaultik) removeSnapshotRemote(
snapshotID string, inLocalIndex bool, markerErr error,
opts *RemoveOptions,
) (bool, error) {
if opts.LocalOnly {
return false, nil
}
log.Info("Removing snapshot metadata from remote storage",
"snapshot_id", snapshotID)
var deleted int
err := markerErr
if err == nil {
deleted, err = v.deleteRemoteSnapshotByKey(
snapshot.RemoteSnapshotKey(snapshotID))
}
if err != nil && !inLocalIndex {
return false, fmt.Errorf(
"removing snapshot metadata from destination store: %w", err)
}
if err != nil {
log.Warn("Could not remove snapshot metadata from remote storage; "+
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
"again once the remote is reachable",
"snapshot_id", snapshotID, "error", err)
// The UI writes to stdout, which under --json holds only the
// document; the log record above is the warning on stderr.
if v.UI != nil && !opts.JSON {
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
"%v. Run '%s %s' again once the remote is reachable.",
err, snapshotRemoveCommandHint, snapshotID)
}
return false, nil
}
return deleted > 0, nil
}
// RemoveAllSnapshots removes every snapshot known to the local
// database from the local index, and (with --remote) every snapshot
// metadata directory in remote storage. Both sides are processed so a
// "remove --all" leaves nothing behind, even when the local DB and
// remote storage have diverged. Unless LocalOnly is set, it fails
// before it removes anything when the destination store cannot be
// listed or is not present by checkDestinationMarker.
func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) {
err := v.EnsureStorageBinding()
if err != nil {
return nil, err
}
localSnaps, err := v.localSnapshotIDs()
if err != nil {
return nil, fmt.Errorf("listing local snapshots: %w", err)
}
// remoteKeys is the set of metadata/<key>/ subdirectories on the
// destination store. Only for a local-only remove is a failure
// downgraded to a warning, so a permission-denied or unreachable
// remote can't block it.
remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys()
if remoteErr != nil {
if !opts.LocalOnly {
return nil, remoteErr
}
log.Warn("Could not list remote snapshots", "error", remoteErr)
v.UI.Warningf("Could not list remote snapshots: %v.", remoteErr)
}
// Anything visible on the remote that doesn't correspond to a
// known local human ID is treated as an orphan key — handled only
// when --remote is in effect.
knownLocalKeys := make(map[string]string, len(localSnaps))
for _, id := range localSnaps {
knownLocalKeys[snapshot.RemoteSnapshotKey(id)] = id
}
var orphanRemoteKeys []string
for _, key := range remoteKeys {
if _, known := knownLocalKeys[key]; !known {
orphanRemoteKeys = append(orphanRemoteKeys, key)
}
}
if len(localSnaps) == 0 && len(orphanRemoteKeys) == 0 {
if !opts.JSON {
v.UI.Infof("No snapshots found.")
}
return &RemoveResult{}, nil
}
if opts.DryRun {
return v.handleRemoveAllDryRun(localSnaps, orphanRemoteKeys, opts)
}
return v.executeRemoveAll(localSnaps, orphanRemoteKeys, opts)
}
// localSnapshotIDs returns every snapshot ID present in the local
// index database, sorted for deterministic iteration. Empty slice if
// the database has no Repositories (e.g. tests).
func (v *Vaultik) localSnapshotIDs() ([]string, error) {
if v.Repositories == nil {
return nil, nil
}
snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return nil, err
}
ids := make([]string, 0, len(snaps))
for _, s := range snaps {
ids = append(ids, s.ID.String())
}
sort.Strings(ids)
return ids, nil
}
// listAllRemoteSnapshotKeys collects the hashed remote keys
// (subdirectories under metadata/) currently present in the
// destination store. Returns (nil, err) when the store cannot be
// listed, or is not present by checkDestinationMarker.
func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) {
log.Info("Listing all remote snapshots")
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
seen := make(map[string]bool)
var keys []string
for object := range objectCh {
if object.Err != nil {
return nil, fmt.Errorf("listing remote snapshots: %w", object.Err)
}
parts := strings.Split(object.Key, "/")
if len(parts) < minSnapshotIDParts ||
parts[0] != metadataDirName || parts[1] == "" {
continue
}
// Skip macOS resource fork files (._*) and other hidden files
if strings.HasPrefix(parts[1], ".") {
continue
}
if !strings.HasSuffix(object.Key, "/") &&
!strings.Contains(object.Key, "/manifest.json.zst") {
continue
}
key := parts[1]
// A remote snapshot key is a SHA-256 hash: 64 lowercase hex
// characters. The listing comes from the untrusted destination,
// so accept a key only in that form.
if !isBlobHash(key) {
log.Warn("Skipping non-conforming key under metadata/",
"key", object.Key)
continue
}
if !seen[key] {
seen[key] = true
keys = append(keys, key)
}
}
err := v.checkDestinationMarker()
if err != nil {
return nil, err
}
return keys, nil
}
// handleRemoveAllDryRun handles the dry-run mode for removing all snapshots
func (v *Vaultik) handleRemoveAllDryRun(
localSnaps, orphanRemoteKeys []string, opts *RemoveOptions,
) (*RemoveResult, error) {
result := &RemoveResult{DryRun: true}
result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...)
if !opts.LocalOnly {
result.SnapshotsRemoved = append(
result.SnapshotsRemoved, orphanRemoteKeys...)
}
if !opts.JSON {
v.UI.Infof("Would remove %d local snapshot(s):", len(localSnaps))
for _, id := range localSnaps {
v.UI.Detailf("%s", v.UI.Snapshot(id))
}
if !opts.LocalOnly {
if len(orphanRemoteKeys) > 0 {
v.UI.Infof("Would also remove %d orphan remote snapshot key(s):",
len(orphanRemoteKeys))
for _, key := range orphanRemoteKeys {
v.UI.Detailf("%s", key)
}
} else {
v.UI.Infof("Would also remove snapshot metadata from " +
"remote storage (blobs untouched).")
}
}
v.UI.Noticef("Dry run: no changes made.")
}
if opts.JSON {
return result, v.outputRemoveJSON(result)
}
return result, nil
}
// executeRemoveAll deletes every local snapshot and, unless LocalOnly
// is set, every corresponding remote metadata directory plus any
// orphan remote keys. Blobs are not touched. It returns an error when
// any snapshot could not be removed.
func (v *Vaultik) executeRemoveAll(
localSnaps, orphanRemoteKeys []string, opts *RemoveOptions,
) (*RemoveResult, error) {
if !opts.Force {
return nil, errRemoveAllNeedsForce
}
log.Info("Removing all snapshots",
"local_count", len(localSnaps),
"orphan_remote_count", len(orphanRemoteKeys))
result := &RemoveResult{}
failures := v.removeAllLocalSnapshots(localSnaps, opts, result)
if !opts.LocalOnly {
failures += v.removeAllOrphanRemotes(orphanRemoteKeys, result)
result.RemoteRemoved = failures == 0
}
if v.SnapshotManager != nil {
err := v.SnapshotManager.CleanupOrphanedData(v.ctx)
if err != nil {
log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err)
}
}
if failures > 0 {
return result, fmt.Errorf("%d %w", failures, errSnapshotRemoveFailed)
}
if opts.JSON {
return result, v.outputRemoveJSON(result)
}
v.UI.Infof("Removed %d snapshot(s).", len(result.SnapshotsRemoved))
if !opts.LocalOnly && result.RemoteRemoved {
v.UI.Infof("Removed snapshot metadata from remote storage.")
}
return result, nil
}
// deleteIncompleteSnapshot deletes one incomplete snapshot and its
// association rows, reporting whether the snapshot row itself was
// deleted.
func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool {
log.Info("Deleting incomplete snapshot", "snapshot_id", id)
// Delete related records first
err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr)
if err != nil {
log.Error("Failed to delete snapshot files",
"snapshot_id", id, "error", err)
}
err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr)
if err != nil {
log.Error("Failed to delete snapshot blobs",
"snapshot_id", id, "error", err)
}
err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr)
if err != nil {
log.Error("Failed to delete snapshot uploads",
"snapshot_id", id, "error", err)
}
err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr)
if err != nil {
log.Error("Failed to delete snapshot",
"snapshot_id", id, "error", err)
return false
}
return true
}
// removeAllLocalSnapshots removes each known local snapshot's remote
// metadata, unless --local-only, and then its local record, returning
// the number of snapshots it could not remove. The local record goes
// only once the metadata is deleted: dropped first, it would leave that
// metadata on the destination store as a remote-only snapshot whose name
// is lost.
func (v *Vaultik) removeAllLocalSnapshots(
localSnaps []string, opts *RemoveOptions, result *RemoveResult,
) int {
failures := 0
for _, snapshotID := range localSnaps {
log.Info("Removing snapshot", "snapshot_id", snapshotID)
if !opts.LocalOnly {
_, err := v.deleteRemoteSnapshotByKey(
snapshot.RemoteSnapshotKey(snapshotID))
if err != nil {
log.Warn("Failed to remove snapshot metadata from remote",
"snapshot_id", snapshotID, "error", err)
failures++
continue
}
}
err := v.deleteSnapshotFromLocalDB(snapshotID)
if err != nil {
log.Error("Failed to remove from local database",
"snapshot_id", snapshotID, "error", err)
failures++
continue
}
result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID)
}
return failures
}
// removeAllOrphanRemotes deletes remote snapshot metadata that has no
// local counterpart, returning the number of deletion failures.
func (v *Vaultik) removeAllOrphanRemotes(
orphanRemoteKeys []string, result *RemoveResult,
) int {
remoteErrors := 0
for _, key := range orphanRemoteKeys {
log.Info("Removing orphan remote snapshot", "remote_key", key)
_, err := v.deleteRemoteSnapshotByKey(key)
if err != nil {
log.Warn("Failed to remove orphan from remote",
"remote_key", key, "error", err)
remoteErrors++
continue
}
result.SnapshotsRemoved = append(result.SnapshotsRemoved, key)
}
return remoteErrors
}
// deleteSnapshotFromLocalDB removes a snapshot from the local database only
func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error {
if v.Repositories == nil {
return nil // No local database
}
// Delete related records first to avoid foreign key constraints
err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err)
}
err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err)
}
err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err)
}
err = v.Repositories.Snapshots.Delete(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err)
}
return nil
}
// deleteRemoteSnapshotByKey removes everything under
// metadata/<remoteKey>/ on the destination store and returns how many
// objects it deleted. The argument is a remote key (double-SHA256
// derivation), not a human snapshot ID; callers that have a human ID
// must hash via snapshot.RemoteSnapshotKey first.
func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) (int, error) {
objectsToDelete, err := v.remoteSnapshotObjects(remoteKey)
if err != nil {
return 0, err
}
for i, key := range objectsToDelete {
err = v.Storage.Delete(v.ctx, key)
if err != nil {
return i, fmt.Errorf("removing %s: %w", key, err)
}
log.Debug("Deleted remote object", "key", key)
}
return len(objectsToDelete), nil
}
// remoteSnapshotObjects lists the keys of every object under
// metadata/<remoteKey>/ on the destination store.
func (v *Vaultik) remoteSnapshotObjects(remoteKey string) ([]string, error) {
prefix := fmt.Sprintf("metadata/%s/", remoteKey)
var keys []string
for object := range v.Storage.ListStream(v.ctx, prefix) {
if object.Err != nil {
return nil, fmt.Errorf("listing objects: %w", object.Err)
}
keys = append(keys, object.Key)
}
return keys, nil
}
// outputRemoveJSON outputs the removal result as JSON
func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error {
encoder := json.NewEncoder(v.Stdout)
encoder.SetIndent("", " ")
return encoder.Encode(result)
}
// PruneResult contains statistics about the prune operation.
// SnapshotsDeleted counts snapshots actually deleted. FilesDeleted,
// ChunksDeleted, and BlobsDeleted are derived from before/after row
// counts of the local index; each is nil when a count could not be read,
// so an unreadable count is reported as unknown rather than silently
// as 0.
type PruneResult struct {
SnapshotsDeleted int64
FilesDeleted *int64
ChunksDeleted *int64
BlobsDeleted *int64
}
// PruneDatabase removes incomplete snapshots and orphaned files, chunks,
// and blobs from the local database. This ensures database consistency
// before starting a new backup or on-demand via the prune command.
func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
log.Info("Pruning local database: " +
"removing incomplete snapshots and orphaned data")
v.UI.Beginf("Pruning local index database " +
"(removing incomplete snapshots and orphaned data).")
result := &PruneResult{}
// Snapshot counts before deletion of incompletes.
snapshotCountBefore := v.tableCountForReport("snapshots")
// First, delete any incomplete snapshots
incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
if err != nil {
return nil, fmt.Errorf("getting incomplete snapshots: %w", err)
}
for _, snap := range incompleteSnapshots {
if v.deleteIncompleteSnapshot(snap.ID.String(), snap.ID) {
result.SnapshotsDeleted++
}
}
// Get counts before cleanup for reporting
fileCountBefore := v.tableCountForReport("files")
chunkCountBefore := v.tableCountForReport("chunks")
blobCountBefore := v.tableCountForReport("blobs")
// Run the cleanup
err = v.SnapshotManager.CleanupOrphanedData(v.ctx)
if err != nil {
return nil, fmt.Errorf("cleanup orphaned data: %w", err)
}
// Get counts after cleanup
fileCountAfter := v.tableCountForReport("files")
chunkCountAfter := v.tableCountForReport("chunks")
blobCountAfter := v.tableCountForReport("blobs")
result.FilesDeleted = countDiff(fileCountBefore, fileCountAfter)
result.ChunksDeleted = countDiff(chunkCountBefore, chunkCountAfter)
result.BlobsDeleted = countDiff(blobCountBefore, blobCountAfter)
log.Info("Local database prune complete",
"incomplete_snapshots", result.SnapshotsDeleted,
"orphaned_files", countText(result.FilesDeleted),
"orphaned_chunks", countText(result.ChunksDeleted),
"orphaned_blobs", countText(result.BlobsDeleted),
)
// Snapshots remaining after removing the incomplete ones; unknown if
// the pre-prune snapshot count could not be read.
snapshotsRemain := countDiff(snapshotCountBefore, &result.SnapshotsDeleted)
v.UI.Completef("Pruned local index database.")
v.UI.Detailf("Incomplete snapshots: %s removed "+
"(total snapshots left in the local index: %s).",
countText(&result.SnapshotsDeleted), countText(snapshotsRemain))
v.UI.Detailf("Orphaned files: %s removed "+
"(total files left in the local index: %s).",
countText(result.FilesDeleted), countText(fileCountAfter))
v.UI.Detailf("Orphaned chunks: %s removed "+
"(total chunks left in the local index: %s).",
countText(result.ChunksDeleted), countText(chunkCountAfter))
v.UI.Detailf("Orphaned blobs: %s removed "+
"(total blobs left in the local index: %s).",
countText(result.BlobsDeleted), countText(blobCountAfter))
return result, nil
}
// unknownText is what a count or size reads as when it could not be
// determined, distinct from "0", which is a real zero.
const unknownText = "unknown"
// tableCountForReport returns the row count of a table for the prune
// summary, or nil if the count could not be read. A read failure is
// logged at warn — visible even under --json, which routes warnings to
// stderr — and then rendered as unknown rather than silently becoming 0,
// so a broken query is a visible failure instead of a plausible wrong
// number.
func (v *Vaultik) tableCountForReport(tableName string) *int64 {
count, err := v.getTableCount(tableName)
if err != nil {
log.Warn("could not read table row count for prune summary",
"table", tableName, "error", err)
return nil
}
return &count
}
// countDiff returns before-after, or nil if either count is unknown so
// that an unreadable count does not collapse into a plausible delta.
func countDiff(before, after *int64) *int64 {
if before == nil || after == nil {
return nil
}
diff := *before - *after
return &diff
}
// countText renders a count that may be unknown: nil (the read failed)
// becomes "unknown", never "0", so a reader can tell an empty table from
// one that could not be queried.
func countText(count *int64) string {
if count == nil {
return unknownText
}
return strconv.FormatInt(*count, 10)
}
// validTableNameRe matches table names containing only lowercase
// alphanumeric characters and underscores.
var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`)
// getTableCount returns the count of rows in a table. The tableName is
// sanitized to only allow [a-z0-9_] characters to prevent SQL
// injection.
func (v *Vaultik) getTableCount(tableName string) (int64, error) {
if v.DB == nil {
return 0, nil
}
if !validTableNameRe.MatchString(tableName) {
return 0, fmt.Errorf("%w: %q", errInvalidTableName, tableName)
}
var count int64
query := "SELECT COUNT(*) FROM " + tableName
err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count)
if err != nil {
return 0, err
}
return count, nil
}