Collapse snapshot prune into vaultik prune; auto-clean on removal

The CLI had two commands named "prune" doing different jobs (local
DB orphan cleanup vs. remote blob garbage collection), which was
confusing and forced a manual two-step workflow after deleting any
snapshot.

Single user-facing prune surface is now `vaultik prune`, which calls
PruneDatabase (local orphan cleanup) then PruneBlobs (remote unref
blob GC). Snapshot deletion paths (snapshot remove, snapshot remove
--all, snapshot purge) auto-run CleanupOrphanedData inline so the
local index database doesn't accumulate ghost rows after every
removal — the user observed ~39k orphaned files and 2 orphaned blobs
after a remove --all because that cleanup was previously a separate
opt-in command. `snapshot prune` is removed.

Also addresses the doc/help-string drift the user audit caught:

  * cli/prune.go help text used to reference a non-existent
    `vaultik purge` command.
  * cli/config.go get/set short/long examples were S3-specific
    (s3.bucket) when the primary storage configuration is
    storage_url.
  * vaultik/info.go printed S3 Bucket/Endpoint/Region labels
    unconditionally; for file:// or rclone:// users those rows
    were empty. The Storage Configuration block now prints the
    storer's Type+Location first, the storage_url string when set,
    and only emits S3 rows that are actually populated.
  * vaultik/info.go's "Run 'vaultik prune --remote'" hint
    referenced a flag that doesn't exist.
  * vaultik/blobcache.go's doc comment claimed LRU eviction, which
    is no longer the restore-time policy (the sweeper drives
    eviction; LRU is the safety-net fallback when maxBytes is
    finite).
  * README.md listed `vaultik restore`, `vaultik snapshot prune`,
    and `s3.bucket` example, all out of date.

README's roadmap section is rewritten with concrete pre-1.0 items
(security audit, error-condition tests, parallel blob downloads,
restart of interrupted restore, …) so the next-steps surface
matches what the project actually still needs.

The cleanup calls are guarded against a nil SnapshotManager so
tests that construct a bare Vaultik struct continue to work.
This commit is contained in:
2026-06-24 08:55:00 +02:00
parent 60abeb636a
commit 1f22b9c603
8 changed files with 179 additions and 103 deletions

View File

@@ -16,14 +16,22 @@ type blobDiskCacheEntry struct {
next *blobDiskCacheEntry
}
// blobDiskCache is an LRU cache that stores blobs on disk instead of in memory.
// Blobs are written to a temp directory keyed by their hash. When total size
// exceeds maxBytes, the least-recently-used entries are evicted (deleted from disk).
// blobDiskCache stores blobs on disk keyed by hash. It exposes ReadAt
// for slice reads (the restore path uses this so chunk extraction
// never reads a whole blob into memory) plus Get/Put for whole-blob
// access.
//
// The Get/ReadAt/peak-Len counters are debugging instrumentation used by
// tests to assert that the restore code path uses ReadAt (which reads
// only the requested slice of a blob) rather than Get (which reads the
// full blob into memory).
// Eviction policy is caller-controlled. The cache keeps an LRU list
// internally and will fall back to LRU eviction if curBytes exceeds
// maxBytes. Restore passes math.MaxInt64 as maxBytes and drives
// eviction itself via Delete() through restoreSweeper, which deletes
// each blob the moment every file that references its chunks has been
// written. LRU never fires under that configuration; it is kept as a
// safety net for callers that don't manage eviction themselves.
//
// Get/ReadAt/peak-Len counters are debugging instrumentation used by
// tests to assert that the restore code path uses ReadAt rather than
// Get and to bound peak disk-cache occupancy.
type blobDiskCache struct {
mu sync.Mutex
dir string

View File

@@ -22,14 +22,29 @@ func (v *Vaultik) ShowInfo() error {
v.printfStdout("Go Version: %s\n", runtime.Version())
v.printlnStdout()
// Storage Configuration
// Storage Configuration. The backend is selected by storage_url
// (s3://, file://, rclone://); the legacy s3.* fields are only
// printed when they're actually populated, since the URL scheme
// is the primary configuration.
v.printfStdout("=== Storage Configuration ===\n")
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
storageInfo := v.Storage.Info()
v.printfStdout("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location)
if v.Config.StorageURL != "" {
v.printfStdout("Storage URL: %s\n", v.Config.StorageURL)
}
if v.Config.S3.Bucket != "" {
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
}
if v.Config.S3.Prefix != "" {
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix)
}
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
if v.Config.S3.Endpoint != "" {
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
}
if v.Config.S3.Region != "" {
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
}
v.printlnStdout()
// Backup Settings
@@ -337,7 +352,7 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize)))
if result.OrphanedBlobCount > 0 {
v.printfStdout("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n")
v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n")
}
}

View File

@@ -48,6 +48,19 @@ type PruneBlobsResult struct {
BytesFreed int64 `json:"bytes_freed"`
}
// Prune removes orphaned data from the local index database AND
// unreferenced blobs from the backup destination store. This is the
// single user-facing prune entry point — the split between local and
// remote cleanup is an implementation detail. Calling code should
// prefer this method over PruneDatabase or PruneBlobs individually
// unless it specifically wants one half.
func (v *Vaultik) Prune(opts *PruneOptions) error {
if _, err := v.PruneDatabase(); err != nil {
return fmt.Errorf("pruning local database: %w", err)
}
return v.PruneBlobs(opts)
}
// PruneBlobs removes unreferenced blobs from storage
func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
log.Info("Starting prune operation")

View File

@@ -768,9 +768,18 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b
}
}
// Tidy up local DB orphans now so users don't have to run a
// separate command after a purge. Guarded against nil for tests
// that don't wire up a SnapshotManager.
if v.SnapshotManager != nil {
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
log.Warn("Failed to clean up orphaned local data after purge", "error", err)
}
}
if !quiet {
v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete))
v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.")
v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.")
}
return nil
@@ -1092,6 +1101,16 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
result.RemoteRemoved = true
}
// Clean up the local rows that just became orphaned (files, chunks,
// blob_chunks, blobs no longer referenced by any snapshot). This
// used to be a separate `vaultik snapshot prune` step; running it
// inline means `snapshot remove` leaves no ghost rows behind.
if v.SnapshotManager != nil {
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
log.Warn("Failed to clean up orphaned local data after removal", "error", err)
}
}
// Output result
if opts.JSON {
return result, v.outputRemoveJSON(result)
@@ -1101,7 +1120,7 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID)
if opts.Remote {
v.printlnStdout("Removed snapshot metadata from remote storage")
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
v.printlnStdout("\nNote: Remote blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
}
return result, nil
@@ -1213,6 +1232,14 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*
result.RemoteRemoved = true
}
// Clean up everything that just became orphaned locally so the
// index database doesn't carry 39k ghost rows after a wipe.
if v.SnapshotManager != nil {
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err)
}
}
if opts.JSON {
return result, v.outputRemoveJSON(result)
}
@@ -1220,7 +1247,7 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*
v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved))
if opts.Remote {
v.printlnStdout("Removed snapshot metadata from remote storage")
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
v.printlnStdout("\nNote: Remote blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
}
return result, nil