Collapse snapshot prune into vaultik prune; auto-clean on removal
The CLI had two commands named "prune" doing different jobs (local
DB orphan cleanup vs. remote blob garbage collection), which was
confusing and forced a manual two-step workflow after deleting any
snapshot.
Single user-facing prune surface is now `vaultik prune`, which calls
PruneDatabase (local orphan cleanup) then PruneBlobs (remote unref
blob GC). Snapshot deletion paths (snapshot remove, snapshot remove
--all, snapshot purge) auto-run CleanupOrphanedData inline so the
local index database doesn't accumulate ghost rows after every
removal — the user observed ~39k orphaned files and 2 orphaned blobs
after a remove --all because that cleanup was previously a separate
opt-in command. `snapshot prune` is removed.
Also addresses the doc/help-string drift the user audit caught:
* cli/prune.go help text used to reference a non-existent
`vaultik purge` command.
* cli/config.go get/set short/long examples were S3-specific
(s3.bucket) when the primary storage configuration is
storage_url.
* vaultik/info.go printed S3 Bucket/Endpoint/Region labels
unconditionally; for file:// or rclone:// users those rows
were empty. The Storage Configuration block now prints the
storer's Type+Location first, the storage_url string when set,
and only emits S3 rows that are actually populated.
* vaultik/info.go's "Run 'vaultik prune --remote'" hint
referenced a flag that doesn't exist.
* vaultik/blobcache.go's doc comment claimed LRU eviction, which
is no longer the restore-time policy (the sweeper drives
eviction; LRU is the safety-net fallback when maxBytes is
finite).
* README.md listed `vaultik restore`, `vaultik snapshot prune`,
and `s3.bucket` example, all out of date.
README's roadmap section is rewritten with concrete pre-1.0 items
(security audit, error-condition tests, parallel blob downloads,
restart of interrupted restore, …) so the next-steps surface
matches what the project actually still needs.
The cleanup calls are guarded against a nil SnapshotManager so
tests that construct a bare Vaultik struct continue to work.
This commit is contained in:
@@ -16,14 +16,22 @@ type blobDiskCacheEntry struct {
|
||||
next *blobDiskCacheEntry
|
||||
}
|
||||
|
||||
// blobDiskCache is an LRU cache that stores blobs on disk instead of in memory.
|
||||
// Blobs are written to a temp directory keyed by their hash. When total size
|
||||
// exceeds maxBytes, the least-recently-used entries are evicted (deleted from disk).
|
||||
// blobDiskCache stores blobs on disk keyed by hash. It exposes ReadAt
|
||||
// for slice reads (the restore path uses this so chunk extraction
|
||||
// never reads a whole blob into memory) plus Get/Put for whole-blob
|
||||
// access.
|
||||
//
|
||||
// The Get/ReadAt/peak-Len counters are debugging instrumentation used by
|
||||
// tests to assert that the restore code path uses ReadAt (which reads
|
||||
// only the requested slice of a blob) rather than Get (which reads the
|
||||
// full blob into memory).
|
||||
// Eviction policy is caller-controlled. The cache keeps an LRU list
|
||||
// internally and will fall back to LRU eviction if curBytes exceeds
|
||||
// maxBytes. Restore passes math.MaxInt64 as maxBytes and drives
|
||||
// eviction itself via Delete() through restoreSweeper, which deletes
|
||||
// each blob the moment every file that references its chunks has been
|
||||
// written. LRU never fires under that configuration; it is kept as a
|
||||
// safety net for callers that don't manage eviction themselves.
|
||||
//
|
||||
// Get/ReadAt/peak-Len counters are debugging instrumentation used by
|
||||
// tests to assert that the restore code path uses ReadAt rather than
|
||||
// Get and to bound peak disk-cache occupancy.
|
||||
type blobDiskCache struct {
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
|
||||
@@ -22,14 +22,29 @@ func (v *Vaultik) ShowInfo() error {
|
||||
v.printfStdout("Go Version: %s\n", runtime.Version())
|
||||
v.printlnStdout()
|
||||
|
||||
// Storage Configuration
|
||||
// Storage Configuration. The backend is selected by storage_url
|
||||
// (s3://, file://, rclone://); the legacy s3.* fields are only
|
||||
// printed when they're actually populated, since the URL scheme
|
||||
// is the primary configuration.
|
||||
v.printfStdout("=== Storage Configuration ===\n")
|
||||
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
|
||||
storageInfo := v.Storage.Info()
|
||||
v.printfStdout("Type: %s\n", storageInfo.Type)
|
||||
v.printfStdout("Location: %s\n", storageInfo.Location)
|
||||
if v.Config.StorageURL != "" {
|
||||
v.printfStdout("Storage URL: %s\n", v.Config.StorageURL)
|
||||
}
|
||||
if v.Config.S3.Bucket != "" {
|
||||
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
|
||||
}
|
||||
if v.Config.S3.Prefix != "" {
|
||||
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix)
|
||||
}
|
||||
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
|
||||
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
|
||||
if v.Config.S3.Endpoint != "" {
|
||||
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
|
||||
}
|
||||
if v.Config.S3.Region != "" {
|
||||
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
|
||||
}
|
||||
v.printlnStdout()
|
||||
|
||||
// Backup Settings
|
||||
@@ -337,7 +352,7 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
||||
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize)))
|
||||
|
||||
if result.OrphanedBlobCount > 0 {
|
||||
v.printfStdout("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n")
|
||||
v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,19 @@ type PruneBlobsResult struct {
|
||||
BytesFreed int64 `json:"bytes_freed"`
|
||||
}
|
||||
|
||||
// Prune removes orphaned data from the local index database AND
|
||||
// unreferenced blobs from the backup destination store. This is the
|
||||
// single user-facing prune entry point — the split between local and
|
||||
// remote cleanup is an implementation detail. Calling code should
|
||||
// prefer this method over PruneDatabase or PruneBlobs individually
|
||||
// unless it specifically wants one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
return fmt.Errorf("pruning local database: %w", err)
|
||||
}
|
||||
return v.PruneBlobs(opts)
|
||||
}
|
||||
|
||||
// PruneBlobs removes unreferenced blobs from storage
|
||||
func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
log.Info("Starting prune operation")
|
||||
|
||||
@@ -768,9 +768,18 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b
|
||||
}
|
||||
}
|
||||
|
||||
// Tidy up local DB orphans now so users don't have to run a
|
||||
// separate command after a purge. Guarded against nil for tests
|
||||
// that don't wire up a SnapshotManager.
|
||||
if v.SnapshotManager != nil {
|
||||
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
|
||||
log.Warn("Failed to clean up orphaned local data after purge", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !quiet {
|
||||
v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete))
|
||||
v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.")
|
||||
v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1092,6 +1101,16 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
|
||||
result.RemoteRemoved = true
|
||||
}
|
||||
|
||||
// Clean up the local rows that just became orphaned (files, chunks,
|
||||
// blob_chunks, blobs no longer referenced by any snapshot). This
|
||||
// used to be a separate `vaultik snapshot prune` step; running it
|
||||
// inline means `snapshot remove` leaves no ghost rows behind.
|
||||
if v.SnapshotManager != nil {
|
||||
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
|
||||
log.Warn("Failed to clean up orphaned local data after removal", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Output result
|
||||
if opts.JSON {
|
||||
return result, v.outputRemoveJSON(result)
|
||||
@@ -1101,7 +1120,7 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
|
||||
v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID)
|
||||
if opts.Remote {
|
||||
v.printlnStdout("Removed snapshot metadata from remote storage")
|
||||
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
|
||||
v.printlnStdout("\nNote: Remote blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
@@ -1213,6 +1232,14 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*
|
||||
result.RemoteRemoved = true
|
||||
}
|
||||
|
||||
// Clean up everything that just became orphaned locally so the
|
||||
// index database doesn't carry 39k ghost rows after a wipe.
|
||||
if v.SnapshotManager != nil {
|
||||
if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil {
|
||||
log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if opts.JSON {
|
||||
return result, v.outputRemoveJSON(result)
|
||||
}
|
||||
@@ -1220,7 +1247,7 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*
|
||||
v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved))
|
||||
if opts.Remote {
|
||||
v.printlnStdout("Removed snapshot metadata from remote storage")
|
||||
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
|
||||
v.printlnStdout("\nNote: Remote blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
|
||||
Reference in New Issue
Block a user