Let prune delete old .partial uploads and remote info count them (closes #466) #474
No files matched your search
@@ -269,9 +269,11 @@ storage_url: ""
|
||||
|
||||
# How old a blob that no snapshot lists must be before prune deletes it,
|
||||
# and how old every object in a directory under metadata/ with no manifest
|
||||
# must be before prune deletes that directory. A backup uploads its blobs
|
||||
# and its db.zst.age before its manifest, so on a destination shared
|
||||
# between hosts or users this must be longer than the longest backup run.
|
||||
# must be before prune deletes that directory. prune also deletes a
|
||||
# .partial object left by a killed upload only once it is this old. A
|
||||
# backup uploads its blobs and its db.zst.age before its manifest, so on
|
||||
# a destination shared between hosts or users this must be longer than
|
||||
# the longest backup run.
|
||||
# It does not protect a blob a backup reuses from an earlier run that
|
||||
# stopped before its manifest: that blob keeps the time of its first
|
||||
# upload, so a prune can delete it.
|
||||
|
||||
@@ -13,9 +13,10 @@ func NewPruneCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Tidy local database and remote storage",
|
||||
Long: `Removes orphaned data from the local index database, and
|
||||
unreferenced blobs and directories under metadata/ with no manifest
|
||||
from the backup destination store.
|
||||
Long: `Removes orphaned data from the local index database. From the
|
||||
backup destination store it removes unreferenced blobs, directories
|
||||
under metadata/ with no manifest, and .partial objects left by killed
|
||||
uploads.
|
||||
|
||||
Local cleanup drops incomplete snapshots and any files, chunks, or
|
||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||
@@ -30,7 +31,8 @@ blob keeps the time of its first upload, so it can be deleted.
|
||||
Remote cleanup also deletes each directory under metadata/ with no
|
||||
manifest, as a backup that stopped before its manifest leaves, once
|
||||
all its objects are older than prune_grace_period and its snapshot is
|
||||
not in the local index.
|
||||
not in the local index. It also deletes each .partial object under
|
||||
metadata/ or blobs/ once it is older than prune_grace_period.
|
||||
|
||||
Snapshot create --prune runs the same cleanup automatically; this
|
||||
command is the manual entry point for the same work (e.g. after a
|
||||
|
||||
@@ -75,7 +75,8 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
- Size of all snapshot metadata (per snapshot and total)
|
||||
- Count and total size of all blobs
|
||||
- Count and size of referenced blobs (from all manifests)
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)`,
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)
|
||||
- Count and size of .partial objects left by killed uploads`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
|
||||
@@ -138,13 +138,11 @@ on Linux), then /etc/vaultik/config.yml.`,
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
||||
"Run in cron mode (silent unless warning or error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||
"After backup, drop older snapshots of the same name, remove "+
|
||||
"orphaned blobs older than prune_grace_period, and delete each "+
|
||||
"directory under metadata/ with no manifest once all its "+
|
||||
"objects are older than prune_grace_period and its snapshot is "+
|
||||
"not in the local index (not after a backup that skipped "+
|
||||
"unreadable files, found a configured path empty, or did not "+
|
||||
"find a mount point an earlier snapshot found)")
|
||||
"After backup, drop older snapshots of the same name, then delete "+
|
||||
"from the destination store what prune deletes there (not "+
|
||||
"after a backup that skipped unreadable files, found a "+
|
||||
"configured path empty, or did not find a mount point an "+
|
||||
"earlier snapshot found)")
|
||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||
"With --prune: keep snapshots newer than this duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||
|
||||
@@ -302,10 +302,10 @@ func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) {
|
||||
}
|
||||
|
||||
// runPostBackupPrune drops older snapshots of the given names and removes
|
||||
// from remote storage the orphan blobs and the directories under
|
||||
// metadata/ with no manifest, as PruneBlobs does. If keepNewerThan is set
|
||||
// (e.g. "4w"), snapshots newer than that duration are kept. Otherwise only
|
||||
// the latest snapshot of each name is kept.
|
||||
// from remote storage the orphan blobs, the directories under metadata/
|
||||
// with no manifest and the `.partial` objects, as PruneBlobs does. If
|
||||
// keepNewerThan is set (e.g. "4w"), snapshots newer than that duration are
|
||||
// kept. Otherwise only the latest snapshot of each name is kept.
|
||||
func (v *Vaultik) runPostBackupPrune(
|
||||
snapshotNames []string, keepNewerThan string,
|
||||
) error {
|
||||
|
||||
Reference in new issue
Block a user