From 52fada52361c21c20b92438bf0ed8bfc01ef8e78 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:31:08 +0000 Subject: [PATCH 1/6] Let prune delete old .partial uploads and remote info count them (closes #466) A killed upload leaves a `.partial` object that only remote nuke removed, and that remote info never showed. The storage interface's DeletePartialUploads becomes ListPartialUploads, which returns each such object's key, size and time; Delete removes it. prune deletes those under metadata/ and blobs/ once older than prune_grace_period, and reports kept, deleted and failed counts. remote info counts them on their own line and in --json. remote nuke lists and deletes them like any other object, and now names any it could not delete. Model: opus-5-5 --- README.md | 96 ++++--- configs/config.example.yml | 2 + docs/DATAMODEL.md | 3 + docs/REPOSTRUCTURE.md | 8 +- internal/config/config.go | 3 +- internal/storage/faultstore/faultstore.go | 8 +- internal/storage/file.go | 39 ++- internal/storage/file_atomic_test.go | 40 ++- internal/storage/file_backend_test.go | 31 +-- internal/storage/rclone.go | 27 +- internal/storage/rclone_test.go | 38 +-- internal/storage/s3.go | 10 +- internal/storage/storer.go | 11 +- .../vaultik/destination_validation_test.go | 6 +- internal/vaultik/info.go | 48 +++- internal/vaultik/integration_test.go | 10 +- internal/vaultik/prune.go | 259 ++++++++++++++---- internal/vaultik/prune_partial_upload_test.go | 126 +++++++++ internal/vaultik/remove_snapshot_test.go | 10 +- 19 files changed, 570 insertions(+), 205 deletions(-) create mode 100644 internal/vaultik/prune_partial_upload_test.go diff --git a/README.md b/README.md index d6d7601..f31480b 100644 --- a/README.md +++ b/README.md @@ -344,27 +344,29 @@ In such a config, an unknown key or a value of the wrong type is refused. orphaned blobs older than `prune_grace_period` from remote storage, and delete each directory under `metadata/` that has no `manifest.json.zst` once every object in it is older than `prune_grace_period` and its snapshot is not in the - local index, as `prune` does. By default keeps only the latest snapshot per - name; use `--keep-newer-than` for a rolling window. When `--skip-errors` - skipped a file or directory it could not read or list, the prune does not run - and every older snapshot is kept, since one of them may be the only one that - holds what was skipped; the command says so and exits non-zero. When the - backup found nothing in a configured path, as in the mount point of a volume - that is not mounted, the prune does not run either: every older snapshot is - kept, and the command names the path and exits non-zero, on every run until - the path holds something again. Take a path that is empty on purpose out of - the config, or remove the older snapshots by hand with `snapshot purge`. When - an earlier snapshot of the same name in the local index found a volume mounted - at or below a configured path (a directory on a different filesystem from its - parent, with an entry in it that the backup does not exclude) and this backup - does not, the prune does not run either: every older snapshot is kept, and the - command names the directory and exits non-zero, on every run until the volume - is mounted again. A backup also does not find the volume when it finds no such - entry in its directory, as when an automounted volume cannot be mounted, or - when that directory is no longer backed up, because it is excluded or its - configured path was removed. If any of this is on purpose, as for a retired - volume, remove the older snapshots by hand with `snapshot purge`. A file - removed during the backup does not stop the prune. + local index, and delete each `.partial` object older than + `prune_grace_period`, as `prune` does. By default keeps only the latest + snapshot per name; use `--keep-newer-than` for a rolling window. When + `--skip-errors` skipped a file or directory it could not read or list, the + prune does not run and every older snapshot is kept, since one of them may be + the only one that holds what was skipped; the command says so and exits + non-zero. When the backup found nothing in a configured path, as in the mount + point of a volume that is not mounted, the prune does not run either: every + older snapshot is kept, and the command names the path and exits non-zero, on + every run until the path holds something again. Take a path that is empty on + purpose out of the config, or remove the older snapshots by hand with + `snapshot purge`. When an earlier snapshot of the same name in the local index + found a volume mounted at or below a configured path (a directory on a + different filesystem from its parent, with an entry in it that the backup does + not exclude) and this backup does not, the prune does not run either: every + older snapshot is kept, and the command names the directory and exits + non-zero, on every run until the volume is mounted again. A backup also does + not find the volume when it finds no such entry in its directory, as when an + automounted volume cannot be mounted, or when that directory is no longer + backed up, because it is excluded or its configured path was removed. If any + of this is on purpose, as for a retired volume, remove the older snapshots by + hand with `snapshot purge`. A file removed during the backup does not stop the + prune. - `--keep-newer-than `: With `--prune`, keep snapshots newer than this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`). Without `--prune`, or with a value that is not a valid duration, the command fails @@ -520,14 +522,16 @@ deleted. Step (3) also deletes each directory under `metadata/` that has no `db.zst.age` but before its manifest, once every object in it is older than `prune_grace_period` and its snapshot is not in the local index. An object the listing gives no time for keeps its directory, and a directory whose name is not -a remote key is left alone. Step (3) reads all remote manifests — network cost +a remote key is left alone. Step (3) also deletes each `.partial` object under +`metadata/` and `blobs/` that an upload cut off part-way left, once it is older +than `prune_grace_period`. Step (3) reads all remote manifests — network cost scales with the number of snapshots. `snapshot create --prune` runs the same cleanup automatically; this is the manual entry point for the same work. If the destination store cannot be listed, or the local index holds snapshots and the destination answers that it has no `vaultik-destination.txt` (an unplugged volume's empty mount point), prune fails and deletes nothing. `snapshot purge` -does the same. If a blob or such a directory cannot be deleted, the command (and -`snapshot create --prune`) exits non-zero. +does the same. If a blob, such a directory or such a `.partial` object cannot be +deleted, the command (and `snapshot create --prune`) exits non-zero. - `--force`: Skip confirmation prompt - `--json`: Output stats as JSON. Also skips the confirmation prompt, as @@ -537,7 +541,10 @@ does the same. If a blob or such a directory cannot be deleted, the command (and gives no time for it, is counted in `blobs_too_new` and not in `blobs_found`. A directory under `metadata/` with no manifest is counted in `metadata_dirs_deleted` once deleted, or in `metadata_dirs_failed` if it - cannot be. + cannot be. A `.partial` object is counted in `partial_uploads_deleted` once + deleted, or in `partial_uploads_failed` if it cannot be. One kept for being + newer than `prune_grace_period`, or of unknown age, is counted in + `partial_uploads_too_new`. **`info`**: Display system configuration, storage settings, encryption recipients, and local database statistics. @@ -552,7 +559,9 @@ in `unreadable_manifests` and counts the manifests under skipped names in `skipped_manifest_count`. An unreadable manifest also leaves its snapshot's blob count and blob size unknown, `null` in `--json`. A file under `blobs/` whose name is not a blob hash is not counted as a blob, since `prune` never deletes -it. +it. A `.partial` object an upload cut off part-way left under `metadata/` or +`blobs/` is not counted as a blob either. Such objects are counted on their own +line, and in `partial_upload_count` and `partial_upload_size` in `--json`. - `--json`: Output as JSON @@ -611,8 +620,9 @@ with a server-side move (local and sftp among them) are written under a temporary name ending in `.partial` and moved into place. Rclone remotes without one are written in place: where such a remote shows a file while it is still being written, a killed upload can leave a truncated object under its name, -which a later backup takes for complete. A leftover `.partial` file is ignored -and can be deleted. +which a later backup takes for complete. A backup ignores a leftover `.partial` +file. `prune` deletes it once it is older than `prune_grace_period`, and +`remote info` counts it. A destination shared between hosts or users, each with its own local index, relies on `prune_grace_period` to keep one's prune from deleting the blobs of @@ -721,6 +731,8 @@ format does and does not protect. 5. Delete each directory under `metadata/` with no manifest whose objects are all older than `prune_grace_period`, unless its snapshot is in the local index +6. Delete each `.partial` object under `metadata/` and `blobs/` older than + `prune_grace_period` ### chunking and deduplication @@ -754,20 +766,20 @@ Run `vaultik config init` to generate a fully commented config file; a complete annotated example also lives in [`configs/config.example.yml`](configs/config.example.yml). Key fields: -| Field | Default | Description | -| -------------------- | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty | -| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot files`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. | -| `snapshots` | (required) | Named snapshot definitions with paths and excludes | -| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) | -| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | -| `exclude` | | Global exclude patterns (applied to all snapshots) | -| `chunk_size` | `10MB` | Average chunk size for content-defined chunking | -| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | -| `compression_level` | `3` | zstd compression level (1-19) | -| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it, and how old every object in a directory under `metadata/` with no manifest must be before `prune` deletes that directory. A backup uploads its blobs and its `db.zst.age` before its manifest, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs and directories at once | -| `hostname` | system hostname | Hostname used in snapshot IDs | -| `index_path` | platform data dir | Local SQLite index path | +| Field | Default | Description | +| -------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty | +| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot files`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. | +| `snapshots` | (required) | Named snapshot definitions with paths and excludes | +| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) | +| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | +| `exclude` | | Global exclude patterns (applied to all snapshots) | +| `chunk_size` | `10MB` | Average chunk size for content-defined chunking | +| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | +| `compression_level` | `3` | zstd compression level (1-19) | +| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it, and how old every object in a directory under `metadata/` with no manifest must be before `prune` deletes that directory. It also sets how old a `.partial` object a killed upload left must be before `prune` deletes it. A backup uploads its blobs and its `db.zst.age` before its manifest, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs and directories at once | +| `hostname` | system hostname | Hostname used in snapshot IDs | +| `index_path` | platform data dir | Local SQLite index path | --- diff --git a/configs/config.example.yml b/configs/config.example.yml index e8fe7b9..a8b0080 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -333,6 +333,8 @@ compression_level: 5 # How old a blob that no snapshot lists must be before prune deletes it, and # how old every object in a directory under metadata/ with no manifest must # be before prune deletes that directory +# It is also how old a .partial object a killed upload left must be before +# prune deletes it # A backup uploads its blobs and its db.zst.age before its manifest, so on a # destination shared between hosts or users this must be longer than the # longest backup run diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index 5dd2af9..c02be9c 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -314,6 +314,9 @@ The restore process doesn't use the local database. Instead: - Delete each directory under `metadata/` that has no `manifest.json.zst` and no local snapshot record, once every object in it is older than `prune_grace_period` +3. **Delete Partial Uploads** + - Delete each `.partial` object a killed upload left under `metadata/` or + `blobs/`, once it is older than `prune_grace_period` ### 6. Incomplete Snapshot Cleanup diff --git a/docs/REPOSTRUCTURE.md b/docs/REPOSTRUCTURE.md index 856d63f..812c594 100644 --- a/docs/REPOSTRUCTURE.md +++ b/docs/REPOSTRUCTURE.md @@ -329,7 +329,8 @@ The prune operation is safe because: 1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references. Under `metadata/` it deletes only a directory that has no `manifest.json.zst`, whose name is a remote key, and that no - local snapshot record claims + local snapshot record claims. It also deletes the `.partial` objects that + uploads cut off part-way leave, which no backup or restore reads 2. Manifests are unencrypted and can be read without keys 3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as @@ -343,8 +344,9 @@ The prune operation is safe because: a backup reuses from an earlier run that stopped before its manifest: the backup skips the upload of a blob already on the destination, so that blob keeps the time of its first upload, and a prune before the backup's manifest - is written can delete it; the new snapshot then cannot be restored. - `remote nuke` has no such check + is written can delete it; the new snapshot then cannot be restored. It + deletes a `.partial` object only once it is older than `prune_grace_period`, + so an upload still running keeps its object. `remote nuke` has no such check ## Restoration Requirements diff --git a/internal/config/config.go b/internal/config/config.go index ba8b259..61faacf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -206,7 +206,8 @@ type Config struct { // directory. A backup writes its manifest only once it has uploaded // all its blobs and its db.zst.age, so a younger blob or directory may // belong to a backup still running, on this host or another sharing - // the destination store. + // the destination store. It is also how old a `.partial` object a + // killed upload left must be before a prune deletes it. PruneGracePeriod time.Duration `yaml:"prune_grace_period"` // StorageURL specifies the storage backend using a URL format. diff --git a/internal/storage/faultstore/faultstore.go b/internal/storage/faultstore/faultstore.go index 10afb4f..9193fe3 100644 --- a/internal/storage/faultstore/faultstore.go +++ b/internal/storage/faultstore/faultstore.go @@ -200,9 +200,11 @@ func (f *Storer) ListStream( return f.inner.ListStream(ctx, prefix) } -// DeletePartialUploads delegates unchanged. -func (f *Storer) DeletePartialUploads(ctx context.Context, prefix string) error { - return f.inner.DeletePartialUploads(ctx, prefix) +// ListPartialUploads delegates unchanged. +func (f *Storer) ListPartialUploads( + ctx context.Context, prefix string, +) ([]storage.ObjectInfo, error) { + return f.inner.ListPartialUploads(ctx, prefix) } // Info delegates unchanged. diff --git a/internal/storage/file.go b/internal/storage/file.go index 734a748..8f50931 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -56,8 +56,8 @@ const storageDirPerm = 0o755 // truncated object at the key a later run would Stat and trust as a complete // blob. The rclone backend's upload does the same on remotes with a // server-side move. List and ListStream skip these files, so a leftover from -// an interrupted write is never listed or trusted as a blob; it is otherwise -// harmless. +// an interrupted write is never listed or trusted as a blob; prune deletes +// it once it is older than the prune grace period. const tempSuffix = ".partial" // Put stores data at the specified key. @@ -248,22 +248,24 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec return ch } -// DeletePartialUploads removes every file under prefix whose name ends in +// ListPartialUploads returns each file under prefix whose name ends in // tempSuffix. Like List, it reads through symlinked directories and fails -// on one whose target is missing. A missing prefix has none to remove. -func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error { - basePath := f.fullPath(prefix) - +// on one whose target is missing. A missing prefix has none. +func (f *FileStorer) ListPartialUploads( + ctx context.Context, prefix string, +) ([]ObjectInfo, error) { exists, err := f.prefixExists(prefix) if err != nil { - return fmt.Errorf("checking path: %w", err) + return nil, fmt.Errorf("checking path: %w", err) } if !exists { - return nil + return nil, nil } - err = f.walk(basePath, func(path string, info os.FileInfo, err error) error { + var partial []ObjectInfo + + err = f.walk(f.fullPath(prefix), func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -276,13 +278,24 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er return nil } - return f.fs.Remove(path) + relPath, err := filepath.Rel(f.basePath, path) + if err != nil { + return fmt.Errorf("computing relative path: %w", err) + } + + partial = append(partial, ObjectInfo{ + Key: strings.ReplaceAll(relPath, string(filepath.Separator), "/"), + Size: info.Size(), + ModTime: info.ModTime(), + }) + + return nil }) if err != nil { - return fmt.Errorf("walking directory: %w", err) + return nil, fmt.Errorf("walking directory: %w", err) } - return nil + return partial, nil } // Info returns human-readable storage location information. diff --git a/internal/storage/file_atomic_test.go b/internal/storage/file_atomic_test.go index 6030ea2..adfe4bc 100644 --- a/internal/storage/file_atomic_test.go +++ b/internal/storage/file_atomic_test.go @@ -121,9 +121,10 @@ func TestFileStorer_ListSkipsPartialFiles(t *testing.T) { } } -// TestFileStorer_DeletePartialUploads checks that a leftover temp file is -// removed and the object at the real key is kept. -func TestFileStorer_DeletePartialUploads(t *testing.T) { +// TestFileStorer_ListPartialUploads checks that a leftover temp file is +// listed with its size and time, that the object at the real key is not, +// and that Delete removes the leftover by the listed key. +func TestFileStorer_ListPartialUploads(t *testing.T) { t.Parallel() base := t.TempDir() @@ -133,23 +134,46 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) { t.Fatalf("NewFileStorer: %v", err) } + checkListPartialUploads(t, f, base) +} + +// checkListPartialUploads stores an object and writes a leftover temp +// file next to it in the directory base that s stores into. It checks +// that ListPartialUploads lists only the leftover, with its size and a +// time, and that Delete removes it by the listed key and keeps the +// object. +func checkListPartialUploads(t *testing.T, s storage.Storer, base string) { + t.Helper() + ctx := context.Background() - err = f.Put(ctx, testBlobKey, strings.NewReader("blob-bytes")) + err := s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes")) if err != nil { t.Fatalf("Put: %v", err) } - leftover := filepath.Join(base, testBlobKey+"-123456.partial") + leftoverKey := testBlobKey + "-123456.partial" + leftover := filepath.Join(base, leftoverKey) err = os.WriteFile(leftover, []byte("half"), 0o600) if err != nil { t.Fatalf("writing leftover temp file: %v", err) } - err = f.DeletePartialUploads(ctx, "blobs/") + partial, err := s.ListPartialUploads(ctx, "blobs/") if err != nil { - t.Fatalf("DeletePartialUploads: %v", err) + t.Fatalf("ListPartialUploads: %v", err) + } + + if len(partial) != 1 || partial[0].Key != leftoverKey || + partial[0].Size != int64(len("half")) || partial[0].ModTime.IsZero() { + t.Fatalf("ListPartialUploads = %+v, want only %s, 4 bytes, "+ + "with a time", partial, leftoverKey) + } + + err = s.Delete(ctx, partial[0].Key) + if err != nil { + t.Fatalf("Delete: %v", err) } _, err = os.Stat(leftover) @@ -157,7 +181,7 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) { t.Errorf("leftover temp file was not removed: %v", err) } - _, err = f.Stat(ctx, testBlobKey) + _, err = s.Stat(ctx, testBlobKey) if err != nil { t.Errorf("Stat of the real key: %v", err) } diff --git a/internal/storage/file_backend_test.go b/internal/storage/file_backend_test.go index 15cf6a8..395cc66 100644 --- a/internal/storage/file_backend_test.go +++ b/internal/storage/file_backend_test.go @@ -239,11 +239,11 @@ func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) { } } -// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that -// DeletePartialUploads, which remote nuke uses, removes a leftover temp -// file behind a symlinked metadata/ or blobs/ directory and keeps the -// object at the real key. -func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) { +// TestFileStorerListPartialUploadsThroughSymlinkedDirectory checks that +// ListPartialUploads, which prune and remote nuke use, lists a leftover +// temp file behind a symlinked metadata/ or blobs/ directory under its +// key, and leaves out the object at the real key. +func TestFileStorerListPartialUploadsThroughSymlinkedDirectory(t *testing.T) { t.Parallel() ctx := context.Background() @@ -265,29 +265,24 @@ func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) { t.Fatalf("Put: %v", err) } - leftover := filepath.Join(base, key+"-123456.partial") + leftoverKey := key + "-123456.partial" - err = os.WriteFile(leftover, []byte("half"), 0o600) + err = os.WriteFile(filepath.Join(base, leftoverKey), + []byte("half"), 0o600) if err != nil { t.Fatalf("writing leftover temp file: %v", err) } moveBehindSymlink(t, base, dir) - err = s.DeletePartialUploads(ctx, dir+"/") + partial, err := s.ListPartialUploads(ctx, dir+"/") if err != nil { - t.Fatalf("DeletePartialUploads: %v", err) + t.Fatalf("ListPartialUploads: %v", err) } - // leftover runs through the link. - _, err = os.Stat(leftover) - if !os.IsNotExist(err) { - t.Errorf("leftover temp file was not removed: %v", err) - } - - _, err = s.Stat(ctx, key) - if err != nil { - t.Errorf("Stat of the real key: %v", err) + if len(partial) != 1 || partial[0].Key != leftoverKey { + t.Errorf("ListPartialUploads = %+v, want only %s", + partial, leftoverKey) } }) } diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 39413cb..99a2174 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -233,29 +233,28 @@ func (r *RcloneStorer) ListStream( return ch } -// DeletePartialUploads removes every object under prefix whose name ends -// in tempSuffix. -func (r *RcloneStorer) DeletePartialUploads(ctx context.Context, prefix string) error { - var partial []fs.Object +// ListPartialUploads returns each object under prefix whose name ends in +// tempSuffix. +func (r *RcloneStorer) ListPartialUploads( + ctx context.Context, prefix string, +) ([]ObjectInfo, error) { + var partial []ObjectInfo err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) { key := obj.Remote() if strings.HasPrefix(key, prefix) && strings.HasSuffix(key, tempSuffix) { - partial = append(partial, obj) + partial = append(partial, ObjectInfo{ + Key: key, + Size: obj.Size(), + ModTime: obj.ModTime(ctx), + }) } }) if err != nil { - return fmt.Errorf("listing objects: %w", err) + return nil, fmt.Errorf("listing objects: %w", err) } - for _, obj := range partial { - err = obj.Remove(ctx) - if err != nil { - return fmt.Errorf("removing object: %w", err) - } - } - - return nil + return partial, nil } // Info returns human-readable storage location information. diff --git a/internal/storage/rclone_test.go b/internal/storage/rclone_test.go index 32ac7a2..6d4c83e 100644 --- a/internal/storage/rclone_test.go +++ b/internal/storage/rclone_test.go @@ -205,45 +205,21 @@ func TestRcloneStorerListSkipsPartialFiles(t *testing.T) { } } -// TestRcloneStorerDeletePartialUploads checks that a temporary file left -// by a killed upload is removed and the object at the real key is kept. +// TestRcloneStorerListPartialUploads checks that a temporary file left by +// a killed upload is listed with its size and time, that the object at +// the real key is not, and that Delete removes the leftover by the listed +// key. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config -func TestRcloneStorerDeletePartialUploads(t *testing.T) { +func TestRcloneStorerListPartialUploads(t *testing.T) { dir := t.TempDir() - ctx := context.Background() - s, err := storage.NewRcloneStorer(ctx, ":local", dir) + s, err := storage.NewRcloneStorer(context.Background(), ":local", dir) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } - err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes")) - if err != nil { - t.Fatalf("Put: %v", err) - } - - leftover := filepath.Join(dir, testBlobKey+"-123456.partial") - - err = os.WriteFile(leftover, []byte("half"), 0o600) - if err != nil { - t.Fatalf("writing leftover temp file: %v", err) - } - - err = s.DeletePartialUploads(ctx, "blobs/") - if err != nil { - t.Fatalf("DeletePartialUploads: %v", err) - } - - _, err = os.Stat(leftover) - if !os.IsNotExist(err) { - t.Errorf("leftover temp file was not removed: %v", err) - } - - _, err = s.Stat(ctx, testBlobKey) - if err != nil { - t.Errorf("Stat of the real key: %v", err) - } + checkListPartialUploads(t, s, dir) } // newRcloneStorerOnWrappedLocal registers name as rclone's local backend diff --git a/internal/storage/s3.go b/internal/storage/s3.go index bbe133b..930e139 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -100,10 +100,12 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI return ch } -// DeletePartialUploads has nothing to remove: S3 shows an object only once -// its upload has completed, so an upload cut off part-way leaves none. -func (s *S3Storer) DeletePartialUploads(_ context.Context, _ string) error { - return nil +// ListPartialUploads finds none: S3 shows an object only once its upload +// has completed, so an upload cut off part-way leaves none. +func (s *S3Storer) ListPartialUploads( + _ context.Context, _ string, +) ([]ObjectInfo, error) { + return nil, nil } // Info returns human-readable storage location information. diff --git a/internal/storage/storer.go b/internal/storage/storer.go index 3417cb7..34d10cb 100644 --- a/internal/storage/storer.go +++ b/internal/storage/storer.go @@ -77,11 +77,12 @@ type Storer interface { // If an error occurs during listing, the final item will have Err set. ListStream(ctx context.Context, prefix string) <-chan ObjectInfo - // DeletePartialUploads removes every object under prefix that an - // upload cut off part-way left under a temporary name ending in - // `.partial`. The file and rclone backends' List and ListStream skip - // such an object. - DeletePartialUploads(ctx context.Context, prefix string) error + // ListPartialUploads returns each object under prefix that an upload + // cut off part-way left under a temporary name ending in `.partial`, + // with its key, size and modification time. The file and rclone + // backends' List and ListStream skip such an object; Delete removes + // it. + ListPartialUploads(ctx context.Context, prefix string) ([]ObjectInfo, error) // Info returns human-readable storage location information. Info() Info diff --git a/internal/vaultik/destination_validation_test.go b/internal/vaultik/destination_validation_test.go index 9bc7d58..22f951c 100644 --- a/internal/vaultik/destination_validation_test.go +++ b/internal/vaultik/destination_validation_test.go @@ -258,8 +258,10 @@ func (s *stubLister) List(_ context.Context, _ string) ([]string, error) { return nil, errStubUnused } -func (s *stubLister) DeletePartialUploads(_ context.Context, _ string) error { - return errStubUnused +func (s *stubLister) ListPartialUploads( + _ context.Context, _ string, +) ([]storage.ObjectInfo, error) { + return nil, errStubUnused } func (s *stubLister) Info() storage.Info { diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index c4cc18c..53721b4 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -216,6 +216,11 @@ type RemoteInfoResult struct { OrphanedBlobCount *int `json:"orphaned_blob_count"` OrphanedBlobSize *int64 `json:"orphaned_blob_size"` + // `.partial` objects under metadata/ and blobs/ that uploads cut off + // part-way left. The blob figures above leave them out. + PartialUploadCount int `json:"partial_upload_count"` + PartialUploadSize int64 `json:"partial_upload_size"` + // Remote key of each snapshot whose manifest could not be read UnreadableManifests []string `json:"unreadable_manifests,omitempty"` @@ -279,6 +284,16 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { return err } + partialUploads, err := v.listPartialUploads() + if err != nil { + return err + } + + for _, object := range partialUploads { + result.PartialUploadCount++ + result.PartialUploadSize += object.Size + } + log.Info("Remote info complete", "snapshots", result.TotalMetadataCount, "total_blobs", result.TotalBlobCount, @@ -539,6 +554,12 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { "", "", ubytes(result.TotalMetadataSize)) } + v.printRemoteBlobStorage(result) +} + +// printRemoteBlobStorage renders the blob storage section of the +// human-readable remote info output. +func (v *Vaultik) printRemoteBlobStorage(result *RemoteInfoResult) { v.stdoutf("\n=== Blob Storage ===\n") v.stdoutf("Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), @@ -552,16 +573,29 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { "(%d manifest(s) could not be read, "+ "%d manifest(s) under a non-conforming name skipped)\n", len(result.UnreadableManifests), result.SkippedManifestCount) - - return + } else { + v.stdoutf("Orphaned (unreferenced): %s (%s)\n", + humanize.Comma(int64(*result.OrphanedBlobCount)), + ubytes(*result.OrphanedBlobSize)) } - v.stdoutf("Orphaned (unreferenced): %s (%s)\n", - humanize.Comma(int64(*result.OrphanedBlobCount)), - ubytes(*result.OrphanedBlobSize)) + v.stdoutf("Partial uploads: %s (%s)\n", + humanize.Comma(int64(result.PartialUploadCount)), + ubytes(result.PartialUploadSize)) - if *result.OrphanedBlobCount > 0 { - v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") + var toRemove []string + + if result.OrphanedBlobCount != nil && *result.OrphanedBlobCount > 0 { + toRemove = append(toRemove, "orphaned blobs") + } + + if result.PartialUploadCount > 0 { + toRemove = append(toRemove, "partial uploads") + } + + if len(toRemove) > 0 { + v.stdoutf("\nRun 'vaultik prune' to remove %s.\n", + strings.Join(toRemove, " and ")) } } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index f9a90b7..05df7d7 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -155,10 +155,12 @@ func (m *MockStorer) ListStream( return ch } -// DeletePartialUploads has nothing to remove: Put stores each object -// under its key at once. -func (m *MockStorer) DeletePartialUploads(_ context.Context, _ string) error { - return nil +// ListPartialUploads finds none: Put stores each object under its key at +// once. +func (m *MockStorer) ListPartialUploads( + _ context.Context, _ string, +) ([]storage.ObjectInfo, error) { + return nil, nil } func (m *MockStorer) Info() storage.Info { diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 453f347..4d9b3c3 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -31,6 +31,11 @@ var errBlobDeleteFailed = errors.New("blob(s) could not be deleted") var errMetadataDirDeleteFailed = errors.New( "metadata director(ies) with no manifest could not be deleted") +// errPartialUploadDeleteFailed reports `.partial` objects prune could not +// delete. +var errPartialUploadDeleteFailed = errors.New( + "partial upload(s) could not be deleted") + // errObjectsLeft reports objects remote nuke still finds on the backup // destination store after deleting. var errObjectsLeft = errors.New( @@ -138,9 +143,9 @@ func (v *Vaultik) emptyDestinationStore() error { // that fails is logged and the others still run; that object then // shows up in the listing afterwards. func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) { - keys, err := v.Storage.List(v.ctx, prefix) + keys, err := v.listEverythingUnder(prefix) if err != nil { - return nil, fmt.Errorf("listing %s: %w", prefix, err) + return nil, err } for _, key := range keys { @@ -150,19 +155,47 @@ func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) { } } - // The file and rclone listings skip `.partial` objects, so the - // deletes above never reach them. - err = v.Storage.DeletePartialUploads(v.ctx, prefix) - if err != nil { - return nil, fmt.Errorf("deleting partial uploads: %w", err) - } + return v.listEverythingUnder(prefix) +} - left, err := v.Storage.List(v.ctx, prefix) +// listEverythingUnder returns the key of every object under prefix, +// including the `.partial` objects uploads cut off part-way left, which +// the file and rclone backends' List skips. +func (v *Vaultik) listEverythingUnder(prefix string) ([]string, error) { + keys, err := v.Storage.List(v.ctx, prefix) if err != nil { return nil, fmt.Errorf("listing %s: %w", prefix, err) } - return left, nil + partialUploads, err := v.Storage.ListPartialUploads(v.ctx, prefix) + if err != nil { + return nil, fmt.Errorf("listing partial uploads under %s: %w", + prefix, err) + } + + for _, object := range partialUploads { + keys = append(keys, object.Key) + } + + return keys, nil +} + +// listPartialUploads returns the `.partial` objects under metadata/ and +// blobs/ that uploads cut off part-way left. +func (v *Vaultik) listPartialUploads() ([]storage.ObjectInfo, error) { + var partialUploads []storage.ObjectInfo + + for _, prefix := range []string{"metadata/", "blobs/"} { + objects, err := v.Storage.ListPartialUploads(v.ctx, prefix) + if err != nil { + return nil, fmt.Errorf("listing partial uploads under %s: %w", + prefix, err) + } + + partialUploads = append(partialUploads, objects...) + } + + return partialUploads, nil } // PruneBlobsResult contains the result of a blob prune operation @@ -182,11 +215,19 @@ type PruneBlobsResult struct { // not delete. MetadataDirsDeleted int `json:"metadata_dirs_deleted"` MetadataDirsFailed int `json:"metadata_dirs_failed,omitempty"` + // PartialUploadsDeleted counts the `.partial` objects uploads cut off + // part-way left that prune deleted, and PartialUploadsFailed those it + // could not delete. PartialUploadsTooNew counts those kept for being + // newer than the prune grace period, or of unknown age. + PartialUploadsDeleted int `json:"partial_uploads_deleted"` + PartialUploadsFailed int `json:"partial_uploads_failed,omitempty"` + PartialUploadsTooNew int `json:"partial_uploads_too_new"` } // Prune removes orphaned data from the local index database AND, from the -// backup destination store, unreferenced blobs and the directories under -// metadata/ with no manifest. This is the single user-facing prune entry +// backup destination store, unreferenced blobs, the directories under +// metadata/ with no manifest and the `.partial` objects uploads cut off +// part-way left. This is the single user-facing prune entry // point — the split between local and remote cleanup is an // implementation detail. Calling code should prefer this method over // PruneDatabase or PruneBlobs individually unless it specifically wants @@ -214,8 +255,9 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { return v.PruneBlobs(opts) } -// PruneBlobs removes unreferenced blobs from storage, and each directory -// under metadata/ that a backup left with no manifest. +// PruneBlobs removes unreferenced blobs from storage, each directory +// under metadata/ that a backup left with no manifest, and each +// `.partial` object an upload cut off part-way left. func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { log.Info("Starting prune operation") @@ -242,14 +284,21 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return err } - result := &PruneBlobsResult{ - BlobsFound: len(unreferencedBlobs), - BlobsTooNew: tooNew, + partialUploads, partialTooNew, err := v.findOldPartialUploads() + if err != nil { + return err } - v.reportBlobsTooNew(tooNew, opts) + result := &PruneBlobsResult{ + BlobsFound: len(unreferencedBlobs), + BlobsTooNew: tooNew, + PartialUploadsTooNew: partialTooNew, + } - if len(unreferencedBlobs) == 0 && len(dirsWithoutManifest) == 0 { + v.reportTooNew(result, opts) + + if len(unreferencedBlobs) == 0 && len(dirsWithoutManifest) == 0 && + len(partialUploads) == 0 { log.Info("No unreferenced blobs found") if opts.JSON { @@ -263,15 +312,17 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", ubytes(totalSize), - "metadata_dirs_without_manifest", len(dirsWithoutManifest)) + "metadata_dirs_without_manifest", len(dirsWithoutManifest), + "partial_uploads", len(partialUploads)) if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize, - dirsWithoutManifest) { + dirsWithoutManifest, partialUploads) { return ErrNotConfirmed } v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result) v.deleteMetadataDirs(dirsWithoutManifest, result) + v.deletePartialUploads(partialUploads, result) if opts.JSON { err = v.outputPruneBlobsJSON(result) @@ -279,9 +330,16 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return err } } else { - v.reportPruneDeletions(result, len(dirsWithoutManifest)) + v.reportPruneDeletions(result, len(dirsWithoutManifest), + len(partialUploads)) } + return pruneDeleteFailures(result) +} + +// pruneDeleteFailures returns an error for each kind of object prune +// could not delete, or nil when every delete succeeded. +func pruneDeleteFailures(result *PruneBlobsResult) error { var failures []error if result.BlobsFailed > 0 { @@ -294,26 +352,41 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { result.MetadataDirsFailed, errMetadataDirDeleteFailed)) } + if result.PartialUploadsFailed > 0 { + failures = append(failures, fmt.Errorf("%d %w", + result.PartialUploadsFailed, errPartialUploadDeleteFailed)) + } + return errors.Join(failures...) } -// confirmPrune states what prune found to delete, unreferenced blobs and +// confirmPrune states what prune found to delete, unreferenced blobs, // directories under metadata/ with no manifest, naming each directory by -// its remote key, and, unless --force is given, asks the operator to -// confirm. Under --json it prints nothing and does not ask. It reports -// whether to go ahead. +// its remote key, and `.partial` objects, and, unless --force is given, +// asks the operator to confirm. Under --json it prints nothing and does +// not ask. It reports whether to go ahead. func (v *Vaultik) confirmPrune( opts *PruneOptions, blobCount int, blobsSize int64, dirs []string, + partialUploads []storage.ObjectInfo, ) bool { if opts.JSON { return true } + var partialSize int64 + + for _, object := range partialUploads { + partialSize += object.Size + } + dirCount := len(dirs) + partialCount := len(partialUploads) blobsFound := fmt.Sprintf("Found %d unreferenced blob(s) totaling %s.", blobCount, ubytes(blobsSize)) dirsFound := fmt.Sprintf("Found %d metadata director(ies) left by "+ "backups that stopped before their manifest:", dirCount) + partialFound := fmt.Sprintf("Found %d partial upload(s) totaling %s, "+ + "left by uploads cut off part-way.", partialCount, ubytes(partialSize)) if opts.Force { if blobCount > 0 { @@ -328,6 +401,10 @@ func (v *Vaultik) confirmPrune( } } + if partialCount > 0 { + v.UI.Infof("%s", partialFound) + } + return true } @@ -346,16 +423,13 @@ func (v *Vaultik) confirmPrune( } } - switch { - case dirCount == 0: - v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", blobCount) - case blobCount == 0: - v.stdoutf("\nDelete %d metadata director(ies)? [y/N] ", dirCount) - default: - v.stdoutf("\nDelete %d unreferenced blob(s) and %d metadata "+ - "director(ies)? [y/N] ", blobCount, dirCount) + if partialCount > 0 { + v.stdoutf("%s\n", partialFound) } + v.stdoutf("\nDelete %s? [y/N] ", + pruneTargets(blobCount, dirCount, partialCount)) + var confirm string // A failure to read the answer means no confirmation. @@ -369,9 +443,36 @@ func (v *Vaultik) confirmPrune( return true } -// reportPruneDeletions says how many blobs and metadata directories -// prune deleted, naming each kind only when it found some to delete. -func (v *Vaultik) reportPruneDeletions(result *PruneBlobsResult, dirCount int) { +// pruneTargets names what the prune prompt offers to delete, such as +// "2 unreferenced blob(s) and 1 partial upload(s)", leaving out each kind +// with a count of zero. +func pruneTargets(blobCount, dirCount, partialCount int) string { + var targets []string + + if blobCount > 0 { + targets = append(targets, + fmt.Sprintf("%d unreferenced blob(s)", blobCount)) + } + + if dirCount > 0 { + targets = append(targets, + fmt.Sprintf("%d metadata director(ies)", dirCount)) + } + + if partialCount > 0 { + targets = append(targets, + fmt.Sprintf("%d partial upload(s)", partialCount)) + } + + return strings.Join(targets, " and ") +} + +// reportPruneDeletions says how many blobs, metadata directories and +// partial uploads prune deleted, naming each kind only when it found some +// to delete. +func (v *Vaultik) reportPruneDeletions( + result *PruneBlobsResult, dirCount, partialCount int, +) { if result.BlobsFound > 0 { v.UI.Infof("Deleted %d blob(s) totaling %s.", result.BlobsDeleted, ubytes(result.BytesFreed)) @@ -381,6 +482,11 @@ func (v *Vaultik) reportPruneDeletions(result *PruneBlobsResult, dirCount int) { v.UI.Infof("Deleted %d metadata director(ies) with no manifest.", result.MetadataDirsDeleted) } + + if partialCount > 0 { + v.UI.Infof("Deleted %d partial upload(s).", + result.PartialUploadsDeleted) + } } // listMetadataObjects lists every object under metadata/ on the @@ -611,18 +717,79 @@ func (v *Vaultik) deleteMetadataDirs( } } -// reportBlobsTooNew says how many unreferenced blobs prune kept for being -// newer than the grace period or of unknown age. Under --json the -// document carries the count instead. -func (v *Vaultik) reportBlobsTooNew(tooNew int, opts *PruneOptions) { - if tooNew == 0 || opts.JSON { +// reportTooNew says how many unreferenced blobs and partial uploads prune +// kept for being newer than the grace period or of unknown age. Under +// --json the document carries the counts instead. +func (v *Vaultik) reportTooNew(result *PruneBlobsResult, opts *PruneOptions) { + if opts.JSON { return } - v.UI.Infof("Kept %d unreferenced blob(s) newer than "+ - "prune_grace_period (%s), or of unknown age: a backup still "+ - "running may need them.", - tooNew, v.Config.PruneGracePeriod) + if result.BlobsTooNew > 0 { + v.UI.Infof("Kept %d unreferenced blob(s) newer than "+ + "prune_grace_period (%s), or of unknown age: a backup still "+ + "running may need them.", + result.BlobsTooNew, v.Config.PruneGracePeriod) + } + + if result.PartialUploadsTooNew > 0 { + v.UI.Infof("Kept %d partial upload(s) newer than "+ + "prune_grace_period (%s), or of unknown age: an upload may "+ + "still be writing them.", + result.PartialUploadsTooNew, v.Config.PruneGracePeriod) + } +} + +// findOldPartialUploads returns the `.partial` objects under metadata/ +// and blobs/ that are older than the prune grace period. It also returns +// how many it left out for being newer than that, or of unknown age +// because the listing gave no time for them. +func (v *Vaultik) findOldPartialUploads() ([]storage.ObjectInfo, int, error) { + partialUploads, err := v.listPartialUploads() + if err != nil { + return nil, 0, err + } + + cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod) + + var ( + old []storage.ObjectInfo + tooNew int + ) + + for _, object := range partialUploads { + if object.ModTime.IsZero() || object.ModTime.After(cutoff) { + tooNew++ + + continue + } + + old = append(old, object) + } + + return old, tooNew, nil +} + +// deletePartialUploads deletes the given `.partial` objects and counts +// in the result those deleted and those it could not delete. +func (v *Vaultik) deletePartialUploads( + partialUploads []storage.ObjectInfo, result *PruneBlobsResult, +) { + for _, object := range partialUploads { + err := v.Storage.Delete(v.ctx, object.Key) + if err != nil { + log.Error("Failed to delete partial upload", + "key", object.Key, "error", err) + + result.PartialUploadsFailed++ + + continue + } + + log.Info("Deleted partial upload", "key", object.Key) + + result.PartialUploadsDeleted++ + } } // deleteUnreferencedBlobs deletes the given blobs from storage and diff --git a/internal/vaultik/prune_partial_upload_test.go b/internal/vaultik/prune_partial_upload_test.go new file mode 100644 index 0000000..b163eb4 --- /dev/null +++ b/internal/vaultik/prune_partial_upload_test.go @@ -0,0 +1,126 @@ +package vaultik_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/config" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/ui" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// partialUploadGracePeriod is the prune grace period of the vaultik +// newPartialUploadStore returns. +const partialUploadGracePeriod = time.Hour + +// newPartialUploadStore returns a vaultik on a file:// destination +// holding the `.partial` files killed uploads leave: one under blobs/ +// and one under metadata/ older than the prune grace period, and one +// under blobs/ newer than that. It also returns the buffer the vaultik +// writes its output to, the paths of the old files and the path of the +// new one. +func newPartialUploadStore( + t *testing.T, +) (*vaultik.Vaultik, *bytes.Buffer, []string, string) { + t.Helper() + + storeDir := t.TempDir() + + store, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + hash := testBlobHashA + oldFiles := []string{ + filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], + hash+"-123456.partial"), + filepath.Join(storeDir, metadataObjectKey( + "testhost_home_2026-01-01T00:00:00Z", "db.zst.age-123456.partial")), + } + newFile := filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], + hash+"-654321.partial") + + beforeGracePeriod := time.Now().Add(-partialUploadGracePeriod - time.Minute) + + for _, path := range append([]string{newFile}, oldFiles...) { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + require.NoError(t, os.WriteFile(path, []byte("half an upload"), 0o600)) + } + + for _, path := range oldFiles { + require.NoError(t, os.Chtimes(path, beforeGracePeriod, beforeGracePeriod)) + } + + stdout := &bytes.Buffer{} + v := &vaultik.Vaultik{ + Config: &config.Config{PruneGracePeriod: partialUploadGracePeriod}, + Storage: store, + Stdout: stdout, + UI: ui.NewWithColor(stdout, false), + } + v.SetContext(context.Background()) + + return v, stdout, oldFiles, newFile +} + +// TestPruneDeletesPartialUploadsOlderThanGracePeriod checks that prune +// deletes a `.partial` file a killed upload left under blobs/ or +// metadata/ once it is older than the grace period, keeps a newer one, +// and counts both in the --json document. +func TestPruneDeletesPartialUploadsOlderThanGracePeriod(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + v, stdout, oldFiles, newFile := newPartialUploadStore(t) + + require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) + + for _, path := range oldFiles { + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "%s must be deleted", path) + } + + _, err := os.Stat(newFile) + require.NoError(t, err, "a partial upload newer than the grace period "+ + "must be kept") + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(stdout.Bytes(), &result)) + assert.Equal(t, 2, result.PartialUploadsDeleted) + assert.Equal(t, 1, result.PartialUploadsTooNew) +} + +// TestRemoteInfoCountsPartialUploads checks that remote info counts +// every `.partial` file under blobs/ and metadata/, whatever its age, and +// advises running prune. +func TestRemoteInfoCountsPartialUploads(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + v, stdout, _, _ := newPartialUploadStore(t) + + require.NoError(t, v.RemoteInfo(true)) + + var result vaultik.RemoteInfoResult + + require.NoError(t, json.Unmarshal(stdout.Bytes(), &result)) + assert.Equal(t, 3, result.PartialUploadCount) + assert.Equal(t, int64(3*len("half an upload")), result.PartialUploadSize) + assert.Equal(t, 0, result.TotalBlobCount) + + stdout.Reset() + require.NoError(t, v.RemoteInfo(false)) + + text := stdout.String() + assert.Contains(t, text, "Partial uploads: 3 (") + assert.Contains(t, text, "Run 'vaultik prune' to remove partial uploads.") +} diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index df0f202..1ae7abd 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -136,10 +136,12 @@ func (s *testStorer) ListStream( return ch } -// DeletePartialUploads has nothing to remove: Put stores each object -// under its key at once. -func (s *testStorer) DeletePartialUploads(_ context.Context, _ string) error { - return nil +// ListPartialUploads finds none: Put stores each object under its key at +// once. +func (s *testStorer) ListPartialUploads( + _ context.Context, _ string, +) ([]storage.ObjectInfo, error) { + return nil, nil } func (s *testStorer) Info() storage.Info { -- 2.54.0 From 3d0222d644f8e21df3edcdec3e2458acee46e009 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:45:34 +0000 Subject: [PATCH 2/6] Reword the .partial sentences that stack clauses The sentences added for .partial objects in README.md, docs/REPOSTRUCTURE.md, the example config and the PruneGracePeriod comment read "a .partial object an upload cut off part-way left" and "how old a .partial object a killed upload left must be": the verb arrives late and "cut off" reads as the verb. They now say "left by a killed upload". Model: opus-5-5 --- README.md | 42 +++++++++++++++++++------------------- configs/config.example.yml | 4 ++-- docs/REPOSTRUCTURE.md | 4 ++-- internal/config/config.go | 4 ++-- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index f31480b..c91ae16 100644 --- a/README.md +++ b/README.md @@ -523,10 +523,10 @@ deleted. Step (3) also deletes each directory under `metadata/` that has no `prune_grace_period` and its snapshot is not in the local index. An object the listing gives no time for keeps its directory, and a directory whose name is not a remote key is left alone. Step (3) also deletes each `.partial` object under -`metadata/` and `blobs/` that an upload cut off part-way left, once it is older -than `prune_grace_period`. Step (3) reads all remote manifests — network cost -scales with the number of snapshots. `snapshot create --prune` runs the same -cleanup automatically; this is the manual entry point for the same work. If the +`metadata/` and `blobs/` left by a killed upload, once it is older than +`prune_grace_period`. Step (3) reads all remote manifests — network cost scales +with the number of snapshots. `snapshot create --prune` runs the same cleanup +automatically; this is the manual entry point for the same work. If the destination store cannot be listed, or the local index holds snapshots and the destination answers that it has no `vaultik-destination.txt` (an unplugged volume's empty mount point), prune fails and deletes nothing. `snapshot purge` @@ -559,9 +559,9 @@ in `unreadable_manifests` and counts the manifests under skipped names in `skipped_manifest_count`. An unreadable manifest also leaves its snapshot's blob count and blob size unknown, `null` in `--json`. A file under `blobs/` whose name is not a blob hash is not counted as a blob, since `prune` never deletes -it. A `.partial` object an upload cut off part-way left under `metadata/` or -`blobs/` is not counted as a blob either. Such objects are counted on their own -line, and in `partial_upload_count` and `partial_upload_size` in `--json`. +it. A leftover `.partial` object under `metadata/` or `blobs/` is not counted as +a blob either. Such objects are counted on their own line, and in +`partial_upload_count` and `partial_upload_size` in `--json`. - `--json`: Output as JSON @@ -766,20 +766,20 @@ Run `vaultik config init` to generate a fully commented config file; a complete annotated example also lives in [`configs/config.example.yml`](configs/config.example.yml). Key fields: -| Field | Default | Description | -| -------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty | -| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot files`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. | -| `snapshots` | (required) | Named snapshot definitions with paths and excludes | -| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) | -| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | -| `exclude` | | Global exclude patterns (applied to all snapshots) | -| `chunk_size` | `10MB` | Average chunk size for content-defined chunking | -| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | -| `compression_level` | `3` | zstd compression level (1-19) | -| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it, and how old every object in a directory under `metadata/` with no manifest must be before `prune` deletes that directory. It also sets how old a `.partial` object a killed upload left must be before `prune` deletes it. A backup uploads its blobs and its `db.zst.age` before its manifest, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs and directories at once | -| `hostname` | system hostname | Hostname used in snapshot IDs | -| `index_path` | platform data dir | Local SQLite index path | +| Field | Default | Description | +| -------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty | +| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot files`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. | +| `snapshots` | (required) | Named snapshot definitions with paths and excludes | +| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) | +| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | +| `exclude` | | Global exclude patterns (applied to all snapshots) | +| `chunk_size` | `10MB` | Average chunk size for content-defined chunking | +| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | +| `compression_level` | `3` | zstd compression level (1-19) | +| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it, and how old every object in a directory under `metadata/` with no manifest must be before `prune` deletes that directory. `prune` also deletes a `.partial` object left by a killed upload only once it is this old. A backup uploads its blobs and its `db.zst.age` before its manifest, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs and directories at once | +| `hostname` | system hostname | Hostname used in snapshot IDs | +| `index_path` | platform data dir | Local SQLite index path | --- diff --git a/configs/config.example.yml b/configs/config.example.yml index a8b0080..8895d5f 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -333,8 +333,8 @@ compression_level: 5 # How old a blob that no snapshot lists must be before prune deletes it, and # how old every object in a directory under metadata/ with no manifest must # be before prune deletes that directory -# It is also how old a .partial object a killed upload left must be before -# prune deletes it +# prune also deletes a .partial object left by a killed upload only once it is +# this old # A backup uploads its blobs and its db.zst.age before its manifest, so on a # destination shared between hosts or users this must be longer than the # longest backup run diff --git a/docs/REPOSTRUCTURE.md b/docs/REPOSTRUCTURE.md index 812c594..b17e955 100644 --- a/docs/REPOSTRUCTURE.md +++ b/docs/REPOSTRUCTURE.md @@ -329,8 +329,8 @@ The prune operation is safe because: 1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references. Under `metadata/` it deletes only a directory that has no `manifest.json.zst`, whose name is a remote key, and that no - local snapshot record claims. It also deletes the `.partial` objects that - uploads cut off part-way leave, which no backup or restore reads + local snapshot record claims. It also deletes the `.partial` objects left by + killed uploads, which no backup or restore reads 2. Manifests are unencrypted and can be read without keys 3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as diff --git a/internal/config/config.go b/internal/config/config.go index 61faacf..1e2e1c3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -206,8 +206,8 @@ type Config struct { // directory. A backup writes its manifest only once it has uploaded // all its blobs and its db.zst.age, so a younger blob or directory may // belong to a backup still running, on this host or another sharing - // the destination store. It is also how old a `.partial` object a - // killed upload left must be before a prune deletes it. + // the destination store. A prune also deletes a `.partial` object left + // by a killed upload only once it is this old. PruneGracePeriod time.Duration `yaml:"prune_grace_period"` // StorageURL specifies the storage backend using a URL format. -- 2.54.0 From 18961fa52f47bc874e7feb93c8f5d38d48fb33a9 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:45:42 +0000 Subject: [PATCH 3/6] Name prune in the docs sentence on .partial age In docs/REPOSTRUCTURE.md the sentence on the age of a .partial object began with "It" right after a sentence about the new snapshot, so it read as the snapshot deleting the object. It now says "Prune". Model: opus-5-5 --- docs/REPOSTRUCTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/REPOSTRUCTURE.md b/docs/REPOSTRUCTURE.md index b17e955..09783ce 100644 --- a/docs/REPOSTRUCTURE.md +++ b/docs/REPOSTRUCTURE.md @@ -344,7 +344,7 @@ The prune operation is safe because: a backup reuses from an earlier run that stopped before its manifest: the backup skips the upload of a blob already on the destination, so that blob keeps the time of its first upload, and a prune before the backup's manifest - is written can delete it; the new snapshot then cannot be restored. It + is written can delete it; the new snapshot then cannot be restored. Prune deletes a `.partial` object only once it is older than `prune_grace_period`, so an upload still running keeps its object. `remote nuke` has no such check -- 2.54.0 From 332c2d30a271de06c96e2cdbc8164a6a79d9945c Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:46:03 +0000 Subject: [PATCH 4/6] Shorten the README's opening sentence for --prune Adding .partial objects made the first sentence of the --prune bullet about 70 words long, chaining three cleanup steps with "and ..., and". It now says the flag deletes from the destination store what prune deletes there; the prune section lists what that is. Model: opus-5-5 --- README.md | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c91ae16..3b3d0b8 100644 --- a/README.md +++ b/README.md @@ -340,23 +340,19 @@ In such a config, an unknown key or a value of the wrong type is refused. error that explains how to fix it - `--cron`: Silent on total success; warnings and errors are still printed (for crontab) -- `--prune`: After backup, drop older snapshots of each backed-up name, remove - orphaned blobs older than `prune_grace_period` from remote storage, and delete - each directory under `metadata/` that has no `manifest.json.zst` once every - object in it is older than `prune_grace_period` and its snapshot is not in the - local index, and delete each `.partial` object older than - `prune_grace_period`, as `prune` does. By default keeps only the latest - snapshot per name; use `--keep-newer-than` for a rolling window. When - `--skip-errors` skipped a file or directory it could not read or list, the - prune does not run and every older snapshot is kept, since one of them may be - the only one that holds what was skipped; the command says so and exits - non-zero. When the backup found nothing in a configured path, as in the mount - point of a volume that is not mounted, the prune does not run either: every - older snapshot is kept, and the command names the path and exits non-zero, on - every run until the path holds something again. Take a path that is empty on - purpose out of the config, or remove the older snapshots by hand with - `snapshot purge`. When an earlier snapshot of the same name in the local index - found a volume mounted at or below a configured path (a directory on a +- `--prune`: After backup, drop older snapshots of each backed-up name, then + delete from the destination store what `prune` deletes there. By default keeps + only the latest snapshot per name; use `--keep-newer-than` for a rolling + window. When `--skip-errors` skipped a file or directory it could not read or + list, the prune does not run and every older snapshot is kept, since one of + them may be the only one that holds what was skipped; the command says so and + exits non-zero. When the backup found nothing in a configured path, as in the + mount point of a volume that is not mounted, the prune does not run either: + every older snapshot is kept, and the command names the path and exits + non-zero, on every run until the path holds something again. Take a path that + is empty on purpose out of the config, or remove the older snapshots by hand + with `snapshot purge`. When an earlier snapshot of the same name in the local + index found a volume mounted at or below a configured path (a directory on a different filesystem from its parent, with an entry in it that the backup does not exclude) and this backup does not, the prune does not run either: every older snapshot is kept, and the command names the directory and exits -- 2.54.0 From 92296daf52d3f93f28c27bdda938e9943440bbdd Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:47:00 +0000 Subject: [PATCH 5/6] Mention .partial objects in the help and config init text prune --help, remote info --help, the snapshot create --prune flag, the config init template's prune_grace_period comment and the runPostBackupPrune comment each listed what prune deletes or remote info counts, and none named the .partial objects this change adds. They do now; the --prune flag refers to prune instead of repeating its list. Model: opus-5-5 --- internal/cli/config.go | 8 +++++--- internal/cli/prune.go | 10 ++++++---- internal/cli/remote.go | 3 ++- internal/cli/snapshot.go | 12 +++++------- internal/vaultik/snapshot.go | 8 ++++---- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/internal/cli/config.go b/internal/cli/config.go index fe0face..c180dce 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -269,9 +269,11 @@ storage_url: "" # How old a blob that no snapshot lists must be before prune deletes it, # and how old every object in a directory under metadata/ with no manifest -# must be before prune deletes that directory. A backup uploads its blobs -# and its db.zst.age before its manifest, so on a destination shared -# between hosts or users this must be longer than the longest backup run. +# must be before prune deletes that directory. prune also deletes a +# .partial object left by a killed upload only once it is this old. A +# backup uploads its blobs and its db.zst.age before its manifest, so on +# a destination shared between hosts or users this must be longer than +# the longest backup run. # It does not protect a blob a backup reuses from an earlier run that # stopped before its manifest: that blob keeps the time of its first # upload, so a prune can delete it. diff --git a/internal/cli/prune.go b/internal/cli/prune.go index 6063e7a..bb1e1e5 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -13,9 +13,10 @@ func NewPruneCommand() *cobra.Command { cmd := &cobra.Command{ Use: "prune", Short: "Tidy local database and remote storage", - Long: `Removes orphaned data from the local index database, and -unreferenced blobs and directories under metadata/ with no manifest -from the backup destination store. + Long: `Removes orphaned data from the local index database. From the +backup destination store it removes unreferenced blobs, directories +under metadata/ with no manifest, and .partial objects left by killed +uploads. Local cleanup drops incomplete snapshots and any files, chunks, or blobs no longer referenced by a completed snapshot. Remote cleanup @@ -30,7 +31,8 @@ blob keeps the time of its first upload, so it can be deleted. Remote cleanup also deletes each directory under metadata/ with no manifest, as a backup that stopped before its manifest leaves, once all its objects are older than prune_grace_period and its snapshot is -not in the local index. +not in the local index. It also deletes each .partial object under +metadata/ or blobs/ once it is older than prune_grace_period. Snapshot create --prune runs the same cleanup automatically; this command is the manual entry point for the same work (e.g. after a diff --git a/internal/cli/remote.go b/internal/cli/remote.go index f46bb26..b7b6085 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -75,7 +75,8 @@ func newRemoteInfoCommand() *cobra.Command { - Size of all snapshot metadata (per snapshot and total) - Count and total size of all blobs - Count and size of referenced blobs (from all manifests) -- Count and size of orphaned blobs (not referenced by any manifest)`, +- Count and size of orphaned blobs (not referenced by any manifest) +- Count and size of .partial objects left by killed uploads`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 86ac329..5e97dd0 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -138,13 +138,11 @@ on Linux), then /etc/vaultik/config.yml.`, cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless warning or error)") cmd.Flags().BoolVar(&opts.Prune, "prune", false, - "After backup, drop older snapshots of the same name, remove "+ - "orphaned blobs older than prune_grace_period, and delete each "+ - "directory under metadata/ with no manifest once all its "+ - "objects are older than prune_grace_period and its snapshot is "+ - "not in the local index (not after a backup that skipped "+ - "unreadable files, found a configured path empty, or did not "+ - "find a mount point an earlier snapshot found)") + "After backup, drop older snapshots of the same name, then delete "+ + "from the destination store what prune deletes there (not "+ + "after a backup that skipped unreadable files, found a "+ + "configured path empty, or did not find a mount point an "+ + "earlier snapshot found)") cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration "+ "(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+ diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index ae95c93..9bef3eb 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -302,10 +302,10 @@ func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) { } // runPostBackupPrune drops older snapshots of the given names and removes -// from remote storage the orphan blobs and the directories under -// metadata/ with no manifest, as PruneBlobs does. If keepNewerThan is set -// (e.g. "4w"), snapshots newer than that duration are kept. Otherwise only -// the latest snapshot of each name is kept. +// from remote storage the orphan blobs, the directories under metadata/ +// with no manifest and the `.partial` objects, as PruneBlobs does. If +// keepNewerThan is set (e.g. "4w"), snapshots newer than that duration are +// kept. Otherwise only the latest snapshot of each name is kept. func (v *Vaultik) runPostBackupPrune( snapshotNames []string, keepNewerThan string, ) error { -- 2.54.0 From 59664f219e95b1303cc69a3115c8feb03d4792c0 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 07:53:03 +0000 Subject: [PATCH 6/6] Test that a failed .partial delete fails prune Nothing tested that prune exits non-zero and counts partial_uploads_failed when a .partial object cannot be deleted, so dropping that branch from pruneDeleteFailures passed the suite. The new test fails every delete through faultstore and checks both. Model: opus-5-5 --- internal/vaultik/prune_partial_upload_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/internal/vaultik/prune_partial_upload_test.go b/internal/vaultik/prune_partial_upload_test.go index b163eb4..c70d173 100644 --- a/internal/vaultik/prune_partial_upload_test.go +++ b/internal/vaultik/prune_partial_upload_test.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/storage/faultstore" "sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/vaultik" ) @@ -99,6 +100,36 @@ func TestPruneDeletesPartialUploadsOlderThanGracePeriod(t *testing.T) { assert.Equal(t, 1, result.PartialUploadsTooNew) } +// TestPruneFailedPartialUploadDeleteReturnsError checks that prune +// returns an error, and counts the failures in the --json document, when +// the `.partial` files it found cannot be deleted. +func TestPruneFailedPartialUploadDeleteReturnsError(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + v, stdout, oldFiles, _ := newPartialUploadStore(t) + + store := faultstore.New(v.Storage) + store.OnDelete = func(string) bool { return true } + v.Storage = store + + err := v.PruneBlobs(&vaultik.PruneOptions{JSON: true}) + + require.ErrorContains(t, err, "2 partial upload(s) could not be deleted") + + for _, path := range oldFiles { + _, statErr := os.Stat(path) + require.NoError(t, statErr, "%s must still exist", path) + } + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(stdout.Bytes(), &result), + "stdout must hold the JSON document, got:\n%s", stdout.String()) + assert.Equal(t, 2, result.PartialUploadsFailed) + assert.Equal(t, 0, result.PartialUploadsDeleted) +} + // TestRemoteInfoCountsPartialUploads checks that remote info counts // every `.partial` file under blobs/ and metadata/, whatever its age, and // advises running prune. -- 2.54.0