Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf2f085e80 |
@@ -96,14 +96,6 @@ vaultik snapshot restore <snapshot-id> ~/vaultik-restore
|
|||||||
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
||||||
```
|
```
|
||||||
|
|
||||||
On a destination shared between hosts or users, the prune in that cron line
|
|
||||||
relies on `prune_grace_period` (default `24h`) to leave the blobs of a backup
|
|
||||||
another one is still running, so it must be longer than the longest backup
|
|
||||||
run. It does not protect a blob that backup reuses from an earlier run that
|
|
||||||
stopped before its manifest: the prune can delete such a blob, and the new
|
|
||||||
snapshot then cannot be restored. To not depend on the grace period, give each
|
|
||||||
host its own prefix or directory. See [locking](#locking).
|
|
||||||
|
|
||||||
## restoring on another machine
|
## restoring on another machine
|
||||||
|
|
||||||
Restoring on a host that never ran the backup — a replacement machine
|
Restoring on a host that never ran the backup — a replacement machine
|
||||||
@@ -200,22 +192,7 @@ process-wide lock at `$XDG_DATA_HOME/vaultik/vaultik.pid`
|
|||||||
(`~/.local/share/vaultik/vaultik.pid` on Linux) for the whole run. Only
|
(`~/.local/share/vaultik/vaultik.pid` on Linux) for the whole run. Only
|
||||||
one of them runs at a time: a second one exits immediately with an
|
one of them runs at a time: a second one exits immediately with an
|
||||||
"already running" error rather than waiting, so two writers can never
|
"already running" error rather than waiting, so two writers can never
|
||||||
corrupt the local index.
|
corrupt the local index or the destination store.
|
||||||
|
|
||||||
The lock covers one user on one machine. It does not stop another user,
|
|
||||||
or another host, from writing to the same destination store at the same
|
|
||||||
time. A backup uploads its blobs before the manifest that lists them, so a
|
|
||||||
prune from elsewhere at that moment finds blobs no snapshot lists. It
|
|
||||||
deletes such a blob only once it is older than `prune_grace_period`
|
|
||||||
(default `24h`), which therefore must be longer than the longest backup run
|
|
||||||
of any host or user sharing the destination. The grace period does not
|
|
||||||
protect a blob a backup reuses from an earlier run that stopped before its
|
|
||||||
manifest: a backup skips the upload of a blob already on the destination, so
|
|
||||||
that blob keeps the time of its first upload, and a prune from elsewhere
|
|
||||||
during the backup can delete it; the new snapshot then cannot be restored.
|
|
||||||
`remote nuke` deletes everything regardless. To not depend on the grace
|
|
||||||
period, give each host or user its own prefix (or directory) on the
|
|
||||||
destination.
|
|
||||||
|
|
||||||
Read-only commands — `info`, `snapshot list`, `snapshot verify`, and
|
Read-only commands — `info`, `snapshot list`, `snapshot verify`, and
|
||||||
`remote info` — do not take the lock and are never blocked, so they run
|
`remote info` — do not take the lock and are never blocked, so they run
|
||||||
@@ -319,8 +296,7 @@ value where a list belongs or a word where a number belongs), is refused.
|
|||||||
* `--cron`: Silent on total success; warnings and errors are still printed
|
* `--cron`: Silent on total success; warnings and errors are still printed
|
||||||
(for crontab)
|
(for crontab)
|
||||||
* `--prune`: After backup, drop older snapshots of each backed-up name and
|
* `--prune`: After backup, drop older snapshots of each backed-up name and
|
||||||
remove orphaned blobs older than `prune_grace_period` from remote storage,
|
remove orphaned blobs from remote storage. By default keeps only the latest
|
||||||
as `prune` does. By default keeps only the latest
|
|
||||||
snapshot per name; use `--keep-newer-than` for a rolling window. When
|
snapshot per name; use `--keep-newer-than` for a rolling window. When
|
||||||
`--skip-errors` skipped a file or directory it could not read or list, the
|
`--skip-errors` skipped a file or directory it could not read or list, the
|
||||||
prune does not run and every older snapshot is kept, since one of them may
|
prune does not run and every older snapshot is kept, since one of them may
|
||||||
@@ -447,15 +423,9 @@ local snapshot whose remote metadata is missing is dropped from the
|
|||||||
local index; (2) delete orphaned local rows (files, chunks, blobs no
|
local index; (2) delete orphaned local rows (files, chunks, blobs no
|
||||||
longer referenced by any completed snapshot); (3) list every remote
|
longer referenced by any completed snapshot); (3) list every remote
|
||||||
manifest on the destination store to compute the still-referenced blob
|
manifest on the destination store to compute the still-referenced blob
|
||||||
set and delete any blob not in that set that is older than
|
set and delete any blob not in that set. Step (3) reads all remote
|
||||||
`prune_grace_period` (default `24h`). A newer one is kept and counted: it
|
manifests — network cost scales with the number of snapshots. `snapshot
|
||||||
may belong to a backup still running, on this host or another sharing the
|
create --prune` runs the same cleanup automatically; this is the
|
||||||
destination store, that has not yet uploaded its manifest. The grace period
|
|
||||||
does not protect a blob such a backup reuses from an earlier run that stopped
|
|
||||||
before its manifest: that blob keeps the time of its first upload, so it can
|
|
||||||
be deleted. Step (3) reads all remote manifests — network cost scales with
|
|
||||||
the number of snapshots.
|
|
||||||
`snapshot create --prune` runs the same cleanup automatically; this is the
|
|
||||||
manual entry point for the same work. If the destination store cannot
|
manual entry point for the same work. If the destination store cannot
|
||||||
be listed, or the local index holds snapshots and the destination has
|
be listed, or the local index holds snapshots and the destination has
|
||||||
no `vaultik-destination.txt` (an unplugged volume's empty mount point),
|
no `vaultik-destination.txt` (an unplugged volume's empty mount point),
|
||||||
@@ -466,9 +436,6 @@ exits non-zero.
|
|||||||
* `--json`: Output stats as JSON. Also skips the confirmation prompt, as
|
* `--json`: Output stats as JSON. Also skips the confirmation prompt, as
|
||||||
`--force` does. A blob that cannot be deleted is counted in
|
`--force` does. A blob that cannot be deleted is counted in
|
||||||
`blobs_failed`, and the document is printed before the non-zero exit.
|
`blobs_failed`, and the document is printed before the non-zero exit.
|
||||||
A blob kept for being newer than `prune_grace_period`, or because the
|
|
||||||
listing of the destination gives no time for it, is counted in
|
|
||||||
`blobs_too_new` and not in `blobs_found`.
|
|
||||||
|
|
||||||
**`info`**: Display system configuration, storage settings, encryption
|
**`info`**: Display system configuration, storage settings, encryption
|
||||||
recipients, and local database statistics.
|
recipients, and local database statistics.
|
||||||
@@ -543,14 +510,6 @@ being written, a killed upload can leave a truncated object under its name,
|
|||||||
which a later backup takes for complete. A leftover `.partial` file is ignored
|
which a later backup takes for complete. A leftover `.partial` file is ignored
|
||||||
and can be deleted.
|
and can be deleted.
|
||||||
|
|
||||||
A destination shared between hosts or users, each with its own local index,
|
|
||||||
relies on `prune_grace_period` to keep one's prune from deleting the blobs of
|
|
||||||
another's backup still running, so it must be longer than the longest backup
|
|
||||||
run. It does not protect a blob that backup reuses from an earlier run that
|
|
||||||
stopped before its manifest, which the prune can delete. To not depend on the
|
|
||||||
grace period, give each host its own prefix (or directory). See
|
|
||||||
[locking](#locking).
|
|
||||||
|
|
||||||
Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is
|
Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is
|
||||||
still supported for backward compatibility. `storage_url` takes precedence if
|
still supported for backward compatibility. `storage_url` takes precedence if
|
||||||
both are set.
|
both are set.
|
||||||
@@ -637,8 +596,7 @@ format does and does not protect.
|
|||||||
1. List all snapshot manifests
|
1. List all snapshot manifests
|
||||||
2. Build set of all referenced blob hashes
|
2. Build set of all referenced blob hashes
|
||||||
3. List all blobs in storage
|
3. List all blobs in storage
|
||||||
4. Delete any blob not in the referenced set that is older than
|
4. Delete any blob not in the referenced set
|
||||||
`prune_grace_period`
|
|
||||||
|
|
||||||
### chunking and deduplication
|
### chunking and deduplication
|
||||||
|
|
||||||
@@ -683,7 +641,6 @@ complete annotated example also lives in
|
|||||||
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
|
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
|
||||||
| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` |
|
| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` |
|
||||||
| `compression_level` | `3` | zstd compression level (1-19) |
|
| `compression_level` | `3` | zstd compression level (1-19) |
|
||||||
| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it. A backup uploads its blobs before the manifest that lists them, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs at once |
|
|
||||||
| `hostname` | system hostname | Hostname used in snapshot IDs |
|
| `hostname` | system hostname | Hostname used in snapshot IDs |
|
||||||
| `index_path` | platform data dir | Local SQLite index path |
|
| `index_path` | platform data dir | Local SQLite index path |
|
||||||
|
|
||||||
|
|||||||
@@ -26,217 +26,395 @@ owner's; a `v*` tag runs the release build in
|
|||||||
|
|
||||||
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
||||||
that skipped files it could not read
|
that skipped files it could not read
|
||||||
([issue #382](https://git.eeqj.de/sneak/vaultik/issues/382)). Such a
|
([issue #382](https://git.eeqj.de/sneak/vaultik/issues/382)). Under
|
||||||
prune could delete the only snapshot still holding those files.
|
`--skip-errors` the prune still ran, so by default it deleted the
|
||||||
|
previous snapshot of the name, the only one still holding a file that
|
||||||
|
had become unreadable, along with blobs only that snapshot used. When
|
||||||
|
`--skip-errors` skipped anything it could not read or list, the prune
|
||||||
|
now does not run and older snapshots are kept, a warning says why, and
|
||||||
|
the run still exits non-zero.
|
||||||
|
|
||||||
- 2026-10-14: Made `snapshot create --skip-errors` exit non-zero,
|
- 2026-10-14: Made `snapshot create --skip-errors` exit non-zero when
|
||||||
instead of 0, when it skips anything it cannot read
|
it skips anything it cannot read
|
||||||
([issue #310](https://git.eeqj.de/sneak/vaultik/issues/310)).
|
([issue #310](https://git.eeqj.de/sneak/vaultik/issues/310)). The run
|
||||||
|
printed its skip errors, then `Finished successfully.`, and exited 0,
|
||||||
|
so a cron wrapper could not tell it from a complete backup. Such a
|
||||||
|
run now keeps its snapshots and then fails with
|
||||||
|
`Backup completed, but N file(s) could not be read`, and an entry the
|
||||||
|
walk could not read, such as a directory it could not list, now
|
||||||
|
counts as failed too.
|
||||||
|
|
||||||
- 2026-10-14: Kept the local index when `remote nuke` cannot list or
|
- 2026-10-14: Kept the local index when `remote nuke` cannot list or
|
||||||
find the destination store
|
find the destination store
|
||||||
([issue #292](https://git.eeqj.de/sneak/vaultik/issues/292)). Nuke
|
([issue #292](https://git.eeqj.de/sneak/vaultik/issues/292)). Nuke
|
||||||
had cleared the local snapshot records anyway; it now fails before
|
took a store it could not list, or one without
|
||||||
removing anything.
|
`vaultik-destination.txt` while the local index held snapshots, as a
|
||||||
|
warning and cleared the local snapshot records anyway; for the empty
|
||||||
|
mount point of an unplugged volume it then reported the store empty
|
||||||
|
and exited 0. It now fails on either before it removes anything, and
|
||||||
|
deletes each snapshot's metadata on the destination store before its
|
||||||
|
local record, so a failed delete keeps the snapshot in the local
|
||||||
|
index.
|
||||||
|
|
||||||
- 2026-10-14: Stopped counting directories and unchanged symlinks as
|
- 2026-10-14: Stopped counting directories and unchanged symlinks as
|
||||||
files to back up
|
files to back up
|
||||||
([issue #305](https://git.eeqj.de/sneak/vaultik/issues/305)).
|
([issue #305](https://git.eeqj.de/sneak/vaultik/issues/305)). Every
|
||||||
|
backup queued every directory and symlink to be backed up, so the
|
||||||
|
line ending the walk counted directories and their sizes as files and
|
||||||
|
data to back up while its enumerated count left them out, and every
|
||||||
|
symlink was reported as backed up on every run. Directories are no
|
||||||
|
longer counted as files or data to back up, and a symlink is compared
|
||||||
|
with its record like a regular file, so an unchanged one counts as
|
||||||
|
unchanged.
|
||||||
|
|
||||||
- 2026-10-14: Refused a backup destination that an unplugged volume
|
- 2026-10-14: Refused a backup destination that an unplugged volume
|
||||||
left empty, as the owner ruled
|
left empty, as the owner ruled
|
||||||
([issue #239](https://git.eeqj.de/sneak/vaultik/issues/239)). Every
|
([issue #239](https://git.eeqj.de/sneak/vaultik/issues/239)). A
|
||||||
backup now writes `vaultik-destination.txt` at the destination root,
|
`file://` destination at the mount point of an unplugged volume listed
|
||||||
and while the local index holds snapshots stored there, a destination
|
as an empty store, so `prune` dropped every local snapshot record, and
|
||||||
without it is treated as not present.
|
`snapshot create` wrote a whole backup to the internal disk under the
|
||||||
|
empty mount point. Every backup now writes `vaultik-destination.txt`
|
||||||
|
at the destination root, and while the local index holds snapshots
|
||||||
|
stored there, a destination without it is treated as not present:
|
||||||
|
`snapshot create` refuses before writing, and `prune` deletes
|
||||||
|
nothing. The `file://` backend also never creates its destination
|
||||||
|
directory.
|
||||||
|
|
||||||
- 2026-10-14: Warned about restored ownership only when an entry's
|
- 2026-10-14: Warned about restored ownership only when an entry's
|
||||||
owner or group differs from the recorded one
|
owner or group differs
|
||||||
([issue #316](https://git.eeqj.de/sneak/vaultik/issues/316)); a
|
([issue #316](https://git.eeqj.de/sneak/vaultik/issues/316)). A
|
||||||
restore by a non-root user had warned even when none did.
|
restore by a normal user warned that file ownership was not preserved
|
||||||
|
even when every entry came out with its recorded owner and group. The
|
||||||
|
owner and group the restored entries have on disk are now compared
|
||||||
|
with the recorded ones, and the warning is printed once if any differ.
|
||||||
|
|
||||||
- 2026-10-14: Made `snapshot list` report a changed destination instead
|
- 2026-10-14: Made `snapshot list` report a changed destination instead
|
||||||
of advising `prune`, which refuses in that case
|
of advising `prune`
|
||||||
([issue #317](https://git.eeqj.de/sneak/vaultik/issues/317)).
|
([issue #317](https://git.eeqj.de/sneak/vaultik/issues/317)). With the
|
||||||
|
local index bound to another destination than the configured one,
|
||||||
|
`snapshot list` reported its snapshots as missing from the destination
|
||||||
|
store and advised `prune`, which then refused. It now gives the
|
||||||
|
warning `prune`, `snapshot purge` and `snapshot remove` refuse with:
|
||||||
|
both destinations, and the choice of reverting the configuration or
|
||||||
|
running `vaultik database delete`.
|
||||||
|
|
||||||
- 2026-10-14: Counted a file as deleted by a backup only if the
|
- 2026-10-14: Counted a file as deleted only if the previous snapshot
|
||||||
previous snapshot of the same name listed it
|
of its name had it
|
||||||
([issue #304](https://git.eeqj.de/sneak/vaultik/issues/304)); files
|
([issue #304](https://git.eeqj.de/sneak/vaultik/issues/304)). A backup
|
||||||
that snapshot did not list had been counted too.
|
counted as deleted every file in the local index under the source
|
||||||
|
path that its walk did not find: files the snapshot excludes but
|
||||||
|
another snapshot backs up, and files removed long ago, which every
|
||||||
|
later backup reported again. It now counts only files that the latest
|
||||||
|
completed snapshot of the same name lists and that are gone from disk.
|
||||||
|
|
||||||
- 2026-10-14: Named each prefix once in restore, verify and purge
|
- 2026-10-14: Named each prefix once in restore, verify and purge errors
|
||||||
errors, some of which repeated it
|
([issue #327](https://git.eeqj.de/sneak/vaultik/issues/327)). A
|
||||||
([issue #327](https://git.eeqj.de/sneak/vaultik/issues/327)). A deep
|
missing blob during restore, a failed shallow verify and an invalid
|
||||||
verify log line that named S3 whatever the destination now names
|
purge duration each printed their prefix twice, and a deep verify log
|
||||||
storage.
|
line named S3 whatever the destination. Each prefix now appears once,
|
||||||
|
and that log line names storage instead.
|
||||||
|
|
||||||
- 2026-10-14: Refused credentials in a storage URL's endpoint or rclone
|
- 2026-10-14: Refused credentials in a storage URL's endpoint or rclone
|
||||||
remote name, which were accepted and could then be printed
|
remote name
|
||||||
([issue #322](https://git.eeqj.de/sneak/vaultik/issues/322)).
|
([issue #322](https://git.eeqj.de/sneak/vaultik/issues/322)). Userinfo
|
||||||
|
inside the `endpoint=` value of an `s3://` URL, and rclone
|
||||||
|
`remote,param=value` parameters, were accepted and then printed by
|
||||||
|
`info`, `remote info` and `snapshot create`. Both are now refused
|
||||||
|
without echoing the value, as is an `s3://` or `rclone://` query that
|
||||||
|
cannot be decoded.
|
||||||
|
|
||||||
- 2026-10-14: Ran `$EDITOR` through the shell in `config edit`, so a
|
- 2026-10-14: Ran `$EDITOR` through the shell in `config edit`
|
||||||
value with arguments, such as `code --wait`, no longer fails
|
([issue #315](https://git.eeqj.de/sneak/vaultik/issues/315)). The
|
||||||
([issue #315](https://git.eeqj.de/sneak/vaultik/issues/315)).
|
whole of `$EDITOR` was taken as the program name, so a value with
|
||||||
|
arguments such as `code --wait` or `emacsclient -t` failed with
|
||||||
|
`executable file not found`. It now runs as git runs it, through
|
||||||
|
`sh -c`, with the config path passed as a separate argument.
|
||||||
|
|
||||||
- 2026-10-14: Made `snapshot remove` fail, instead of exiting 0, when
|
- 2026-10-14: Made `snapshot remove` fail when it cannot delete the
|
||||||
it cannot delete the metadata of a snapshot that is only on the
|
metadata of a snapshot that is only on the destination store
|
||||||
destination store
|
([issue #340](https://git.eeqj.de/sneak/vaultik/issues/340)). It
|
||||||
([issue #340](https://git.eeqj.de/sneak/vaultik/issues/340)). The
|
warned and exited 0 having removed nothing. That failure is now an
|
||||||
README and the help now say when it prints the `vaultik prune`
|
error; for a snapshot in the local index, which is still removed
|
||||||
follow-up.
|
there, it stays a warning. The README and the help now say the
|
||||||
|
`vaultik prune` follow-up is printed only when metadata was removed
|
||||||
|
from the destination store, and never under `--json`.
|
||||||
|
|
||||||
- 2026-10-14: Labelled the totals in the local index cleanup before
|
- 2026-10-14: Labelled the totals in the local index cleanup before
|
||||||
each backup as totals
|
each backup as totals
|
||||||
([issue #306](https://git.eeqj.de/sneak/vaultik/issues/306)). Each
|
([issue #306](https://git.eeqj.de/sneak/vaultik/issues/306)). Each
|
||||||
was shown as `(N remain)`, which read as rows left to clean up.
|
removed count was followed by `(N remain)`, which read as incomplete
|
||||||
|
or orphaned rows left over but counted every snapshot, file, chunk or
|
||||||
|
blob left in the local index, so removing the only incomplete snapshot
|
||||||
|
printed `1 removed (1 remain)`. Each line now names that figure as the
|
||||||
|
total left in the local index.
|
||||||
|
|
||||||
- 2026-10-13: Freed a cached blob once no file still to restore needs
|
- 2026-10-13: Freed a cached blob once no file still to restore needs
|
||||||
it ([issue #324](https://git.eeqj.de/sneak/vaultik/issues/324)); one
|
it ([issue #324](https://git.eeqj.de/sneak/vaultik/issues/324)). A
|
||||||
also used by a file outside the selected paths had stayed in the
|
blob was freed only once every file in the snapshot that uses it had
|
||||||
temporary cache until the restore ended.
|
been restored, which a file outside the selected paths, or one that
|
||||||
|
failed under `--skip-errors`, never is, so such a restore kept those
|
||||||
|
blobs in the temporary cache until it ended. A cached blob is now
|
||||||
|
freed once no file still pending in this restore uses it.
|
||||||
|
|
||||||
- 2026-10-13: Kept the whole startup error when it contains `): `
|
- 2026-10-13: Kept the whole startup error when it contains `): `
|
||||||
([issue #320](https://git.eeqj.de/sneak/vaultik/issues/320)). Such
|
([issue #320](https://git.eeqj.de/sneak/vaultik/issues/320)). A
|
||||||
an error was cut at its last `): `, so the `blob_size_limit` one lost
|
startup error was cut at its last `): ` to drop the annotations `fx`
|
||||||
the setting's name.
|
adds, so the `blob_size_limit` message, which itself contains `): `,
|
||||||
|
lost everything but `need at least N bytes`, the setting's name
|
||||||
|
included. Only those annotations are now removed, and the error
|
||||||
|
underneath is shown whole.
|
||||||
|
|
||||||
- 2026-10-13: Silenced the status lines of `prune` and snapshot removal
|
- 2026-10-13: Silenced the status lines of `prune` and snapshot removal
|
||||||
under `--cron` and `--quiet`
|
under `--cron` and `--quiet`
|
||||||
([issue #294](https://git.eeqj.de/sneak/vaultik/issues/294)).
|
([issue #294](https://git.eeqj.de/sneak/vaultik/issues/294)). `prune`,
|
||||||
|
`snapshot purge`, `snapshot remove`, `remote nuke` and the prune step
|
||||||
|
of `snapshot create --prune` wrote their status lines straight to
|
||||||
|
stdout, so the README's cron line printed on every successful run and
|
||||||
|
`--quiet` did not silence them. Those lines are now dropped under both
|
||||||
|
flags; warnings, errors, `--json` documents and confirmation prompts
|
||||||
|
still print.
|
||||||
|
|
||||||
- 2026-10-13: Read the rclone config file that `RCLONE_CONFIG` names
|
- 2026-10-13: Read the rclone config file that `RCLONE_CONFIG` names
|
||||||
([issue #321](https://git.eeqj.de/sneak/vaultik/issues/321)); a
|
([issue #321](https://git.eeqj.de/sneak/vaultik/issues/321)). rclone
|
||||||
remote defined only there had been reported as not found.
|
reads that variable only through its `--config` flag, which vaultik
|
||||||
|
does not have, so a remote defined only in that file was reported as
|
||||||
|
not found. vaultik now sets rclone's config path from `RCLONE_CONFIG`
|
||||||
|
when it is set.
|
||||||
|
|
||||||
- 2026-10-13: Made `/**/` in an exclude pattern match zero directories,
|
- 2026-10-13: Made `/**/` in an exclude pattern match zero directories
|
||||||
as in gitignore
|
([issue #338](https://git.eeqj.de/sneak/vaultik/issues/338)). The
|
||||||
([issue #338](https://git.eeqj.de/sneak/vaultik/issues/338));
|
anchored `/**/x` missed an `x` directly under the snapshot path, and
|
||||||
`a/**/*.log` had missed `a/x.log`.
|
`a/**/*.log` missed `a/x.log`. As in gitignore, a leading `**/` now
|
||||||
|
makes an anchored pattern match at every depth too, and each `/**/`
|
||||||
|
inside a pattern also matches a single `/`.
|
||||||
|
|
||||||
- 2026-10-13: Made the `chunk_size` and `blob_size_limit` defaults and
|
- 2026-10-13: Made the `chunk_size` and `blob_size_limit` defaults and
|
||||||
the `chunk_size` minimum 1000-based, like the size parser and the
|
the `chunk_size` minimum 1000-based
|
||||||
docs ([issue #313](https://git.eeqj.de/sneak/vaultik/issues/313)).
|
([issue #313](https://git.eeqj.de/sneak/vaultik/issues/313)). They
|
||||||
|
were 1024-based while the size parser, the docs and `info` use
|
||||||
|
1000-based units, so `chunk_size: 1MB` was rejected, writing out a
|
||||||
|
documented default changed it, and `info` showed an 11 GB blob size
|
||||||
|
limit. They are now 1000-based like the parser, and
|
||||||
|
`config.example.yml` and the `config init` template state the
|
||||||
|
minimum.
|
||||||
|
|
||||||
- 2026-10-13: Kept a snapshot's remote metadata when `snapshot purge`
|
- 2026-10-13: Kept a snapshot's remote metadata when `snapshot purge`
|
||||||
cannot remove it from the local index
|
cannot remove it from the local index
|
||||||
([issue #337](https://git.eeqj.de/sneak/vaultik/issues/337)); the
|
([issue #337](https://git.eeqj.de/sneak/vaultik/issues/337)).
|
||||||
purge had deleted the metadata anyway and exited 0.
|
`snapshot purge`, and the purge step of `snapshot create --prune`,
|
||||||
|
logged that failure, deleted the snapshot's metadata from the
|
||||||
|
destination store anyway, and exited 0. The purge now stops at that
|
||||||
|
snapshot with an error before touching its remote metadata.
|
||||||
|
|
||||||
- 2026-10-13: Made `snapshot restore` fail for a requested path that
|
- 2026-10-13: Made `snapshot restore` fail for a requested path that
|
||||||
matches nothing, which it had ignored
|
matches nothing
|
||||||
([issue #309](https://git.eeqj.de/sneak/vaultik/issues/309)).
|
([issue #309](https://git.eeqj.de/sneak/vaultik/issues/309)). A path
|
||||||
|
that matched no entry in the snapshot, such as a typo or a relative
|
||||||
|
path, was ignored and the command exited 0, even when nothing matched.
|
||||||
|
Each such path is now named in a warning, the paths that do match are
|
||||||
|
still restored, and the command exits non-zero.
|
||||||
|
|
||||||
- 2026-10-13: Made `remote nuke` delete every object under `metadata/`
|
- 2026-10-13: Made `remote nuke` delete every object under `metadata/`
|
||||||
and `blobs/`
|
and `blobs/`
|
||||||
([issue #291](https://git.eeqj.de/sneak/vaultik/issues/291)); it had
|
([issue #291](https://git.eeqj.de/sneak/vaultik/issues/291)). It
|
||||||
left some, such as a stray `.DS_Store`, while reporting the store
|
deleted only the snapshot metadata and blobs it recognised, so a
|
||||||
empty.
|
snapshot database whose manifest was never uploaded, or a stray file
|
||||||
|
such as `.DS_Store` under `blobs/`, survived while nuke reported the
|
||||||
|
store empty. It now deletes everything listed under both, reports the
|
||||||
|
store empty only when a listing afterwards finds nothing, and
|
||||||
|
otherwise names each object left and fails.
|
||||||
|
|
||||||
- 2026-10-13: Made `config set` create a list for an index under a
|
- 2026-10-13: Made `config set` create a list for an index under a
|
||||||
missing key
|
missing key
|
||||||
([issue #308](https://git.eeqj.de/sneak/vaultik/issues/308)); it had
|
([issue #308](https://git.eeqj.de/sneak/vaultik/issues/308)).
|
||||||
written a map, after which no command could load the config.
|
`config set age_recipients.0 age1...` on a config without
|
||||||
|
`age_recipients` wrote a map keyed `0`, after which no command could
|
||||||
|
load the config. A missing key is now created as a list when the next
|
||||||
|
path element is a number, and `config set` refuses a change that
|
||||||
|
would stop a config that loaded from loading.
|
||||||
|
|
||||||
- 2026-10-13: Listed a `file://` destination through a symlinked
|
- 2026-10-13: Listed a `file://` destination through a symlinked
|
||||||
`metadata/` or `blobs/`, which had listed as empty
|
directory
|
||||||
([issue #296](https://git.eeqj.de/sneak/vaultik/issues/296)). A link
|
([issue #296](https://git.eeqj.de/sneak/vaultik/issues/296)). The
|
||||||
to a missing target now fails the listing
|
listing did not follow a symlinked `metadata/` or `blobs/`, while
|
||||||
|
reads and writes did, so what was behind the link listed as empty;
|
||||||
|
with `metadata/` linked, `prune` then dropped the local snapshot
|
||||||
|
records and deleted every blob. The listings now read through such a
|
||||||
|
link, and fail when its target is missing
|
||||||
([issue #332](https://git.eeqj.de/sneak/vaultik/issues/332)).
|
([issue #332](https://git.eeqj.de/sneak/vaultik/issues/332)).
|
||||||
|
|
||||||
- 2026-10-13: Made `snapshot verify --deep --json` report its blob
|
- 2026-10-13: Made `snapshot verify --deep --json` report its blob
|
||||||
counts ([issue #314](https://git.eeqj.de/sneak/vaultik/issues/314));
|
counts ([issue #314](https://git.eeqj.de/sneak/vaultik/issues/314)).
|
||||||
a failed run had reported 0 missing and 0 verified.
|
Deep verify set `verified` only on success and never set `missing`,
|
||||||
|
so a failed run reported 0 for both. It now counts each blob whose
|
||||||
|
contents pass, and before reading any, counts missing and wrong-size
|
||||||
|
blobs as shallow verify does.
|
||||||
|
|
||||||
- 2026-10-13: Limited and retried the module download in the Docker
|
- 2026-10-13: Limited and retried the module download in the Docker
|
||||||
build ([issue #351](https://git.eeqj.de/sneak/vaultik/issues/351)),
|
build ([issue #351](https://git.eeqj.de/sneak/vaultik/issues/351)).
|
||||||
which had hung for as long as a stalled connection to the module
|
`go mod download` in the `Dockerfile` had no time limit, so a stalled
|
||||||
proxy lasted.
|
connection to the module proxy held the step, and every build on the
|
||||||
|
host sharing it, until the kernel gave up on the connection, once
|
||||||
|
after about 55 minutes. Each such step now runs under a five-minute
|
||||||
|
limit, is tried up to three times, and then fails the build with
|
||||||
|
`module download failed`.
|
||||||
|
|
||||||
- 2026-10-13: Recorded each snapshot's uncompressed size when its
|
- 2026-10-13: Recorded each snapshot's uncompressed size when its
|
||||||
backup runs
|
backup runs
|
||||||
([issue #360](https://git.eeqj.de/sneak/vaultik/issues/360)). A later
|
([issue #360](https://git.eeqj.de/sneak/vaultik/issues/360)).
|
||||||
backup could change an older snapshot's UNCOMPRESSED SIZE in
|
`snapshot list` summed the chunks a snapshot reaches through file
|
||||||
`snapshot list`.
|
rows shared by every snapshot that lists the file and that a changed
|
||||||
|
file rewrites, so backing up a changed file changed the UNCOMPRESSED
|
||||||
|
SIZE of every older snapshot that lists that file. The backup now
|
||||||
|
stores that total in the `snapshots` table's
|
||||||
|
`uncompressed_chunk_size`, which `snapshot list` reads.
|
||||||
|
|
||||||
- 2026-10-13: Stopped `snapshot restore --skip-errors --verify`
|
- 2026-10-13: Stopped `snapshot restore --verify` reporting a file that
|
||||||
reporting a file that failed to restore again as a checksum failure
|
failed to restore again as a checksum failure
|
||||||
([issue #323](https://git.eeqj.de/sneak/vaultik/issues/323)).
|
([issue #323](https://git.eeqj.de/sneak/vaultik/issues/323)). Under
|
||||||
|
`--skip-errors`, a file that failed to restore was verified anyway,
|
||||||
|
failed again because it was never written, and was listed twice as a
|
||||||
|
checksum mismatch. Verification now skips the files that failed to
|
||||||
|
restore and reports its own failures apart, each path once.
|
||||||
|
|
||||||
- 2026-10-13: Recorded the stored size of a blob a backup finds already
|
- 2026-10-13: Recorded the stored size of a blob a backup finds already
|
||||||
stored, not the size of its local copy, which can differ, so
|
stored ([issue #300](https://git.eeqj.de/sneak/vaultik/issues/300)).
|
||||||
`snapshot verify` could fail on a good snapshot
|
The upload was skipped, but the local index and the manifest kept the
|
||||||
([issue #300](https://git.eeqj.de/sneak/vaultik/issues/300)).
|
size of the local copy, which can differ from the stored object when
|
||||||
|
`compression_level` or the recipients changed, so `snapshot verify`
|
||||||
|
failed on a good snapshot. The size the destination store reports for
|
||||||
|
the existing object is now recorded.
|
||||||
|
|
||||||
- 2026-10-13: Stopped the local index and the blob upload step ending
|
- 2026-10-13: Stopped the local index and the blob upload step ending
|
||||||
the process on a failed close of query results or a failed temp file
|
the process on a rollback after Ctrl-C, a failed close of query
|
||||||
cleanup ([issue #345](https://git.eeqj.de/sneak/vaultik/issues/345));
|
results, or a failed temp file cleanup
|
||||||
both are now returned as errors.
|
([issue #345](https://git.eeqj.de/sneak/vaultik/issues/345)). Each
|
||||||
|
ended the process at once and skipped the run's cleanup.
|
||||||
|
Read-only transactions now handle the rollback as other transactions
|
||||||
|
do, the query functions report a close failure through the error they
|
||||||
|
already return, and a failed close or removal of a blob temp file
|
||||||
|
fails the backup.
|
||||||
|
|
||||||
- 2026-10-13: Closed the snapshot database's storage reader too when
|
- 2026-10-13: Stopped a stalled snapshot database read when restore or
|
||||||
restore or deep verify is interrupted
|
deep verify is interrupted
|
||||||
([issue #356](https://git.eeqj.de/sneak/vaultik/issues/356)).
|
([issue #356](https://git.eeqj.de/sneak/vaultik/issues/356)). Both
|
||||||
|
read the snapshot database without the fix for stalled blob reads, so
|
||||||
|
after Ctrl-C a database read stalled on a slow device still left the
|
||||||
|
decrypted database in the temporary directory. That read is now
|
||||||
|
closed on cancel too.
|
||||||
|
|
||||||
- 2026-10-13: Made `prune` and `remote nuke` fail when a delete on the
|
- 2026-10-13: Made `prune` and `remote nuke` fail when a delete fails
|
||||||
backup destination store fails
|
([issue #290](https://git.eeqj.de/sneak/vaultik/issues/290)). `prune`,
|
||||||
([issue #290](https://git.eeqj.de/sneak/vaultik/issues/290)).
|
and the prune step of `snapshot create --prune`, counted a blob it
|
||||||
|
could not delete and still exited 0, and `remote nuke` only warned
|
||||||
|
when it could not delete a snapshot's metadata, advised
|
||||||
|
`vaultik prune`, which never removes metadata, and still reported the
|
||||||
|
store empty. Both now exit non-zero, and nuke says to run
|
||||||
|
`vaultik remote nuke --force` again.
|
||||||
|
|
||||||
- 2026-10-13: Made an S3 listing that the server cuts short an error
|
- 2026-10-13: Made an S3 listing that the server cuts short an error
|
||||||
([issue #297](https://git.eeqj.de/sneak/vaultik/issues/297)); `prune`
|
([issue #297](https://git.eeqj.de/sneak/vaultik/issues/297)). A
|
||||||
had taken it as complete and could delete blobs still in use.
|
listing page marked truncated but carrying no continuation token
|
||||||
|
ended the listing without an error, so `prune` saw only the snapshots
|
||||||
|
on the first page and treated the blobs of the rest as unreferenced.
|
||||||
|
Such a page now fails the listing, so `prune` stops before deleting
|
||||||
|
any blob.
|
||||||
|
|
||||||
- 2026-10-13: Made shallow `snapshot verify` report a blob it could not
|
- 2026-10-13: Made shallow `snapshot verify` report a blob it could not
|
||||||
check apart from a missing one
|
check apart from a missing one
|
||||||
([issue #318](https://git.eeqj.de/sneak/vaultik/issues/318)); any
|
([issue #318](https://git.eeqj.de/sneak/vaultik/issues/318)). Every
|
||||||
error checking a blob had counted as missing, its cause dropped.
|
error from checking a blob or the encrypted database counted as
|
||||||
|
missing and was printed nowhere, so a permission error or an S3 403
|
||||||
|
read as data loss. Only a not-found answer now counts as missing; any
|
||||||
|
other error is reported with its cause as could not be checked, in
|
||||||
|
the text report and under `--json`, and the verify fails.
|
||||||
|
|
||||||
- 2026-10-13: Restored the other files when a file's chunk is in no
|
- 2026-10-13: Restored the other files when a file's chunk is in no
|
||||||
blob under `--skip-errors`
|
blob under `--skip-errors`
|
||||||
([issue #326](https://git.eeqj.de/sneak/vaultik/issues/326)); that
|
([issue #326](https://git.eeqj.de/sneak/vaultik/issues/326)). Restore
|
||||||
file had stopped the restore before anything was restored.
|
planning stopped at the first such file, before the flag could skip
|
||||||
|
it, so nothing was restored. That file now fails like any other file
|
||||||
|
that cannot be restored: skipped under the flag, and fatal without it.
|
||||||
|
|
||||||
- 2026-10-13: Recorded each snapshot's new chunk size when its backup
|
- 2026-10-13: Recorded each snapshot's new chunk size when its backup
|
||||||
runs ([issue #311](https://git.eeqj.de/sneak/vaultik/issues/311)).
|
runs ([issue #311](https://git.eeqj.de/sneak/vaultik/issues/311)).
|
||||||
`snapshot list` could show a NEW CHUNK SIZE of 0 B for a snapshot
|
`snapshot list` worked out NEW CHUNK SIZE from file rows shared by
|
||||||
whose change was a modified file.
|
every snapshot that lists the file and that a changed file rewrites,
|
||||||
|
so a snapshot whose change was a modified file showed 0 B and older
|
||||||
|
snapshots' figures moved after later backups. The backup now stores
|
||||||
|
the size of the chunks it newly stored in the `snapshots` table's
|
||||||
|
`new_chunk_size`, which `snapshot list` reads.
|
||||||
|
|
||||||
- 2026-10-13: Made a backup interrupted during a local index write exit
|
- 2026-10-13: Made a backup interrupted during a local index write exit
|
||||||
as interrupted
|
as interrupted
|
||||||
([issue #301](https://git.eeqj.de/sneak/vaultik/issues/301)); it could
|
([issue #301](https://git.eeqj.de/sneak/vaultik/issues/301)). After
|
||||||
end with `FATAL: failed to rollback transaction` and exit 1 instead of
|
Ctrl-C the transaction was already rolled back, so the explicit
|
||||||
130.
|
rollback failed and was treated as fatal: the run printed
|
||||||
|
`FATAL: failed to rollback transaction`, exited 1 instead of 130, and
|
||||||
|
skipped its cleanup. The rollback's report that the transaction was
|
||||||
|
already over is now ignored, and a rollback that really fails is
|
||||||
|
returned with the original error instead of ending the process.
|
||||||
|
|
||||||
- 2026-10-13: Printed `snapshot list` and `snapshot purge` sizes like
|
- 2026-10-13: Printed `snapshot list` and `snapshot purge` sizes like
|
||||||
every other command
|
every other command
|
||||||
([issue #312](https://git.eeqj.de/sneak/vaultik/issues/312)). They,
|
([issue #312](https://git.eeqj.de/sneak/vaultik/issues/312)). They,
|
||||||
and the `bytes` value in terminal log lines, were 1024-based but
|
and the `bytes` value in terminal log lines, divided by 1024 but
|
||||||
labelled KB or MB, and now use 1000-based units.
|
labelled the result KB or MB, so 6,329,198 bytes read 6.0 MB there and
|
||||||
|
6.3 MB in other commands. All three now use 1000-based units, as other
|
||||||
|
commands do.
|
||||||
|
|
||||||
- 2026-10-13: Closed each blob's storage reader when restore or deep
|
- 2026-10-13: Stopped a stalled blob read when restore or deep verify
|
||||||
verify is interrupted, which ends a stalled read unless it is stuck in
|
is interrupted
|
||||||
the kernel ([issue #325](https://git.eeqj.de/sneak/vaultik/issues/325)).
|
([issue #325](https://git.eeqj.de/sneak/vaultik/issues/325)). Both
|
||||||
|
checked for a cancel only between blobs, and the `file://` backend's
|
||||||
|
reader ignores it, so after Ctrl-C a blob read stalled on a slow
|
||||||
|
device outlasted the 30-second shutdown timeout and the decrypted
|
||||||
|
snapshot database and blob cache were left in the temporary
|
||||||
|
directory. Each blob's storage reader is now closed as soon as the
|
||||||
|
command is cancelled, which ends a stalled read unless it is stuck in
|
||||||
|
the kernel.
|
||||||
|
|
||||||
- 2026-10-13: Made a mistyped subcommand under `snapshot`, `config`,
|
- 2026-10-13: Made a mistyped subcommand under a group command an error
|
||||||
`remote`, `database` or `completion` an error
|
([issue #319](https://git.eeqj.de/sneak/vaultik/issues/319)).
|
||||||
([issue #319](https://git.eeqj.de/sneak/vaultik/issues/319)); each had
|
`snapshot`, `config`, `remote`, `database` and `completion` printed
|
||||||
printed its help and exited 0.
|
their help and exited 0 for a word that is not one of their
|
||||||
|
subcommands, so a mistyped cron line made no backup and reported
|
||||||
|
nothing. Each now prints its help only when given no argument, and
|
||||||
|
otherwise fails as an unknown command, as the top level does.
|
||||||
|
|
||||||
- 2026-10-13: Made an exclude pattern starting with `**/` match
|
- 2026-10-13: Made an exclude pattern starting with `**/` match
|
||||||
directly under the snapshot path too, as in gitignore
|
directly under the snapshot path
|
||||||
([issue #303](https://git.eeqj.de/sneak/vaultik/issues/303)).
|
([issue #303](https://git.eeqj.de/sneak/vaultik/issues/303)).
|
||||||
|
`**/node_modules` and `**/*.pyc` excluded `sub/node_modules` and
|
||||||
|
`sub/y.pyc` but backed up `node_modules/` and `x.pyc` at the top of
|
||||||
|
the path. The leading `**/` is now dropped when the pattern is
|
||||||
|
compiled, so the pattern matches at every depth, the top level
|
||||||
|
included, as in gitignore.
|
||||||
|
|
||||||
- 2026-10-13: Refused a relative `file://` storage URL and decoded its
|
- 2026-10-13: Refused a relative `file://` storage URL and decoded its
|
||||||
percent-escapes
|
percent-escapes
|
||||||
([issue #299](https://git.eeqj.de/sneak/vaultik/issues/299)); a
|
([issue #299](https://git.eeqj.de/sneak/vaultik/issues/299)). The path
|
||||||
relative path had moved with the working directory.
|
after `file://` was used as written, so `file://dest` followed the
|
||||||
|
working directory and a backup run from elsewhere exited 0 and could
|
||||||
|
not be restored, and `my%20stick` made a directory of that literal
|
||||||
|
name. Config load and the storage URL parser now refuse a path that
|
||||||
|
is not absolute, and the parser decodes escapes.
|
||||||
|
|
||||||
- 2026-10-13: Removed the blob temp file when a backup stops part-way
|
- 2026-10-13: Removed the blob temp file when a backup stops part-way
|
||||||
([issue #302](https://git.eeqj.de/sneak/vaultik/issues/302)); the
|
([issue #302](https://git.eeqj.de/sneak/vaultik/issues/302)). When
|
||||||
half-written blob had stayed in `$TMPDIR`.
|
`snapshot create` stopped on a file error, a full temporary directory
|
||||||
|
or Ctrl-C, the half-written `vaultik-blob-*.tmp` of the blob being
|
||||||
|
packed, up to `blob_size_limit` of encrypted data, stayed in
|
||||||
|
`$TMPDIR`. The backup now closes and removes it on the way out.
|
||||||
|
|
||||||
- 2026-10-13: Checked `--keep-newer-than` before a backup starts
|
- 2026-10-13: Checked `--keep-newer-than` before a backup starts
|
||||||
([issue #307](https://git.eeqj.de/sneak/vaultik/issues/307)); an
|
([issue #307](https://git.eeqj.de/sneak/vaultik/issues/307)). An
|
||||||
invalid value had been read only by the prune after the backup, so
|
invalid value was read only by the prune after the backup, so the
|
||||||
retention never ran.
|
snapshot was uploaded, the run reported it as failed and retention
|
||||||
|
never ran; without `--prune` the flag was ignored. `snapshot create`
|
||||||
|
now refuses an invalid duration, or the flag without `--prune`,
|
||||||
|
before it starts, and a failed post-backup prune is reported as
|
||||||
|
`Backup completed, but post-backup prune failed`.
|
||||||
|
|
||||||
- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that
|
- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that
|
||||||
is not a blob as an orphaned blob
|
is not a blob as an orphaned blob
|
||||||
|
|||||||
@@ -328,18 +328,6 @@ storage_url: "rclone://myremote/path/to/backups"
|
|||||||
# Higher = better compression but slower
|
# Higher = better compression but slower
|
||||||
# Default: 3
|
# Default: 3
|
||||||
compression_level: 5
|
compression_level: 5
|
||||||
|
|
||||||
# How old a blob that no snapshot lists must be before prune deletes it
|
|
||||||
# A backup uploads its blobs before the manifest that lists them, so on a
|
|
||||||
# destination shared between hosts or users this must be longer than the
|
|
||||||
# longest backup run
|
|
||||||
# It does not protect a blob a backup reuses from an earlier run that stopped
|
|
||||||
# before its manifest: that blob keeps the time of its first upload, so a
|
|
||||||
# prune can delete it
|
|
||||||
# Supports: 36h, 90m, etc.; 0s deletes them at once
|
|
||||||
# Default: 24h
|
|
||||||
#prune_grace_period: 24h
|
|
||||||
|
|
||||||
# Hostname to use in backup metadata
|
# Hostname to use in backup metadata
|
||||||
# Default: system hostname
|
# Default: system hostname
|
||||||
#hostname: myserver
|
#hostname: myserver
|
||||||
|
|||||||
+1
-2
@@ -228,8 +228,7 @@ The restore process doesn't use the local database. Instead:
|
|||||||
|
|
||||||
1. **Identify Unreferenced Blobs**
|
1. **Identify Unreferenced Blobs**
|
||||||
- Query blobs not referenced by any remaining snapshot
|
- Query blobs not referenced by any remaining snapshot
|
||||||
- Delete from S3, once older than `prune_grace_period`, and from the local
|
- Delete from S3 and local database
|
||||||
database
|
|
||||||
|
|
||||||
### 6. Incomplete Snapshot Cleanup
|
### 6. Incomplete Snapshot Cleanup
|
||||||
|
|
||||||
|
|||||||
@@ -166,7 +166,7 @@ The prune operation is safe because:
|
|||||||
1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references
|
1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references
|
||||||
2. Manifests are unencrypted and can be read without keys
|
2. Manifests are unencrypted and can be read without keys
|
||||||
3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as unreferenced
|
3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as unreferenced
|
||||||
4. It deletes an unreferenced blob only once the blob is older than `prune_grace_period` (default `24h`). A backup uploads its blobs before the manifest that lists them, so a prune running meanwhile, on this host or another sharing the destination, leaves them as long as no backup run takes longer than the grace period. The grace period does not protect a blob a backup reuses from an earlier run that stopped before its manifest: the backup skips the upload of a blob already on the destination, so that blob keeps the time of its first upload, and a prune before the backup's manifest is written can delete it; the new snapshot then cannot be restored. `remote nuke` has no such check
|
4. Prune requires exclusive access to the destination: running it during a concurrent backup can race a snapshot whose manifest is not yet written, so do not prune while a backup is in progress
|
||||||
|
|
||||||
## Restoration Requirements
|
## Restoration Requirements
|
||||||
|
|
||||||
|
|||||||
@@ -239,16 +239,6 @@ storage_url: ""
|
|||||||
# Default: 3
|
# Default: 3
|
||||||
# compression_level: 3
|
# compression_level: 3
|
||||||
|
|
||||||
# How old a blob that no snapshot lists must be before prune deletes it.
|
|
||||||
# A backup uploads its blobs before the manifest that lists them, so on a
|
|
||||||
# destination shared between hosts or users this must be longer than the
|
|
||||||
# longest backup run. It does not protect a blob a backup reuses from an
|
|
||||||
# earlier run that stopped before its manifest: that blob keeps the time of
|
|
||||||
# its first upload, so a prune can delete it.
|
|
||||||
# Accepts: 36h, 90m, etc. 0s deletes them at once.
|
|
||||||
# Default: 24h
|
|
||||||
# prune_grace_period: 24h
|
|
||||||
|
|
||||||
# Hostname used in snapshot IDs. Default: system hostname.
|
# Hostname used in snapshot IDs. Default: system hostname.
|
||||||
# hostname: myserver
|
# hostname: myserver
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -15,8 +14,8 @@ import (
|
|||||||
const flagCron = "--cron"
|
const flagCron = "--cron"
|
||||||
|
|
||||||
// plantUnreferencedBlob writes a blob that no snapshot references to the
|
// plantUnreferencedBlob writes a blob that no snapshot references to the
|
||||||
// destination store at storeDir, dated long before any prune grace period,
|
// destination store at storeDir, so that a prune deletes it. Returns its
|
||||||
// so that a prune deletes it. Returns its path.
|
// path.
|
||||||
func plantUnreferencedBlob(t *testing.T, storeDir string) string {
|
func plantUnreferencedBlob(t *testing.T, storeDir string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -26,9 +25,6 @@ func plantUnreferencedBlob(t *testing.T, storeDir string) string {
|
|||||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
|
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
|
||||||
require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
|
require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
|
||||||
|
|
||||||
longAgo := time.Unix(0, 0)
|
|
||||||
require.NoError(t, os.Chtimes(path, longAgo, longAgo))
|
|
||||||
|
|
||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,12 +20,7 @@ Local cleanup drops incomplete snapshots and any files, chunks, or
|
|||||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||||
scans every snapshot manifest in the destination store, builds the
|
scans every snapshot manifest in the destination store, builds the
|
||||||
set of still-referenced blob hashes, and deletes any blob not in that
|
set of still-referenced blob hashes, and deletes any blob not in that
|
||||||
set that is older than prune_grace_period (default 24h). A newer one
|
set.
|
||||||
is kept: it may belong to a backup still running, on this host or
|
|
||||||
another sharing the destination store, that has not yet uploaded its
|
|
||||||
manifest. The grace period does not protect a blob such a backup
|
|
||||||
reuses from an earlier run that stopped before its manifest: that
|
|
||||||
blob keeps the time of its first upload, so it can be deleted.
|
|
||||||
|
|
||||||
Snapshot create --prune runs the same cleanup automatically; this
|
Snapshot create --prune runs the same cleanup automatically; this
|
||||||
command is the manual entry point for the same work (e.g. after a
|
command is the manual entry point for the same work (e.g. after a
|
||||||
|
|||||||
@@ -132,8 +132,8 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
"Run in cron mode (silent unless warning or error)")
|
"Run in cron mode (silent unless warning or error)")
|
||||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||||
"After backup, drop older snapshots of the same name and remove "+
|
"After backup, drop older snapshots of the same name and remove "+
|
||||||
"orphaned blobs older than prune_grace_period (not after a "+
|
"orphaned blobs (not after a backup that skipped unreadable "+
|
||||||
"backup that skipped unreadable files)")
|
"files)")
|
||||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||||
"With --prune: keep snapshots newer than this duration "+
|
"With --prune: keep snapshots newer than this duration "+
|
||||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"filippo.io/age"
|
"filippo.io/age"
|
||||||
"git.eeqj.de/sneak/smartconfig"
|
"git.eeqj.de/sneak/smartconfig"
|
||||||
@@ -39,7 +38,6 @@ const (
|
|||||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||||
defaultCompressionLevel = 3
|
defaultCompressionLevel = 3
|
||||||
defaultPruneGracePeriod = 24 * time.Hour
|
|
||||||
minChunkSize = 1000 * 1000 // 1MB
|
minChunkSize = 1000 * 1000 // 1MB
|
||||||
minCompressionLevel = 1
|
minCompressionLevel = 1
|
||||||
maxCompressionLevel = 19
|
maxCompressionLevel = 19
|
||||||
@@ -69,8 +67,6 @@ var (
|
|||||||
errFileURLNotAbsolute = errors.New(
|
errFileURLNotAbsolute = errors.New(
|
||||||
"a file:// storage_url needs an absolute path, " +
|
"a file:// storage_url needs an absolute path, " +
|
||||||
"as in file:///path/to/backup")
|
"as in file:///path/to/backup")
|
||||||
errNegativePruneGracePeriod = errors.New(
|
|
||||||
"prune_grace_period must not be negative")
|
|
||||||
errStorageNotConfigured = errors.New(
|
errStorageNotConfigured = errors.New(
|
||||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||||
"s3.bucket + credentials")
|
"s3.bucket + credentials")
|
||||||
@@ -191,13 +187,6 @@ type Config struct {
|
|||||||
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
||||||
CompressionLevel int `yaml:"compression_level"`
|
CompressionLevel int `yaml:"compression_level"`
|
||||||
|
|
||||||
// PruneGracePeriod is how old a blob no manifest lists must be before
|
|
||||||
// a prune deletes it. A backup lists its blobs in a manifest only
|
|
||||||
// once it has uploaded them all, so a younger one may belong to a
|
|
||||||
// backup still running, on this host or another sharing the
|
|
||||||
// destination store.
|
|
||||||
PruneGracePeriod time.Duration `yaml:"prune_grace_period"`
|
|
||||||
|
|
||||||
// StorageURL specifies the storage backend using a URL format.
|
// StorageURL specifies the storage backend using a URL format.
|
||||||
// Takes precedence over S3Config if set.
|
// Takes precedence over S3Config if set.
|
||||||
// Supported formats:
|
// Supported formats:
|
||||||
@@ -299,7 +288,6 @@ func Parse(data []byte) (*Config, error) {
|
|||||||
ChunkSize: defaultChunkSize,
|
ChunkSize: defaultChunkSize,
|
||||||
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
||||||
CompressionLevel: defaultCompressionLevel,
|
CompressionLevel: defaultCompressionLevel,
|
||||||
PruneGracePeriod: defaultPruneGracePeriod,
|
|
||||||
S3: S3Config{PartSize: defaultS3PartSize},
|
S3: S3Config{PartSize: defaultS3PartSize},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -368,7 +356,6 @@ func Parse(data []byte) (*Config, error) {
|
|||||||
// exceeds the configured limit
|
// exceeds the configured limit
|
||||||
// - Compression level must be between 1 and 19
|
// - Compression level must be between 1 and 19
|
||||||
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
|
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
|
||||||
// - Prune grace period must not be negative
|
|
||||||
//
|
//
|
||||||
// Returns an error describing the first validation failure encountered.
|
// Returns an error describing the first validation failure encountered.
|
||||||
func (c *Config) Validate() error {
|
func (c *Config) Validate() error {
|
||||||
@@ -418,10 +405,6 @@ func (c *Config) Validate() error {
|
|||||||
return errBadS3PartSize
|
return errBadS3PartSize
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.PruneGracePeriod < 0 {
|
|
||||||
return errNegativePruneGracePeriod
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
@@ -413,73 +412,6 @@ func TestDocumentedSizeExamples(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestValidatePruneGracePeriod checks that a negative prune_grace_period
|
|
||||||
// is refused rather than acting like 0s, which deletes unreferenced blobs
|
|
||||||
// at once, and that 0s and the documented examples are accepted.
|
|
||||||
func TestValidatePruneGracePeriod(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const withoutGracePeriod = "snapshots:\n" +
|
|
||||||
" test:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" +
|
|
||||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
setting string
|
|
||||||
wantErr error
|
|
||||||
}{
|
|
||||||
{"prune_grace_period: 0s", nil},
|
|
||||||
{"prune_grace_period: 90m", nil},
|
|
||||||
{"prune_grace_period: 36h", nil},
|
|
||||||
{"prune_grace_period: -1h", errNegativePruneGracePeriod},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.setting, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := Parse([]byte(withoutGracePeriod + tt.setting + "\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cfg.Validate()
|
|
||||||
if !errors.Is(err, tt.wantErr) {
|
|
||||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoadPruneGracePeriod checks that a config file without
|
|
||||||
// prune_grace_period loads with the documented 24h. The value is written out
|
|
||||||
// here rather than taken from defaultPruneGracePeriod, so that a change of the
|
|
||||||
// default fails this test.
|
|
||||||
func TestLoadPruneGracePeriod(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const withoutGracePeriod = "snapshots:\n" +
|
|
||||||
" test:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" +
|
|
||||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte(withoutGracePeriod), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := Load(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Load() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.PruneGracePeriod != 24*time.Hour {
|
|
||||||
t.Errorf("prune_grace_period = %s, want 24h", cfg.PruneGracePeriod)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||||
// relative path fails at load. Accepted, it would resolve against the
|
// relative path fails at load. Accepted, it would resolve against the
|
||||||
// working directory, so a backup run from another directory would write
|
// working directory, so a backup run from another directory would write
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
"github.com/aws/aws-sdk-go-v2/config"
|
"github.com/aws/aws-sdk-go-v2/config"
|
||||||
@@ -303,10 +302,7 @@ func IsNotFound(err error) bool {
|
|||||||
type ObjectInfo struct {
|
type ObjectInfo struct {
|
||||||
Key string
|
Key string
|
||||||
Size int64
|
Size int64
|
||||||
// ModTime is the object's LastModified time. Only ListObjectsStream
|
Err error
|
||||||
// sets it.
|
|
||||||
ModTime time.Time
|
|
||||||
Err error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectsStream lists objects with the given prefix and returns a channel.
|
// ListObjectsStream lists objects with the given prefix and returns a channel.
|
||||||
@@ -347,9 +343,8 @@ func (c *Client) ListObjectsStream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
ch <- ObjectInfo{
|
ch <- ObjectInfo{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: *obj.Size,
|
Size: *obj.Size,
|
||||||
ModTime: aws.ToTime(obj.LastModified),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
)
|
)
|
||||||
@@ -20,17 +19,15 @@ import (
|
|||||||
// constructor, so the contract is defined once.
|
// constructor, so the contract is defined once.
|
||||||
//
|
//
|
||||||
// It exercises the public Storer interface: round-trip, stat, list with
|
// It exercises the public Storer interface: round-trip, stat, list with
|
||||||
// prefix filtering, the modification time ListStream reports, overwrite,
|
// prefix filtering, overwrite, delete, delete-of-missing, and not-found on
|
||||||
// delete, delete-of-missing, and not-found on Get and Stat. Each section
|
// Get and Stat. Each section takes its own fresh backend instance, so the
|
||||||
// takes its own fresh backend instance, so the order of sections never
|
// order of sections never matters and no section sees another's objects.
|
||||||
// matters and no section sees another's objects.
|
|
||||||
func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) {
|
func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
conformanceRoundTrip(t, newStorer(t))
|
conformanceRoundTrip(t, newStorer(t))
|
||||||
conformanceOverwrite(t, newStorer(t))
|
conformanceOverwrite(t, newStorer(t))
|
||||||
conformanceList(t, newStorer(t))
|
conformanceList(t, newStorer(t))
|
||||||
conformanceListStreamModTime(t, newStorer(t))
|
|
||||||
conformanceDelete(t, newStorer(t))
|
conformanceDelete(t, newStorer(t))
|
||||||
conformanceNotFound(t, newStorer(t))
|
conformanceNotFound(t, newStorer(t))
|
||||||
}
|
}
|
||||||
@@ -121,35 +118,6 @@ func conformanceList(t *testing.T, s storage.Storer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// conformanceListStreamModTime checks that ListStream gives an object
|
|
||||||
// just written a modification time of about now, which prune compares
|
|
||||||
// with its grace period.
|
|
||||||
func conformanceListStreamModTime(t *testing.T, s storage.Storer) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
err := s.Put(ctx, "blobs/dd/written-now", bytes.NewReader([]byte("data")))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Put: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var listed []storage.ObjectInfo
|
|
||||||
|
|
||||||
for info := range s.ListStream(ctx, "blobs/") {
|
|
||||||
listed = append(listed, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(listed) != 1 || listed[0].Err != nil {
|
|
||||||
t.Fatalf("ListStream(\"blobs/\") = %v, want the one object", listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Since(listed[0].ModTime).Abs() > time.Minute {
|
|
||||||
t.Errorf("ListStream ModTime = %v, want within a minute of now",
|
|
||||||
listed[0].ModTime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// conformanceDelete checks that Delete removes an object and that deleting
|
// conformanceDelete checks that Delete removes an object and that deleting
|
||||||
// a missing key is not an error.
|
// a missing key is not an error.
|
||||||
func conformanceDelete(t *testing.T, s storage.Storer) {
|
func conformanceDelete(t *testing.T, s storage.Storer) {
|
||||||
|
|||||||
@@ -235,9 +235,8 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
|||||||
// Normalize path separators
|
// Normalize path separators
|
||||||
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
|
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
|
||||||
ch <- ObjectInfo{
|
ch <- ObjectInfo{
|
||||||
Key: relPath,
|
Key: relPath,
|
||||||
Size: info.Size(),
|
Size: info.Size(),
|
||||||
ModTime: info.ModTime(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -53,14 +53,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
|
|||||||
rclonePath += path
|
rclonePath += path
|
||||||
}
|
}
|
||||||
|
|
||||||
// On S3, Swift and some other remotes rclone keeps a time of its own in
|
|
||||||
// each object's metadata, which costs one request per object to read.
|
|
||||||
// rclone's use_server_modtime setting makes ModTime the time the
|
|
||||||
// listing gives instead: when the object was stored, which is what
|
|
||||||
// prune compares with prune_grace_period.
|
|
||||||
ctx, ci := fs.AddConfig(ctx)
|
|
||||||
ci.UseServerModTime = true
|
|
||||||
|
|
||||||
// Create the rclone filesystem
|
// Create the rclone filesystem
|
||||||
fsys, err := fs.NewFs(ctx, rclonePath)
|
fsys, err := fs.NewFs(ctx, rclonePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -219,9 +211,8 @@ func (r *RcloneStorer) ListStream(
|
|||||||
|
|
||||||
if prefix == "" || strings.HasPrefix(key, prefix) {
|
if prefix == "" || strings.HasPrefix(key, prefix) {
|
||||||
ch <- ObjectInfo{
|
ch <- ObjectInfo{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: obj.Size(),
|
Size: obj.Size(),
|
||||||
ModTime: obj.ModTime(ctx),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -5,17 +5,11 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
|
||||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configmap"
|
"github.com/rclone/rclone/fs/config/configmap"
|
||||||
@@ -435,72 +429,3 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
|
|||||||
t.Errorf("object is not in the remote's directory: %v", err)
|
t.Errorf("object is not in the remote's directory: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
|
|
||||||
// rclone S3 remote takes each object's time from the listing itself. By
|
|
||||||
// default rclone reads the time it keeps in each object's metadata, one
|
|
||||||
// request per object, and every command that lists the destination store
|
|
||||||
// would slow down as it grows.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
|
|
||||||
func TestRcloneListStreamOnS3MakesNoRequestPerObject(t *testing.T) {
|
|
||||||
backend := s3mem.New()
|
|
||||||
|
|
||||||
err := backend.CreateBucket(s3TestBucket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create bucket: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const body = "blob-bytes"
|
|
||||||
|
|
||||||
_, err = backend.PutObject(s3TestBucket, testBlobKey, nil,
|
|
||||||
strings.NewReader(body), int64(len(body)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("seed object: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var headRequests atomic.Int64
|
|
||||||
|
|
||||||
fakeS3 := gofakes3.New(backend).Server()
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == http.MethodHead {
|
|
||||||
headRequests.Add(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
fakeS3.ServeHTTP(w, r)
|
|
||||||
}))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
remote := ":s3,provider=Other" +
|
|
||||||
",access_key_id=" + s3TestAccessKeyID +
|
|
||||||
",secret_access_key=" + s3TestSecretAccessKey +
|
|
||||||
",endpoint='" + srv.URL + "'"
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
s, err := storage.NewRcloneStorer(ctx, remote, s3TestBucket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRcloneStorer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
headRequests.Store(0)
|
|
||||||
|
|
||||||
var listed []storage.ObjectInfo
|
|
||||||
|
|
||||||
for info := range s.ListStream(ctx, "blobs/") {
|
|
||||||
listed = append(listed, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(listed) != 1 || listed[0].Err != nil {
|
|
||||||
t.Fatalf("ListStream(\"blobs/\") = %v, want the one object", listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Since(listed[0].ModTime).Abs() > time.Minute {
|
|
||||||
t.Errorf("ListStream ModTime = %v, want within a minute of now",
|
|
||||||
listed[0].ModTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
if n := headRequests.Load(); n != 0 {
|
|
||||||
t.Errorf("ListStream made %d HEAD request(s), want none", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -89,10 +89,9 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
|
|||||||
|
|
||||||
for info := range s.client.ListObjectsStream(ctx, prefix, false) {
|
for info := range s.client.ListObjectsStream(ctx, prefix, false) {
|
||||||
ch <- ObjectInfo{
|
ch <- ObjectInfo{
|
||||||
Key: info.Key,
|
Key: info.Key,
|
||||||
Size: info.Size,
|
Size: info.Size,
|
||||||
ModTime: info.ModTime,
|
Err: info.Err,
|
||||||
Err: info.Err,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrNotFound is returned when an object does not exist.
|
// ErrNotFound is returned when an object does not exist.
|
||||||
@@ -28,10 +27,7 @@ type ProgressCallback func(bytesTransferred int64) error
|
|||||||
type ObjectInfo struct {
|
type ObjectInfo struct {
|
||||||
Key string // Object key/path
|
Key string // Object key/path
|
||||||
Size int64 // Size in bytes
|
Size int64 // Size in bytes
|
||||||
// ModTime is when the object was written, as the store reports it.
|
Err error // Error for streaming results (nil on success)
|
||||||
// Only ListStream sets it; Stat leaves it zero.
|
|
||||||
ModTime time.Time
|
|
||||||
Err error // Error for streaming results (nil on success)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Info provides human-readable storage configuration.
|
// Info provides human-readable storage configuration.
|
||||||
@@ -71,8 +67,7 @@ type Storer interface {
|
|||||||
// For large result sets, prefer ListStream.
|
// For large result sets, prefer ListStream.
|
||||||
List(ctx context.Context, prefix string) ([]string, error)
|
List(ctx context.Context, prefix string) ([]string, error)
|
||||||
|
|
||||||
// ListStream returns a channel of ObjectInfo, with each object's key,
|
// ListStream returns a channel of ObjectInfo for large result sets.
|
||||||
// size and modification time, for large result sets.
|
|
||||||
// The channel is closed when listing completes.
|
// The channel is closed when listing completes.
|
||||||
// If an error occurs during listing, the final item will have Err set.
|
// If an error occurs during listing, the final item will have Err set.
|
||||||
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
||||||
|
|||||||
@@ -457,9 +457,9 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, blob := range allBlobs {
|
for _, size := range allBlobs {
|
||||||
result.TotalBlobCount++
|
result.TotalBlobCount++
|
||||||
result.TotalBlobSize += blob.Size
|
result.TotalBlobSize += size
|
||||||
}
|
}
|
||||||
|
|
||||||
// A blob named only by a manifest that could not be read, or by one
|
// A blob named only by a manifest that could not be read, or by one
|
||||||
@@ -474,10 +474,10 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
orphanedSize int64
|
orphanedSize int64
|
||||||
)
|
)
|
||||||
|
|
||||||
for hash, blob := range allBlobs {
|
for hash, size := range allBlobs {
|
||||||
if _, referenced := referencedBlobs[hash]; !referenced {
|
if _, referenced := referencedBlobs[hash]; !referenced {
|
||||||
orphanedCount++
|
orphanedCount++
|
||||||
orphanedSize += blob.Size
|
orphanedSize += size
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+17
-59
@@ -5,10 +5,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// PruneOptions contains options for the prune command
|
// PruneOptions contains options for the prune command
|
||||||
@@ -165,10 +163,6 @@ type PruneBlobsResult struct {
|
|||||||
BlobsDeleted int `json:"blobs_deleted"`
|
BlobsDeleted int `json:"blobs_deleted"`
|
||||||
BlobsFailed int `json:"blobs_failed,omitempty"`
|
BlobsFailed int `json:"blobs_failed,omitempty"`
|
||||||
BytesFreed int64 `json:"bytes_freed"`
|
BytesFreed int64 `json:"bytes_freed"`
|
||||||
// BlobsTooNew counts the unreferenced blobs kept because they are
|
|
||||||
// newer than the prune grace period, or of unknown age. BlobsFound
|
|
||||||
// does not include them.
|
|
||||||
BlobsTooNew int `json:"blobs_too_new"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prune removes orphaned data from the local index database AND
|
// Prune removes orphaned data from the local index database AND
|
||||||
@@ -214,15 +208,9 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs(
|
unreferencedBlobs, totalSize := v.findUnreferencedBlobs(allBlobs, allBlobsReferenced)
|
||||||
allBlobs, allBlobsReferenced)
|
|
||||||
|
|
||||||
result := &PruneBlobsResult{
|
result := &PruneBlobsResult{BlobsFound: len(unreferencedBlobs)}
|
||||||
BlobsFound: len(unreferencedBlobs),
|
|
||||||
BlobsTooNew: tooNew,
|
|
||||||
}
|
|
||||||
|
|
||||||
v.reportBlobsTooNew(tooNew, opts)
|
|
||||||
|
|
||||||
if len(unreferencedBlobs) == 0 {
|
if len(unreferencedBlobs) == 0 {
|
||||||
log.Info("No unreferenced blobs found")
|
log.Info("No unreferenced blobs found")
|
||||||
@@ -356,12 +344,11 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
|
|||||||
return snapshotIDs, nil
|
return snapshotIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// listAllRemoteBlobs returns a map of all blob hashes in remote storage
|
// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
|
||||||
// to their listing entries.
|
func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
|
||||||
func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) {
|
|
||||||
log.Info("Listing all blobs in storage")
|
log.Info("Listing all blobs in storage")
|
||||||
|
|
||||||
allBlobs := make(map[string]storage.ObjectInfo)
|
allBlobs := make(map[string]int64)
|
||||||
blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
|
blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
|
||||||
|
|
||||||
for object := range blobObjectCh {
|
for object := range blobObjectCh {
|
||||||
@@ -385,7 +372,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
allBlobs[parts[3]] = object
|
allBlobs[parts[3]] = object.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Found blobs in storage", "count", len(allBlobs))
|
log.Info("Found blobs in storage", "count", len(allBlobs))
|
||||||
@@ -393,59 +380,30 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) {
|
|||||||
return allBlobs, nil
|
return allBlobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// findUnreferencedBlobs returns the hashes of the blobs no manifest
|
// findUnreferencedBlobs returns blob hashes not referenced by any
|
||||||
// references that are older than the prune grace period, and their total
|
// manifest and their total size.
|
||||||
// size. It also returns how many unreferenced blobs it left out for
|
|
||||||
// being newer than that, or of unknown age because the listing gave no
|
|
||||||
// time for them.
|
|
||||||
func (v *Vaultik) findUnreferencedBlobs(
|
func (v *Vaultik) findUnreferencedBlobs(
|
||||||
allBlobs map[string]storage.ObjectInfo, referenced map[string]bool,
|
allBlobs map[string]int64, referenced map[string]bool,
|
||||||
) ([]string, int64, int) {
|
) ([]string, int64) {
|
||||||
var (
|
var (
|
||||||
unreferenced []string
|
unreferenced []string
|
||||||
totalSize int64
|
totalSize int64
|
||||||
tooNew int
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod)
|
for hash, size := range allBlobs {
|
||||||
|
if !referenced[hash] {
|
||||||
for hash, blob := range allBlobs {
|
unreferenced = append(unreferenced, hash)
|
||||||
if referenced[hash] {
|
totalSize += size
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if blob.ModTime.IsZero() || blob.ModTime.After(cutoff) {
|
|
||||||
tooNew++
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
unreferenced = append(unreferenced, hash)
|
|
||||||
totalSize += blob.Size
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return unreferenced, totalSize, tooNew
|
return unreferenced, totalSize
|
||||||
}
|
|
||||||
|
|
||||||
// reportBlobsTooNew says how many unreferenced blobs prune kept for being
|
|
||||||
// newer than the grace period or of unknown age. Under --json the
|
|
||||||
// document carries the count instead.
|
|
||||||
func (v *Vaultik) reportBlobsTooNew(tooNew int, opts *PruneOptions) {
|
|
||||||
if tooNew == 0 || opts.JSON {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
v.UI.Infof("Kept %d unreferenced blob(s) newer than "+
|
|
||||||
"prune_grace_period (%s), or of unknown age: a backup still "+
|
|
||||||
"running may need them.",
|
|
||||||
tooNew, v.Config.PruneGracePeriod)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteUnreferencedBlobs deletes the given blobs from storage and
|
// deleteUnreferencedBlobs deletes the given blobs from storage and
|
||||||
// populates the result.
|
// populates the result.
|
||||||
func (v *Vaultik) deleteUnreferencedBlobs(
|
func (v *Vaultik) deleteUnreferencedBlobs(
|
||||||
unreferencedBlobs []string, allBlobs map[string]storage.ObjectInfo,
|
unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult,
|
||||||
result *PruneBlobsResult,
|
|
||||||
) {
|
) {
|
||||||
log.Info("Deleting unreferenced blobs")
|
log.Info("Deleting unreferenced blobs")
|
||||||
|
|
||||||
@@ -460,7 +418,7 @@ func (v *Vaultik) deleteUnreferencedBlobs(
|
|||||||
}
|
}
|
||||||
|
|
||||||
result.BlobsDeleted++
|
result.BlobsDeleted++
|
||||||
result.BytesFreed += allBlobs[hash].Size
|
result.BytesFreed += allBlobs[hash]
|
||||||
|
|
||||||
if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 {
|
if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 {
|
||||||
log.Info("Deletion progress",
|
log.Info("Deletion progress",
|
||||||
|
|||||||
@@ -1,166 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestPruneDuringBackupKeepsItsBlobs runs a prune from another host
|
|
||||||
// sharing the destination store at the moment a backup has uploaded its
|
|
||||||
// blobs but not yet its manifest, so no manifest lists those blobs. The
|
|
||||||
// prune must leave them, and the backup must then restore.
|
|
||||||
func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := t.TempDir()
|
|
||||||
restoreDir := filepath.Join(tempDir, "restored")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
otherHostCfg := faultTestConfig()
|
|
||||||
otherHostCfg.PruneGracePeriod = time.Hour
|
|
||||||
otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs)
|
|
||||||
|
|
||||||
var (
|
|
||||||
pruned bool
|
|
||||||
pruneErr error
|
|
||||||
)
|
|
||||||
|
|
||||||
store := faultstore.New(inner)
|
|
||||||
store.OnPut = func(key string) faultstore.PutAction {
|
|
||||||
if strings.HasSuffix(key, "/manifest.json.zst") {
|
|
||||||
pruned = true
|
|
||||||
pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true})
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
id := fullFaultBackup(ctx, t, fs, store, cfg, repos, dataDir, dbPath, "shared")
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
require.True(t, pruned, "the prune must run before the manifest upload")
|
|
||||||
require.NoError(t, pruneErr)
|
|
||||||
|
|
||||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
|
||||||
require.NoError(t, v.Restore(&vaultik.RestoreOptions{
|
|
||||||
SnapshotID: id,
|
|
||||||
TargetDir: restoreDir,
|
|
||||||
Verify: true,
|
|
||||||
}), "a prune during the backup must not delete the backup's blobs")
|
|
||||||
|
|
||||||
assertRestoredTree(t, fs, restoreDir, testFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPruneKeepsUnreferencedBlobWithNoTime checks that a blob no
|
|
||||||
// manifest lists, whose listing gives no modification time (as an S3
|
|
||||||
// listing entry without LastModified does), is kept and counted with
|
|
||||||
// the blobs too new to delete: its age is unknown.
|
|
||||||
func TestPruneKeepsUnreferencedBlobWithNoTime(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newListEnv(t)
|
|
||||||
|
|
||||||
// Written into the mock store's map directly rather than through
|
|
||||||
// Put, so its listing gives no time.
|
|
||||||
blobKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] +
|
|
||||||
"/" + testBlobHashA
|
|
||||||
env.store.data[blobKey] = []byte("blob")
|
|
||||||
|
|
||||||
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))
|
|
||||||
|
|
||||||
assert.True(t, env.store.hasKey(blobKey),
|
|
||||||
"a blob of unknown age must be kept")
|
|
||||||
|
|
||||||
var result vaultik.PruneBlobsResult
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result))
|
|
||||||
assert.Equal(t, 0, result.BlobsDeleted)
|
|
||||||
assert.Equal(t, 1, result.BlobsTooNew)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPruneDeletesUnreferencedBlobOlderThanGracePeriod checks that the
|
|
||||||
// grace period only delays deletion: a blob no manifest lists is deleted
|
|
||||||
// once it is older than the grace period, and kept and counted until
|
|
||||||
// then.
|
|
||||||
func TestPruneDeletesUnreferencedBlobOlderThanGracePeriod(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
storeDir := t.TempDir()
|
|
||||||
|
|
||||||
store, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
oldKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] +
|
|
||||||
"/" + testBlobHashA
|
|
||||||
newKey := "blobs/" + testBlobHashB[:2] + "/" + testBlobHashB[2:4] +
|
|
||||||
"/" + testBlobHashB
|
|
||||||
|
|
||||||
for _, key := range []string{oldKey, newKey} {
|
|
||||||
require.NoError(t, store.Put(ctx, key, strings.NewReader("blob")))
|
|
||||||
}
|
|
||||||
|
|
||||||
gracePeriod := time.Hour
|
|
||||||
beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute)
|
|
||||||
require.NoError(t, os.Chtimes(filepath.Join(storeDir, oldKey),
|
|
||||||
beforeGracePeriod, beforeGracePeriod))
|
|
||||||
|
|
||||||
stdout := &bytes.Buffer{}
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: &config.Config{PruneGracePeriod: gracePeriod},
|
|
||||||
Storage: store,
|
|
||||||
Stdout: stdout,
|
|
||||||
UI: ui.NewWithColor(stdout, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))
|
|
||||||
|
|
||||||
_, err = store.Stat(ctx, oldKey)
|
|
||||||
require.ErrorIs(t, err, storage.ErrNotFound,
|
|
||||||
"a blob older than the grace period must be deleted")
|
|
||||||
|
|
||||||
_, err = store.Stat(ctx, newKey)
|
|
||||||
require.NoError(t, err, "a blob newer than the grace period must be kept")
|
|
||||||
|
|
||||||
var result vaultik.PruneBlobsResult
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(stdout.Bytes(), &result))
|
|
||||||
assert.Equal(t, 1, result.BlobsDeleted)
|
|
||||||
assert.Equal(t, 1, result.BlobsTooNew)
|
|
||||||
}
|
|
||||||
@@ -26,14 +26,11 @@ import (
|
|||||||
type testStorer struct {
|
type testStorer struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
data map[string][]byte
|
data map[string][]byte
|
||||||
// putTimes holds when Put stored each object; ListStream reports it.
|
|
||||||
putTimes map[string]time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestStorer() *testStorer {
|
func newTestStorer() *testStorer {
|
||||||
return &testStorer{
|
return &testStorer{
|
||||||
data: make(map[string][]byte),
|
data: make(map[string][]byte),
|
||||||
putTimes: make(map[string]time.Time),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,7 +44,6 @@ func (s *testStorer) Put(_ context.Context, key string, reader io.Reader) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.data[key] = data
|
s.data[key] = data
|
||||||
s.putTimes[key] = time.Now().UTC()
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -91,7 +87,6 @@ func (s *testStorer) Delete(_ context.Context, key string) error {
|
|||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
delete(s.data, key)
|
delete(s.data, key)
|
||||||
delete(s.putTimes, key)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -125,9 +120,8 @@ func (s *testStorer) ListStream(
|
|||||||
for key, data := range s.data {
|
for key, data := range s.data {
|
||||||
if prefix == "" || strings.HasPrefix(key, prefix) {
|
if prefix == "" || strings.HasPrefix(key, prefix) {
|
||||||
ch <- storage.ObjectInfo{
|
ch <- storage.ObjectInfo{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
ModTime: s.putTimes[key],
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user