Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49e12daad2 |
@@ -98,12 +98,11 @@ vaultik snapshot restore <snapshot-id> ~/vaultik-restore
|
||||
|
||||
On a destination shared between hosts or users, the prune in that cron line
|
||||
relies on `prune_grace_period` (default `24h`) to leave the blobs of a backup
|
||||
another one is still running, and its directory under `metadata/` that has no
|
||||
manifest yet, so it must be longer than the longest backup run. It does not
|
||||
protect a blob that backup reuses from an earlier run that stopped before its
|
||||
manifest: the prune can delete such a blob, and the new snapshot then cannot
|
||||
be restored. To not depend on the grace period, give each host its own prefix
|
||||
or directory. See [locking](#locking).
|
||||
another one is still running, so it must be longer than the longest backup
|
||||
run. It does not protect a blob that backup reuses from an earlier run that
|
||||
stopped before its manifest: the prune can delete such a blob, and the new
|
||||
snapshot then cannot be restored. To not depend on the grace period, give each
|
||||
host its own prefix or directory. See [locking](#locking).
|
||||
|
||||
## restoring on another machine
|
||||
|
||||
@@ -205,19 +204,18 @@ corrupt the local index.
|
||||
|
||||
The lock covers one user on one machine. It does not stop another user,
|
||||
or another host, from writing to the same destination store at the same
|
||||
time. A backup uploads its blobs, and then its `db.zst.age`, before its
|
||||
manifest, so a prune from elsewhere at that moment finds blobs no snapshot
|
||||
lists and a directory under `metadata/` with no manifest. It
|
||||
deletes such a blob only once it is older than `prune_grace_period` (default
|
||||
`24h`), and such a directory only once every object in it is, so the grace
|
||||
period must be longer than the longest backup run of any host or user sharing
|
||||
the destination. The grace period does not protect a blob a backup reuses
|
||||
from an earlier run that stopped before its manifest: a backup skips the
|
||||
upload of a blob already on the destination, so that blob keeps the time of
|
||||
its first upload, and a prune from elsewhere during the backup can delete it;
|
||||
the new snapshot then cannot be restored. `remote nuke` deletes everything
|
||||
regardless. To not depend on the grace period, give each host or user its own
|
||||
prefix (or directory) on the destination.
|
||||
time. A backup uploads its blobs before the manifest that lists them, so a
|
||||
prune from elsewhere at that moment finds blobs no snapshot lists. It
|
||||
deletes such a blob only once it is older than `prune_grace_period`
|
||||
(default `24h`), which therefore must be longer than the longest backup run
|
||||
of any host or user sharing the destination. The grace period does not
|
||||
protect a blob a backup reuses from an earlier run that stopped before its
|
||||
manifest: a backup skips the upload of a blob already on the destination, so
|
||||
that blob keeps the time of its first upload, and a prune from elsewhere
|
||||
during the backup can delete it; the new snapshot then cannot be restored.
|
||||
`remote nuke` deletes everything regardless. To not depend on the grace
|
||||
period, give each host or user its own prefix (or directory) on the
|
||||
destination.
|
||||
|
||||
Read-only commands — `info`, `snapshot list`, `snapshot verify`, and
|
||||
`remote info` — do not take the lock and are never blocked, so they run
|
||||
@@ -320,12 +318,9 @@ value where a list belongs or a word where a number belongs), is refused.
|
||||
error that explains how to fix it
|
||||
* `--cron`: Silent on total success; warnings and errors are still printed
|
||||
(for crontab)
|
||||
* `--prune`: After backup, drop older snapshots of each backed-up name,
|
||||
* `--prune`: After backup, drop older snapshots of each backed-up name and
|
||||
remove orphaned blobs older than `prune_grace_period` from remote storage,
|
||||
and delete each directory under `metadata/` that has no
|
||||
`manifest.json.zst` once every object in it is older than
|
||||
`prune_grace_period` and its snapshot is not in the local index, as `prune`
|
||||
does. By default keeps only the latest
|
||||
as `prune` does. By default keeps only the latest
|
||||
snapshot per name; use `--keep-newer-than` for a rolling window. When
|
||||
`--skip-errors` skipped a file or directory it could not read or list, the
|
||||
prune does not run and every older snapshot is kept, since one of them may
|
||||
@@ -337,17 +332,14 @@ value where a list belongs or a word where a number belongs), is refused.
|
||||
path that is empty on purpose out of the config, or remove the older
|
||||
snapshots by hand with `snapshot purge`. When an earlier snapshot of the
|
||||
same name in the local index found a volume mounted at or below a
|
||||
configured path (a directory on a different filesystem from its parent,
|
||||
with an entry in it that the backup does not exclude) and this backup does
|
||||
not, the prune does not run either: every older snapshot is kept, and the
|
||||
command names the directory and exits non-zero, on every run until the
|
||||
volume is mounted again. A backup also does not find the volume when it
|
||||
finds no such entry in its directory, as when an automounted volume cannot
|
||||
be mounted, or when that directory is no longer backed up, because it is
|
||||
excluded or its configured path was removed. If any of this is on purpose,
|
||||
as for a retired volume, remove the older snapshots by hand with
|
||||
`snapshot purge`. A file removed during the backup does not stop the
|
||||
prune.
|
||||
configured path (a directory on a different filesystem from its parent)
|
||||
and this backup finds none mounted there, the prune does not run either:
|
||||
every older snapshot is kept, and the command names the directory and
|
||||
exits non-zero, on every run until the volume is mounted again. The same
|
||||
happens when that directory is no longer backed up, because it is excluded
|
||||
or its configured path was removed. For a volume retired on purpose, remove
|
||||
the older snapshots by hand with `snapshot purge`. A file removed during
|
||||
the backup does not stop the prune.
|
||||
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`).
|
||||
Without `--prune`, or with a value that is not a valid duration, the
|
||||
@@ -479,29 +471,22 @@ may belong to a backup still running, on this host or another sharing the
|
||||
destination store, that has not yet uploaded its manifest. The grace period
|
||||
does not protect a blob such a backup reuses from an earlier run that stopped
|
||||
before its manifest: that blob keeps the time of its first upload, so it can
|
||||
be deleted. Step (3) also deletes each directory under `metadata/` that has
|
||||
no `manifest.json.zst`, as a backup leaves when it stops after uploading
|
||||
`db.zst.age` but before its manifest, once every object in it is older than
|
||||
`prune_grace_period` and its snapshot is not in the local index. An object the
|
||||
listing gives no time for keeps its directory, and a directory whose name is
|
||||
not a remote key is left alone. Step (3) reads all remote manifests — network
|
||||
cost scales with the number of snapshots.
|
||||
be deleted. Step (3) reads all remote manifests — network cost scales with
|
||||
the number of snapshots.
|
||||
`snapshot create --prune` runs the same cleanup automatically; this is the
|
||||
manual entry point for the same work. If the destination store cannot
|
||||
be listed, or the local index holds snapshots and the destination answers
|
||||
that it has no `vaultik-destination.txt` (an unplugged volume's empty mount
|
||||
point), prune fails and deletes nothing. `snapshot purge` does the same. If a
|
||||
blob or such a directory cannot be deleted, the command (and
|
||||
`snapshot create --prune`) exits non-zero.
|
||||
blob cannot be deleted, the command (and `snapshot create --prune`)
|
||||
exits non-zero.
|
||||
* `--force`: Skip confirmation prompt
|
||||
* `--json`: Output stats as JSON. Also skips the confirmation prompt, as
|
||||
`--force` does. A blob that cannot be deleted is counted in
|
||||
`blobs_failed`, and the document is printed before the non-zero exit.
|
||||
A blob kept for being newer than `prune_grace_period`, or because the
|
||||
listing of the destination gives no time for it, is counted in
|
||||
`blobs_too_new` and not in `blobs_found`. A directory under `metadata/`
|
||||
with no manifest is counted in `metadata_dirs_deleted` once deleted, or
|
||||
in `metadata_dirs_failed` if it cannot be.
|
||||
`blobs_too_new` and not in `blobs_found`.
|
||||
|
||||
**`info`**: Display system configuration, storage settings, encryption
|
||||
recipients, and local database statistics.
|
||||
@@ -578,9 +563,8 @@ and can be deleted.
|
||||
|
||||
A destination shared between hosts or users, each with its own local index,
|
||||
relies on `prune_grace_period` to keep one's prune from deleting the blobs of
|
||||
another's backup still running, or its directory under `metadata/` that has
|
||||
no manifest yet, so it must be longer than the longest backup run. It does not
|
||||
protect a blob that backup reuses from an earlier run that
|
||||
another's backup still running, so it must be longer than the longest backup
|
||||
run. It does not protect a blob that backup reuses from an earlier run that
|
||||
stopped before its manifest, which the prune can delete. To not depend on the
|
||||
grace period, give each host its own prefix (or directory). See
|
||||
[locking](#locking).
|
||||
@@ -680,9 +664,6 @@ format does and does not protect.
|
||||
3. List all blobs in storage
|
||||
4. Delete any blob not in the referenced set that is older than
|
||||
`prune_grace_period`
|
||||
5. Delete each directory under `metadata/` with no manifest whose objects
|
||||
are all older than `prune_grace_period`, unless its snapshot is in the
|
||||
local index
|
||||
|
||||
### chunking and deduplication
|
||||
|
||||
@@ -727,7 +708,7 @@ complete annotated example also lives in
|
||||
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
|
||||
| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` |
|
||||
| `compression_level` | `3` | zstd compression level (1-19) |
|
||||
| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it, and how old every object in a directory under `metadata/` with no manifest must be before `prune` deletes that directory. A backup uploads its blobs and its `db.zst.age` before its manifest, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs and directories at once |
|
||||
| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it. A backup uploads its blobs before the manifest that lists them, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). It does not protect a blob a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so a prune can delete it. A duration such as `36h` or `90m`; `0s` deletes such blobs at once |
|
||||
| `hostname` | system hostname | Hostname used in snapshot IDs |
|
||||
| `index_path` | platform data dir | Local SQLite index path |
|
||||
|
||||
|
||||
@@ -30,8 +30,7 @@ owner's; a `v*` tag runs the release build in
|
||||
([issue #395](https://git.eeqj.de/sneak/vaultik/issues/395)). Such a
|
||||
prune could delete every snapshot still holding the volume's files.
|
||||
Each snapshot now records in the local index the directories it found
|
||||
on a different filesystem from their parent and found an entry in that
|
||||
the backup does not exclude.
|
||||
on a different filesystem from their parent.
|
||||
|
||||
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
||||
that found nothing in a configured path
|
||||
|
||||
+3
-5
@@ -329,16 +329,14 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
# Default: 3
|
||||
compression_level: 5
|
||||
|
||||
# How old a blob that no snapshot lists must be before prune deletes it, and
|
||||
# how old every object in a directory under metadata/ with no manifest must
|
||||
# be before prune deletes that directory
|
||||
# A backup uploads its blobs and its db.zst.age before its manifest, so on a
|
||||
# How old a blob that no snapshot lists must be before prune deletes it
|
||||
# A backup uploads its blobs before the manifest that lists them, so on a
|
||||
# destination shared between hosts or users this must be longer than the
|
||||
# longest backup run
|
||||
# It does not protect a blob a backup reuses from an earlier run that stopped
|
||||
# before its manifest: that blob keeps the time of its first upload, so a
|
||||
# prune can delete it
|
||||
# Supports: 36h, 90m, etc.; 0s deletes such blobs and directories at once
|
||||
# Supports: 36h, 90m, etc.; 0s deletes them at once
|
||||
# Default: 24h
|
||||
#prune_grace_period: 24h
|
||||
|
||||
|
||||
+3
-7
@@ -154,14 +154,14 @@ Maps snapshots to the blobs they reference.
|
||||
**Purpose:** Tracks blob dependencies for snapshots and enables manifest generation.
|
||||
|
||||
### 10. `snapshot_mount_points`
|
||||
The directories at or below a configured path, the configured path itself included, that a snapshot's backup found an entry in and, once it had listed them, found on a different filesystem from their parent directory (a different device number), as a mount point is. An automount point whose volume cannot be mounted is not recorded, because the backup finds no entry in it.
|
||||
The directories at or below a configured path that a snapshot's backup found on a different filesystem from their parent directory (a different device number), as a mount point is, the configured path itself included.
|
||||
|
||||
**Columns:**
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `path` (TEXT) - Absolute path of the directory
|
||||
- PRIMARY KEY (`snapshot_id`, `path`)
|
||||
|
||||
**Purpose:** Lets `snapshot create --prune` skip the prune when a backup does not find a mount point that an earlier snapshot of the same name found, as when the volume is not mounted.
|
||||
**Purpose:** Lets `snapshot create --prune` skip the prune when a volume an earlier snapshot of the same name found mounted is not mounted now.
|
||||
|
||||
### 11. `uploads`
|
||||
Tracks blob upload metrics.
|
||||
@@ -183,7 +183,7 @@ Tracks blob upload metrics.
|
||||
- `INSERT OR REPLACE INTO files` - Update file metadata
|
||||
- `SELECT * FROM files WHERE path = ?` - Check if file has changed
|
||||
- `INSERT INTO snapshot_files` - Add file to current snapshot
|
||||
- `INSERT OR IGNORE INTO snapshot_mount_points` - Record each directory on a different filesystem from its parent that the walk found an entry in
|
||||
- `INSERT INTO snapshot_mount_points` - Record each directory on a different filesystem from its parent
|
||||
|
||||
2. **Chunking** (for changed files)
|
||||
- `INSERT OR IGNORE INTO chunks` - Store new chunks
|
||||
@@ -241,10 +241,6 @@ The restore process doesn't use the local database. Instead:
|
||||
- Query blobs not referenced by any remaining snapshot
|
||||
- Delete from S3, once older than `prune_grace_period`, and from the local
|
||||
database
|
||||
2. **Delete Metadata Directories With No Manifest**
|
||||
- Delete each directory under `metadata/` that has no `manifest.json.zst`
|
||||
and no local snapshot record, once every object in it is older than
|
||||
`prune_grace_period`
|
||||
|
||||
### 6. Incomplete Snapshot Cleanup
|
||||
|
||||
|
||||
@@ -165,10 +165,10 @@ These are known, deliberate properties of the format and the tooling, recorded s
|
||||
## Pruning Safety
|
||||
|
||||
The prune operation is safe because:
|
||||
1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references. Under `metadata/` it deletes only a directory that has no `manifest.json.zst`, whose name is a remote key, and that no local snapshot record claims
|
||||
1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references
|
||||
2. Manifests are unencrypted and can be read without keys
|
||||
3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as unreferenced
|
||||
4. It deletes an unreferenced blob only once the blob is older than `prune_grace_period` (default `24h`), and a directory under `metadata/` with no manifest only once every object in it is. A backup uploads its blobs and its `db.zst.age` before its manifest, so a prune running meanwhile, on this host or another sharing the destination, leaves them as long as no backup run takes longer than the grace period. The grace period does not protect a blob a backup reuses from an earlier run that stopped before its manifest: the backup skips the upload of a blob already on the destination, so that blob keeps the time of its first upload, and a prune before the backup's manifest is written can delete it; the new snapshot then cannot be restored. `remote nuke` has no such check
|
||||
4. It deletes an unreferenced blob only once the blob is older than `prune_grace_period` (default `24h`). A backup uploads its blobs before the manifest that lists them, so a prune running meanwhile, on this host or another sharing the destination, leaves them as long as no backup run takes longer than the grace period. The grace period does not protect a blob a backup reuses from an earlier run that stopped before its manifest: the backup skips the upload of a blob already on the destination, so that blob keeps the time of its first upload, and a prune before the backup's manifest is written can delete it; the new snapshot then cannot be restored. `remote nuke` has no such check
|
||||
|
||||
## Restoration Requirements
|
||||
|
||||
|
||||
@@ -239,15 +239,13 @@ storage_url: ""
|
||||
# Default: 3
|
||||
# compression_level: 3
|
||||
|
||||
# How old a blob that no snapshot lists must be before prune deletes it,
|
||||
# and how old every object in a directory under metadata/ with no manifest
|
||||
# must be before prune deletes that directory. A backup uploads its blobs
|
||||
# and its db.zst.age before its manifest, so on a destination shared
|
||||
# between hosts or users this must be longer than the longest backup run.
|
||||
# It does not protect a blob a backup reuses from an earlier run that
|
||||
# stopped before its manifest: that blob keeps the time of its first
|
||||
# upload, so a prune can delete it.
|
||||
# Accepts: 36h, 90m, etc. 0s deletes such blobs and directories at once.
|
||||
# How old a blob that no snapshot lists must be before prune deletes it.
|
||||
# A backup uploads its blobs before the manifest that lists them, so on a
|
||||
# destination shared between hosts or users this must be longer than the
|
||||
# longest backup run. It does not protect a blob a backup reuses from an
|
||||
# earlier run that stopped before its manifest: that blob keeps the time of
|
||||
# its first upload, so a prune can delete it.
|
||||
# Accepts: 36h, 90m, etc. 0s deletes them at once.
|
||||
# Default: 24h
|
||||
# prune_grace_period: 24h
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -9,7 +8,6 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
// Flags of `snapshot create` used to build argument vectors below.
|
||||
@@ -119,51 +117,6 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing backs
|
||||
// up, records in the local index that this backup found the source
|
||||
// directory as a mount point, and runs `snapshot create --prune`, which
|
||||
// does not find it as one. The command exits non-zero, reports the
|
||||
// backup as completed, not as a failed snapshot creation, and keeps the
|
||||
// older snapshot. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
||||
t *testing.T,
|
||||
) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
require.Equal(t, 0, code)
|
||||
|
||||
// writeBackupConfig puts the source directory and the local index
|
||||
// beside the config file.
|
||||
dir := filepath.Dir(configPath)
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, filepath.Join(dir, "index.sqlite"))
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
backedUp, err := repos.Snapshots.GetOnlySnapshot(ctx)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, repos.Snapshots.AddMountPoint(ctx, nil,
|
||||
backedUp.ID.String(), filepath.Join(dir, "source")))
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "Backup completed, but 1 mount point(s) "+
|
||||
"an earlier snapshot recorded were not found")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
}
|
||||
|
||||
// writeBackupConfig builds a hermetic config with one small file in its
|
||||
// source directory and an empty destination store directory. Returns the
|
||||
// config path and the store directory.
|
||||
|
||||
@@ -13,9 +13,8 @@ func NewPruneCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Tidy local database and remote storage",
|
||||
Long: `Removes orphaned data from the local index database, and
|
||||
unreferenced blobs and directories under metadata/ with no manifest
|
||||
from the backup destination store.
|
||||
Long: `Removes orphaned data from both the local index database and
|
||||
unreferenced blobs from the backup destination store.
|
||||
|
||||
Local cleanup drops incomplete snapshots and any files, chunks, or
|
||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||
@@ -27,10 +26,6 @@ another sharing the destination store, that has not yet uploaded its
|
||||
manifest. The grace period does not protect a blob such a backup
|
||||
reuses from an earlier run that stopped before its manifest: that
|
||||
blob keeps the time of its first upload, so it can be deleted.
|
||||
Remote cleanup also deletes each directory under metadata/ with no
|
||||
manifest, as a backup that stopped before its manifest leaves, once
|
||||
all its objects are older than prune_grace_period and its snapshot is
|
||||
not in the local index.
|
||||
|
||||
Snapshot create --prune runs the same cleanup automatically; this
|
||||
command is the manual entry point for the same work (e.g. after a
|
||||
|
||||
@@ -133,13 +133,11 @@ on Linux), then /etc/vaultik/config.yml.`,
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
||||
"Run in cron mode (silent unless warning or error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||
"After backup, drop older snapshots of the same name, remove "+
|
||||
"orphaned blobs older than prune_grace_period, and delete each "+
|
||||
"directory under metadata/ with no manifest once all its "+
|
||||
"objects are older than prune_grace_period and its snapshot is "+
|
||||
"not in the local index (not after a backup that skipped "+
|
||||
"unreadable files, found a configured path empty, or did not "+
|
||||
"find a mount point an earlier snapshot found)")
|
||||
"After backup, drop older snapshots of the same name and remove "+
|
||||
"orphaned blobs older than prune_grace_period (not after a "+
|
||||
"backup that skipped unreadable files, found a configured "+
|
||||
"path empty, or found no volume where an earlier snapshot "+
|
||||
"found one mounted)")
|
||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||
"With --prune: keep snapshots newer than this duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||
|
||||
@@ -192,12 +192,10 @@ type Config struct {
|
||||
CompressionLevel int `yaml:"compression_level"`
|
||||
|
||||
// PruneGracePeriod is how old a blob no manifest lists must be before
|
||||
// a prune deletes it, and how old every object in a directory under
|
||||
// metadata/ with no manifest must be before a prune deletes that
|
||||
// directory. A backup writes its manifest only once it has uploaded
|
||||
// all its blobs and its db.zst.age, so a younger blob or directory may
|
||||
// belong to a backup still running, on this host or another sharing
|
||||
// the destination store.
|
||||
// a prune deletes it. A backup lists its blobs in a manifest only
|
||||
// once it has uploaded them all, so a younger one may belong to a
|
||||
// backup still running, on this host or another sharing the
|
||||
// destination store.
|
||||
PruneGracePeriod time.Duration `yaml:"prune_grace_period"`
|
||||
|
||||
// StorageURL specifies the storage backend using a URL format.
|
||||
|
||||
@@ -415,8 +415,7 @@ func TestDocumentedSizeExamples(t *testing.T) {
|
||||
|
||||
// TestValidatePruneGracePeriod checks that a negative prune_grace_period
|
||||
// is refused rather than acting like 0s, which deletes unreferenced blobs
|
||||
// and directories under metadata/ with no manifest at once, and that 0s
|
||||
// and the documented examples are accepted.
|
||||
// at once, and that 0s and the documented examples are accepted.
|
||||
func TestValidatePruneGracePeriod(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ CREATE INDEX IF NOT EXISTS idx_snapshot_blobs_blob_id ON snapshot_blobs(blob_id)
|
||||
|
||||
-- Snapshot mount points table: the directories at or below a configured
|
||||
-- path that the snapshot's backup found on a different filesystem from
|
||||
-- their parent directory, as a mount point is, and found an entry in
|
||||
-- their parent directory, as a mount point is
|
||||
CREATE TABLE IF NOT EXISTS snapshot_mount_points (
|
||||
snapshot_id TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
//nolint:testpackage // exercises the unexported deviceNumberFromStat
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The device number a real backup reads is the same for a directory and
|
||||
// a file in it, and differs between the temp directory and /dev.
|
||||
// Without it no mount point is ever found. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
func TestDeviceNumberFromStat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "file.txt")
|
||||
require.NoError(t, os.WriteFile(file, nil, 0o600))
|
||||
|
||||
deviceOf := func(path string) uint64 {
|
||||
t.Helper()
|
||||
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
device, ok := deviceNumberFromStat(info)
|
||||
require.True(t, ok, "no device number read for %s", path)
|
||||
|
||||
return device
|
||||
}
|
||||
|
||||
assert.Equal(t, deviceOf(dir), deviceOf(file))
|
||||
|
||||
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
|
||||
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
|
||||
}
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
@@ -17,10 +15,9 @@ import (
|
||||
)
|
||||
|
||||
// A scan records with the snapshot each directory on a different device
|
||||
// from its parent directory that the walk found an entry in, as a mount
|
||||
// point is, the scanned path included. A filesystem that gives no device
|
||||
// numbers, as the in-memory one, records none. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
// from its parent directory, as a mount point is, the scanned path
|
||||
// included. A filesystem that gives no device numbers, as the in-memory
|
||||
// one, records none. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
func TestScanRecordsMountPoints(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -98,148 +95,3 @@ func TestScanRecordsMountPoints(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// automountFs is an in-memory filesystem with a volume at mountPoint
|
||||
// that is mounted on first access: opening mountPoint to list it mounts
|
||||
// the volume or, if it cannot be mounted, fails with "no such file or
|
||||
// directory", as with the autofs daemon. Stat records the device an
|
||||
// entry is on at that moment, as a real stat does: unmountedDevice for
|
||||
// mountPoint until the volume is mounted; the volume's, 2, for
|
||||
// mountPoint once it is and for everything in it; 1 for everything else.
|
||||
type automountFs struct {
|
||||
afero.Fs
|
||||
|
||||
mountPoint string
|
||||
unmountedDevice uint64
|
||||
canMount bool
|
||||
mounted bool
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Open is defined to return the interface.
|
||||
func (fs *automountFs) Open(name string) (afero.File, error) {
|
||||
if name == fs.mountPoint && !fs.canMount {
|
||||
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
|
||||
}
|
||||
|
||||
if name == fs.mountPoint {
|
||||
fs.mounted = true
|
||||
}
|
||||
|
||||
return fs.Fs.Open(name)
|
||||
}
|
||||
|
||||
func (fs *automountFs) Stat(name string) (os.FileInfo, error) {
|
||||
info, err := fs.Fs.Stat(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
device := uint64(1)
|
||||
|
||||
switch {
|
||||
case name == fs.mountPoint && !fs.mounted:
|
||||
device = fs.unmountedDevice
|
||||
case name == fs.mountPoint || strings.HasPrefix(name, fs.mountPoint+"/"):
|
||||
device = 2
|
||||
}
|
||||
|
||||
return statWithDevice{FileInfo: info, device: device}, nil
|
||||
}
|
||||
|
||||
// statWithDevice is an entry automountFs.Stat looked up, with the device
|
||||
// it was on then.
|
||||
type statWithDevice struct {
|
||||
os.FileInfo
|
||||
|
||||
device uint64
|
||||
}
|
||||
|
||||
// deviceAtLookup returns the device automountFs.Stat recorded for info.
|
||||
func deviceAtLookup(info os.FileInfo) (uint64, bool) {
|
||||
stat, ok := info.(statWithDevice)
|
||||
|
||||
return stat.device, ok
|
||||
}
|
||||
|
||||
// A volume mounted on first access is mounted when the walk lists its
|
||||
// mount point. Until then the mount point is on the automounter's own
|
||||
// device (a systemd automount) or on its parent's (an autofs map that
|
||||
// shows its mount points before mounting them). Either way only the
|
||||
// mount point is recorded, by the backup that mounts the volume and by a
|
||||
// later one that finds it mounted. Listing an automount point whose
|
||||
// volume cannot be mounted fails; it is not recorded, because the walk
|
||||
// finds no entry in it. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
func TestScanRecordsOnlyTheMountPointOfAnAutomountedVolume(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const scannedPath = "/data/source"
|
||||
|
||||
mountPoint := scannedPath + "/nas"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
unmountedDevice uint64
|
||||
canMount bool
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "on the automounter's device until listed",
|
||||
unmountedDevice: 3, canMount: true, want: []string{mountPoint},
|
||||
},
|
||||
{
|
||||
name: "on its parent's device until listed",
|
||||
unmountedDevice: 1, canMount: true, want: []string{mountPoint},
|
||||
},
|
||||
{
|
||||
name: "volume that cannot be mounted",
|
||||
unmountedDevice: 1, canMount: false, want: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
fs := &automountFs{
|
||||
Fs: afero.NewMemMapFs(), mountPoint: mountPoint,
|
||||
unmountedDevice: tt.unmountedDevice, canMount: tt.canMount,
|
||||
}
|
||||
|
||||
require.NoError(t, fs.MkdirAll(mountPoint+"/photos", 0o755))
|
||||
|
||||
for _, path := range []string{
|
||||
scannedPath + "/notes.txt", mountPoint + "/photos/a.jpg",
|
||||
} {
|
||||
require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0o644))
|
||||
}
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
for _, snapshotID := range []string{"first", "later"} {
|
||||
createTestSnapshotRecord(ctx, t, repos, snapshotID)
|
||||
|
||||
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
||||
FS: fs,
|
||||
ChunkSize: int64(1024 * 16),
|
||||
Repositories: repos,
|
||||
MaxBlobSize: int64(1024 * 1024),
|
||||
CompressionLevel: 3,
|
||||
AgeRecipients: []string{testAgePublicKey},
|
||||
DeviceNumber: deviceAtLookup,
|
||||
})
|
||||
|
||||
_, err = scanner.Scan(ctx, scannedPath, snapshotID)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := repos.Snapshots.GetMountPoints(ctx, snapshotID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got, "mount points of the %s backup", snapshotID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,7 +310,7 @@ func (s *Scanner) Scan(
|
||||
}
|
||||
}
|
||||
|
||||
err = s.recordDirectories(ctx, scanResult)
|
||||
err = s.recordDirectories(ctx, scanResult.Directories)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -888,11 +888,6 @@ type ScanPhaseResult struct {
|
||||
FilesToProcess []*FileToProcess
|
||||
UnchangedFileIDs []types.FileID // IDs of unchanged files to associate with snapshot
|
||||
Directories []*FileToProcess // Every directory found, to record
|
||||
// MountPoints are the directories, the scanned path included, that
|
||||
// the walk found an entry in and, once it had listed them, on a
|
||||
// different device from their parent directory (see
|
||||
// recordIfMountPoint).
|
||||
MountPoints []string
|
||||
}
|
||||
|
||||
// scanPhase performs the initial directory scan to identify files to process
|
||||
@@ -912,7 +907,7 @@ func (s *Scanner) scanPhase(
|
||||
// estimate leaves them out.
|
||||
estimatedTotal := countNonDirectories(knownFiles)
|
||||
|
||||
collector := &scanCollector{dirsWithEntries: make(map[string]struct{})}
|
||||
collector := &scanCollector{}
|
||||
|
||||
// Set up periodic status output
|
||||
startTime := time.Now()
|
||||
@@ -957,8 +952,6 @@ func (s *Scanner) scanPhase(
|
||||
result.EntriesFound++
|
||||
}
|
||||
|
||||
s.recordIfMountPoint(filePath, path, collector)
|
||||
|
||||
// Handle symlinks and directories
|
||||
handled, err := s.recordSpecialEntry(
|
||||
filePath, info, existingFiles, knownFiles, collector, result)
|
||||
@@ -993,7 +986,6 @@ func (s *Scanner) scanPhase(
|
||||
FilesToProcess: collector.filesToProcess,
|
||||
UnchangedFileIDs: collector.unchangedFileIDs,
|
||||
Directories: collector.directories,
|
||||
MountPoints: collector.mountPoints,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1120,29 +1112,6 @@ type scanCollector struct {
|
||||
unchangedFileIDs []types.FileID // Just IDs - no new records needed
|
||||
directories []*FileToProcess
|
||||
filesScanned int64
|
||||
// dirsWithEntries holds each directory the walk has found an entry in.
|
||||
dirsWithEntries map[string]struct{}
|
||||
mountPoints []string
|
||||
}
|
||||
|
||||
// firstEntryIn notes that the walk found an entry in directory dir and
|
||||
// reports whether it is the first one.
|
||||
func (c *scanCollector) firstEntryIn(dir string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
_, seen := c.dirsWithEntries[dir]
|
||||
c.dirsWithEntries[dir] = struct{}{}
|
||||
|
||||
return !seen
|
||||
}
|
||||
|
||||
// addMountPoint sets a mount point aside to be recorded.
|
||||
func (c *scanCollector) addMountPoint(dir string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.mountPoints = append(c.mountPoints, dir)
|
||||
}
|
||||
|
||||
// addDirectory sets a directory aside to be recorded. It is not counted
|
||||
@@ -1338,23 +1307,29 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
|
||||
// mount points (see recordMountPoints). A directory has no data to back
|
||||
// up, so it is not among the files counted and backed up.
|
||||
func (s *Scanner) recordDirectories(
|
||||
ctx context.Context, scanned *ScanPhaseResult,
|
||||
ctx context.Context, directories []*FileToProcess,
|
||||
) error {
|
||||
for _, dir := range scanned.Directories {
|
||||
for _, dir := range directories {
|
||||
err := s.recordNonRegularFile(ctx, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording directory %s: %w", dir.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
return s.recordMountPoints(ctx, scanned.MountPoints)
|
||||
return s.recordMountPoints(ctx, directories)
|
||||
}
|
||||
|
||||
// recordMountPoints records with the snapshot the mount points the walk
|
||||
// found (see ScanPhaseResult.MountPoints).
|
||||
// recordMountPoints records with the snapshot each directory the walk
|
||||
// found on a different device from its parent directory, as a mount
|
||||
// point is, the scanned path included.
|
||||
func (s *Scanner) recordMountPoints(
|
||||
ctx context.Context, mountPoints []string,
|
||||
ctx context.Context, directories []*FileToProcess,
|
||||
) error {
|
||||
mountPoints, err := s.findMountPoints(directories)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
for _, dir := range mountPoints {
|
||||
log.Info("Found a mount point", "path", dir)
|
||||
@@ -1369,53 +1344,57 @@ func (s *Scanner) recordMountPoints(
|
||||
})
|
||||
}
|
||||
|
||||
// recordIfMountPoint sets the directory that holds entry, a path the
|
||||
// walk found, aside to be recorded as a mount point if it is on a
|
||||
// different device from its parent directory. The walk calls it for each
|
||||
// entry it finds, and it decides on the first entry in a directory, so a
|
||||
// directory the walk found no entry in is never recorded. By then the
|
||||
// walk has listed the directory, which mounts a volume mounted there on
|
||||
// first access, so the directory and its parent are looked up again:
|
||||
// before the listing, the directory can be on the automounter's own
|
||||
// device (a systemd automount) or on its parent's (an autofs map that
|
||||
// shows its mount points before mounting them). The scanned path is in
|
||||
// no listing the walk made, so it is skipped as an entry. A directory
|
||||
// that cannot be looked up then, or whose parent cannot, is not
|
||||
// recorded, so this snapshot does not stop the prune of a later backup
|
||||
// that misses its volume.
|
||||
func (s *Scanner) recordIfMountPoint(
|
||||
entry, scannedPath string, collector *scanCollector,
|
||||
) {
|
||||
dir := filepath.Dir(entry)
|
||||
|
||||
if entry == scannedPath || !collector.firstEntryIn(dir) {
|
||||
return
|
||||
// findMountPoints returns the directories, among those the walk found,
|
||||
// that are on a different device from their parent directory. The
|
||||
// scanned path is compared with the directory that holds it, which the
|
||||
// walk did not reach.
|
||||
func (s *Scanner) findMountPoints(
|
||||
directories []*FileToProcess,
|
||||
) ([]string, error) {
|
||||
walked := make(map[string]os.FileInfo, len(directories))
|
||||
for _, dir := range directories {
|
||||
walked[dir.Path] = dir.FileInfo
|
||||
}
|
||||
|
||||
dirDevice, ok := s.deviceNumberAt(dir)
|
||||
if !ok {
|
||||
return
|
||||
var mountPoints []string
|
||||
|
||||
for _, dir := range directories {
|
||||
parentPath := filepath.Dir(dir.Path)
|
||||
|
||||
parent, ok := walked[parentPath]
|
||||
if !ok {
|
||||
var err error
|
||||
|
||||
parent, err = s.fs.Stat(parentPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"looking up the parent of %s: %w", dir.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.onOtherDevice(dir.FileInfo, parent) {
|
||||
mountPoints = append(mountPoints, dir.Path)
|
||||
}
|
||||
}
|
||||
|
||||
parentDevice, ok := s.deviceNumberAt(filepath.Dir(dir))
|
||||
if ok && dirDevice != parentDevice {
|
||||
collector.addMountPoint(dir)
|
||||
}
|
||||
return mountPoints, nil
|
||||
}
|
||||
|
||||
// deviceNumberAt looks path up now and returns the number of the device
|
||||
// that holds it, and false when the lookup fails (logged) or gives no
|
||||
// device number.
|
||||
func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
|
||||
info, err := s.fs.Stat(path)
|
||||
if err != nil {
|
||||
log.Warn("Could not look up a directory to check for a mount point",
|
||||
"path", path, "error", err)
|
||||
|
||||
return 0, false
|
||||
// onOtherDevice reports whether dir is on a different device from
|
||||
// parent. A file whose device number cannot be read counts as on the
|
||||
// same device as its parent.
|
||||
func (s *Scanner) onOtherDevice(dir, parent os.FileInfo) bool {
|
||||
dirDevice, ok := s.deviceNumber(dir)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return s.deviceNumber(info)
|
||||
parentDevice, ok := s.deviceNumber(parent)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return dirDevice != parentDevice
|
||||
}
|
||||
|
||||
// deviceNumberFromStat returns the number of the device that holds the
|
||||
|
||||
@@ -100,9 +100,8 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
|
||||
// TestS3ListingCutShortIsAnError checks that List and ListStream fail on a
|
||||
// listing page marked truncated that carries no continuation token, as a
|
||||
// server answering V2 listings V1-style sends it. Taken as the last page,
|
||||
// it would hide every later key: prune would treat the blobs of the
|
||||
// snapshots listed there as unreferenced, and a directory under metadata/
|
||||
// whose manifest is listed there as one with no manifest.
|
||||
// it would hide every later key, and prune would treat the blobs of the
|
||||
// snapshots listed there as unreferenced.
|
||||
func TestS3ListingCutShortIsAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
|
||||
// TestNukeRemoteFailsWhenADeleteFails checks that remote nuke returns an
|
||||
// error when a snapshot's metadata or a blob cannot be deleted, does not
|
||||
// report the store empty, and advises running nuke again: prune deletes
|
||||
// no snapshot metadata that has its manifest.
|
||||
// no metadata.
|
||||
func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
+59
-243
@@ -4,12 +4,10 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
@@ -26,11 +24,6 @@ var errNukeRequiresForce = errors.New(
|
||||
// errBlobDeleteFailed reports unreferenced blobs prune could not delete.
|
||||
var errBlobDeleteFailed = errors.New("blob(s) could not be deleted")
|
||||
|
||||
// errMetadataDirDeleteFailed reports directories under metadata/ with no
|
||||
// manifest that prune could not delete.
|
||||
var errMetadataDirDeleteFailed = errors.New(
|
||||
"metadata director(ies) with no manifest could not be deleted")
|
||||
|
||||
// errObjectsLeft reports objects remote nuke still finds on the backup
|
||||
// destination store after deleting.
|
||||
var errObjectsLeft = errors.New(
|
||||
@@ -38,8 +31,7 @@ var errObjectsLeft = errors.New(
|
||||
|
||||
// nukeCommandHint is the command suggested when a remote nuke could not
|
||||
// delete everything: running it again deletes what is left, which
|
||||
// `vaultik prune` does not do for snapshot metadata that has its
|
||||
// manifest.
|
||||
// `vaultik prune` does not do for snapshot metadata.
|
||||
const nukeCommandHint = "vaultik remote nuke --force"
|
||||
|
||||
// metadataDirName is the top-level remote directory holding snapshot
|
||||
@@ -177,20 +169,14 @@ type PruneBlobsResult struct {
|
||||
// newer than the prune grace period, or of unknown age. BlobsFound
|
||||
// does not include them.
|
||||
BlobsTooNew int `json:"blobs_too_new"`
|
||||
// MetadataDirsDeleted counts the directories under metadata/ with no
|
||||
// manifest that prune deleted, and MetadataDirsFailed those it could
|
||||
// not delete.
|
||||
MetadataDirsDeleted int `json:"metadata_dirs_deleted"`
|
||||
MetadataDirsFailed int `json:"metadata_dirs_failed,omitempty"`
|
||||
}
|
||||
|
||||
// Prune removes orphaned data from the local index database AND, from the
|
||||
// backup destination store, unreferenced blobs and the directories under
|
||||
// metadata/ with no manifest. This is the single user-facing prune entry
|
||||
// point — the split between local and remote cleanup is an
|
||||
// implementation detail. Calling code should prefer this method over
|
||||
// PruneDatabase or PruneBlobs individually unless it specifically wants
|
||||
// one half.
|
||||
// Prune removes orphaned data from the local index database AND
|
||||
// unreferenced blobs from the backup destination store. This is the
|
||||
// single user-facing prune entry point — the split between local and
|
||||
// remote cleanup is an implementation detail. Calling code should
|
||||
// prefer this method over PruneDatabase or PruneBlobs individually
|
||||
// unless it specifically wants one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
@@ -214,17 +200,11 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
return v.PruneBlobs(opts)
|
||||
}
|
||||
|
||||
// PruneBlobs removes unreferenced blobs from storage, and each directory
|
||||
// under metadata/ that a backup left with no manifest.
|
||||
// PruneBlobs removes unreferenced blobs from storage
|
||||
func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
log.Info("Starting prune operation")
|
||||
|
||||
metadataObjects, err := v.listMetadataObjects()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
allBlobsReferenced, err := v.collectReferencedBlobs(metadataObjects)
|
||||
allBlobsReferenced, err := v.collectReferencedBlobs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -237,11 +217,6 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs(
|
||||
allBlobs, allBlobsReferenced)
|
||||
|
||||
dirsWithoutManifest, err := v.findMetadataDirsWithoutManifest(metadataObjects)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := &PruneBlobsResult{
|
||||
BlobsFound: len(unreferencedBlobs),
|
||||
BlobsTooNew: tooNew,
|
||||
@@ -249,7 +224,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
|
||||
v.reportBlobsTooNew(tooNew, opts)
|
||||
|
||||
if len(unreferencedBlobs) == 0 && len(dirsWithoutManifest) == 0 {
|
||||
if len(unreferencedBlobs) == 0 {
|
||||
log.Info("No unreferenced blobs found")
|
||||
|
||||
if opts.JSON {
|
||||
@@ -262,16 +237,38 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
}
|
||||
|
||||
log.Info("Found unreferenced blobs",
|
||||
"count", len(unreferencedBlobs), "total_size", ubytes(totalSize),
|
||||
"metadata_dirs_without_manifest", len(dirsWithoutManifest))
|
||||
"count", len(unreferencedBlobs), "total_size", ubytes(totalSize))
|
||||
|
||||
if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize,
|
||||
dirsWithoutManifest) {
|
||||
return nil
|
||||
if opts.Force && !opts.JSON {
|
||||
v.UI.Infof("Found %d unreferenced blob(s) totaling %s.",
|
||||
len(unreferencedBlobs), ubytes(totalSize))
|
||||
}
|
||||
|
||||
if !opts.Force && !opts.JSON {
|
||||
// The count, the prompt and its answer bypass the UI: they are an
|
||||
// interactive exchange the operator must see, so --quiet does not
|
||||
// silence them.
|
||||
v.stdoutf("Found %d unreferenced blob(s) totaling %s.\n",
|
||||
len(unreferencedBlobs), ubytes(totalSize))
|
||||
v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
|
||||
|
||||
var confirm string
|
||||
|
||||
_, err = v.scanStdin(&confirm)
|
||||
if err != nil {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil //nolint:nilerr // read failure means no confirmation
|
||||
}
|
||||
|
||||
if strings.ToLower(confirm) != "y" {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
|
||||
v.deleteMetadataDirs(dirsWithoutManifest, result)
|
||||
|
||||
if opts.JSON {
|
||||
err = v.outputPruneBlobsJSON(result)
|
||||
@@ -279,142 +276,33 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
v.reportPruneDeletions(result, len(dirsWithoutManifest))
|
||||
}
|
||||
|
||||
var failures []error
|
||||
|
||||
if result.BlobsFailed > 0 {
|
||||
failures = append(failures,
|
||||
fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed))
|
||||
}
|
||||
|
||||
if result.MetadataDirsFailed > 0 {
|
||||
failures = append(failures, fmt.Errorf("%d %w",
|
||||
result.MetadataDirsFailed, errMetadataDirDeleteFailed))
|
||||
}
|
||||
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
// confirmPrune states what prune found to delete, unreferenced blobs and
|
||||
// directories under metadata/ with no manifest, naming each directory by
|
||||
// its remote key, and, unless --force is given, asks the operator to
|
||||
// confirm. Under --json it prints nothing and does not ask. It reports
|
||||
// whether to go ahead.
|
||||
func (v *Vaultik) confirmPrune(
|
||||
opts *PruneOptions, blobCount int, blobsSize int64, dirs []string,
|
||||
) bool {
|
||||
if opts.JSON {
|
||||
return true
|
||||
}
|
||||
|
||||
dirCount := len(dirs)
|
||||
blobsFound := fmt.Sprintf("Found %d unreferenced blob(s) totaling %s.",
|
||||
blobCount, ubytes(blobsSize))
|
||||
dirsFound := fmt.Sprintf("Found %d metadata director(ies) left by "+
|
||||
"backups that stopped before their manifest:", dirCount)
|
||||
|
||||
if opts.Force {
|
||||
if blobCount > 0 {
|
||||
v.UI.Infof("%s", blobsFound)
|
||||
}
|
||||
|
||||
if dirCount > 0 {
|
||||
v.UI.Infof("%s", dirsFound)
|
||||
|
||||
for _, remoteKey := range dirs {
|
||||
v.UI.Detailf("metadata/%s/", remoteKey)
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// The counts, the names, the prompt and its answer bypass the UI:
|
||||
// they are an interactive exchange the operator must see, so --quiet
|
||||
// does not silence them.
|
||||
if blobCount > 0 {
|
||||
v.stdoutf("%s\n", blobsFound)
|
||||
}
|
||||
|
||||
if dirCount > 0 {
|
||||
v.stdoutf("%s\n", dirsFound)
|
||||
|
||||
for _, remoteKey := range dirs {
|
||||
v.stdoutf(" metadata/%s/\n", remoteKey)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case dirCount == 0:
|
||||
v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", blobCount)
|
||||
case blobCount == 0:
|
||||
v.stdoutf("\nDelete %d metadata director(ies)? [y/N] ", dirCount)
|
||||
default:
|
||||
v.stdoutf("\nDelete %d unreferenced blob(s) and %d metadata "+
|
||||
"director(ies)? [y/N] ", blobCount, dirCount)
|
||||
}
|
||||
|
||||
var confirm string
|
||||
|
||||
// A failure to read the answer means no confirmation.
|
||||
_, err := v.scanStdin(&confirm)
|
||||
if err != nil || strings.ToLower(confirm) != "y" {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// reportPruneDeletions says how many blobs and metadata directories
|
||||
// prune deleted, naming each kind only when it found some to delete.
|
||||
func (v *Vaultik) reportPruneDeletions(result *PruneBlobsResult, dirCount int) {
|
||||
if result.BlobsFound > 0 {
|
||||
v.UI.Infof("Deleted %d blob(s) totaling %s.",
|
||||
result.BlobsDeleted, ubytes(result.BytesFreed))
|
||||
}
|
||||
|
||||
if dirCount > 0 {
|
||||
v.UI.Infof("Deleted %d metadata director(ies) with no manifest.",
|
||||
result.MetadataDirsDeleted)
|
||||
}
|
||||
}
|
||||
|
||||
// listMetadataObjects lists every object under metadata/ on the
|
||||
// destination store.
|
||||
func (v *Vaultik) listMetadataObjects() ([]storage.ObjectInfo, error) {
|
||||
log.Info("Listing remote snapshots")
|
||||
|
||||
var objects []storage.ObjectInfo
|
||||
|
||||
for object := range v.Storage.ListStream(v.ctx, "metadata/") {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
|
||||
}
|
||||
|
||||
objects = append(objects, object)
|
||||
if result.BlobsFailed > 0 {
|
||||
return fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed)
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectReferencedBlobs downloads every manifest among the listed
|
||||
// metadata objects and returns the set of referenced blob hashes.
|
||||
// collectReferencedBlobs downloads all manifests and returns the set of
|
||||
// referenced blob hashes.
|
||||
//
|
||||
// Every manifest must be read successfully. A manifest that cannot be
|
||||
// downloaded or decoded means its snapshot's blobs are unknown, so
|
||||
// treating them as unreferenced would let prune delete data a snapshot
|
||||
// still needs. Rather than risk that silent loss, any failure returns an
|
||||
// error naming the remote key and prune deletes nothing.
|
||||
func (v *Vaultik) collectReferencedBlobs(
|
||||
metadataObjects []storage.ObjectInfo,
|
||||
) (map[string]bool, error) {
|
||||
// IDs returned by uniqueSnapshotIDs are remote keys (hashed
|
||||
func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
|
||||
log.Info("Listing remote snapshots")
|
||||
// IDs returned by listUniqueSnapshotIDs are remote keys (hashed
|
||||
// subdirectories under metadata/), not human snapshot IDs.
|
||||
remoteKeys := uniqueSnapshotIDs(metadataObjects)
|
||||
remoteKeys, err := v.listUniqueSnapshotIDs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing snapshot keys: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Found manifests in remote storage", "count", len(remoteKeys))
|
||||
|
||||
@@ -439,14 +327,18 @@ func (v *Vaultik) collectReferencedBlobs(
|
||||
return allBlobsReferenced, nil
|
||||
}
|
||||
|
||||
// uniqueSnapshotIDs returns deduplicated snapshot IDs from the listed
|
||||
// metadata objects.
|
||||
func uniqueSnapshotIDs(metadataObjects []storage.ObjectInfo) []string {
|
||||
// listUniqueSnapshotIDs returns deduplicated snapshot IDs from remote metadata
|
||||
func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
|
||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
seen := make(map[string]bool)
|
||||
|
||||
var snapshotIDs []string
|
||||
|
||||
for _, object := range metadataObjects {
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
|
||||
}
|
||||
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= minSnapshotIDParts &&
|
||||
parts[0] == metadataDirName && parts[1] != "" {
|
||||
@@ -461,7 +353,7 @@ func uniqueSnapshotIDs(metadataObjects []storage.ObjectInfo) []string {
|
||||
}
|
||||
}
|
||||
|
||||
return snapshotIDs
|
||||
return snapshotIDs, nil
|
||||
}
|
||||
|
||||
// listAllRemoteBlobs returns a map of all blob hashes in remote storage
|
||||
@@ -535,82 +427,6 @@ func (v *Vaultik) findUnreferencedBlobs(
|
||||
return unreferenced, totalSize, tooNew
|
||||
}
|
||||
|
||||
// findMetadataDirsWithoutManifest returns the remote key of each
|
||||
// directory under metadata/ that has no manifest.json.zst, such as one a
|
||||
// backup leaves when its manifest upload fails after db.zst.age was
|
||||
// uploaded. It leaves out a directory a local snapshot record claims,
|
||||
// and one holding an object newer than the prune grace period, or of
|
||||
// unknown age because the listing gave no time for it: a backup still
|
||||
// running may be writing there. A name that is not a remote key is
|
||||
// never returned.
|
||||
func (v *Vaultik) findMetadataDirsWithoutManifest(
|
||||
metadataObjects []storage.ObjectInfo,
|
||||
) ([]string, error) {
|
||||
localIDs, err := v.localSnapshotIDs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing local snapshots: %w", err)
|
||||
}
|
||||
|
||||
keep := make(map[string]bool, len(localIDs))
|
||||
for _, id := range localIDs {
|
||||
keep[snapshot.RemoteSnapshotKey(id)] = true
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod)
|
||||
listed := make(map[string]bool)
|
||||
|
||||
for _, object := range metadataObjects {
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) < metadataKeyParts || !isBlobHash(parts[1]) {
|
||||
continue
|
||||
}
|
||||
|
||||
remoteKey := parts[1]
|
||||
listed[remoteKey] = true
|
||||
|
||||
if parts[2] == "manifest.json.zst" ||
|
||||
object.ModTime.IsZero() || object.ModTime.After(cutoff) {
|
||||
keep[remoteKey] = true
|
||||
}
|
||||
}
|
||||
|
||||
var remoteKeys []string
|
||||
|
||||
for remoteKey := range listed {
|
||||
if !keep[remoteKey] {
|
||||
remoteKeys = append(remoteKeys, remoteKey)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(remoteKeys)
|
||||
|
||||
return remoteKeys, nil
|
||||
}
|
||||
|
||||
// deleteMetadataDirs deletes everything in the given directories under
|
||||
// metadata/ and counts in the result the directories deleted and those
|
||||
// it could not delete.
|
||||
func (v *Vaultik) deleteMetadataDirs(
|
||||
remoteKeys []string, result *PruneBlobsResult,
|
||||
) {
|
||||
for _, remoteKey := range remoteKeys {
|
||||
_, err := v.deleteRemoteSnapshotByKey(remoteKey)
|
||||
if err != nil {
|
||||
log.Error("Failed to delete metadata directory with no manifest",
|
||||
"remote_key", remoteKey, "error", err)
|
||||
|
||||
result.MetadataDirsFailed++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
log.Info("Deleted metadata directory with no manifest",
|
||||
"remote_key", remoteKey)
|
||||
|
||||
result.MetadataDirsDeleted++
|
||||
}
|
||||
}
|
||||
|
||||
// reportBlobsTooNew says how many unreferenced blobs prune kept for being
|
||||
// newer than the grace period or of unknown age. Under --json the
|
||||
// document carries the count instead.
|
||||
|
||||
@@ -24,9 +24,8 @@ import (
|
||||
|
||||
// TestPruneDuringBackupKeepsItsBlobs runs a prune from another host
|
||||
// sharing the destination store at the moment a backup has uploaded its
|
||||
// blobs and its db.zst.age but not yet its manifest, so no manifest lists
|
||||
// those blobs. The prune must leave the blobs and the db.zst.age, and the
|
||||
// backup must then restore.
|
||||
// blobs but not yet its manifest, so no manifest lists those blobs. The
|
||||
// prune must leave them, and the backup must then restore.
|
||||
func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
@@ -80,8 +79,7 @@ func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) {
|
||||
SnapshotID: id,
|
||||
TargetDir: restoreDir,
|
||||
Verify: true,
|
||||
}), "a prune during the backup must not delete the backup's blobs "+
|
||||
"or its db.zst.age")
|
||||
}), "a prune during the backup must not delete the backup's blobs")
|
||||
|
||||
assertRestoredTree(t, fs, restoreDir, testFiles)
|
||||
}
|
||||
|
||||
@@ -1,225 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// metadataObjectKey returns the key of the named file in the metadata
|
||||
// directory of the snapshot with the given human ID.
|
||||
func metadataObjectKey(snapshotID, name string) string {
|
||||
return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + name
|
||||
}
|
||||
|
||||
// TestPruneDeletesMetadataDirWithoutManifestOlderThanGracePeriod checks
|
||||
// that prune deletes a directory under metadata/ holding only
|
||||
// db.zst.age, as a backup leaves when its manifest upload fails, once it
|
||||
// is older than the grace period, and keeps one newer than that. A
|
||||
// directory with a manifest is never deleted, however old.
|
||||
func TestPruneDeletesMetadataDirWithoutManifestOlderThanGracePeriod(
|
||||
t *testing.T,
|
||||
) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := t.TempDir()
|
||||
|
||||
store, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
const (
|
||||
oldID = "testhost_home_2026-01-01T00:00:00Z"
|
||||
newID = "testhost_home_2026-01-02T00:00:00Z"
|
||||
completeID = "testhost_home_2026-01-03T00:00:00Z"
|
||||
)
|
||||
|
||||
oldDB := metadataObjectKey(oldID, "db.zst.age")
|
||||
newDB := metadataObjectKey(newID, "db.zst.age")
|
||||
completeDB := metadataObjectKey(completeID, "db.zst.age")
|
||||
completeManifest := metadataObjectKey(completeID, "manifest.json.zst")
|
||||
|
||||
manifest, err := snapshot.EncodeManifest(&snapshot.Manifest{
|
||||
SnapshotID: snapshot.RemoteSnapshotKey(completeID),
|
||||
}, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, key := range []string{oldDB, newDB, completeDB} {
|
||||
require.NoError(t, store.Put(ctx, key, strings.NewReader("db")))
|
||||
}
|
||||
|
||||
require.NoError(t, store.Put(ctx, completeManifest,
|
||||
bytes.NewReader(manifest)))
|
||||
|
||||
gracePeriod := time.Hour
|
||||
beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute)
|
||||
|
||||
for _, key := range []string{oldDB, completeDB, completeManifest} {
|
||||
require.NoError(t, os.Chtimes(filepath.Join(storeDir, key),
|
||||
beforeGracePeriod, beforeGracePeriod))
|
||||
}
|
||||
|
||||
stdout := &bytes.Buffer{}
|
||||
v := &vaultik.Vaultik{
|
||||
Config: &config.Config{PruneGracePeriod: gracePeriod},
|
||||
Storage: store,
|
||||
Stdout: stdout,
|
||||
UI: ui.NewWithColor(stdout, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))
|
||||
|
||||
_, err = store.Stat(ctx, oldDB)
|
||||
require.ErrorIs(t, err, storage.ErrNotFound,
|
||||
"a directory with no manifest older than the grace period "+
|
||||
"must be deleted")
|
||||
|
||||
for _, key := range []string{newDB, completeDB, completeManifest} {
|
||||
_, err = store.Stat(ctx, key)
|
||||
require.NoError(t, err, "%s must be kept", key)
|
||||
}
|
||||
|
||||
var result vaultik.PruneBlobsResult
|
||||
|
||||
require.NoError(t, json.Unmarshal(stdout.Bytes(), &result))
|
||||
assert.Equal(t, 1, result.MetadataDirsDeleted)
|
||||
}
|
||||
|
||||
// TestPruneKeepsMetadataDirClaimedUnknownAgeOrNotRemoteKey checks that
|
||||
// prune --force keeps a directory under metadata/ with no manifest when a
|
||||
// local snapshot record claims it, when the listing gives no time for an
|
||||
// object in it, or when its name is not a remote key. A fourth such
|
||||
// directory is named, deleted and reported, although there are no
|
||||
// unreferenced blobs to delete.
|
||||
func TestPruneKeepsMetadataDirClaimedUnknownAgeOrNotRemoteKey(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
env := newListEnv(t)
|
||||
|
||||
const (
|
||||
claimedID = "testhost_home_2026-01-01T00:00:00Z"
|
||||
leftID = "testhost_home_2026-01-03T00:00:00Z"
|
||||
)
|
||||
|
||||
env.addLocal(t, claimedID, time.Now().Add(-time.Hour))
|
||||
|
||||
claimedDB := metadataObjectKey(claimedID, "db.zst.age")
|
||||
unknownAgeDB := metadataObjectKey("testhost_home_2026-01-02T00:00:00Z",
|
||||
"db.zst.age")
|
||||
leftDB := metadataObjectKey(leftID, "db.zst.age")
|
||||
notRemoteKeyDB := "metadata/not-a-remote-key/db.zst.age"
|
||||
|
||||
for _, key := range []string{claimedDB, leftDB, notRemoteKeyDB} {
|
||||
require.NoError(t, env.store.Put(ctx, key, strings.NewReader("db")))
|
||||
}
|
||||
|
||||
// Written into the mock store's map directly rather than through
|
||||
// Put, so its listing gives no time.
|
||||
env.store.data[unknownAgeDB] = []byte("db")
|
||||
|
||||
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{Force: true}))
|
||||
|
||||
assert.True(t, env.store.hasKey(claimedDB),
|
||||
"a directory a local snapshot record claims must be kept")
|
||||
assert.True(t, env.store.hasKey(unknownAgeDB),
|
||||
"a directory holding an object of unknown age must be kept")
|
||||
assert.True(t, env.store.hasKey(notRemoteKeyDB),
|
||||
"a directory whose name is not a remote key must be kept")
|
||||
assert.False(t, env.store.hasKey(leftDB),
|
||||
"a directory with no manifest, unclaimed and old, must be deleted")
|
||||
|
||||
output := env.stdout.String()
|
||||
|
||||
assert.Contains(t, output, "Found 1 metadata director(ies) left by "+
|
||||
"backups that stopped before their manifest:")
|
||||
assert.Contains(t, output,
|
||||
"metadata/"+snapshot.RemoteSnapshotKey(leftID)+"/")
|
||||
assert.NotContains(t, output, "not-a-remote-key")
|
||||
assert.Contains(t, output,
|
||||
"Deleted 1 metadata director(ies) with no manifest.")
|
||||
assert.NotContains(t, output, "No unreferenced blobs to remove.")
|
||||
}
|
||||
|
||||
// TestPruneDeclinedKeepsMetadataDirWithoutManifest checks that the prompt
|
||||
// names a directory under metadata/ with no manifest that prune is about
|
||||
// to delete, and that answering no keeps it.
|
||||
func TestPruneDeclinedKeepsMetadataDirWithoutManifest(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
env := newListEnv(t)
|
||||
|
||||
const leftID = "testhost_home_2026-01-01T00:00:00Z"
|
||||
|
||||
leftDB := metadataObjectKey(leftID, "db.zst.age")
|
||||
require.NoError(t, env.store.Put(context.Background(), leftDB,
|
||||
strings.NewReader("db")))
|
||||
|
||||
env.v.Stdin = strings.NewReader("n\n")
|
||||
|
||||
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
|
||||
|
||||
assert.True(t, env.store.hasKey(leftDB),
|
||||
"a declined prune must keep the directory")
|
||||
|
||||
output := env.stdout.String()
|
||||
|
||||
assert.Contains(t, output,
|
||||
" metadata/"+snapshot.RemoteSnapshotKey(leftID)+"/\n")
|
||||
assert.Contains(t, output, "Delete 1 metadata director(ies)? [y/N] ")
|
||||
assert.Contains(t, output, "Cancelled")
|
||||
assert.NotContains(t, output, "Deleted")
|
||||
}
|
||||
|
||||
// TestPruneFailedMetadataDirDeleteReturnsError checks that prune --json
|
||||
// counts a directory under metadata/ with no manifest that it cannot
|
||||
// delete, and then returns an error, so the command exits non-zero.
|
||||
func TestPruneFailedMetadataDirDeleteReturnsError(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
env := newListEnv(t)
|
||||
|
||||
leftDB := metadataObjectKey("testhost_home_2026-01-01T00:00:00Z",
|
||||
"db.zst.age")
|
||||
require.NoError(t, env.store.Put(context.Background(), leftDB,
|
||||
strings.NewReader("db")))
|
||||
|
||||
store := faultstore.New(env.store)
|
||||
store.OnDelete = func(key string) bool {
|
||||
return strings.HasPrefix(key, "metadata/")
|
||||
}
|
||||
env.v.Storage = store
|
||||
|
||||
err := env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true})
|
||||
|
||||
require.ErrorContains(t, err,
|
||||
"1 metadata director(ies) with no manifest could not be deleted")
|
||||
assert.True(t, env.store.hasKey(leftDB))
|
||||
|
||||
var result vaultik.PruneBlobsResult
|
||||
|
||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result),
|
||||
"stdout must hold the JSON document, got:\n%s", env.stdout.String())
|
||||
assert.Equal(t, 1, result.MetadataDirsFailed)
|
||||
assert.Equal(t, 0, result.MetadataDirsDeleted)
|
||||
}
|
||||
@@ -55,14 +55,12 @@ var ErrSourcePathEmpty = errors.New("source path(s) were empty")
|
||||
|
||||
// ErrVolumeNotMounted is returned when --prune skipped the post-backup
|
||||
// prune because a directory that an earlier snapshot of the same name
|
||||
// found as a mount point was not found as one in this run: the volume
|
||||
// mounted there is not mounted, the run found no entry in it, or the
|
||||
// directory is no longer backed up. The snapshots are complete and
|
||||
// kept, so the caller can report it apart from a failed backup. A run
|
||||
// that also skipped unreadable files or found a configured path empty
|
||||
// returns ErrFilesUnreadable or ErrSourcePathEmpty instead.
|
||||
var ErrVolumeNotMounted = errors.New(
|
||||
"mount point(s) an earlier snapshot recorded were not found")
|
||||
// found as a mount point was not one in this run, as when the volume
|
||||
// mounted there is not mounted. The snapshots are complete and kept, so
|
||||
// the caller can report it apart from a failed backup. A run that also
|
||||
// skipped unreadable files or found a configured path empty returns
|
||||
// ErrFilesUnreadable or ErrSourcePathEmpty instead.
|
||||
var ErrVolumeNotMounted = errors.New("volume(s) were not mounted")
|
||||
|
||||
// listRecentLimit caps how many snapshot rows are fetched from the
|
||||
// local index for listing and purge operations.
|
||||
@@ -235,12 +233,10 @@ func (v *Vaultik) skipPostBackupPrune(
|
||||
}
|
||||
|
||||
if len(unmountedPaths) > 0 {
|
||||
v.UI.Warningf("Skipped the post-backup prune because this backup "+
|
||||
"did not find a mount point at %s, where an earlier snapshot "+
|
||||
"found one: the volume is not mounted, this backup found no "+
|
||||
"entry in it, or the directory is no longer backed up. Older "+
|
||||
"snapshots are kept. Mount the volume again, or, if that is on "+
|
||||
"purpose, remove the older snapshots by hand with "+
|
||||
v.UI.Warningf("Skipped the post-backup prune because no volume "+
|
||||
"is mounted at %s, where an earlier snapshot found one; older "+
|
||||
"snapshots are kept. If a volume was retired on purpose, "+
|
||||
"remove the older snapshots by hand with "+
|
||||
"'vaultik snapshot purge'.",
|
||||
strings.Join(unmountedPaths, ", "))
|
||||
}
|
||||
@@ -307,10 +303,9 @@ func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) {
|
||||
}
|
||||
|
||||
// runPostBackupPrune drops older snapshots of the given names and removes
|
||||
// from remote storage the orphan blobs and the directories under
|
||||
// metadata/ with no manifest, as PruneBlobs does. If keepNewerThan is set
|
||||
// (e.g. "4w"), snapshots newer than that duration are kept. Otherwise only
|
||||
// the latest snapshot of each name is kept.
|
||||
// orphan blobs from remote storage. If keepNewerThan is set (e.g. "4w"),
|
||||
// snapshots newer than that duration are kept. Otherwise only the latest
|
||||
// snapshot of each name is kept.
|
||||
func (v *Vaultik) runPostBackupPrune(
|
||||
snapshotNames []string, keepNewerThan string,
|
||||
) error {
|
||||
@@ -337,7 +332,7 @@ func (v *Vaultik) runPostBackupPrune(
|
||||
|
||||
err = v.PruneBlobs(&PruneOptions{Force: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("pruning the destination store: %w", err)
|
||||
return fmt.Errorf("pruning orphaned blobs: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1455,7 +1450,7 @@ const pruneCommandHint = "vaultik prune"
|
||||
// snapshotRemoveCommandHint is the command suggested, with the
|
||||
// snapshot's ID, when a remove could not reach the destination store:
|
||||
// running it again removes the snapshot's metadata there, which
|
||||
// `vaultik prune` never does while the manifest is there.
|
||||
// `vaultik prune` never does.
|
||||
const snapshotRemoveCommandHint = "vaultik snapshot remove"
|
||||
|
||||
// RemoveSnapshot removes a snapshot from the local index database and,
|
||||
@@ -1658,8 +1653,7 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
|
||||
// the local index, its removal there has already happened, so a failure
|
||||
// is only a warning telling the user to run `vaultik snapshot remove`
|
||||
// for the snapshot again once the destination store is reachable
|
||||
// (`vaultik prune` never removes snapshot metadata that has its
|
||||
// manifest).
|
||||
// (`vaultik prune` never removes snapshot metadata).
|
||||
func (v *Vaultik) removeSnapshotRemote(
|
||||
snapshotID string, inLocalIndex bool, markerErr error,
|
||||
opts *RemoveOptions,
|
||||
|
||||
@@ -4,10 +4,8 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
@@ -263,69 +261,15 @@ func TestBackupWithNoEmptySourcePathPrunes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// automountFs is the real filesystem with a volume at volumeDir that is
|
||||
// mounted on first access, as by an autofs map that shows its mount
|
||||
// points before mounting them: listing volumeDir mounts the volume or,
|
||||
// while cannotMount, fails with "no such file or directory".
|
||||
type automountFs struct {
|
||||
afero.OsFs
|
||||
|
||||
volumeDir string
|
||||
cannotMount bool
|
||||
mounted bool
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Open is defined to return the interface.
|
||||
func (f *automountFs) Open(name string) (afero.File, error) {
|
||||
if name == f.volumeDir && f.cannotMount {
|
||||
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
|
||||
}
|
||||
|
||||
if name == f.volumeDir {
|
||||
f.mounted = true
|
||||
}
|
||||
|
||||
return f.OsFs.Open(name)
|
||||
}
|
||||
|
||||
// A backup under --prune in which a volume that an earlier snapshot found
|
||||
// mounted below the configured path is missing does not prune: run after
|
||||
// run, the snapshot that holds the volume's files is kept with everything
|
||||
// it stored, and the run fails. Once the volume is back, the prune runs.
|
||||
// The volume is mounted on first access and every backup starts with it
|
||||
// not mounted, so its directory is on its parent's device until the walk
|
||||
// lists it. While the volume is missing, its directory is left empty, or
|
||||
// listing it fails with "no such file or directory", as when an autofs
|
||||
// volume cannot be mounted, which the walk counts as a directory removed
|
||||
// during the backup. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
// mounted below the configured path is not mounted does not prune: run
|
||||
// after run, the snapshot that holds the volume's files is kept with
|
||||
// everything it stored, and the run fails. Once the volume is mounted
|
||||
// again, the prune runs. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
func TestBackupWithVolumeNotMountedDoesNotPrune(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
listingFails bool
|
||||
}{
|
||||
{name: "directory left empty", listingFails: false},
|
||||
{name: "listing fails", listingFails: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
backUpWithVolumeNotMounted(t, tt.listingFails)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// backUpWithVolumeNotMounted runs the backups of
|
||||
// TestBackupWithVolumeNotMountedDoesNotPrune. While the volume is
|
||||
// missing, listing its directory fails if listingFails; otherwise the
|
||||
// directory is empty.
|
||||
func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
|
||||
t.Helper()
|
||||
|
||||
// volume is the directory, below the configured path, where the
|
||||
// volume is mounted.
|
||||
const (
|
||||
@@ -338,21 +282,20 @@ func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
|
||||
volume + "/a.txt": []byte("file a"),
|
||||
"notes.txt": []byte("notes"),
|
||||
})
|
||||
env.setSnapshot(snapshotName, "")
|
||||
env.v.Config.Snapshots[snapshotName] = config.SnapshotConfig{
|
||||
Paths: []string{env.src},
|
||||
}
|
||||
prune := &vaultik.SnapshotCreateOptions{
|
||||
Prune: true, Snapshots: []string{snapshotName},
|
||||
}
|
||||
|
||||
// The volume's directory is on the volume's device, 2, while the
|
||||
// volume is mounted. Everything else is on device 1.
|
||||
volumeDir := filepath.Join(env.src, volume)
|
||||
fs := &automountFs{volumeDir: volumeDir}
|
||||
// While the volume is mounted, its directory is on a device of its
|
||||
// own.
|
||||
mounted := true
|
||||
newScanner := env.v.ScannerFactory
|
||||
env.v.ScannerFactory = func(params snapshot.ScannerParams) *snapshot.Scanner {
|
||||
fs.mounted = false
|
||||
params.Fs = fs
|
||||
params.DeviceNumber = func(info os.FileInfo) (uint64, bool) {
|
||||
if info.Name() == volume && fs.mounted {
|
||||
if mounted && info.Name() == volume {
|
||||
return 2, true
|
||||
}
|
||||
|
||||
@@ -368,13 +311,13 @@ func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
|
||||
stored, err := env.v.Storage.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// The volume goes missing.
|
||||
fs.cannotMount = listingFails
|
||||
// A volume that is not mounted leaves its mount point behind as an
|
||||
// empty directory on its parent's device.
|
||||
volumeDir := filepath.Join(env.src, volume)
|
||||
require.NoError(t, os.RemoveAll(volumeDir))
|
||||
require.NoError(t, os.Mkdir(volumeDir, 0o750))
|
||||
|
||||
if !listingFails {
|
||||
require.NoError(t, os.RemoveAll(volumeDir))
|
||||
require.NoError(t, os.Mkdir(volumeDir, 0o750))
|
||||
}
|
||||
mounted = false
|
||||
|
||||
for range 2 {
|
||||
env.out.Reset()
|
||||
@@ -382,7 +325,7 @@ func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
|
||||
err := env.v.CreateSnapshot(prune)
|
||||
require.ErrorIs(t, err, vaultik.ErrVolumeNotMounted)
|
||||
assert.Contains(t, env.out.String(),
|
||||
"did not find a mount point at "+volumeDir+" of snapshot "+snapshotName)
|
||||
"no volume is mounted at "+volumeDir+" of snapshot "+snapshotName)
|
||||
assert.Contains(t, env.out.String(), "'vaultik snapshot purge'")
|
||||
assert.NotContains(t, env.out.String(), "Running post-backup prune.")
|
||||
}
|
||||
@@ -399,11 +342,12 @@ func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
|
||||
require.NoError(t, err, "%s is kept", key)
|
||||
}
|
||||
|
||||
// The volume is back.
|
||||
fs.cannotMount = false
|
||||
|
||||
// The volume is mounted again.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(volumeDir, "a.txt"), []byte("file a"), 0o600))
|
||||
|
||||
mounted = true
|
||||
|
||||
require.NoError(t, env.v.CreateSnapshot(prune))
|
||||
|
||||
snapshots, err = env.v.Repositories.Snapshots.ListRecent(
|
||||
|
||||
Reference in New Issue
Block a user