Compare commits

..
Author SHA1 Message Date
sneak 2aa6fc8f66 Report snapshot blobs missing from the destination in remote info (closes #425)
check / check (push) Waiting to run
remote info now counts the blobs that a manifest lists but the
destination does not hold, with their total size, and names each
snapshot that lists one, since it cannot be restored. When any blob is
missing the command exits non-zero after writing its report. --json
gives the figures in missing_blob_count, missing_blob_size and
snapshots_missing_blobs.

Model: opus-5-5
2026-11-19 08:24:27 +00:00
5 changed files with 147 additions and 22 deletions

No files matched your search

+11 -16
View File
@@ -127,11 +127,6 @@ vaultik config set s3.secret_access_key "..."
vaultik snapshot list vaultik snapshot list
``` ```
For an rclone destination, the storage credentials are in rclone's config file
instead. Copy that file to the new host, or recreate it with `rclone config`,
and set `storage_url` to the original `rclone://remote/path`. See
[storage backends](#storage-backends).
`snapshot list` reads the destination store without the private key. A snapshot `snapshot list` reads the destination store without the private key. A snapshot
that is not in this host's (empty) local index is shown as remote-only: its row that is not in this host's (empty) local index is shown as remote-only: its row
is identified by `<remote only:...>` rather than by a `hostname_name_timestamp` is identified by `<remote only:...>` rather than by a `hostname_name_timestamp`
@@ -286,8 +281,6 @@ work as written.
the key is not typed into your shell history. the key is not typed into your shell history.
- `VAULTIK_CONFIG`: Path to config file (overridden by `--config`) - `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
- `VAULTIK_INDEX_PATH`: Override local SQLite index path - `VAULTIK_INDEX_PATH`: Override local SQLite index path
- `RCLONE_CONFIG`: Path to the rclone config file used by an `rclone://`
destination (default `~/.config/rclone/rclone.conf`)
- `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the - `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the
run (development/debugging) run (development/debugging)
- `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits - `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits
@@ -561,6 +554,12 @@ count and blob size unknown, `null` in `--json`. A file under `blobs/` whose
name is not a blob hash is not counted as a blob, since `prune` never deletes name is not a blob hash is not counted as a blob, since `prune` never deletes
it. it.
A blob that a manifest lists but the destination does not hold is reported as
missing, with a count and size. The report names each snapshot that lists a
missing blob, since that snapshot cannot be restored. The command then exits
non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and
`snapshots_missing_blobs`.
- `--json`: Output as JSON - `--json`: Output as JSON
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the **`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
@@ -609,15 +608,11 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. `remote` is the name of a remote in rclone's config file, which holds providers. Requires rclone to be configured separately (`rclone config`). The
that remote's credentials. vaultik reads the same file the `rclone` tool reads, path after the remote name is relative: rclone resolves it on the remote, for
by default `~/.config/rclone/rclone.conf`. Create it with `rclone config`, or example against the login directory on sftp. A second slash makes it absolute,
copy it from another machine; vaultik does not need the `rclone` tool once the as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
file exists. The `RCLONE_CONFIG` environment variable names a different file. path; a relative one is refused.
The path after the remote name is relative: rclone resolves it on the remote,
for example against the login directory on sftp. A second slash makes it
absolute, as in `rclone://remote//srv/backup`. A remote of type `local` needs an
absolute path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+3 -1
View File
@@ -75,7 +75,9 @@ func newRemoteInfoCommand() *cobra.Command {
- Size of all snapshot metadata (per snapshot and total) - Size of all snapshot metadata (per snapshot and total)
- Count and total size of all blobs - Count and total size of all blobs
- Count and size of referenced blobs (from all manifests) - Count and size of referenced blobs (from all manifests)
- Count and size of orphaned blobs (not referenced by any manifest)`, - Count and size of orphaned blobs (not referenced by any manifest)
- Count and size of referenced blobs missing from remote storage, and the
snapshots that list them; the command exits non-zero when any are missing`,
Args: cobra.NoArgs, Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error { RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution // Use unified config resolution
+72 -5
View File
@@ -9,6 +9,7 @@ import (
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
) )
// ShowInfo displays system and configuration information // ShowInfo displays system and configuration information
@@ -187,6 +188,9 @@ type SnapshotMetadataInfo struct {
// backup interrupted before its manifest upload leaves a directory // backup interrupted before its manifest upload leaves a directory
// without one, which prune does not treat as a snapshot. // without one, which prune does not treat as a snapshot.
hasManifest bool hasManifest bool
// Hashes of the blobs the snapshot's manifest lists.
blobHashes []string
} }
// RemoteInfoResult contains all remote storage information // RemoteInfoResult contains all remote storage information
@@ -216,6 +220,13 @@ type RemoteInfoResult struct {
OrphanedBlobCount *int `json:"orphaned_blob_count"` OrphanedBlobCount *int `json:"orphaned_blob_count"`
OrphanedBlobSize *int64 `json:"orphaned_blob_size"` OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
// Blobs that a manifest lists but the destination does not hold,
// and the remote key of each snapshot whose manifest lists one.
// Those snapshots cannot be restored.
MissingBlobCount int `json:"missing_blob_count"`
MissingBlobSize int64 `json:"missing_blob_size"`
SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"`
// Remote key of each snapshot whose manifest could not be read // Remote key of each snapshot whose manifest could not be read
UnreadableManifests []string `json:"unreadable_manifests,omitempty"` UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
@@ -283,19 +294,30 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
"snapshots", result.TotalMetadataCount, "snapshots", result.TotalMetadataCount,
"total_blobs", result.TotalBlobCount, "total_blobs", result.TotalBlobCount,
"referenced_blobs", result.ReferencedBlobCount, "referenced_blobs", result.ReferencedBlobCount,
"missing_blobs", result.MissingBlobCount,
"unreadable_manifests", len(result.UnreadableManifests)) "unreadable_manifests", len(result.UnreadableManifests))
if jsonOutput { if jsonOutput {
enc := json.NewEncoder(v.Stdout) enc := json.NewEncoder(v.Stdout)
enc.SetIndent("", " ") enc.SetIndent("", " ")
return enc.Encode(result) err = enc.Encode(result)
if err != nil {
return fmt.Errorf("encoding JSON: %w", err)
}
} }
if showText { if showText {
v.printRemoteInfoTable(result) v.printRemoteInfoTable(result)
} }
// The report is written first, so the exit status adds to it.
if result.MissingBlobCount > 0 {
return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+
"%d snapshot(s) cannot be restored", errProblemsInStorage,
result.MissingBlobCount, len(result.SnapshotsMissingBlobs))
}
return nil return nil
} }
@@ -405,6 +427,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
for _, blob := range manifest.Blobs { for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = blob.CompressedSize referencedBlobs[blob.Hash] = blob.CompressedSize
blobsSize += blob.CompressedSize blobsSize += blob.CompressedSize
info.blobHashes = append(info.blobHashes, blob.Hash)
} }
info.BlobCount = &blobCount info.BlobCount = &blobCount
@@ -462,6 +485,8 @@ func (v *Vaultik) scanRemoteBlobStorage(
result.TotalBlobSize += blob.Size result.TotalBlobSize += blob.Size
} }
countMissingBlobs(result, referencedBlobs, allBlobs)
// A blob named only by a manifest that could not be read, or by one // A blob named only by a manifest that could not be read, or by one
// under a skipped name, would be counted as orphaned, so the orphan // under a skipped name, would be counted as orphaned, so the orphan
// figures stay unknown. // figures stay unknown.
@@ -487,6 +512,33 @@ func (v *Vaultik) scanRemoteBlobStorage(
return nil return nil
} }
// countMissingBlobs records in result the referenced blobs that
// allBlobs, the listing of the destination, does not hold, and the
// snapshots whose manifests list one of them. It reads the snapshots
// from result.Snapshots, so populateRemoteInfoResult must run first.
func countMissingBlobs(
result *RemoteInfoResult, referencedBlobs map[string]int64,
allBlobs map[string]storage.ObjectInfo,
) {
for hash, size := range referencedBlobs {
if _, present := allBlobs[hash]; !present {
result.MissingBlobCount++
result.MissingBlobSize += size
}
}
for _, info := range result.Snapshots {
for _, hash := range info.blobHashes {
if _, present := allBlobs[hash]; !present {
result.SnapshotsMissingBlobs = append(
result.SnapshotsMissingBlobs, info.SnapshotID)
break
}
}
}
}
// printRemoteInfoTable renders the human-readable remote info output // printRemoteInfoTable renders the human-readable remote info output
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
const ( const (
@@ -539,6 +591,12 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
"", "", ubytes(result.TotalMetadataSize)) "", "", ubytes(result.TotalMetadataSize))
} }
v.printBlobStorageSummary(result)
}
// printBlobStorageSummary renders the blob storage section that ends
// the human-readable remote info output.
func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
v.stdoutf("\n=== Blob Storage ===\n") v.stdoutf("\n=== Blob Storage ===\n")
v.stdoutf("Total blobs on remote: %s (%s)\n", v.stdoutf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)), humanize.Comma(int64(result.TotalBlobCount)),
@@ -546,16 +604,16 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.stdoutf("Referenced by snapshots: %s (%s)\n", v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Comma(int64(result.ReferencedBlobCount)),
ubytes(result.ReferencedBlobSize)) ubytes(result.ReferencedBlobSize))
v.stdoutf("Missing from remote: %s (%s)\n",
humanize.Comma(int64(result.MissingBlobCount)),
ubytes(result.MissingBlobSize))
if result.OrphanedBlobCount == nil { if result.OrphanedBlobCount == nil {
v.stdoutf("Orphaned (unreferenced): unknown "+ v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+ "(%d manifest(s) could not be read, "+
"%d manifest(s) under a non-conforming name skipped)\n", "%d manifest(s) under a non-conforming name skipped)\n",
len(result.UnreadableManifests), result.SkippedManifestCount) len(result.UnreadableManifests), result.SkippedManifestCount)
} else {
return
}
v.stdoutf("Orphaned (unreferenced): %s (%s)\n", v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(*result.OrphanedBlobCount)), humanize.Comma(int64(*result.OrphanedBlobCount)),
ubytes(*result.OrphanedBlobSize)) ubytes(*result.OrphanedBlobSize))
@@ -563,6 +621,15 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
if *result.OrphanedBlobCount > 0 { if *result.OrphanedBlobCount > 0 {
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
} }
}
if len(result.SnapshotsMissingBlobs) > 0 {
v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n")
for _, id := range result.SnapshotsMissingBlobs {
v.stdoutf(" %s\n", id)
}
}
} }
// ellipsis is appended by truncateString when it shortens a string. // ellipsis is appended by truncateString when it shortens a string.
+60
View File
@@ -83,6 +83,7 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) {
env := newListEnv(t) env := newListEnv(t)
readableKey := env.addRemote(t, listRemoteID, readableKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID) unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
require.NoError(t, env.store.Put(context.Background(), require.NoError(t, env.store.Put(context.Background(),
@@ -270,6 +271,7 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
env := newListEnv(t) env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
const statusLine = ui.Marker + " Scanning snapshot metadata...\n" const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
@@ -281,3 +283,61 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata") assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
} }
// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a
// manifest lists but the destination does not hold is counted as
// missing, that the snapshot listing it is named, and that remote info
// then fails after writing its report.
func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
// The first snapshot lists blob A, which is not stored. The second
// lists blob B, which is.
brokenKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB})
addBlob(t, env.store.testStorer, testBlobHashB)
const wantError = "1 blob(s) that snapshots list are missing; " +
"1 snapshot(s) cannot be restored"
err := env.v.RemoteInfo(false)
require.ErrorContains(t, err, wantError)
text := env.stdout.String()
assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)")
assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+
" "+brokenKey+"\n")
assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n")
env.stdout.Reset()
require.ErrorContains(t, env.v.RemoteInfo(true), wantError)
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["missing_blob_count"], 0)
assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0)
assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"])
// --quiet drops the report, not the failure.
env.stdout.Reset()
env.v.UI.SetQuiet(true)
require.ErrorContains(t, env.v.RemoteInfo(false), wantError)
assert.Empty(t, env.stdout.String())
env.v.UI.SetQuiet(false)
// With blob A stored again nothing is missing.
addBlob(t, env.store.testStorer, testBlobHashA)
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
doc = nil
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 0, doc["missing_blob_count"], 0)
assert.InDelta(t, 0, doc["missing_blob_size"], 0)
assert.NotContains(t, doc, "snapshots_missing_blobs")
}
+1
View File
@@ -349,6 +349,7 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
env.addRemote(t, listLocalID, start) env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts( require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes)) context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
addBlob(t, env.store.testStorer, testBlobHashA)
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]), assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),