Report snapshot blobs missing from the destination in remote info (closes #425)
check / check (push) Waiting to run
check / check (push) Waiting to run
remote info now counts the blobs that a manifest lists but the destination does not hold, with their total size, and names each snapshot that lists one, since it cannot be restored. When any blob is missing the command exits non-zero after writing its report. --json gives the figures in missing_blob_count, missing_blob_size and snapshots_missing_blobs. Model: opus-5-5
This commit is contained in:
1 parent
be362080d3
commit
041e7642a2
5 files changed
+140
-1
No files matched your search
@@ -560,6 +560,12 @@ it. A leftover `.partial` object under `metadata/` or `blobs/` is not counted as
|
||||
a blob either. Such objects are counted on their own line, and in
|
||||
`partial_upload_count` and `partial_upload_size` in `--json`.
|
||||
|
||||
A blob that a manifest lists but the destination does not hold is reported as
|
||||
missing, with a count and size. The report names each snapshot that lists a
|
||||
missing blob, since that snapshot cannot be restored. The command then exits
|
||||
non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and
|
||||
`snapshots_missing_blobs`.
|
||||
|
||||
- `--json`: Output as JSON
|
||||
|
||||
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
|
||||
|
||||
@@ -76,6 +76,8 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
- Count and total size of all blobs
|
||||
- Count and size of referenced blobs (from all manifests)
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)
|
||||
- Count and size of referenced blobs missing from remote storage, and the
|
||||
snapshots that list them; the command exits non-zero when any are missing
|
||||
- Count and size of .partial objects left by killed uploads`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// ShowInfo displays system and configuration information
|
||||
@@ -187,6 +188,9 @@ type SnapshotMetadataInfo struct {
|
||||
// backup interrupted before its manifest upload leaves a directory
|
||||
// without one, which prune does not treat as a snapshot.
|
||||
hasManifest bool
|
||||
|
||||
// Hashes of the blobs the snapshot's manifest lists.
|
||||
blobHashes []string
|
||||
}
|
||||
|
||||
// RemoteInfoResult contains all remote storage information
|
||||
@@ -216,6 +220,13 @@ type RemoteInfoResult struct {
|
||||
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
||||
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
||||
|
||||
// Blobs that a manifest lists but the destination does not hold,
|
||||
// and the remote key of each snapshot whose manifest lists one.
|
||||
// Those snapshots cannot be restored.
|
||||
MissingBlobCount int `json:"missing_blob_count"`
|
||||
MissingBlobSize int64 `json:"missing_blob_size"`
|
||||
SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"`
|
||||
|
||||
// `.partial` objects under metadata/ and blobs/ left by killed
|
||||
// uploads. The blob figures above leave them out.
|
||||
PartialUploadCount int `json:"partial_upload_count"`
|
||||
@@ -298,20 +309,38 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
"snapshots", result.TotalMetadataCount,
|
||||
"total_blobs", result.TotalBlobCount,
|
||||
"referenced_blobs", result.ReferencedBlobCount,
|
||||
"missing_blobs", result.MissingBlobCount,
|
||||
"unreadable_manifests", len(result.UnreadableManifests))
|
||||
|
||||
if jsonOutput {
|
||||
enc := json.NewEncoder(v.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
|
||||
return enc.Encode(result)
|
||||
err = enc.Encode(result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding JSON: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if showText {
|
||||
v.printRemoteInfoTable(result)
|
||||
}
|
||||
|
||||
// The report is written first, so the exit status adds to it.
|
||||
return missingBlobsError(result)
|
||||
}
|
||||
|
||||
// missingBlobsError returns the error remote info exits with when a
|
||||
// snapshot's manifest lists blobs the destination does not hold, or nil
|
||||
// when none are missing.
|
||||
func missingBlobsError(result *RemoteInfoResult) error {
|
||||
if result.MissingBlobCount == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+
|
||||
"%d snapshot(s) cannot be restored", errProblemsInStorage,
|
||||
result.MissingBlobCount, len(result.SnapshotsMissingBlobs))
|
||||
}
|
||||
|
||||
// collectSnapshotMetadata scans remote metadata and returns
|
||||
@@ -420,6 +449,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
|
||||
for _, blob := range manifest.Blobs {
|
||||
referencedBlobs[blob.Hash] = blob.CompressedSize
|
||||
blobsSize += blob.CompressedSize
|
||||
info.blobHashes = append(info.blobHashes, blob.Hash)
|
||||
}
|
||||
|
||||
info.BlobCount = &blobCount
|
||||
@@ -477,6 +507,8 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
||||
result.TotalBlobSize += blob.Size
|
||||
}
|
||||
|
||||
countMissingBlobs(result, referencedBlobs, allBlobs)
|
||||
|
||||
// A blob named only by a manifest that could not be read, or by one
|
||||
// under a skipped name, would be counted as orphaned, so the orphan
|
||||
// figures stay unknown.
|
||||
@@ -502,6 +534,33 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
||||
return nil
|
||||
}
|
||||
|
||||
// countMissingBlobs records in result the referenced blobs that
|
||||
// allBlobs, the listing of the destination, does not hold, and the
|
||||
// snapshots whose manifests list one of them. It reads the snapshots
|
||||
// from result.Snapshots, so populateRemoteInfoResult must run first.
|
||||
func countMissingBlobs(
|
||||
result *RemoteInfoResult, referencedBlobs map[string]int64,
|
||||
allBlobs map[string]storage.ObjectInfo,
|
||||
) {
|
||||
for hash, size := range referencedBlobs {
|
||||
if _, present := allBlobs[hash]; !present {
|
||||
result.MissingBlobCount++
|
||||
result.MissingBlobSize += size
|
||||
}
|
||||
}
|
||||
|
||||
for _, info := range result.Snapshots {
|
||||
for _, hash := range info.blobHashes {
|
||||
if _, present := allBlobs[hash]; !present {
|
||||
result.SnapshotsMissingBlobs = append(
|
||||
result.SnapshotsMissingBlobs, info.SnapshotID)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printRemoteInfoTable renders the human-readable remote info output
|
||||
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
||||
const (
|
||||
@@ -567,6 +626,9 @@ func (v *Vaultik) printRemoteBlobStorage(result *RemoteInfoResult) {
|
||||
v.stdoutf("Referenced by snapshots: %s (%s)\n",
|
||||
humanize.Comma(int64(result.ReferencedBlobCount)),
|
||||
ubytes(result.ReferencedBlobSize))
|
||||
v.stdoutf("Missing from remote: %s (%s)\n",
|
||||
humanize.Comma(int64(result.MissingBlobCount)),
|
||||
ubytes(result.MissingBlobSize))
|
||||
|
||||
if result.OrphanedBlobCount == nil {
|
||||
v.stdoutf("Orphaned (unreferenced): unknown "+
|
||||
@@ -597,6 +659,14 @@ func (v *Vaultik) printRemoteBlobStorage(result *RemoteInfoResult) {
|
||||
v.stdoutf("\nRun 'vaultik prune' to remove %s.\n",
|
||||
strings.Join(toRemove, " and "))
|
||||
}
|
||||
|
||||
if len(result.SnapshotsMissingBlobs) > 0 {
|
||||
v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n")
|
||||
|
||||
for _, id := range result.SnapshotsMissingBlobs {
|
||||
v.stdoutf(" %s\n", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ellipsis is appended by truncateString when it shortens a string.
|
||||
|
||||
@@ -83,6 +83,7 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) {
|
||||
env := newListEnv(t)
|
||||
readableKey := env.addRemote(t, listRemoteID,
|
||||
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||
|
||||
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
|
||||
require.NoError(t, env.store.Put(context.Background(),
|
||||
@@ -270,6 +271,7 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
||||
|
||||
env := newListEnv(t)
|
||||
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||
|
||||
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
|
||||
|
||||
@@ -281,3 +283,61 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
||||
require.NoError(t, env.v.RemoteInfo(false))
|
||||
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
|
||||
}
|
||||
|
||||
// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a
|
||||
// manifest lists but the destination does not hold is counted as
|
||||
// missing, that the snapshot listing it is named, and that remote info
|
||||
// then fails after writing its report.
|
||||
func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
env := newListEnv(t)
|
||||
|
||||
// The first snapshot lists blob A, which is not stored. The second
|
||||
// lists blob B, which is.
|
||||
brokenKey := env.addRemote(t, listRemoteID,
|
||||
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||
addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB})
|
||||
addBlob(t, env.store.testStorer, testBlobHashB)
|
||||
|
||||
const wantError = "1 blob(s) that snapshots list are missing; " +
|
||||
"1 snapshot(s) cannot be restored"
|
||||
|
||||
err := env.v.RemoteInfo(false)
|
||||
require.ErrorContains(t, err, wantError)
|
||||
|
||||
text := env.stdout.String()
|
||||
assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)")
|
||||
assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+
|
||||
" "+brokenKey+"\n")
|
||||
assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n")
|
||||
|
||||
env.stdout.Reset()
|
||||
require.ErrorContains(t, env.v.RemoteInfo(true), wantError)
|
||||
|
||||
var doc map[string]any
|
||||
|
||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
|
||||
assert.InDelta(t, 1, doc["missing_blob_count"], 0)
|
||||
assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0)
|
||||
assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"])
|
||||
|
||||
// --quiet drops the report, not the failure.
|
||||
env.stdout.Reset()
|
||||
env.v.UI.SetQuiet(true)
|
||||
require.ErrorContains(t, env.v.RemoteInfo(false), wantError)
|
||||
assert.Empty(t, env.stdout.String())
|
||||
env.v.UI.SetQuiet(false)
|
||||
|
||||
// With blob A stored again nothing is missing.
|
||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||
env.stdout.Reset()
|
||||
require.NoError(t, env.v.RemoteInfo(true))
|
||||
|
||||
doc = nil
|
||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
|
||||
assert.InDelta(t, 0, doc["missing_blob_count"], 0)
|
||||
assert.InDelta(t, 0, doc["missing_blob_size"], 0)
|
||||
assert.NotContains(t, doc, "snapshots_missing_blobs")
|
||||
}
|
||||
@@ -349,6 +349,7 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
|
||||
env.addRemote(t, listLocalID, start)
|
||||
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
|
||||
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
|
||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||
|
||||
require.NoError(t, env.v.RemoteInfo(false))
|
||||
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
|
||||
|
||||
Reference in new issue
Block a user