diff --git a/README.md b/README.md index 8d9e77e..cbc4df1 100644 --- a/README.md +++ b/README.md @@ -554,6 +554,12 @@ count and blob size unknown, `null` in `--json`. A file under `blobs/` whose name is not a blob hash is not counted as a blob, since `prune` never deletes it. +A blob that a manifest lists but the destination does not hold is reported as +missing, with a count and size. The report names each snapshot that lists a +missing blob, since that snapshot cannot be restored. The command then exits +non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and +`snapshots_missing_blobs`. + - `--json`: Output as JSON **`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the diff --git a/internal/cli/remote.go b/internal/cli/remote.go index f46bb26..28cc2d8 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -75,7 +75,9 @@ func newRemoteInfoCommand() *cobra.Command { - Size of all snapshot metadata (per snapshot and total) - Count and total size of all blobs - Count and size of referenced blobs (from all manifests) -- Count and size of orphaned blobs (not referenced by any manifest)`, +- Count and size of orphaned blobs (not referenced by any manifest) +- Count and size of referenced blobs missing from remote storage, and the + snapshots that list them; the command exits non-zero when any are missing`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index c4cc18c..0e3b8c1 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -9,6 +9,7 @@ import ( "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" ) // ShowInfo displays system and configuration information @@ -187,6 +188,9 @@ type SnapshotMetadataInfo struct { // backup interrupted before its manifest upload leaves a directory // without one, which prune does not treat as a snapshot. hasManifest bool + + // Hashes of the blobs the snapshot's manifest lists. + blobHashes []string } // RemoteInfoResult contains all remote storage information @@ -216,6 +220,13 @@ type RemoteInfoResult struct { OrphanedBlobCount *int `json:"orphaned_blob_count"` OrphanedBlobSize *int64 `json:"orphaned_blob_size"` + // Blobs that a manifest lists but the destination does not hold, + // and the remote key of each snapshot whose manifest lists one. + // Those snapshots cannot be restored. + MissingBlobCount int `json:"missing_blob_count"` + MissingBlobSize int64 `json:"missing_blob_size"` + SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"` + // Remote key of each snapshot whose manifest could not be read UnreadableManifests []string `json:"unreadable_manifests,omitempty"` @@ -283,19 +294,30 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { "snapshots", result.TotalMetadataCount, "total_blobs", result.TotalBlobCount, "referenced_blobs", result.ReferencedBlobCount, + "missing_blobs", result.MissingBlobCount, "unreadable_manifests", len(result.UnreadableManifests)) if jsonOutput { enc := json.NewEncoder(v.Stdout) enc.SetIndent("", " ") - return enc.Encode(result) + err = enc.Encode(result) + if err != nil { + return fmt.Errorf("encoding JSON: %w", err) + } } if showText { v.printRemoteInfoTable(result) } + // The report is written first, so the exit status adds to it. + if result.MissingBlobCount > 0 { + return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+ + "%d snapshot(s) cannot be restored", errProblemsInStorage, + result.MissingBlobCount, len(result.SnapshotsMissingBlobs)) + } + return nil } @@ -405,6 +427,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests( for _, blob := range manifest.Blobs { referencedBlobs[blob.Hash] = blob.CompressedSize blobsSize += blob.CompressedSize + info.blobHashes = append(info.blobHashes, blob.Hash) } info.BlobCount = &blobCount @@ -462,6 +485,8 @@ func (v *Vaultik) scanRemoteBlobStorage( result.TotalBlobSize += blob.Size } + countMissingBlobs(result, referencedBlobs, allBlobs) + // A blob named only by a manifest that could not be read, or by one // under a skipped name, would be counted as orphaned, so the orphan // figures stay unknown. @@ -487,6 +512,33 @@ func (v *Vaultik) scanRemoteBlobStorage( return nil } +// countMissingBlobs records in result the referenced blobs that +// allBlobs, the listing of the destination, does not hold, and the +// snapshots whose manifests list one of them. It reads the snapshots +// from result.Snapshots, so populateRemoteInfoResult must run first. +func countMissingBlobs( + result *RemoteInfoResult, referencedBlobs map[string]int64, + allBlobs map[string]storage.ObjectInfo, +) { + for hash, size := range referencedBlobs { + if _, present := allBlobs[hash]; !present { + result.MissingBlobCount++ + result.MissingBlobSize += size + } + } + + for _, info := range result.Snapshots { + for _, hash := range info.blobHashes { + if _, present := allBlobs[hash]; !present { + result.SnapshotsMissingBlobs = append( + result.SnapshotsMissingBlobs, info.SnapshotID) + + break + } + } + } +} + // printRemoteInfoTable renders the human-readable remote info output func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { const ( @@ -539,6 +591,12 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { "", "", ubytes(result.TotalMetadataSize)) } + v.printBlobStorageSummary(result) +} + +// printBlobStorageSummary renders the blob storage section that ends +// the human-readable remote info output. +func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) { v.stdoutf("\n=== Blob Storage ===\n") v.stdoutf("Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), @@ -546,22 +604,31 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { v.stdoutf("Referenced by snapshots: %s (%s)\n", humanize.Comma(int64(result.ReferencedBlobCount)), ubytes(result.ReferencedBlobSize)) + v.stdoutf("Missing from remote: %s (%s)\n", + humanize.Comma(int64(result.MissingBlobCount)), + ubytes(result.MissingBlobSize)) if result.OrphanedBlobCount == nil { v.stdoutf("Orphaned (unreferenced): unknown "+ "(%d manifest(s) could not be read, "+ "%d manifest(s) under a non-conforming name skipped)\n", len(result.UnreadableManifests), result.SkippedManifestCount) + } else { + v.stdoutf("Orphaned (unreferenced): %s (%s)\n", + humanize.Comma(int64(*result.OrphanedBlobCount)), + ubytes(*result.OrphanedBlobSize)) - return + if *result.OrphanedBlobCount > 0 { + v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") + } } - v.stdoutf("Orphaned (unreferenced): %s (%s)\n", - humanize.Comma(int64(*result.OrphanedBlobCount)), - ubytes(*result.OrphanedBlobSize)) + if len(result.SnapshotsMissingBlobs) > 0 { + v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n") - if *result.OrphanedBlobCount > 0 { - v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") + for _, id := range result.SnapshotsMissingBlobs { + v.stdoutf(" %s\n", id) + } } } diff --git a/internal/vaultik/remote_info_test.go b/internal/vaultik/remote_info_test.go index 5d68a03..e01ab44 100644 --- a/internal/vaultik/remote_info_test.go +++ b/internal/vaultik/remote_info_test.go @@ -83,6 +83,7 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) { env := newListEnv(t) readableKey := env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addBlob(t, env.store.testStorer, testBlobHashA) unreadableKey := snapshot.RemoteSnapshotKey(listLocalID) require.NoError(t, env.store.Put(context.Background(), @@ -270,6 +271,7 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) { env := newListEnv(t) env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addBlob(t, env.store.testStorer, testBlobHashA) const statusLine = ui.Marker + " Scanning snapshot metadata...\n" @@ -281,3 +283,61 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) { require.NoError(t, env.v.RemoteInfo(false)) assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata") } + +// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a +// manifest lists but the destination does not hold is counted as +// missing, that the snapshot listing it is named, and that remote info +// then fails after writing its report. +func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + + // The first snapshot lists blob A, which is not stored. The second + // lists blob B, which is. + brokenKey := env.addRemote(t, listRemoteID, + time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB}) + addBlob(t, env.store.testStorer, testBlobHashB) + + const wantError = "1 blob(s) that snapshots list are missing; " + + "1 snapshot(s) cannot be restored" + + err := env.v.RemoteInfo(false) + require.ErrorContains(t, err, wantError) + + text := env.stdout.String() + assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)") + assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+ + " "+brokenKey+"\n") + assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n") + + env.stdout.Reset() + require.ErrorContains(t, env.v.RemoteInfo(true), wantError) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) + assert.InDelta(t, 1, doc["missing_blob_count"], 0) + assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0) + assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"]) + + // --quiet drops the report, not the failure. + env.stdout.Reset() + env.v.UI.SetQuiet(true) + require.ErrorContains(t, env.v.RemoteInfo(false), wantError) + assert.Empty(t, env.stdout.String()) + env.v.UI.SetQuiet(false) + + // With blob A stored again nothing is missing. + addBlob(t, env.store.testStorer, testBlobHashA) + env.stdout.Reset() + require.NoError(t, env.v.RemoteInfo(true)) + + doc = nil + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) + assert.InDelta(t, 0, doc["missing_blob_count"], 0) + assert.InDelta(t, 0, doc["missing_blob_size"], 0) + assert.NotContains(t, doc, "snapshots_missing_blobs") +} diff --git a/internal/vaultik/snapshot_list_test.go b/internal/vaultik/snapshot_list_test.go index 0e468a0..86dcbaf 100644 --- a/internal/vaultik/snapshot_list_test.go +++ b/internal/vaultik/snapshot_list_test.go @@ -349,6 +349,7 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) { env.addRemote(t, listLocalID, start) require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts( context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes)) + addBlob(t, env.store.testStorer, testBlobHashA) require.NoError(t, env.v.RemoteInfo(false)) assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),