From 72a30582e8b16dd9d431990586f39abdd4882c2b Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 12 Oct 2026 23:55:43 +0000 Subject: [PATCH] Stop remote info counting a non-blob file under blobs/ (closes #295) remote info counted every four-part key under blobs/ as a blob, while prune skips a name that is not a blob hash. A stray file such as .DS_Store was reported as an orphaned blob, with advice to run prune, which then deleted nothing. remote info now takes its blob list from prune's listAllRemoteBlobs, so its blob and orphan figures, in the table and in --json, cover exactly what prune would consider. blobKeyParts moves to prune.go, its only remaining user. Model: opus-5-5 --- README.md | 4 +++- TODO.md | 8 +++++++ internal/vaultik/info.go | 27 +++++++---------------- internal/vaultik/prune.go | 4 ++++ internal/vaultik/remote_info_test.go | 33 ++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5c58a8f..7def62f 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,9 @@ recipients, and local database statistics. **`remote info`**: Show storage backend type and location plus detailed remote storage inventory: per-snapshot metadata sizes, blob counts, and orphaned blob detection. A name under `metadata/` that is not a remote -key is skipped with a warning and is not printed. If a listed +key is skipped with a warning and is not printed. A file under `blobs/` +whose name is not a blob hash is not counted as a blob, since `prune` +never deletes it. If a listed `manifest.json.zst` cannot be read, or sits under a skipped name, the orphaned blob figures are reported as unknown; `--json` gives them as `null`, lists the remote key of each unreadable manifest in diff --git a/TODO.md b/TODO.md index 018c80c..2675416 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,14 @@ the tag exists and is exercised; what is left is merging `next` to # Completed Steps +- 2026-10-12: Stopped `remote info` counting a file under `blobs/` that + is not a blob as an orphaned blob + ([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray + file such as `.DS_Store` was reported as an orphaned blob and the + report advised running `prune`, which skips such a file and deleted + nothing. `remote info` now takes its blob list from the same listing + `prune` uses, in the table and in `--json`. + - 2026-10-12: Rejected a retention duration longer than about 292 years ([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The number given to `--older-than` or `--keep-newer-than` was multiplied diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index e587e27..64656d8 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -167,10 +167,6 @@ const ( // metadataKeyParts is the minimum "/"-separated segment count of a // metadata object key (metadata//). metadataKeyParts = 3 - - // blobKeyParts is the minimum "/"-separated segment count of a blob - // object key (blobs///). - blobKeyParts = 4 ) // SnapshotMetadataInfo contains information about a single snapshot's metadata @@ -454,23 +450,16 @@ func (v *Vaultik) scanRemoteBlobStorage( v.stdoutf("Scanning blobs...\n") } - blobCh := v.Storage.ListStream(v.ctx, "blobs/") - allBlobs := make(map[string]int64) + // Use prune's listing, so a file under blobs/ that prune would never + // delete is not counted as a blob, and so not as an orphaned one. + allBlobs, err := v.listAllRemoteBlobs() + if err != nil { + return err + } - for obj := range blobCh { - if obj.Err != nil { - return fmt.Errorf("listing blobs: %w", obj.Err) - } - - parts := strings.Split(obj.Key, "/") - if len(parts) < blobKeyParts { - continue - } - - hash := parts[3] - allBlobs[hash] = obj.Size + for _, size := range allBlobs { result.TotalBlobCount++ - result.TotalBlobSize += obj.Size + result.TotalBlobSize += size } // A blob named only by a manifest that could not be read, or by one diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 6bd737a..31949c9 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -23,6 +23,10 @@ var errNukeRequiresForce = errors.New( // metadata. const metadataDirName = "metadata" +// blobKeyParts is the "/"-separated segment count of a blob object key +// (blobs///). +const blobKeyParts = 4 + // NukeRemote deletes every snapshot's metadata and every blob from remote // storage, along with any object an upload cut off part-way left under a // temporary `.partial` name. After this returns successfully the bucket diff --git a/internal/vaultik/remote_info_test.go b/internal/vaultik/remote_info_test.go index ceb5f11..76bac72 100644 --- a/internal/vaultik/remote_info_test.go +++ b/internal/vaultik/remote_info_test.go @@ -185,6 +185,39 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) { assert.NotContains(t, doc, "unreadable_manifests") } +// TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs checks that a file +// under blobs/ whose name is not a blob hash is not counted as a blob. +// prune never deletes such a file, so counting it as an orphaned blob +// would advise a prune that removes nothing. +func TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addBlob(t, env.store.testStorer, testBlobHashA) + require.NoError(t, env.store.Put(context.Background(), + "blobs/"+testBlobHashA[:2]+"/"+testBlobHashA[2:4]+"/.DS_Store", + bytes.NewReader([]byte("junk")))) + + require.NoError(t, env.v.RemoteInfo(false)) + + text := env.stdout.String() + assert.Contains(t, text, "Total blobs on remote: 1 (") + assert.Contains(t, text, "Orphaned (unreferenced): 0 (") + assert.NotContains(t, text, "vaultik prune") + + env.stdout.Reset() + require.NoError(t, env.v.RemoteInfo(true)) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) + assert.InDelta(t, 1, doc["total_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_size"], 0) +} + // TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that // a directory under metadata/ holding no manifest.json.zst, such as one // left by a backup interrupted before its manifest upload, leaves the