From 402201eb8bf72b94cde7703c8af242eccd1ae85e Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 12 Oct 2026 23:55:43 +0000 Subject: [PATCH] Stop remote info counting a non-blob file under blobs/ (closes #295) remote info counted every key under blobs/ with four or more path parts as a blob, while prune skips a name that is not a blob hash. A stray file such as .DS_Store was reported as an orphaned blob, with advice to run prune, which then deleted nothing. remote info now takes its blob list from prune's listAllRemoteBlobs, so its blob and orphan figures, in the table and in --json, cover exactly what prune would consider. blobKeyParts moves to prune.go, its only remaining user. Model: opus-5-5 --- README.md | 4 +++- TODO.md | 8 +++++++ internal/vaultik/info.go | 27 +++++++---------------- internal/vaultik/prune.go | 4 ++++ internal/vaultik/remote_info_test.go | 33 ++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c1d87ad..1451d0a 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,9 @@ orphaned blob figures are reported as unknown; `--json` gives them as `null`, lists the remote key of each unreadable manifest in `unreadable_manifests` and counts the manifests under skipped names in `skipped_manifest_count`. An unreadable manifest also leaves its -snapshot's blob count and blob size unknown, `null` in `--json`. +snapshot's blob count and blob size unknown, `null` in `--json`. A file +under `blobs/` whose name is not a blob hash is not counted as a blob, +since `prune` never deletes it. * `--json`: Output as JSON **`remote nuke`**: Delete every snapshot's metadata and every blob from the diff --git a/TODO.md b/TODO.md index e124826..9160574 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,14 @@ the tag exists and is exercised; what is left is merging `next` to # Completed Steps +- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that + is not a blob as an orphaned blob + ([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray + file such as `.DS_Store` was reported as an orphaned blob and the + report advised running `prune`, which skips such a file and deleted + nothing. `remote info` now takes its blob list from the same listing + `prune` uses, in the table and in `--json`. + - 2026-10-13: Made `snapshot remove` fail for a snapshot that does not exist ([issue #293](https://git.eeqj.de/sneak/vaultik/issues/293)). The command never checked the ID: it reported the snapshot removed diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index e587e27..64656d8 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -167,10 +167,6 @@ const ( // metadataKeyParts is the minimum "/"-separated segment count of a // metadata object key (metadata//). metadataKeyParts = 3 - - // blobKeyParts is the minimum "/"-separated segment count of a blob - // object key (blobs///). - blobKeyParts = 4 ) // SnapshotMetadataInfo contains information about a single snapshot's metadata @@ -454,23 +450,16 @@ func (v *Vaultik) scanRemoteBlobStorage( v.stdoutf("Scanning blobs...\n") } - blobCh := v.Storage.ListStream(v.ctx, "blobs/") - allBlobs := make(map[string]int64) + // Use prune's listing, so a file under blobs/ that prune would never + // delete is not counted as a blob, and so not as an orphaned one. + allBlobs, err := v.listAllRemoteBlobs() + if err != nil { + return err + } - for obj := range blobCh { - if obj.Err != nil { - return fmt.Errorf("listing blobs: %w", obj.Err) - } - - parts := strings.Split(obj.Key, "/") - if len(parts) < blobKeyParts { - continue - } - - hash := parts[3] - allBlobs[hash] = obj.Size + for _, size := range allBlobs { result.TotalBlobCount++ - result.TotalBlobSize += obj.Size + result.TotalBlobSize += size } // A blob named only by a manifest that could not be read, or by one diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 6bd737a..31949c9 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -23,6 +23,10 @@ var errNukeRequiresForce = errors.New( // metadata. const metadataDirName = "metadata" +// blobKeyParts is the "/"-separated segment count of a blob object key +// (blobs///). +const blobKeyParts = 4 + // NukeRemote deletes every snapshot's metadata and every blob from remote // storage, along with any object an upload cut off part-way left under a // temporary `.partial` name. After this returns successfully the bucket diff --git a/internal/vaultik/remote_info_test.go b/internal/vaultik/remote_info_test.go index ceb5f11..76bac72 100644 --- a/internal/vaultik/remote_info_test.go +++ b/internal/vaultik/remote_info_test.go @@ -185,6 +185,39 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) { assert.NotContains(t, doc, "unreadable_manifests") } +// TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs checks that a file +// under blobs/ whose name is not a blob hash is not counted as a blob. +// prune never deletes such a file, so counting it as an orphaned blob +// would advise a prune that removes nothing. +func TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addBlob(t, env.store.testStorer, testBlobHashA) + require.NoError(t, env.store.Put(context.Background(), + "blobs/"+testBlobHashA[:2]+"/"+testBlobHashA[2:4]+"/.DS_Store", + bytes.NewReader([]byte("junk")))) + + require.NoError(t, env.v.RemoteInfo(false)) + + text := env.stdout.String() + assert.Contains(t, text, "Total blobs on remote: 1 (") + assert.Contains(t, text, "Orphaned (unreferenced): 0 (") + assert.NotContains(t, text, "vaultik prune") + + env.stdout.Reset() + require.NoError(t, env.v.RemoteInfo(true)) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) + assert.InDelta(t, 1, doc["total_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_size"], 0) +} + // TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that // a directory under metadata/ holding no manifest.json.zst, such as one // left by a backup interrupted before its manifest upload, leaves the