Stop remote info counting a non-blob file under blobs/ (closes #295)
check / check (push) Waiting to run

remote info counted every four-part key under blobs/ as a blob, while
prune skips a name that is not a blob hash. A stray file such as
.DS_Store was reported as an orphaned blob, with advice to run prune,
which then deleted nothing. remote info now takes its blob list from
prune's listAllRemoteBlobs, so its blob and orphan figures, in the
table and in --json, cover exactly what prune would consider.

blobKeyParts moves to prune.go, its only remaining user.

Model: opus-5-5
This commit is contained in:
2026-10-12 23:56:07 +00:00
parent c9a519c233
commit 72a30582e8
5 changed files with 56 additions and 20 deletions
+3 -1
View File
@@ -393,7 +393,9 @@ recipients, and local database statistics.
**`remote info`**: Show storage backend type and location plus detailed **`remote info`**: Show storage backend type and location plus detailed
remote storage inventory: per-snapshot metadata sizes, blob counts, and remote storage inventory: per-snapshot metadata sizes, blob counts, and
orphaned blob detection. A name under `metadata/` that is not a remote orphaned blob detection. A name under `metadata/` that is not a remote
key is skipped with a warning and is not printed. If a listed key is skipped with a warning and is not printed. A file under `blobs/`
whose name is not a blob hash is not counted as a blob, since `prune`
never deletes it. If a listed
`manifest.json.zst` cannot be read, or sits under a skipped name, the `manifest.json.zst` cannot be read, or sits under a skipped name, the
orphaned blob figures are reported as unknown; `--json` gives them as orphaned blob figures are reported as unknown; `--json` gives them as
`null`, lists the remote key of each unreadable manifest in `null`, lists the remote key of each unreadable manifest in
+8
View File
@@ -22,6 +22,14 @@ the tag exists and is exercised; what is left is merging `next` to
# Completed Steps # Completed Steps
- 2026-10-12: Stopped `remote info` counting a file under `blobs/` that
is not a blob as an orphaned blob
([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray
file such as `.DS_Store` was reported as an orphaned blob and the
report advised running `prune`, which skips such a file and deleted
nothing. `remote info` now takes its blob list from the same listing
`prune` uses, in the table and in `--json`.
- 2026-10-12: Rejected a retention duration longer than about 292 years - 2026-10-12: Rejected a retention duration longer than about 292 years
([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The ([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The
number given to `--older-than` or `--keep-newer-than` was multiplied number given to `--older-than` or `--keep-newer-than` was multiplied
+7 -18
View File
@@ -167,10 +167,6 @@ const (
// metadataKeyParts is the minimum "/"-separated segment count of a // metadataKeyParts is the minimum "/"-separated segment count of a
// metadata object key (metadata/<snapshot-id>/<filename>). // metadata object key (metadata/<snapshot-id>/<filename>).
metadataKeyParts = 3 metadataKeyParts = 3
// blobKeyParts is the minimum "/"-separated segment count of a blob
// object key (blobs/<aa>/<bb>/<hash>).
blobKeyParts = 4
) )
// SnapshotMetadataInfo contains information about a single snapshot's metadata // SnapshotMetadataInfo contains information about a single snapshot's metadata
@@ -454,23 +450,16 @@ func (v *Vaultik) scanRemoteBlobStorage(
v.stdoutf("Scanning blobs...\n") v.stdoutf("Scanning blobs...\n")
} }
blobCh := v.Storage.ListStream(v.ctx, "blobs/") // Use prune's listing, so a file under blobs/ that prune would never
allBlobs := make(map[string]int64) // delete is not counted as a blob, and so not as an orphaned one.
allBlobs, err := v.listAllRemoteBlobs()
for obj := range blobCh { if err != nil {
if obj.Err != nil { return err
return fmt.Errorf("listing blobs: %w", obj.Err)
} }
parts := strings.Split(obj.Key, "/") for _, size := range allBlobs {
if len(parts) < blobKeyParts {
continue
}
hash := parts[3]
allBlobs[hash] = obj.Size
result.TotalBlobCount++ result.TotalBlobCount++
result.TotalBlobSize += obj.Size result.TotalBlobSize += size
} }
// A blob named only by a manifest that could not be read, or by one // A blob named only by a manifest that could not be read, or by one
+4
View File
@@ -23,6 +23,10 @@ var errNukeRequiresForce = errors.New(
// metadata. // metadata.
const metadataDirName = "metadata" const metadataDirName = "metadata"
// blobKeyParts is the "/"-separated segment count of a blob object key
// (blobs/<aa>/<bb>/<hash>).
const blobKeyParts = 4
// NukeRemote deletes every snapshot's metadata and every blob from remote // NukeRemote deletes every snapshot's metadata and every blob from remote
// storage, along with any object an upload cut off part-way left under a // storage, along with any object an upload cut off part-way left under a
// temporary `.partial` name. After this returns successfully the bucket // temporary `.partial` name. After this returns successfully the bucket
+33
View File
@@ -185,6 +185,39 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
assert.NotContains(t, doc, "unreadable_manifests") assert.NotContains(t, doc, "unreadable_manifests")
} }
// TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs checks that a file
// under blobs/ whose name is not a blob hash is not counted as a blob.
// prune never deletes such a file, so counting it as an orphaned blob
// would advise a prune that removes nothing.
func TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
require.NoError(t, env.store.Put(context.Background(),
"blobs/"+testBlobHashA[:2]+"/"+testBlobHashA[2:4]+"/.DS_Store",
bytes.NewReader([]byte("junk"))))
require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String()
assert.Contains(t, text, "Total blobs on remote: 1 (")
assert.Contains(t, text, "Orphaned (unreferenced): 0 (")
assert.NotContains(t, text, "vaultik prune")
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["total_blob_count"], 0)
assert.InDelta(t, 0, doc["orphaned_blob_count"], 0)
assert.InDelta(t, 0, doc["orphaned_blob_size"], 0)
}
// TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that // TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that
// a directory under metadata/ holding no manifest.json.zst, such as one // a directory under metadata/ holding no manifest.json.zst, such as one
// left by a backup interrupted before its manifest upload, leaves the // left by a backup interrupted before its manifest upload, leaves the