diff --git a/README.md b/README.md index c1d87ad..1451d0a 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,9 @@ orphaned blob figures are reported as unknown; `--json` gives them as `null`, lists the remote key of each unreadable manifest in `unreadable_manifests` and counts the manifests under skipped names in `skipped_manifest_count`. An unreadable manifest also leaves its -snapshot's blob count and blob size unknown, `null` in `--json`. +snapshot's blob count and blob size unknown, `null` in `--json`. A file +under `blobs/` whose name is not a blob hash is not counted as a blob, +since `prune` never deletes it. * `--json`: Output as JSON **`remote nuke`**: Delete every snapshot's metadata and every blob from the diff --git a/TODO.md b/TODO.md index e124826..9160574 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,14 @@ the tag exists and is exercised; what is left is merging `next` to # Completed Steps +- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that + is not a blob as an orphaned blob + ([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray + file such as `.DS_Store` was reported as an orphaned blob and the + report advised running `prune`, which skips such a file and deleted + nothing. `remote info` now takes its blob list from the same listing + `prune` uses, in the table and in `--json`. + - 2026-10-13: Made `snapshot remove` fail for a snapshot that does not exist ([issue #293](https://git.eeqj.de/sneak/vaultik/issues/293)). The command never checked the ID: it reported the snapshot removed diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index e587e27..64656d8 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -167,10 +167,6 @@ const ( // metadataKeyParts is the minimum "/"-separated segment count of a // metadata object key (metadata//). metadataKeyParts = 3 - - // blobKeyParts is the minimum "/"-separated segment count of a blob - // object key (blobs///). - blobKeyParts = 4 ) // SnapshotMetadataInfo contains information about a single snapshot's metadata @@ -454,23 +450,16 @@ func (v *Vaultik) scanRemoteBlobStorage( v.stdoutf("Scanning blobs...\n") } - blobCh := v.Storage.ListStream(v.ctx, "blobs/") - allBlobs := make(map[string]int64) + // Use prune's listing, so a file under blobs/ that prune would never + // delete is not counted as a blob, and so not as an orphaned one. + allBlobs, err := v.listAllRemoteBlobs() + if err != nil { + return err + } - for obj := range blobCh { - if obj.Err != nil { - return fmt.Errorf("listing blobs: %w", obj.Err) - } - - parts := strings.Split(obj.Key, "/") - if len(parts) < blobKeyParts { - continue - } - - hash := parts[3] - allBlobs[hash] = obj.Size + for _, size := range allBlobs { result.TotalBlobCount++ - result.TotalBlobSize += obj.Size + result.TotalBlobSize += size } // A blob named only by a manifest that could not be read, or by one diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 6bd737a..31949c9 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -23,6 +23,10 @@ var errNukeRequiresForce = errors.New( // metadata. const metadataDirName = "metadata" +// blobKeyParts is the "/"-separated segment count of a blob object key +// (blobs///). +const blobKeyParts = 4 + // NukeRemote deletes every snapshot's metadata and every blob from remote // storage, along with any object an upload cut off part-way left under a // temporary `.partial` name. After this returns successfully the bucket diff --git a/internal/vaultik/remote_info_test.go b/internal/vaultik/remote_info_test.go index ceb5f11..76bac72 100644 --- a/internal/vaultik/remote_info_test.go +++ b/internal/vaultik/remote_info_test.go @@ -185,6 +185,39 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) { assert.NotContains(t, doc, "unreadable_manifests") } +// TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs checks that a file +// under blobs/ whose name is not a blob hash is not counted as a blob. +// prune never deletes such a file, so counting it as an orphaned blob +// would advise a prune that removes nothing. +func TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) + addBlob(t, env.store.testStorer, testBlobHashA) + require.NoError(t, env.store.Put(context.Background(), + "blobs/"+testBlobHashA[:2]+"/"+testBlobHashA[2:4]+"/.DS_Store", + bytes.NewReader([]byte("junk")))) + + require.NoError(t, env.v.RemoteInfo(false)) + + text := env.stdout.String() + assert.Contains(t, text, "Total blobs on remote: 1 (") + assert.Contains(t, text, "Orphaned (unreferenced): 0 (") + assert.NotContains(t, text, "vaultik prune") + + env.stdout.Reset() + require.NoError(t, env.v.RemoteInfo(true)) + + var doc map[string]any + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) + assert.InDelta(t, 1, doc["total_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_count"], 0) + assert.InDelta(t, 0, doc["orphaned_blob_size"], 0) +} + // TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that // a directory under metadata/ holding no manifest.json.zst, such as one // left by a backup interrupted before its manifest upload, leaves the