Hash snapshot IDs at the storage boundary; make snapshot list resilient

Two related changes, both addressing leakage and brittleness around
the public bytes the destination store sees.

First, every remote storage path that previously embedded a human
snapshot ID (e.g. metadata/heraklion_berlin.sneak.fs.photos.2026.
catalog_2026-06-24T07:00:15Z/...) now uses the hashed remote key:

  RemoteSnapshotKey(id) = hex(SHA256(SHA256("vaultik|" + id)))

Applied at:

  * uploadSnapshotArtifacts (snapshot create write path)
  * the manifest.json.zst snapshot_id field — manifest is
    unencrypted, so the human ID would otherwise be readable to
    anyone with bucket-list permission
  * cleanupIncompleteSnapshots metadata-existence probe
  * snapshot restore / verify (downloadSnapshotDB,
    loadVerificationData)
  * downloadManifestByKey, deleteRemoteSnapshotByKey
  * CleanupLocalSnapshots reconciliation
  * the locally-driven removal paths (RemoveSnapshot,
    RemoveAllSnapshots, confirmAndExecutePurge)

The local index database keeps human IDs everywhere — the hash is a
boundary translation, not a rename. A directory listing of the
backup destination now looks like
"metadata/<64-hex>/{db.zst.age,manifest.json.zst}" with no host,
snapshot-name, or timestamp information visible.

Second, snapshot list no longer fails just because remote storage is
unreachable, and only consults the remote when the local machine can
plausibly decrypt:

  * Listing is always driven by the local index database — that's
    what holds the human IDs, timestamps, and per-snapshot stats
    that the table actually shows.
  * If no age secret key is configured, we skip remote listing
    entirely (the box is treated as a write-only backup machine —
    there's no value showing it remote-only keys it could never
    restore).
  * If a key IS configured, we try the remote listing; failures
    (volume unmounted, permission denied, network error) downgrade
    to a warning instead of aborting the command.
  * When the remote listing succeeds, we cross-reference by hashing
    each local human ID and diffing against the returned key set.
    Local-only snapshots get the existing "stale local record"
    cleanup hint; remote-only keys are surfaced as a single
    "NOTE: N remote snapshot(s) found in backup destination store
    but not in local database" line.

FileStorer construction also no longer does an eager mkdir — the
basePath is recorded and the directory is created lazily on first
write. A missing or unmounted destination during `snapshot list`
should NOT block the command, and now it doesn't.

RemoveAllSnapshots is rewritten to drive deletion from the local
index instead of from a remote listing, hashing each local ID to
find the corresponding remote key. Orphan remote keys (no matching
local snapshot) are handled separately and only deleted when
--remote is set. Existing tests are updated to hash storage paths
through the new RemoteSnapshotKey helper.

The hash format is a hard pre-1.0 break: existing remote snapshots
written under the human-ID path scheme are no longer readable; they
need to be either re-uploaded under the new scheme or manually
renamed. There is no fallback path; matching the project policy of
"no migrations pre-1.0."
This commit is contained in:
2026-06-26 01:54:35 +02:00
parent a84b911155
commit fd759a921a
9 changed files with 328 additions and 237 deletions
+23 -9
View File
@@ -132,7 +132,9 @@ func (s *testStorer) Info() storage.StorageInfo {
}
}
// addManifest creates a compressed manifest in storage
// addManifest creates a compressed manifest in storage at the same
// hashed path the production code uses. snapshotID is the human ID;
// the storage path uses RemoteSnapshotKey(id).
func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes []string) {
t.Helper()
@@ -144,8 +146,9 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes
}
}
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
manifest := &snapshot.Manifest{
SnapshotID: snapshotID,
SnapshotID: remoteKey,
BlobCount: len(blobs),
Blobs: blobs,
}
@@ -153,11 +156,19 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes
data, err := snapshot.EncodeManifest(manifest, 3)
require.NoError(t, err)
key := "metadata/" + snapshotID + "/manifest.json.zst"
key := "metadata/" + remoteKey + "/manifest.json.zst"
err = store.Put(context.Background(), key, bytes.NewReader(data))
require.NoError(t, err)
}
// remoteKeyPath returns the storage-relative path to a snapshot's
// metadata directory or manifest under the hashed remote-key scheme.
// Tests use this in hasKey/asserts to avoid scattering RemoteSnapshotKey
// calls throughout.
func remoteKeyPath(snapshotID, suffix string) string {
return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + suffix
}
// addBlob adds a fake blob to storage
func addBlob(t *testing.T, store *testStorer, hash string) {
t.Helper()
@@ -198,7 +209,7 @@ func TestRemoveSnapshot_LocalOnly(t *testing.T) {
// Blobs should NOT be deleted (that's what prune is for)
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
// Remote metadata should NOT be deleted (no --remote flag)
assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst")))
// Verify output
assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database")
@@ -225,7 +236,7 @@ func TestRemoveSnapshot_WithRemote(t *testing.T) {
// Blobs should NOT be deleted
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
// Remote metadata SHOULD be deleted
assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst")))
// Verify output mentions prune
assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database")
@@ -255,7 +266,7 @@ func TestRemoveSnapshot_DryRun(t *testing.T) {
// Nothing should be deleted
assert.Equal(t, initialCount, store.keyCount())
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst")))
// Verify dry run message
assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
@@ -299,8 +310,8 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) {
// Blobs should NOT be deleted
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
// Remote metadata SHOULD be deleted
assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
assert.False(t, store.hasKey("metadata/snapshot-002/manifest.json.zst"))
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst")))
assert.False(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst")))
// Verify output
assert.Contains(t, tv.Stdout.String(), "Removed 2 snapshot(s)")
@@ -318,7 +329,10 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) {
tv := vaultik.NewForTesting(store)
opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true}
// --remote is required to enumerate orphan remote keys; without
// it, RemoveAll only acts on local snapshots, and NewForTesting
// has no local DB.
opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true, Remote: true}
result, err := tv.RemoveAllSnapshots(opts)
require.NoError(t, err)