diff --git a/internal/snapshot/remotekey.go b/internal/snapshot/remotekey.go new file mode 100644 index 0000000..031b9ce --- /dev/null +++ b/internal/snapshot/remotekey.go @@ -0,0 +1,40 @@ +package snapshot + +import ( + "crypto/sha256" + "encoding/hex" +) + +// remoteKeyPrefix is mixed into the snapshot ID hash so the resulting +// hex digest is domain-separated from any other "double SHA256 of a +// string" identifier the user might also use. Keeping this stable is a +// hard compatibility requirement: changing it invalidates every +// existing snapshot's remote storage path. +const remoteKeyPrefix = "vaultik|" + +// RemoteSnapshotKey returns the storage-side identifier for a snapshot +// given its human snapshot ID. It is hex(SHA256(SHA256(prefix + id))). +// The two SHA256 rounds match Bitcoin's "hash256" convention so the +// output looks like a 64-character hex blob with no exploitable +// structure visible to a remote observer. +// +// We use this in three places: +// +// - the "metadata//..." subdirectory on the storage +// backend so a directory listing of the bucket / file:// dest +// doesn't reveal hostnames, configured snapshot names, or backup +// timestamps; +// - the `snapshot_id` field of the unencrypted manifest.json.zst +// for the same reason; +// - any code path that needs to translate a known local snapshot ID +// into the path it would occupy on remote storage. +// +// The human ID stays the user-visible handle everywhere else — local +// database joins, CLI arguments, summary lines, log fields — because +// it's never written to the public bytes once this function gates +// every storage-path construction. +func RemoteSnapshotKey(snapshotID string) string { + first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID)) + second := sha256.Sum256(first[:]) + return hex.EncodeToString(second[:]) +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 431d5bf..f3f26d6 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -314,10 +314,17 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot return finalData, tempDBPath, nil } -// uploadSnapshotArtifacts uploads the database backup and blob manifest to S3 +// uploadSnapshotArtifacts uploads the database backup and blob manifest +// to remote storage at metadata//, where remote-key is the +// double-SHA256 derivation of the snapshot ID (see RemoteSnapshotKey). +// We never write the human-readable snapshot ID into any unencrypted +// part of remote storage so a listing of the destination bucket leaks +// no host, configuration, or scheduling information. func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error { + remoteKey := RemoteSnapshotKey(snapshotID) + // Upload database backup (compressed and encrypted) - dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) + dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) dbUploadStart := time.Now() if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil { @@ -332,7 +339,7 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot "speed", humanize.SI(dbUploadSpeed, "bps")) // Upload blob manifest (compressed only, not encrypted) - manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) + manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) manifestUploadStart := time.Now() if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil { return fmt.Errorf("uploading blob manifest: %w", err) @@ -607,9 +614,11 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri } } - // Create manifest + // Create manifest. SnapshotID in the unencrypted manifest is the + // double-SHA256 remote key, not the human ID, so the public bytes + // don't reveal hostname/snapshot-name/timestamp metadata. manifest := &Manifest{ - SnapshotID: snapshotID, + SnapshotID: RemoteSnapshotKey(snapshotID), Timestamp: time.Now().UTC().Format(time.RFC3339), BlobCount: len(blobs), TotalCompressedSize: totalCompressedSize, @@ -680,8 +689,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // Check each incomplete snapshot for metadata in storage for _, snapshot := range incompleteSnapshots { - // Check if metadata exists in storage - metadataKey := fmt.Sprintf("metadata/%s/db.zst", snapshot.ID) + // Check if metadata exists in storage (paths use the hashed + // remote key so we don't leak host info to the listing). + metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) _, err := sm.storage.Stat(ctx, metadataKey) if err != nil { diff --git a/internal/storage/file.go b/internal/storage/file.go index deb8c0a..1c1fcf6 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -19,15 +19,20 @@ type FileStorer struct { } // NewFileStorer creates a new filesystem storage backend. -// The basePath directory will be created if it doesn't exist. -// Uses the real OS filesystem by default; call SetFilesystem to override for testing. +// +// Construction is intentionally cheap and does not touch the filesystem. +// The basePath is recorded; the directory is created lazily on first +// write. Reads (Get/Stat/List) tolerate a missing basePath — a missing +// or unmounted destination during `snapshot list` should NOT block the +// command, it should degrade to "no remote snapshots reachable" with a +// warning. Write operations (Put/PutWithProgress) call MkdirAll for the +// per-blob parent directory, which also covers basePath on first use. +// +// Uses the real OS filesystem by default; call SetFilesystem to +// override for testing. func NewFileStorer(basePath string) (*FileStorer, error) { - fs := afero.NewOsFs() - if err := fs.MkdirAll(basePath, 0755); err != nil { - return nil, fmt.Errorf("file:// storage: cannot create or access %s: %w (check that the volume is mounted and writable)", basePath, err) - } return &FileStorer{ - fs: fs, + fs: afero.NewOsFs(), basePath: basePath, }, nil } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index 5fd4b2c..945d448 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -651,10 +651,12 @@ func TestEndToEndFileStorage(t *testing.T) { require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) // Verify the backup actually landed on disk under blobs/ and metadata/. + // The metadata subdirectory uses the hashed remote key, not the human + // snapshot ID, so the on-disk structure doesn't leak hostname/name/time. blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs")) require.NoError(t, err) require.True(t, blobInfo.IsDir()) - metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshotID)) + metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) require.NoError(t, err) require.True(t, metaInfo.IsDir()) diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index ef86027..90f503e 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -123,20 +123,22 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { // collectReferencedBlobs downloads all manifests and returns the set of referenced blob hashes func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { log.Info("Listing remote snapshots") - snapshotIDs, err := v.listUniqueSnapshotIDs() + // IDs returned by listUniqueSnapshotIDs are remote keys (hashed + // subdirectories under metadata/), not human snapshot IDs. + remoteKeys, err := v.listUniqueSnapshotIDs() if err != nil { - return nil, fmt.Errorf("listing snapshot IDs: %w", err) + return nil, fmt.Errorf("listing snapshot keys: %w", err) } - log.Info("Found manifests in remote storage", "count", len(snapshotIDs)) + log.Info("Found manifests in remote storage", "count", len(remoteKeys)) allBlobsReferenced := make(map[string]bool) manifestCount := 0 - for _, snapshotID := range snapshotIDs { - log.Debug("Processing manifest", "snapshot_id", snapshotID) - manifest, err := v.downloadManifest(snapshotID) + for _, remoteKey := range remoteKeys { + log.Debug("Processing manifest", "remote_key", remoteKey) + manifest, err := v.downloadManifestByKey(remoteKey) if err != nil { - log.Error("Failed to download manifest", "snapshot_id", snapshotID, "error", err) + log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err) continue } for _, blob := range manifest.Blobs { diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index 6db8d66..8164c81 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -132,7 +132,9 @@ func (s *testStorer) Info() storage.StorageInfo { } } -// addManifest creates a compressed manifest in storage +// addManifest creates a compressed manifest in storage at the same +// hashed path the production code uses. snapshotID is the human ID; +// the storage path uses RemoteSnapshotKey(id). func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes []string) { t.Helper() @@ -144,8 +146,9 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes } } + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) manifest := &snapshot.Manifest{ - SnapshotID: snapshotID, + SnapshotID: remoteKey, BlobCount: len(blobs), Blobs: blobs, } @@ -153,11 +156,19 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes data, err := snapshot.EncodeManifest(manifest, 3) require.NoError(t, err) - key := "metadata/" + snapshotID + "/manifest.json.zst" + key := "metadata/" + remoteKey + "/manifest.json.zst" err = store.Put(context.Background(), key, bytes.NewReader(data)) require.NoError(t, err) } +// remoteKeyPath returns the storage-relative path to a snapshot's +// metadata directory or manifest under the hashed remote-key scheme. +// Tests use this in hasKey/asserts to avoid scattering RemoteSnapshotKey +// calls throughout. +func remoteKeyPath(snapshotID, suffix string) string { + return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + suffix +} + // addBlob adds a fake blob to storage func addBlob(t *testing.T, store *testStorer, hash string) { t.Helper() @@ -198,7 +209,7 @@ func TestRemoveSnapshot_LocalOnly(t *testing.T) { // Blobs should NOT be deleted (that's what prune is for) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) // Remote metadata should NOT be deleted (no --remote flag) - assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst")) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) // Verify output assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database") @@ -225,7 +236,7 @@ func TestRemoveSnapshot_WithRemote(t *testing.T) { // Blobs should NOT be deleted assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) // Remote metadata SHOULD be deleted - assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst")) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) // Verify output mentions prune assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database") @@ -255,7 +266,7 @@ func TestRemoveSnapshot_DryRun(t *testing.T) { // Nothing should be deleted assert.Equal(t, initialCount, store.keyCount()) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst")) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) // Verify dry run message assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]") @@ -299,8 +310,8 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { // Blobs should NOT be deleted assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) // Remote metadata SHOULD be deleted - assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst")) - assert.False(t, store.hasKey("metadata/snapshot-002/manifest.json.zst")) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst"))) // Verify output assert.Contains(t, tv.Stdout.String(), "Removed 2 snapshot(s)") @@ -318,7 +329,10 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) { tv := vaultik.NewForTesting(store) - opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true} + // --remote is required to enumerate orphan remote keys; without + // it, RemoveAll only acts on local snapshots, and NewForTesting + // has no local DB. + opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true, Remote: true} result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index e7ac019..3499e13 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -18,6 +18,7 @@ import ( "sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/types" ) @@ -394,10 +395,12 @@ func (v *Vaultik) handleRestoreVerification( return nil } -// downloadSnapshotDB downloads and decrypts the snapshot metadata database +// downloadSnapshotDB downloads and decrypts the snapshot metadata +// database. The snapshotID is the human ID; we hash it to the remote +// key for the storage path. func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (*database.DB, error) { // Download encrypted database from storage - dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) + dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshot.RemoteSnapshotKey(snapshotID)) reader, err := v.Storage.Get(v.ctx, dbKey) if err != nil { diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index ca3fcb7..2c6f56a 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -8,16 +8,13 @@ import ( "regexp" "sort" "strings" - "sync" "text/tabwriter" "time" "github.com/dustin/go-humanize" - "golang.org/x/sync/errgroup" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" - "sneak.berlin/go/vaultik/internal/types" ) // SnapshotCreateOptions contains options for the snapshot create command @@ -383,25 +380,43 @@ func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, unc return compressed, uncompressed } -// ListSnapshots lists all snapshots +// ListSnapshots prints the table of snapshots, plus any reconciliation +// warnings/notes between the local index and the backup destination +// store. +// +// The local index database is always the primary source for the +// table — it has the human snapshot IDs, timestamps, and per-snapshot +// stats. +// +// If an age secret key is configured AND remote listing succeeds, we +// cross-reference: any local snapshot whose hashed key isn't visible +// remotely gets a "local-only" cleanup hint, and any remote key that +// doesn't correspond to a known local snapshot gets reported in a +// NOTE. +// +// If no age key is set the local machine is assumed write-only +// (backup-only), so we skip remote listing entirely — there's no +// value showing keys the user couldn't restore anyway. +// +// If remote listing fails (unmounted volume, permission denied, +// network), we degrade to local-only with a warning. List never +// fails just because the destination is unreachable. func (v *Vaultik) ListSnapshots(jsonOutput bool) error { log.Info("Listing snapshots") - remoteSnapshots, err := v.listRemoteSnapshotIDs() + + localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { - return err + return fmt.Errorf("listing local snapshots: %w", err) } - localSnapshotMap, err := v.reconcileLocalWithRemote(remoteSnapshots) - if err != nil { - return err + snapshots := make([]SnapshotInfo, 0, len(localSnaps)) + for _, ls := range localSnaps { + if ls.CompletedAt == nil { + continue + } + snapshots = append(snapshots, v.snapshotInfoFromLocal(ls)) } - snapshots, err := v.buildSnapshotInfoList(remoteSnapshots, localSnapshotMap) - if err != nil { - return err - } - - // Sort by timestamp (newest first) sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) @@ -416,173 +431,85 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { return err } - // Warn about local snapshots that don't exist in remote storage. - var stale []string - for id := range localSnapshotMap { - if !remoteSnapshots[id] { - stale = append(stale, id) + if v.Config.AgeSecretKey == "" { + return nil + } + + remoteKeys, err := v.listAllRemoteSnapshotKeys() + if err != nil { + v.UI.Warning("Could not list backup destination store: %v.", err) + return nil + } + + localKeys := make(map[string]string, len(localSnaps)) + for _, ls := range localSnaps { + if ls.CompletedAt == nil { + continue + } + localKeys[snapshot.RemoteSnapshotKey(ls.ID.String())] = ls.ID.String() + } + remoteSet := make(map[string]bool, len(remoteKeys)) + for _, k := range remoteKeys { + remoteSet[k] = true + } + + var localOnly []string + for key, humanID := range localKeys { + if !remoteSet[key] { + localOnly = append(localOnly, humanID) } } - if len(stale) > 0 { - v.UI.Warning("%d local snapshot record(s) not found in backup destination store:", len(stale)) - for _, id := range stale { + var remoteOnlyCount int + for key := range remoteSet { + if _, ok := localKeys[key]; !ok { + remoteOnlyCount++ + } + } + + if len(localOnly) > 0 { + v.UI.Warning("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + for _, id := range localOnly { v.UI.Info("%s", v.UI.Snapshot(id)) } v.UI.Info("Run 'vaultik snapshot cleanup' to remove stale local records.") } + if remoteOnlyCount > 0 { + v.UI.Notice("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) + } return nil } -// listRemoteSnapshotIDs returns a set of snapshot IDs found in remote storage -func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) { - remoteSnapshots := make(map[string]bool) - objectCh := v.Storage.ListStream(v.ctx, "metadata/") +// snapshotInfoFromLocal builds a SnapshotInfo row from a local snapshot +// record. Failures from any per-snapshot stat query degrade that +// column to its snapshot-row fallback but never fail the listing. +func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { + idStr := ls.ID.String() - for object := range objectCh { - if object.Err != nil { - return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) - } - - parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - if strings.HasPrefix(parts[1], ".") { - continue - } - remoteSnapshots[parts[1]] = true - } - } - - return remoteSnapshots, nil -} - -// reconcileLocalWithRemote builds a map of local snapshots keyed by ID for cross-referencing with remote -func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) { - localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, idStr) if err != nil { - return nil, fmt.Errorf("listing local snapshots: %w", err) + log.Warn("Failed to get total compressed size", "id", idStr, "error", err) + totalSize = ls.BlobSize } - localSnapshotMap := make(map[string]*database.Snapshot) - for _, s := range localSnapshots { - localSnapshotMap[s.ID.String()] = s + uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(v.ctx, idStr) + if err != nil { + log.Warn("Failed to get uncompressed chunk size", "id", idStr, "error", err) } - return localSnapshotMap, nil -} - -// buildSnapshotInfoList constructs SnapshotInfo entries from remote IDs and local data -func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSnapshotMap map[string]*database.Snapshot) ([]SnapshotInfo, error) { - snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots)) - - // remoteOnly collects snapshot IDs that need a manifest download. - var remoteOnly []string - - for snapshotID := range remoteSnapshots { - if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil { - totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, snapshotID) - if err != nil { - log.Warn("Failed to get total compressed size", "id", snapshotID, "error", err) - totalSize = localSnap.BlobSize - } - - uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(v.ctx, snapshotID) - if err != nil { - log.Warn("Failed to get uncompressed chunk size", "id", snapshotID, "error", err) - } - - newChunkSize, err := v.Repositories.Snapshots.GetSnapshotNewChunkSize(v.ctx, snapshotID) - if err != nil { - log.Warn("Failed to get new chunk size", "id", snapshotID, "error", err) - } - - snapshots = append(snapshots, SnapshotInfo{ - ID: localSnap.ID, - Timestamp: localSnap.StartedAt, - CompressedSize: totalSize, - UncompressedSize: uncompressedSize, - NewChunkSize: newChunkSize, - LocallyTracked: true, - }) - } else { - timestamp, err := parseSnapshotTimestamp(snapshotID) - if err != nil { - log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err) - continue - } - - // Pre-add with zero size; will be filled by concurrent downloads. - snapshots = append(snapshots, SnapshotInfo{ - ID: types.SnapshotID(snapshotID), - Timestamp: timestamp, - CompressedSize: 0, - LocallyTracked: false, - }) - remoteOnly = append(remoteOnly, snapshotID) - } + newChunkSize, err := v.Repositories.Snapshots.GetSnapshotNewChunkSize(v.ctx, idStr) + if err != nil { + log.Warn("Failed to get new chunk size", "id", idStr, "error", err) } - // Download manifests concurrently for remote-only snapshots. - if len(remoteOnly) > 0 { - // maxConcurrentManifestDownloads bounds parallel manifest fetches to - // avoid overwhelming the S3 endpoint while still being much faster - // than serial downloads. - const maxConcurrentManifestDownloads = 10 - - type manifestResult struct { - snapshotID string - size int64 - } - - var ( - mu sync.Mutex - results []manifestResult - ) - - g, gctx := errgroup.WithContext(v.ctx) - g.SetLimit(maxConcurrentManifestDownloads) - - for _, sid := range remoteOnly { - g.Go(func() error { - manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", sid) - reader, err := v.Storage.Get(gctx, manifestPath) - if err != nil { - return fmt.Errorf("downloading manifest for %s: %w", sid, err) - } - defer func() { _ = reader.Close() }() - - manifest, err := snapshot.DecodeManifest(reader) - if err != nil { - return fmt.Errorf("decoding manifest for %s: %w", sid, err) - } - - mu.Lock() - results = append(results, manifestResult{ - snapshotID: sid, - size: manifest.TotalCompressedSize, - }) - mu.Unlock() - return nil - }) - } - - if err := g.Wait(); err != nil { - return nil, fmt.Errorf("fetching manifest sizes: %w", err) - } - - // Build a lookup from results and patch the pre-added entries. - sizeMap := make(map[string]int64, len(results)) - for _, r := range results { - sizeMap[r.snapshotID] = r.size - } - for i := range snapshots { - if sz, ok := sizeMap[string(snapshots[i].ID)]; ok { - snapshots[i].CompressedSize = sz - } - } + return SnapshotInfo{ + ID: ls.ID, + Timestamp: ls.StartedAt, + CompressedSize: totalSize, + UncompressedSize: uncompressedSize, + NewChunkSize: newChunkSize, + LocallyTracked: true, } - - return snapshots, nil } // printSnapshotTable renders the snapshot list as a formatted table @@ -768,7 +695,7 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) } - if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { + if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err) } } @@ -813,8 +740,9 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio v.printVerifyHeader(snapshotID, opts) - // Download and parse manifest - manifest, err := v.downloadManifest(snapshotID) + // Download and parse manifest. The caller supplies a human + // snapshot ID; we hash it to address remote storage. + manifest, err := v.downloadManifestByKey(snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { if opts.JSON { result.Status = "failed" @@ -929,12 +857,18 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { // CleanupLocalSnapshots removes local snapshot records that have no // corresponding metadata in remote storage. These are typically left -// behind by incomplete or interrupted backups. +// behind by incomplete or interrupted backups. Each local snapshot's +// human ID is hashed via RemoteSnapshotKey and compared against the +// remote listing. func (v *Vaultik) CleanupLocalSnapshots() error { - remoteSnapshots, err := v.listRemoteSnapshotIDs() + remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { return err } + remoteSet := make(map[string]bool, len(remoteKeys)) + for _, k := range remoteKeys { + remoteSet[k] = true + } localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { @@ -944,7 +878,7 @@ func (v *Vaultik) CleanupLocalSnapshots() error { var removed int for _, snap := range localSnapshots { id := snap.ID.String() - if !remoteSnapshots[id] { + if !remoteSet[snapshot.RemoteSnapshotKey(id)] { v.printfStdout("Removing stale local record: %s\n", id) if err := v.deleteSnapshotFromLocalDB(id); err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) @@ -964,8 +898,12 @@ func (v *Vaultik) CleanupLocalSnapshots() error { // Helper methods that were previously on SnapshotApp -func (v *Vaultik) downloadManifest(snapshotID string) (*snapshot.Manifest, error) { - manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) +// downloadManifestByKey fetches the manifest at +// metadata//manifest.json.zst. The remoteKey is the double- +// SHA256 derivation produced by snapshot.RemoteSnapshotKey, not the +// human snapshot ID. Callers that have a human ID must hash first. +func (v *Vaultik) downloadManifestByKey(remoteKey string) (*snapshot.Manifest, error) { + manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) reader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { @@ -1100,7 +1038,7 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov // If --remote, also remove from remote storage if opts.Remote { log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) - if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { + if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { return result, fmt.Errorf("removing from remote storage: %w", err) } result.RemoteRemoved = true @@ -1131,14 +1069,42 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov return result, nil } -// RemoveAllSnapshots removes all snapshots from local database and optionally from remote +// RemoveAllSnapshots removes every snapshot known to the local +// database from the local index, and (with --remote) every snapshot +// metadata directory in remote storage. Both sides are processed so a +// "remove --all" leaves nothing behind, even when the local DB and +// remote storage have diverged. func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) { - snapshotIDs, err := v.listAllRemoteSnapshotIDs() + localSnaps, err := v.localSnapshotIDs() if err != nil { - return nil, err + return nil, fmt.Errorf("listing local snapshots: %w", err) } - if len(snapshotIDs) == 0 { + // remoteKeys is the set of metadata// subdirectories on the + // destination store; failures are downgraded to a warning so a + // permission-denied or unreachable remote can't block a local-only + // remove. + remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys() + if remoteErr != nil { + log.Warn("Could not list remote snapshots", "error", remoteErr) + v.UI.Warning("Could not list remote snapshots: %v.", remoteErr) + } + + // Anything visible on the remote that doesn't correspond to a + // known local human ID is treated as an orphan key — handled only + // when --remote is in effect. + knownLocalKeys := make(map[string]string, len(localSnaps)) + for _, id := range localSnaps { + knownLocalKeys[snapshot.RemoteSnapshotKey(id)] = id + } + var orphanRemoteKeys []string + for _, key := range remoteKeys { + if _, known := knownLocalKeys[key]; !known { + orphanRemoteKeys = append(orphanRemoteKeys, key) + } + } + + if len(localSnaps) == 0 && len(orphanRemoteKeys) == 0 { if !opts.JSON { v.printlnStdout("No snapshots found") } @@ -1146,19 +1112,42 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) } if opts.DryRun { - return v.handleRemoveAllDryRun(snapshotIDs, opts) + return v.handleRemoveAllDryRun(localSnaps, orphanRemoteKeys, opts) } - return v.executeRemoveAll(snapshotIDs, opts) + return v.executeRemoveAll(localSnaps, orphanRemoteKeys, opts) } -// listAllRemoteSnapshotIDs collects all unique snapshot IDs from remote storage -func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { - log.Info("Listing all snapshots") +// localSnapshotIDs returns every snapshot ID present in the local +// index database, sorted for deterministic iteration. Empty slice if +// the database has no Repositories (e.g. tests). +func (v *Vaultik) localSnapshotIDs() ([]string, error) { + if v.Repositories == nil { + return nil, nil + } + snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 100000) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(snaps)) + for _, s := range snaps { + ids = append(ids, s.ID.String()) + } + sort.Strings(ids) + return ids, nil +} + +// listAllRemoteSnapshotKeys collects the hashed remote keys +// (subdirectories under metadata/) currently present in the +// destination store. Returns (nil, err) when the store cannot be +// listed; callers must treat that as "no remote info available," not +// fatal. +func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { + log.Info("Listing all remote snapshots") objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) - var snapshotIDs []string + var keys []string for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) @@ -1171,30 +1160,36 @@ func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { continue } if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { - sid := parts[1] - if !seen[sid] { - seen[sid] = true - snapshotIDs = append(snapshotIDs, sid) + key := parts[1] + if !seen[key] { + seen[key] = true + keys = append(keys, key) } } } } - return snapshotIDs, nil + return keys, nil } // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots -func (v *Vaultik) handleRemoveAllDryRun(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { - result := &RemoveResult{ - DryRun: true, - SnapshotsRemoved: snapshotIDs, +func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { + result := &RemoveResult{DryRun: true} + result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) + if opts.Remote { + result.SnapshotsRemoved = append(result.SnapshotsRemoved, orphanRemoteKeys...) } if !opts.JSON { - v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) - for _, id := range snapshotIDs { + v.printfStdout("Would remove %d local snapshot(s):\n", len(localSnaps)) + for _, id := range localSnaps { v.printfStdout(" %s\n", id) } - if opts.Remote { + if opts.Remote && len(orphanRemoteKeys) > 0 { + v.printfStdout("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + for _, key := range orphanRemoteKeys { + v.printfStdout(" %s\n", key) + } + } else if opts.Remote { v.printlnStdout("Would also remove from remote storage") } v.printlnStdout("[Dry run - no changes made]") @@ -1205,17 +1200,19 @@ func (v *Vaultik) handleRemoveAllDryRun(snapshotIDs []string, opts *RemoveOption return result, nil } -// executeRemoveAll removes all snapshots from local database and optionally from remote storage -func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { +// executeRemoveAll deletes every local snapshot (and, with --remote, +// every corresponding remote metadata directory plus any orphan remote +// keys that don't match a local snapshot). +func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { // --all requires --force if !opts.Force { return nil, fmt.Errorf("--all requires --force") } - log.Info("Removing all snapshots", "count", len(snapshotIDs)) + log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} - for _, snapshotID := range snapshotIDs { + for _, snapshotID := range localSnaps { log.Info("Removing snapshot", "snapshot_id", snapshotID) if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { @@ -1224,7 +1221,7 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (* } if opts.Remote { - if err := v.deleteSnapshotFromRemote(snapshotID); err != nil { + if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { log.Error("Failed to remove from remote", "snapshot_id", snapshotID, "error", err) continue } @@ -1233,6 +1230,17 @@ func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (* result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) } + if opts.Remote { + for _, key := range orphanRemoteKeys { + log.Info("Removing orphan remote snapshot", "remote_key", key) + if err := v.deleteRemoteSnapshotByKey(key); err != nil { + log.Error("Failed to remove orphan from remote", "remote_key", key, "error", err) + continue + } + result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) + } + } + if opts.Remote { result.RemoteRemoved = true } @@ -1281,9 +1289,13 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { return nil } -// deleteSnapshotFromRemote removes snapshot metadata files from remote storage -func (v *Vaultik) deleteSnapshotFromRemote(snapshotID string) error { - prefix := fmt.Sprintf("metadata/%s/", snapshotID) +// deleteRemoteSnapshotByKey removes everything under +// metadata// on the destination store. The argument is a +// remote key (double-SHA256 derivation), not a human snapshot ID; +// callers that have a human ID must hash via snapshot.RemoteSnapshotKey +// first. +func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error { + prefix := fmt.Sprintf("metadata/%s/", remoteKey) objectCh := v.Storage.ListStream(v.ctx, prefix) var objectsToDelete []string diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 5ee20eb..9ad2f42 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -106,8 +106,11 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { // loadVerificationData downloads manifest, database, and blob list for verification func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, result *VerifyResult) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { + // All remote paths use the hashed key derived from the human ID. + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) + // Download manifest - manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) + manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) log.Info("Downloading manifest", "path", manifestPath) if !opts.JSON { v.printfStdout("Downloading manifest...\n") @@ -136,7 +139,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r } // Download and decrypt database - dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) + dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) log.Info("Downloading encrypted database", "path", dbPath) dbReader, err := v.Storage.Get(v.ctx, dbPath) if err != nil {