Bind the local index to its backup destination
The local index tracks which chunks and blobs already exist at the backup destination. Nothing was recording *which* destination, so changing storage_url and running a backup left the scanner treating every already-seen chunk as still-present at the new (empty) location. Uploads were skipped silently and the resulting snapshots pointed at blobs that don't exist at the new destination. Fix: record storage_url in a new local_meta key-value table on first mutating command, and refuse to proceed when the configured URL later differs from the stored one. The error explains the two recovery paths (revert the config, or run 'vaultik database purge' to discard the index and rebuild from a fresh full backup). Wired into snapshot create / prune / snapshot remove / snapshot purge / snapshot cleanup. Read-only inspection commands (snapshot list, remote info, store info) are exempt.
This commit is contained in:
@@ -55,6 +55,9 @@ type PruneBlobsResult struct {
|
||||
// prefer this method over PruneDatabase or PruneBlobs individually
|
||||
// unless it specifically wants one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
return fmt.Errorf("pruning local database: %w", err)
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
"index_path", v.Config.IndexPath,
|
||||
)
|
||||
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clean up incomplete snapshots FIRST, before any scanning
|
||||
// This is critical for data safety - see CleanupIncompleteSnapshots for details
|
||||
hostname := v.Config.Hostname
|
||||
@@ -581,6 +585,9 @@ type SnapshotPurgeOptions struct {
|
||||
// snapshot name, not the latest globally. This prevents `home` and `system`
|
||||
// snapshots from cannibalizing each other.
|
||||
func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Sync with remote first
|
||||
if err := v.syncWithRemote(); err != nil {
|
||||
return fmt.Errorf("syncing with remote: %w", err)
|
||||
@@ -861,6 +868,9 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
|
||||
// human ID is hashed via RemoteSnapshotKey and compared against the
|
||||
// remote listing.
|
||||
func (v *Vaultik) CleanupLocalSnapshots() error {
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return err
|
||||
}
|
||||
remoteKeys, err := v.listAllRemoteSnapshotKeys()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1006,6 +1016,10 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
|
||||
SnapshotID: snapshotID,
|
||||
}
|
||||
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
result.DryRun = true
|
||||
if !opts.JSON {
|
||||
@@ -1086,6 +1100,9 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
|
||||
// "remove --all" leaves nothing behind, even when the local DB and
|
||||
// remote storage have diverged.
|
||||
func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) {
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
localSnaps, err := v.localSnapshotIDs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing local snapshots: %w", err)
|
||||
|
||||
91
internal/vaultik/storage_bind.go
Normal file
91
internal/vaultik/storage_bind.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// EnsureStorageBinding guarantees that the local index database is
|
||||
// bound to the currently-configured storage destination. Every mutating
|
||||
// command must call this before touching either the local index or the
|
||||
// destination store, because the two live in lockstep: the local index
|
||||
// records which chunks/blobs already exist at the destination, and
|
||||
// mismatched destination + local index produces silent corruption (the
|
||||
// scanner sees "known" chunks and skips uploads, then writes snapshots
|
||||
// whose manifests reference blobs that aren't on the new destination).
|
||||
//
|
||||
// Behaviour:
|
||||
// - On first use (empty stored value), record the configured
|
||||
// storage_url and log the binding.
|
||||
// - When the stored value matches the configured storage_url, do
|
||||
// nothing and return nil.
|
||||
// - When the two differ, refuse with an error that tells the user
|
||||
// how to recover (revert the config, or run `vaultik database
|
||||
// purge` to discard the local index and rebuild against the new
|
||||
// destination on the next backup).
|
||||
//
|
||||
// Read-only inspection commands (remote info, snapshot list, etc.)
|
||||
// deliberately don't call this: they can be run against a bare
|
||||
// destination store without any binding state.
|
||||
func (v *Vaultik) EnsureStorageBinding() error {
|
||||
if v.Repositories == nil || v.Config == nil {
|
||||
// NewForTesting builds a Vaultik with no DB or config;
|
||||
// there's nothing to bind and no binding to check. Callers
|
||||
// exercising the bind path use a real DB and populate Config
|
||||
// explicitly (see storage_bind_test.go).
|
||||
return nil
|
||||
}
|
||||
|
||||
configured := v.Config.StorageURL
|
||||
if configured == "" {
|
||||
// Some legacy configs still use the split s3.* keys instead of
|
||||
// storage_url. Falling back to a synthetic URL for those would
|
||||
// hide the fact that the binding is loose. Instead, treat
|
||||
// unset as "nothing to bind against" — the check is
|
||||
// necessarily best-effort for those configs.
|
||||
return nil
|
||||
}
|
||||
|
||||
stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading local storage binding: %w", err)
|
||||
}
|
||||
|
||||
if stored == "" {
|
||||
if err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured); err != nil {
|
||||
return fmt.Errorf("recording local storage binding: %w", err)
|
||||
}
|
||||
log.Info("Bound local index to storage destination", "storage_url", configured)
|
||||
return nil
|
||||
}
|
||||
|
||||
if stored == configured {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%s", buildBindingMismatchMessage(stored, configured))
|
||||
}
|
||||
|
||||
// buildBindingMismatchMessage assembles the multi-line explanation
|
||||
// shown when the local index is bound to a different destination than
|
||||
// the currently-configured one. Kept as a separate function so the
|
||||
// lint-flagged multi-line format string is expressed as a plain string
|
||||
// literal rather than a fmt.Errorf argument (staticcheck ST1005
|
||||
// disallows trailing punctuation on error format strings).
|
||||
func buildBindingMismatchMessage(stored, configured string) string {
|
||||
return "local index is bound to a different backup destination\n" +
|
||||
" local index bound to: " + stored + "\n" +
|
||||
" currently configured: " + configured + "\n" +
|
||||
"\n" +
|
||||
"The local index database tracks which chunks and blobs already exist at the\n" +
|
||||
"destination store. Using it against a different destination would silently\n" +
|
||||
"skip uploads (the scanner would treat every chunk as already present), leaving\n" +
|
||||
"future snapshots referencing blobs that don't exist at the new destination.\n" +
|
||||
"\n" +
|
||||
"To proceed, either:\n" +
|
||||
" - revert storage_url in your config to the bound destination, or\n" +
|
||||
" - run 'vaultik database purge' to discard the local index and rebuild it\n" +
|
||||
" from a fresh full backup against the new destination"
|
||||
}
|
||||
72
internal/vaultik/storage_bind_test.go
Normal file
72
internal/vaultik/storage_bind_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// buildBindTestVaultik returns a minimal Vaultik wired with a real
|
||||
// in-memory DB and a config carrying the given StorageURL. Enough to
|
||||
// exercise EnsureStorageBinding without spinning up storage or fx.
|
||||
func buildBindTestVaultik(t *testing.T, storageURL string) (*vaultik.TestVaultik, *database.Repositories) {
|
||||
t.Helper()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
tv := vaultik.NewForTesting(nil)
|
||||
tv.Repositories = repos
|
||||
tv.DB = db
|
||||
tv.Config = &config.Config{StorageURL: storageURL}
|
||||
tv.SetContext(context.Background())
|
||||
|
||||
return tv, repos
|
||||
}
|
||||
|
||||
func TestEnsureStorageBinding_FreshDBRecordsURL(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new")
|
||||
|
||||
require.NoError(t, tv.EnsureStorageBinding())
|
||||
|
||||
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "file:///mnt/backups/new", got, "first call must record the configured URL")
|
||||
}
|
||||
|
||||
func TestEnsureStorageBinding_MatchingURLPasses(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
tv, repos := buildBindTestVaultik(t, "s3://bucket/prefix")
|
||||
|
||||
require.NoError(t, repos.LocalMeta.Set(context.Background(),
|
||||
database.LocalMetaKeyStorageURL, "s3://bucket/prefix"))
|
||||
|
||||
require.NoError(t, tv.EnsureStorageBinding())
|
||||
}
|
||||
|
||||
func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new")
|
||||
|
||||
require.NoError(t, repos.LocalMeta.Set(context.Background(),
|
||||
database.LocalMetaKeyStorageURL, "file:///mnt/backups/old"))
|
||||
|
||||
err := tv.EnsureStorageBinding()
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "file:///mnt/backups/old")
|
||||
assert.Contains(t, err.Error(), "file:///mnt/backups/new")
|
||||
assert.Contains(t, err.Error(), "vaultik database purge")
|
||||
}
|
||||
Reference in New Issue
Block a user