Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b42c4bb325 |
No files matched your search
@@ -24,12 +24,6 @@ owner's; a `v*` tag runs the release build in
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-11-10: Printed the status and progress lines of `remote info`
|
|
||||||
and `snapshot verify` through `internal/ui`, with its marker and
|
|
||||||
color, as the README describes
|
|
||||||
([issue #410](https://git.eeqj.de/sneak/vaultik/issues/410)). Their
|
|
||||||
reports stay plain.
|
|
||||||
|
|
||||||
- 2026-10-15: Stopped `snapshot create --prune` pruning after a backup
|
- 2026-10-15: Stopped `snapshot create --prune` pruning after a backup
|
||||||
in which a volume mounted below a configured path in an earlier
|
in which a volume mounted below a configured path in an earlier
|
||||||
snapshot of the same name was not mounted
|
snapshot of the same name was not mounted
|
||||||
|
|||||||
@@ -271,18 +271,16 @@ func (sm *SnapshotManager) CompleteSnapshot(
|
|||||||
return sm.MarkSnapshotComplete(ctx, snapshotID)
|
return sm.MarkSnapshotComplete(ctx, snapshotID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSnapshotMetadata exports snapshot metadata to the backup destination store
|
// ExportSnapshotMetadata exports snapshot metadata to S3
|
||||||
//
|
//
|
||||||
// This method executes the complete snapshot metadata export process:
|
// This method executes the complete snapshot metadata export process:
|
||||||
// 1. Creates a temporary directory for working files
|
// 1. Creates a temporary directory for working files
|
||||||
// 2. Copies the main database to preserve its state
|
// 2. Copies the main database to preserve its state
|
||||||
// 3. Cleans the copy to contain only current snapshot data
|
// 3. Cleans the copy to contain only current snapshot data
|
||||||
// 4. Vacuums the copy to remove the deleted data
|
// 4. Dumps the cleaned database to SQL
|
||||||
// 5. Compresses the copy with zstd
|
// 5. Compresses the SQL dump with zstd
|
||||||
// 6. Encrypts the compressed copy with age (always; recipients are required)
|
// 6. Encrypts the compressed data with age (always; recipients are required)
|
||||||
// 7. Builds the blob manifest from the copy, compressed but not encrypted
|
// 7. Uploads to S3 at: snapshots/{snapshot-id}.sql.zst[.age]
|
||||||
// 8. Uploads the database to metadata/{remote-key}/db.zst.age
|
|
||||||
// 9. Uploads the manifest to metadata/{remote-key}/manifest.json.zst
|
|
||||||
//
|
//
|
||||||
// The only caller (finalizeSnapshotMetadata) does not close the main database
|
// The only caller (finalizeSnapshotMetadata) does not close the main database
|
||||||
// before calling this method: the index is copied at dbPath while it is still
|
// before calling this method: the index is copied at dbPath while it is still
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
v.stdoutf("Type: %s\n", storageInfo.Type)
|
v.stdoutf("Type: %s\n", storageInfo.Type)
|
||||||
v.stdoutf("Location: %s\n", storageInfo.Location)
|
v.stdoutf("Location: %s\n", storageInfo.Location)
|
||||||
v.printlnStdout()
|
v.printlnStdout()
|
||||||
v.UI.Beginf("Scanning snapshot metadata...")
|
v.stdoutf("Scanning snapshot metadata...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
|
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
|
||||||
@@ -265,7 +265,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
v.UI.Beginf("Downloading %d manifest(s)...", manifestCount)
|
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
|
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
|
||||||
@@ -447,7 +447,7 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
||||||
) error {
|
) error {
|
||||||
if showText {
|
if showText {
|
||||||
v.UI.Beginf("Scanning blobs...")
|
v.stdoutf("Scanning blobs...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use prune's listing, so a file under blobs/ that prune would never
|
// Use prune's listing, so a file under blobs/ that prune would never
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// testBlobHashB is a blob that the manifest written by addRemote does
|
// testBlobHashB is a blob that the manifest written by addRemote does
|
||||||
@@ -260,24 +259,3 @@ func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
|
|||||||
assert.NotContains(t, doc, "unreadable_manifests")
|
assert.NotContains(t, doc, "unreadable_manifests")
|
||||||
assert.NotContains(t, doc, "skipped_manifest_count")
|
assert.NotContains(t, doc, "skipped_manifest_count")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRemoteInfo_StatusLineUsesUIMarker checks that the remote info
|
|
||||||
// status lines go through internal/ui: marker-prefixed, and silenced by
|
|
||||||
// --quiet.
|
|
||||||
func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newListEnv(t)
|
|
||||||
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
|
||||||
|
|
||||||
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
|
|
||||||
|
|
||||||
require.NoError(t, env.v.RemoteInfo(false))
|
|
||||||
assert.Contains(t, env.stdout.String(), statusLine)
|
|
||||||
|
|
||||||
env.stdout.Reset()
|
|
||||||
env.v.UI.SetQuiet(true)
|
|
||||||
require.NoError(t, env.v.RemoteInfo(false))
|
|
||||||
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
|
|
||||||
}
|
|
||||||
@@ -31,10 +31,9 @@ var (
|
|||||||
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
||||||
errFilesFailedVerify = errors.New("files failed verification")
|
errFilesFailedVerify = errors.New("files failed verification")
|
||||||
errDecryptionKeyRequired = errors.New(
|
errDecryptionKeyRequired = errors.New(
|
||||||
"decryption key required for restore\n\n" +
|
"age secret key is missing; set it in the " +
|
||||||
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
|
"VAULTIK_AGE_SECRET_KEY environment variable or the " +
|
||||||
"age private key file:\n" +
|
"age_secret_key config file field")
|
||||||
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
|
|
||||||
// errInvalidAgeSecretKey is returned when the configured key does not
|
// errInvalidAgeSecretKey is returned when the configured key does not
|
||||||
// parse as any age identity. It names the source but never the value,
|
// parse as any age identity. It names the source but never the value,
|
||||||
// which is secret, so the message is safe to print and log.
|
// which is secret, so the message is safe to print and log.
|
||||||
|
|||||||
@@ -106,3 +106,23 @@ func TestRestoreIdentitiesMissingKey(t *testing.T) {
|
|||||||
_, err := v.restoreIdentities()
|
_, err := v.restoreIdentities()
|
||||||
require.ErrorIs(t, err, errDecryptionKeyRequired)
|
require.ErrorIs(t, err, errDecryptionKeyRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestMissingKeyErrorsNameBothSources asserts restore and deep verify
|
||||||
|
// return the same missing-key error, whose text names both places the key
|
||||||
|
// can be set: the VAULTIK_AGE_SECRET_KEY environment variable first since
|
||||||
|
// it is preferred, then the age_secret_key config file field.
|
||||||
|
func TestMissingKeyErrorsNameBothSources(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const want = "age secret key is missing; set it in " +
|
||||||
|
"the VAULTIK_AGE_SECRET_KEY environment variable or the " +
|
||||||
|
"age_secret_key config file field"
|
||||||
|
|
||||||
|
v := &Vaultik{Config: &config.Config{}}
|
||||||
|
|
||||||
|
_, err := v.restoreIdentities()
|
||||||
|
require.EqualError(t, err, want)
|
||||||
|
|
||||||
|
err = v.RunDeepVerify("any-snapshot", &VerifyOptions{})
|
||||||
|
require.EqualError(t, err, want)
|
||||||
|
}
|
||||||
@@ -1019,7 +1019,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
|||||||
v.printlnStdout()
|
v.printlnStdout()
|
||||||
|
|
||||||
// Check each blob is present with the size the manifest records.
|
// Check each blob is present with the size the manifest records.
|
||||||
v.UI.Beginf("Checking blob presence and sizes...")
|
v.stdoutf("Checking blob presence and sizes...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
v.checkSnapshotDatabase(remoteKey, result)
|
v.checkSnapshotDatabase(remoteKey, result)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ var errStorageBindingMismatch = errors.New(
|
|||||||
// nothing and return nil.
|
// nothing and return nil.
|
||||||
// - When the two differ, refuse with an error that tells the user
|
// - When the two differ, refuse with an error that tells the user
|
||||||
// how to recover (revert the config, or run `vaultik database
|
// how to recover (revert the config, or run `vaultik database
|
||||||
// delete` to discard the local index and rebuild against the new
|
// purge` to discard the local index and rebuild against the new
|
||||||
// destination on the next backup).
|
// destination on the next backup).
|
||||||
//
|
//
|
||||||
// Read-only inspection commands (remote info, snapshot list, etc.)
|
// Read-only inspection commands (remote info, snapshot list, etc.)
|
||||||
|
|||||||
@@ -162,11 +162,6 @@ func (v *Vaultik) StartOperation(fn func()) func(context.Context) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CanDecrypt returns true if this Vaultik instance has decryption capabilities
|
|
||||||
func (v *Vaultik) CanDecrypt() bool {
|
|
||||||
return v.Config.AgeSecretKey != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetFilesystem returns the filesystem instance used by Vaultik
|
// GetFilesystem returns the filesystem instance used by Vaultik
|
||||||
//
|
//
|
||||||
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
|
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
|
||||||
|
|||||||
+12
-18
@@ -23,8 +23,6 @@ import (
|
|||||||
// Sentinel errors for snapshot verification failures.
|
// Sentinel errors for snapshot verification failures.
|
||||||
var (
|
var (
|
||||||
errVerificationFailed = errors.New("verification failed")
|
errVerificationFailed = errors.New("verification failed")
|
||||||
errSecretKeyRequired = errors.New(
|
|
||||||
"VAULTIK_AGE_SECRET_KEY not set; required for deep verification")
|
|
||||||
errChunksOutOfOrder = errors.New("chunks out of order")
|
errChunksOutOfOrder = errors.New("chunks out of order")
|
||||||
errChunkHashMismatch = errors.New("chunk hash mismatch")
|
errChunkHashMismatch = errors.New("chunk hash mismatch")
|
||||||
errNegativeChunkLength = errors.New("chunk length is negative")
|
errNegativeChunkLength = errors.New("chunk length is negative")
|
||||||
@@ -106,13 +104,9 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
Mode: "deep",
|
Mode: "deep",
|
||||||
}
|
}
|
||||||
|
|
||||||
if !v.CanDecrypt() {
|
|
||||||
return v.deepVerifyFailure(result, opts,
|
|
||||||
errSecretKeyRequired.Error(), errSecretKeyRequired)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the age secret key once, the same way restore does, and reuse
|
// Parse the age secret key once, the same way restore does, and reuse
|
||||||
// the identities for the database and every blob.
|
// the identities for the database and every blob. A missing key fails
|
||||||
|
// here, before anything is downloaded.
|
||||||
identities, err := v.restoreIdentities()
|
identities, err := v.restoreIdentities()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
@@ -190,7 +184,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
log.Info("Downloading manifest", "remote_key", remoteKey)
|
log.Info("Downloading manifest", "remote_key", remoteKey)
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Beginf("Downloading manifest...")
|
v.stdoutf("Downloading manifest...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||||
@@ -205,9 +199,9 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
"manifest_total_size", ubytes(manifest.TotalCompressedSize))
|
"manifest_total_size", ubytes(manifest.TotalCompressedSize))
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Completef("Manifest loaded: %d blobs (%s)",
|
v.stdoutf("Manifest loaded: %d blobs (%s)\n",
|
||||||
manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
|
manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
|
||||||
v.UI.Beginf("Downloading and decrypting database...")
|
v.stdoutf("Downloading and decrypting database...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
tdb, err := v.downloadVerifiedSnapshotDB(
|
tdb, err := v.downloadVerifiedSnapshotDB(
|
||||||
@@ -235,7 +229,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
"db_total_size", ubytes(dbTotalSize))
|
"db_total_size", ubytes(dbTotalSize))
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Completef("Database loaded: %d blobs (%s)",
|
v.stdoutf("Database loaded: %d blobs (%s)\n",
|
||||||
len(dbBlobs), ubytes(dbTotalSize))
|
len(dbBlobs), ubytes(dbTotalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -295,7 +289,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
identities []age.Identity,
|
identities []age.Identity,
|
||||||
) error {
|
) error {
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Beginf("Verifying manifest against database...")
|
v.stdoutf("Verifying manifest against database...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
err := v.verifyManifestAgainstDatabase(manifest, dbBlobs)
|
err := v.verifyManifestAgainstDatabase(manifest, dbBlobs)
|
||||||
@@ -304,8 +298,8 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Completef("Manifest verified.")
|
v.stdoutf("Manifest verified.\n")
|
||||||
v.UI.Beginf("Checking blob existence in remote storage...")
|
v.stdoutf("Checking blob existence in remote storage...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.verifyBlobExistenceFromDB(dbBlobs, result)
|
err = v.verifyBlobExistenceFromDB(dbBlobs, result)
|
||||||
@@ -314,8 +308,8 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Completef("All blobs exist.")
|
v.stdoutf("All blobs exist.\n")
|
||||||
v.UI.Beginf("Downloading and verifying blob contents (%d blobs, %s)...",
|
v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n",
|
||||||
len(dbBlobs), ubytes(totalSize))
|
len(dbBlobs), ubytes(totalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -790,7 +784,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.UI.Progressf("Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s",
|
v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n",
|
||||||
i+1, len(blobs), remaining,
|
i+1, len(blobs), remaining,
|
||||||
ubytes(bytesProcessed),
|
ubytes(bytesProcessed),
|
||||||
ubytes(totalBytesExpected),
|
ubytes(totalBytesExpected),
|
||||||
|
|||||||
@@ -100,51 +100,3 @@ func TestVerify_QuietSuppressesReport(t *testing.T) {
|
|||||||
require.Empty(t, stdout.String(),
|
require.Empty(t, stdout.String(),
|
||||||
"a failed verify must write no report under --quiet")
|
"a failed verify must write no report under --quiet")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestVerify_DeepStatusLineUsesUIMarker checks that the deep verify
|
|
||||||
// status lines go through internal/ui: marker-prefixed, and silenced by
|
|
||||||
// --quiet.
|
|
||||||
func TestVerify_DeepStatusLineUsesUIMarker(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
|
|
||||||
dataDir := filepath.Join(tempDir, "source")
|
|
||||||
storeDir := t.TempDir()
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs,
|
|
||||||
filepath.Join(dataDir, "data.bin"),
|
|
||||||
bytesPattern("marker-", 64*1024), 0o644))
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
cfg, storer, snapshotID := runFileStorageBackup(
|
|
||||||
ctx, t, fs, dataDir, storeDir, dbPath, 32*1024, 128*1024)
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
|
||||||
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: cfg,
|
|
||||||
Storage: storer,
|
|
||||||
Fs: fs,
|
|
||||||
Stdout: &stdout,
|
|
||||||
Stderr: io.Discard,
|
|
||||||
UI: ui.NewWithColor(&stdout, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
opts := &vaultik.VerifyOptions{Deep: true}
|
|
||||||
|
|
||||||
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
|
|
||||||
require.Contains(t, stdout.String(),
|
|
||||||
"\n"+ui.Marker+" Downloading manifest...\n")
|
|
||||||
|
|
||||||
stdout.Reset()
|
|
||||||
v.UI.SetQuiet(true)
|
|
||||||
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
|
|
||||||
require.NotContains(t, stdout.String(), "Downloading manifest")
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user