Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bbe39fa073 | ||
|
|
13a35a986a |
No files matched your search
@@ -24,6 +24,12 @@ owner's; a `v*` tag runs the release build in
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-11-10: Printed the status and progress lines of `remote info`
|
||||||
|
and `snapshot verify` through `internal/ui`, with its marker and
|
||||||
|
color, as the README describes
|
||||||
|
([issue #410](https://git.eeqj.de/sneak/vaultik/issues/410)). Their
|
||||||
|
reports stay plain.
|
||||||
|
|
||||||
- 2026-10-15: Stopped `snapshot create --prune` pruning after a backup
|
- 2026-10-15: Stopped `snapshot create --prune` pruning after a backup
|
||||||
in which a volume mounted below a configured path in an earlier
|
in which a volume mounted below a configured path in an earlier
|
||||||
snapshot of the same name was not mounted
|
snapshot of the same name was not mounted
|
||||||
|
|||||||
@@ -271,16 +271,18 @@ func (sm *SnapshotManager) CompleteSnapshot(
|
|||||||
return sm.MarkSnapshotComplete(ctx, snapshotID)
|
return sm.MarkSnapshotComplete(ctx, snapshotID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSnapshotMetadata exports snapshot metadata to S3
|
// ExportSnapshotMetadata exports snapshot metadata to the backup destination store
|
||||||
//
|
//
|
||||||
// This method executes the complete snapshot metadata export process:
|
// This method executes the complete snapshot metadata export process:
|
||||||
// 1. Creates a temporary directory for working files
|
// 1. Creates a temporary directory for working files
|
||||||
// 2. Copies the main database to preserve its state
|
// 2. Copies the main database to preserve its state
|
||||||
// 3. Cleans the copy to contain only current snapshot data
|
// 3. Cleans the copy to contain only current snapshot data
|
||||||
// 4. Dumps the cleaned database to SQL
|
// 4. Vacuums the copy to remove the deleted data
|
||||||
// 5. Compresses the SQL dump with zstd
|
// 5. Compresses the copy with zstd
|
||||||
// 6. Encrypts the compressed data with age (always; recipients are required)
|
// 6. Encrypts the compressed copy with age (always; recipients are required)
|
||||||
// 7. Uploads to S3 at: snapshots/{snapshot-id}.sql.zst[.age]
|
// 7. Builds the blob manifest from the copy, compressed but not encrypted
|
||||||
|
// 8. Uploads the database to metadata/{remote-key}/db.zst.age
|
||||||
|
// 9. Uploads the manifest to metadata/{remote-key}/manifest.json.zst
|
||||||
//
|
//
|
||||||
// The only caller (finalizeSnapshotMetadata) does not close the main database
|
// The only caller (finalizeSnapshotMetadata) does not close the main database
|
||||||
// before calling this method: the index is copied at dbPath while it is still
|
// before calling this method: the index is copied at dbPath while it is still
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
v.stdoutf("Type: %s\n", storageInfo.Type)
|
v.stdoutf("Type: %s\n", storageInfo.Type)
|
||||||
v.stdoutf("Location: %s\n", storageInfo.Location)
|
v.stdoutf("Location: %s\n", storageInfo.Location)
|
||||||
v.printlnStdout()
|
v.printlnStdout()
|
||||||
v.stdoutf("Scanning snapshot metadata...\n")
|
v.UI.Beginf("Scanning snapshot metadata...")
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
|
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
|
||||||
@@ -265,7 +265,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount)
|
v.UI.Beginf("Downloading %d manifest(s)...", manifestCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
|
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
|
||||||
@@ -447,7 +447,7 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
||||||
) error {
|
) error {
|
||||||
if showText {
|
if showText {
|
||||||
v.stdoutf("Scanning blobs...\n")
|
v.UI.Beginf("Scanning blobs...")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use prune's listing, so a file under blobs/ that prune would never
|
// Use prune's listing, so a file under blobs/ that prune would never
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
)
|
)
|
||||||
|
|
||||||
// testBlobHashB is a blob that the manifest written by addRemote does
|
// testBlobHashB is a blob that the manifest written by addRemote does
|
||||||
@@ -259,3 +260,24 @@ func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
|
|||||||
assert.NotContains(t, doc, "unreadable_manifests")
|
assert.NotContains(t, doc, "unreadable_manifests")
|
||||||
assert.NotContains(t, doc, "skipped_manifest_count")
|
assert.NotContains(t, doc, "skipped_manifest_count")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRemoteInfo_StatusLineUsesUIMarker checks that the remote info
|
||||||
|
// status lines go through internal/ui: marker-prefixed, and silenced by
|
||||||
|
// --quiet.
|
||||||
|
func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newListEnv(t)
|
||||||
|
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||||
|
|
||||||
|
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
|
||||||
|
|
||||||
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
|
assert.Contains(t, env.stdout.String(), statusLine)
|
||||||
|
|
||||||
|
env.stdout.Reset()
|
||||||
|
env.v.UI.SetQuiet(true)
|
||||||
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
|
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
|
||||||
|
}
|
||||||
@@ -1019,7 +1019,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
|||||||
v.printlnStdout()
|
v.printlnStdout()
|
||||||
|
|
||||||
// Check each blob is present with the size the manifest records.
|
// Check each blob is present with the size the manifest records.
|
||||||
v.stdoutf("Checking blob presence and sizes...\n")
|
v.UI.Beginf("Checking blob presence and sizes...")
|
||||||
}
|
}
|
||||||
|
|
||||||
v.checkSnapshotDatabase(remoteKey, result)
|
v.checkSnapshotDatabase(remoteKey, result)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ var errStorageBindingMismatch = errors.New(
|
|||||||
// nothing and return nil.
|
// nothing and return nil.
|
||||||
// - When the two differ, refuse with an error that tells the user
|
// - When the two differ, refuse with an error that tells the user
|
||||||
// how to recover (revert the config, or run `vaultik database
|
// how to recover (revert the config, or run `vaultik database
|
||||||
// purge` to discard the local index and rebuild against the new
|
// delete` to discard the local index and rebuild against the new
|
||||||
// destination on the next backup).
|
// destination on the next backup).
|
||||||
//
|
//
|
||||||
// Read-only inspection commands (remote info, snapshot list, etc.)
|
// Read-only inspection commands (remote info, snapshot list, etc.)
|
||||||
|
|||||||
+10
-10
@@ -190,7 +190,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
log.Info("Downloading manifest", "remote_key", remoteKey)
|
log.Info("Downloading manifest", "remote_key", remoteKey)
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("Downloading manifest...\n")
|
v.UI.Beginf("Downloading manifest...")
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||||
@@ -205,9 +205,9 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
"manifest_total_size", ubytes(manifest.TotalCompressedSize))
|
"manifest_total_size", ubytes(manifest.TotalCompressedSize))
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("Manifest loaded: %d blobs (%s)\n",
|
v.UI.Completef("Manifest loaded: %d blobs (%s)",
|
||||||
manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
|
manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
|
||||||
v.stdoutf("Downloading and decrypting database...\n")
|
v.UI.Beginf("Downloading and decrypting database...")
|
||||||
}
|
}
|
||||||
|
|
||||||
tdb, err := v.downloadVerifiedSnapshotDB(
|
tdb, err := v.downloadVerifiedSnapshotDB(
|
||||||
@@ -235,7 +235,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
"db_total_size", ubytes(dbTotalSize))
|
"db_total_size", ubytes(dbTotalSize))
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("Database loaded: %d blobs (%s)\n",
|
v.UI.Completef("Database loaded: %d blobs (%s)",
|
||||||
len(dbBlobs), ubytes(dbTotalSize))
|
len(dbBlobs), ubytes(dbTotalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -295,7 +295,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
identities []age.Identity,
|
identities []age.Identity,
|
||||||
) error {
|
) error {
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("Verifying manifest against database...\n")
|
v.UI.Beginf("Verifying manifest against database...")
|
||||||
}
|
}
|
||||||
|
|
||||||
err := v.verifyManifestAgainstDatabase(manifest, dbBlobs)
|
err := v.verifyManifestAgainstDatabase(manifest, dbBlobs)
|
||||||
@@ -304,8 +304,8 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("Manifest verified.\n")
|
v.UI.Completef("Manifest verified.")
|
||||||
v.stdoutf("Checking blob existence in remote storage...\n")
|
v.UI.Beginf("Checking blob existence in remote storage...")
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.verifyBlobExistenceFromDB(dbBlobs, result)
|
err = v.verifyBlobExistenceFromDB(dbBlobs, result)
|
||||||
@@ -314,8 +314,8 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf("All blobs exist.\n")
|
v.UI.Completef("All blobs exist.")
|
||||||
v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n",
|
v.UI.Beginf("Downloading and verifying blob contents (%d blobs, %s)...",
|
||||||
len(dbBlobs), ubytes(totalSize))
|
len(dbBlobs), ubytes(totalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -790,7 +790,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if !opts.JSON && !v.UI.Quiet() {
|
if !opts.JSON && !v.UI.Quiet() {
|
||||||
v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n",
|
v.UI.Progressf("Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s",
|
||||||
i+1, len(blobs), remaining,
|
i+1, len(blobs), remaining,
|
||||||
ubytes(bytesProcessed),
|
ubytes(bytesProcessed),
|
||||||
ubytes(totalBytesExpected),
|
ubytes(totalBytesExpected),
|
||||||
|
|||||||
@@ -100,3 +100,51 @@ func TestVerify_QuietSuppressesReport(t *testing.T) {
|
|||||||
require.Empty(t, stdout.String(),
|
require.Empty(t, stdout.String(),
|
||||||
"a failed verify must write no report under --quiet")
|
"a failed verify must write no report under --quiet")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestVerify_DeepStatusLineUsesUIMarker checks that the deep verify
|
||||||
|
// status lines go through internal/ui: marker-prefixed, and silenced by
|
||||||
|
// --quiet.
|
||||||
|
func TestVerify_DeepStatusLineUsesUIMarker(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
dataDir := filepath.Join(tempDir, "source")
|
||||||
|
storeDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs,
|
||||||
|
filepath.Join(dataDir, "data.bin"),
|
||||||
|
bytesPattern("marker-", 64*1024), 0o644))
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
cfg, storer, snapshotID := runFileStorageBackup(
|
||||||
|
ctx, t, fs, dataDir, storeDir, dbPath, 32*1024, 128*1024)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
v := &vaultik.Vaultik{
|
||||||
|
Config: cfg,
|
||||||
|
Storage: storer,
|
||||||
|
Fs: fs,
|
||||||
|
Stdout: &stdout,
|
||||||
|
Stderr: io.Discard,
|
||||||
|
UI: ui.NewWithColor(&stdout, false),
|
||||||
|
}
|
||||||
|
v.SetContext(ctx)
|
||||||
|
|
||||||
|
opts := &vaultik.VerifyOptions{Deep: true}
|
||||||
|
|
||||||
|
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
|
||||||
|
require.Contains(t, stdout.String(),
|
||||||
|
"\n"+ui.Marker+" Downloading manifest...\n")
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
v.UI.SetQuiet(true)
|
||||||
|
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
|
||||||
|
require.NotContains(t, stdout.String(), "Downloading manifest")
|
||||||
|
}
|
||||||
Reference in new issue
Block a user