Author SHA1 Message Date
sneak 28597dcff8 Print remote info and verify status lines through internal/ui (closes #410)
check / check (push) Failing after 33m7s
The status and progress lines of `remote info` and `snapshot verify`
(shallow and deep) now use the `internal/ui` Begin, Complete and
Progress methods, so they carry its marker and, on a terminal, color.
The reports themselves stay plain on stdout.

Model: opus-5-5
2026-11-10 20:53:51 +00:00
clawbot 5a0da6da12 Write the apps snapshot in config init only on macOS (closes #408)
check / check (push) Failing after 35m29s
The default config backed up /Applications on every system. That directory exists only on macOS, so on Linux the first snapshot create stopped on the missing path. config init now writes the apps snapshot only when runtime.GOOS is darwin; the macOS file is unchanged.

Model: opus-5-5 (implementation, review); opus-5-5 (merge)
2026-11-10 21:52:03 +01:00
clawbot 13a35a986a Correct two false doc comments (closes #403)
check / check (push) Successful in 13m0s
Two doc comments described behaviour that no longer exists. The comment on `ExportSnapshotMetadata` described an SQL dump uploaded under `snapshots/`; it now lists the steps the function performs and the two keys it writes. The comment on `EnsureStorageBinding` named `vaultik database purge`; the command is `vaultik database delete`. Comments only.

Model: opus-5-5
2026-11-09 07:43:24 +01:00
11 changed files with 173 additions and 33 deletions

No files matched your search

+2 -2
View File
@@ -32,8 +32,8 @@ mkdir /Volumes/usbstick/mybackup
# (System Settings → Privacy & Security → Full Disk Access), otherwise # (System Settings → Privacy & Security → Full Disk Access), otherwise
# the backup will abort with a permission error on protected directories # the backup will abort with a permission error on protected directories
# run your first backup (the default config backs up ~ and /Applications # run your first backup (the default config backs up ~, and /Applications
# with sensible excludes) # on macOS only, with sensible excludes)
vaultik snapshot create vaultik snapshot create
# see what you have # see what you have
+11
View File
@@ -24,6 +24,17 @@ owner's; a `v*` tag runs the release build in
# Completed Steps # Completed Steps
- 2026-11-10: Printed the status and progress lines of `remote info`
and `snapshot verify` through `internal/ui`, with its marker and
color, as the README describes
([issue #410](https://git.eeqj.de/sneak/vaultik/issues/410)). Their
reports stay plain.
- 2026-11-10: Made `config init` write the `apps` snapshot, which backs
up `/Applications`, only on macOS
([issue #408](https://git.eeqj.de/sneak/vaultik/issues/408)). On
Linux the first `snapshot create` stopped on the missing directory.
- 2026-10-15: Stopped `snapshot create --prune` pruning after a backup - 2026-10-15: Stopped `snapshot create --prune` pruning after a backup
in which a volume mounted below a configured path in an earlier in which a volume mounted below a configured path in an earlier
snapshot of the same name was not mounted snapshot of the same name was not mounted
+27 -4
View File
@@ -8,6 +8,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"reflect" "reflect"
"runtime"
"slices" "slices"
"strconv" "strconv"
"strings" "strings"
@@ -30,7 +31,7 @@ const configSetArgs = 2
// parent config dirs (e.g. ~/.config) are conventionally traversable. // parent config dirs (e.g. ~/.config) are conventionally traversable.
const configDirMode = 0o755 const configDirMode = 0o755
// configYAMLIndent matches the 2-space indentation of defaultConfigTemplate, // configYAMLIndent matches the 2-space indentation of the default config,
// so `config set` writes the file back with the same indentation rather than // so `config set` writes the file back with the same indentation rather than
// yaml.Marshal's 4-space default. // yaml.Marshal's 4-space default.
const configYAMLIndent = 2 const configYAMLIndent = 2
@@ -50,7 +51,22 @@ var (
"so the file was not changed") "so the file was not changed")
) )
const defaultConfigTemplate = `# vaultik configuration // macOS is the value of runtime.GOOS on macOS.
const macOS = "darwin"
// defaultConfigText returns the file `config init` writes on the operating
// system goos, named as runtime.GOOS names it. Only macOS gets the apps
// snapshot, since /Applications exists nowhere else.
func defaultConfigText(goos string) string {
if goos == macOS {
return defaultConfigHead + appsSnapshotConfig + defaultConfigTail
}
return defaultConfigHead + defaultConfigTail
}
// defaultConfigHead is the default config up to the end of the home snapshot.
const defaultConfigHead = `# vaultik configuration
# Documentation: https://sneak.berlin/go/vaultik # Documentation: https://sneak.berlin/go/vaultik
# ─── REQUIRED ──────────────────────────────────────────────────────────────── # ─── REQUIRED ────────────────────────────────────────────────────────────────
@@ -168,7 +184,10 @@ snapshots:
- "/Movies/ProxyMedia" - "/Movies/ProxyMedia"
- "/Music/iTunes/Album Artwork" - "/Music/iTunes/Album Artwork"
- "/Pictures/iPod Photo Cache" - "/Pictures/iPod Photo Cache"
`
// appsSnapshotConfig is the apps snapshot of the default config.
const appsSnapshotConfig = `
# Third-party applications. OS-provided apps live in /System/Applications # Third-party applications. OS-provided apps live in /System/Applications
# on modern macOS and are never in /Applications, but Apple-installed # on modern macOS and are never in /Applications, but Apple-installed
# App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since # App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since
@@ -189,7 +208,10 @@ snapshots:
- "/Steam.app" - "/Steam.app"
- "/VirtualBox.app" - "/VirtualBox.app"
- "/Utilities/Adobe Installers" - "/Utilities/Adobe Installers"
`
// defaultConfigTail is the default config after the snapshots.
const defaultConfigTail = `
# Storage backend (pick ONE of the three forms below). # Storage backend (pick ONE of the three forms below).
# #
# S3-compatible: # S3-compatible:
@@ -311,7 +333,8 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
return fmt.Errorf("creating config directory %s: %w", dir, err) return fmt.Errorf("creating config directory %s: %w", dir, err)
} }
err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode) err = os.WriteFile(path, []byte(defaultConfigText(runtime.GOOS)),
configFileMode)
if err != nil { if err != nil {
return fmt.Errorf("writing config file: %w", err) return fmt.Errorf("writing config file: %w", err)
} }
@@ -560,7 +583,7 @@ func yamlPathOnly(root *yaml.Node, keys []string) (*yaml.Node, error) {
} }
// marshalConfigYAML renders a config document tree with 2-space indentation, // marshalConfigYAML renders a config document tree with 2-space indentation,
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which // matching the default config. yaml.Marshal defaults to 4 spaces, which
// would reindent the whole file on the first `config set` despite the promise // would reindent the whole file on the first `config set` despite the promise
// to preserve formatting. // to preserve formatting.
func marshalConfigYAML(root *yaml.Node) ([]byte, error) { func marshalConfigYAML(root *yaml.Node) ([]byte, error) {
+41 -7
View File
@@ -13,14 +13,15 @@ import (
"sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/ui"
) )
// TestDefaultConfigTemplateParses ensures the init template is valid YAML // TestDefaultConfigParsesOnMacOS ensures the config `config init` writes on
// that unmarshals into the Config struct with the expected snapshots. // macOS is valid YAML that unmarshals into the Config struct with the
func TestDefaultConfigTemplateParses(t *testing.T) { // expected snapshots.
func TestDefaultConfigParsesOnMacOS(t *testing.T) {
t.Parallel() t.Parallel()
var cfg config.Config var cfg config.Config
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg) err := yaml.Unmarshal([]byte(defaultConfigText("darwin")), &cfg)
if err != nil { if err != nil {
t.Fatalf("default config template is not valid YAML: %v", err) t.Fatalf("default config template is not valid YAML: %v", err)
} }
@@ -58,6 +59,39 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
} }
} }
// TestDefaultConfigOnLinux checks that the config `config init` writes on
// Linux loads with its home snapshot and without the apps snapshot, whose
// /Applications exists only on macOS.
func TestDefaultConfigOnLinux(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigText("linux")), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, "storage_url", "file:///mnt/backups")
if err != nil {
t.Fatalf("config set storage_url: %v", err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
if _, ok := cfg.Snapshots["home"]; !ok {
t.Error("expected 'home' snapshot in the Linux default config")
}
if _, ok := cfg.Snapshots["apps"]; ok {
t.Error("the Linux default config should have no 'apps' snapshot")
}
}
// testRecipient is a valid age public key. // testRecipient is a valid age public key.
const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj" const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
@@ -71,7 +105,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.yml") path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode) err := os.WriteFile(path, []byte(defaultConfigText(macOS)), configFileMode)
if err != nil { if err != nil {
t.Fatalf("write config: %v", err) t.Fatalf("write config: %v", err)
} }
@@ -220,7 +254,7 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
{levelReference, "blob_size_limit", "1MB", {levelReference, "blob_size_limit", "1MB",
"blob_size_limit must be at least"}, "blob_size_limit must be at least"},
// The type check on a config that does not load yet. // The type check on a config that does not load yet.
{defaultConfigTemplate, "snapshots.apps.paths", "/srv", {defaultConfigText(macOS), "snapshots.apps.paths", "/srv",
"into []string"}, "into []string"},
// The type check on a config that cannot be read here. // The type check on a config that cannot be read here.
{configWithUnsetReference, "snapshots.home.paths", "/x", {configWithUnsetReference, "snapshots.home.paths", "/x",
@@ -380,7 +414,7 @@ func loadAfterConfigSet(t *testing.T, key, value string) *config.Config {
path := filepath.Join(t.TempDir(), "config.yml") path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode) err := os.WriteFile(path, []byte(defaultConfigText(macOS)), configFileMode)
if err != nil { if err != nil {
t.Fatalf("write config: %v", err) t.Fatalf("write config: %v", err)
} }
+7 -5
View File
@@ -271,16 +271,18 @@ func (sm *SnapshotManager) CompleteSnapshot(
return sm.MarkSnapshotComplete(ctx, snapshotID) return sm.MarkSnapshotComplete(ctx, snapshotID)
} }
// ExportSnapshotMetadata exports snapshot metadata to S3 // ExportSnapshotMetadata exports snapshot metadata to the backup destination store
// //
// This method executes the complete snapshot metadata export process: // This method executes the complete snapshot metadata export process:
// 1. Creates a temporary directory for working files // 1. Creates a temporary directory for working files
// 2. Copies the main database to preserve its state // 2. Copies the main database to preserve its state
// 3. Cleans the copy to contain only current snapshot data // 3. Cleans the copy to contain only current snapshot data
// 4. Dumps the cleaned database to SQL // 4. Vacuums the copy to remove the deleted data
// 5. Compresses the SQL dump with zstd // 5. Compresses the copy with zstd
// 6. Encrypts the compressed data with age (always; recipients are required) // 6. Encrypts the compressed copy with age (always; recipients are required)
// 7. Uploads to S3 at: snapshots/{snapshot-id}.sql.zst[.age] // 7. Builds the blob manifest from the copy, compressed but not encrypted
// 8. Uploads the database to metadata/{remote-key}/db.zst.age
// 9. Uploads the manifest to metadata/{remote-key}/manifest.json.zst
// //
// The only caller (finalizeSnapshotMetadata) does not close the main database // The only caller (finalizeSnapshotMetadata) does not close the main database
// before calling this method: the index is copied at dbPath while it is still // before calling this method: the index is copied at dbPath while it is still
+3 -3
View File
@@ -246,7 +246,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
v.stdoutf("Type: %s\n", storageInfo.Type) v.stdoutf("Type: %s\n", storageInfo.Type)
v.stdoutf("Location: %s\n", storageInfo.Location) v.stdoutf("Location: %s\n", storageInfo.Location)
v.printlnStdout() v.printlnStdout()
v.stdoutf("Scanning snapshot metadata...\n") v.UI.Beginf("Scanning snapshot metadata...")
} }
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata() snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
@@ -265,7 +265,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
} }
} }
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount) v.UI.Beginf("Downloading %d manifest(s)...", manifestCount)
} }
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests( referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
@@ -447,7 +447,7 @@ func (v *Vaultik) scanRemoteBlobStorage(
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool, result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
) error { ) error {
if showText { if showText {
v.stdoutf("Scanning blobs...\n") v.UI.Beginf("Scanning blobs...")
} }
// Use prune's listing, so a file under blobs/ that prune would never // Use prune's listing, so a file under blobs/ that prune would never
+22
View File
@@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/ui"
) )
// testBlobHashB is a blob that the manifest written by addRemote does // testBlobHashB is a blob that the manifest written by addRemote does
@@ -259,3 +260,24 @@ func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
assert.NotContains(t, doc, "unreadable_manifests") assert.NotContains(t, doc, "unreadable_manifests")
assert.NotContains(t, doc, "skipped_manifest_count") assert.NotContains(t, doc, "skipped_manifest_count")
} }
// TestRemoteInfo_StatusLineUsesUIMarker checks that the remote info
// status lines go through internal/ui: marker-prefixed, and silenced by
// --quiet.
func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, env.stdout.String(), statusLine)
env.stdout.Reset()
env.v.UI.SetQuiet(true)
require.NoError(t, env.v.RemoteInfo(false))
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
}
+1 -1
View File
@@ -1019,7 +1019,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(
v.printlnStdout() v.printlnStdout()
// Check each blob is present with the size the manifest records. // Check each blob is present with the size the manifest records.
v.stdoutf("Checking blob presence and sizes...\n") v.UI.Beginf("Checking blob presence and sizes...")
} }
v.checkSnapshotDatabase(remoteKey, result) v.checkSnapshotDatabase(remoteKey, result)
+1 -1
View File
@@ -30,7 +30,7 @@ var errStorageBindingMismatch = errors.New(
// nothing and return nil. // nothing and return nil.
// - When the two differ, refuse with an error that tells the user // - When the two differ, refuse with an error that tells the user
// how to recover (revert the config, or run `vaultik database // how to recover (revert the config, or run `vaultik database
// purge` to discard the local index and rebuild against the new // delete` to discard the local index and rebuild against the new
// destination on the next backup). // destination on the next backup).
// //
// Read-only inspection commands (remote info, snapshot list, etc.) // Read-only inspection commands (remote info, snapshot list, etc.)
+10 -10
View File
@@ -190,7 +190,7 @@ func (v *Vaultik) loadVerificationData(
log.Info("Downloading manifest", "remote_key", remoteKey) log.Info("Downloading manifest", "remote_key", remoteKey)
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Downloading manifest...\n") v.UI.Beginf("Downloading manifest...")
} }
manifest, err := v.downloadManifestByKey(remoteKey) manifest, err := v.downloadManifestByKey(remoteKey)
@@ -205,9 +205,9 @@ func (v *Vaultik) loadVerificationData(
"manifest_total_size", ubytes(manifest.TotalCompressedSize)) "manifest_total_size", ubytes(manifest.TotalCompressedSize))
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Manifest loaded: %d blobs (%s)\n", v.UI.Completef("Manifest loaded: %d blobs (%s)",
manifest.BlobCount, ubytes(manifest.TotalCompressedSize)) manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
v.stdoutf("Downloading and decrypting database...\n") v.UI.Beginf("Downloading and decrypting database...")
} }
tdb, err := v.downloadVerifiedSnapshotDB( tdb, err := v.downloadVerifiedSnapshotDB(
@@ -235,7 +235,7 @@ func (v *Vaultik) loadVerificationData(
"db_total_size", ubytes(dbTotalSize)) "db_total_size", ubytes(dbTotalSize))
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Database loaded: %d blobs (%s)\n", v.UI.Completef("Database loaded: %d blobs (%s)",
len(dbBlobs), ubytes(dbTotalSize)) len(dbBlobs), ubytes(dbTotalSize))
} }
@@ -295,7 +295,7 @@ func (v *Vaultik) runVerificationSteps(
identities []age.Identity, identities []age.Identity,
) error { ) error {
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Verifying manifest against database...\n") v.UI.Beginf("Verifying manifest against database...")
} }
err := v.verifyManifestAgainstDatabase(manifest, dbBlobs) err := v.verifyManifestAgainstDatabase(manifest, dbBlobs)
@@ -304,8 +304,8 @@ func (v *Vaultik) runVerificationSteps(
} }
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Manifest verified.\n") v.UI.Completef("Manifest verified.")
v.stdoutf("Checking blob existence in remote storage...\n") v.UI.Beginf("Checking blob existence in remote storage...")
} }
err = v.verifyBlobExistenceFromDB(dbBlobs, result) err = v.verifyBlobExistenceFromDB(dbBlobs, result)
@@ -314,8 +314,8 @@ func (v *Vaultik) runVerificationSteps(
} }
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("All blobs exist.\n") v.UI.Completef("All blobs exist.")
v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", v.UI.Beginf("Downloading and verifying blob contents (%d blobs, %s)...",
len(dbBlobs), ubytes(totalSize)) len(dbBlobs), ubytes(totalSize))
} }
@@ -790,7 +790,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
) )
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", v.UI.Progressf("Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s",
i+1, len(blobs), remaining, i+1, len(blobs), remaining,
ubytes(bytesProcessed), ubytes(bytesProcessed),
ubytes(totalBytesExpected), ubytes(totalBytesExpected),
+48
View File
@@ -100,3 +100,51 @@ func TestVerify_QuietSuppressesReport(t *testing.T) {
require.Empty(t, stdout.String(), require.Empty(t, stdout.String(),
"a failed verify must write no report under --quiet") "a failed verify must write no report under --quiet")
} }
// TestVerify_DeepStatusLineUsesUIMarker checks that the deep verify
// status lines go through internal/ui: marker-prefixed, and silenced by
// --quiet.
func TestVerify_DeepStatusLineUsesUIMarker(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "source")
storeDir := t.TempDir()
dbPath := filepath.Join(tempDir, "index.sqlite")
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
require.NoError(t, afero.WriteFile(fs,
filepath.Join(dataDir, "data.bin"),
bytesPattern("marker-", 64*1024), 0o644))
ctx := context.Background()
cfg, storer, snapshotID := runFileStorageBackup(
ctx, t, fs, dataDir, storeDir, dbPath, 32*1024, 128*1024)
var stdout bytes.Buffer
v := &vaultik.Vaultik{
Config: cfg,
Storage: storer,
Fs: fs,
Stdout: &stdout,
Stderr: io.Discard,
UI: ui.NewWithColor(&stdout, false),
}
v.SetContext(ctx)
opts := &vaultik.VerifyOptions{Deep: true}
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
require.Contains(t, stdout.String(),
"\n"+ui.Marker+" Downloading manifest...\n")
stdout.Reset()
v.UI.SetQuiet(true)
require.NoError(t, v.VerifySnapshotWithOptions(snapshotID, opts))
require.NotContains(t, stdout.String(), "Downloading manifest")
}