Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
1a972ee0a1 |
No files matched your search
+17
-55
@@ -1,32 +1,16 @@
|
||||
# vaultik configuration file example
|
||||
# This file shows every configuration option. The values are examples, not
|
||||
# defaults; where an option has a default, its comment gives it.
|
||||
# Copy this file and change the values you need
|
||||
# This file shows all available configuration options with their default values
|
||||
# Copy this file and uncomment/modify the values you need
|
||||
|
||||
# Age recipient public keys for encryption (age: https://age-encryption.org/)
|
||||
# Age recipient public keys for encryption
|
||||
# Backups are encrypted to these public keys. snapshot create needs at least
|
||||
# one; listing, verifying and restoring do not
|
||||
# Generate a key pair, and show its public key, with:
|
||||
# age-keygen -o vaultik_backup_private_key.txt
|
||||
# grep 'public key' vaultik_backup_private_key.txt
|
||||
# Keep the private key file: restoring needs it. Do not pipe age-keygen
|
||||
# straight into grep, which would discard the private key
|
||||
# Default: none
|
||||
# Generate with: age-keygen | grep "public key"
|
||||
age_recipients:
|
||||
- age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz
|
||||
|
||||
# Age private key for decryption
|
||||
# Only snapshot restore and snapshot verify --deep need it. The
|
||||
# VAULTIK_AGE_SECRET_KEY environment variable overrides this setting
|
||||
# Do not set it on a machine you back up. Supply the environment variable
|
||||
# only on a machine where you run snapshot restore or snapshot verify --deep
|
||||
# Default: not set
|
||||
#age_secret_key: YOUR_AGE_PRIVATE_KEY
|
||||
|
||||
# Named snapshots - each snapshot can contain multiple paths
|
||||
# Each snapshot gets its own ID and can have snapshot-specific excludes
|
||||
# Required: at least one snapshot with at least one path. Only the config
|
||||
# commands, version and completion run without one
|
||||
snapshots:
|
||||
testing:
|
||||
paths:
|
||||
@@ -269,18 +253,11 @@ snapshots:
|
||||
- "**/vendor"
|
||||
|
||||
# Global patterns to exclude from all backups
|
||||
# Default: none
|
||||
exclude:
|
||||
- "*.tmp"
|
||||
|
||||
# Storage URL. When it is not set, the endpoint, bucket and prefix in the s3
|
||||
# section below are used instead.
|
||||
# Storage URL - use either this OR the s3 section below
|
||||
# Supports: s3://bucket/prefix, file:///path, rclone://remote/path
|
||||
# An s3:// URL also needs s3.access_key_id and s3.secret_access_key below
|
||||
# A file:// path must be absolute, and the directory must already exist:
|
||||
# vaultik never creates it
|
||||
# An rclone:// URL names an rclone remote (https://rclone.org/), a storage
|
||||
# service set up in rclone's own config (rclone config)
|
||||
storage_url: "rclone://myremote/path/to/backups"
|
||||
|
||||
# S3-compatible storage configuration
|
||||
@@ -306,30 +283,25 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
# #region: us-east-1
|
||||
#
|
||||
# # Use SSL/TLS for S3 connections
|
||||
# # Only read when storage_url is not set and the endpoint has no scheme;
|
||||
# # with an s3:// storage_url, use ?ssl=false in the URL instead
|
||||
# # Default: true
|
||||
# #use_ssl: true
|
||||
#
|
||||
# # Part size for multipart uploads
|
||||
# # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
|
||||
# # An object too large for 10,000 parts of this size gets larger parts
|
||||
# # A blob too large for 10,000 parts of this size gets larger parts
|
||||
# # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
|
||||
# # Default: 5MiB
|
||||
# #part_size: 5MiB
|
||||
|
||||
# Path to local SQLite index database
|
||||
# This database tracks file state for incremental backups
|
||||
# The VAULTIK_INDEX_PATH environment variable overrides this setting
|
||||
# Both must be absolute paths; a leading ~ is expanded
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
#index_path: /path/to/index.sqlite
|
||||
|
||||
# Average size of a chunk, a piece of a file's content. Chunk boundaries
|
||||
# depend on the file's content, not on fixed offsets
|
||||
# Smaller chunks find more duplicate content and make the index larger
|
||||
# Average chunk size for content-defined chunking
|
||||
# Smaller chunks = better deduplication but more metadata
|
||||
# Minimum 1MB
|
||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
@@ -339,11 +311,9 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
# Multiple chunks are packed into blobs up to this size
|
||||
# Must be at least four times chunk_size (the largest chunk the chunker can
|
||||
# emit); a smaller limit would let a single-chunk blob exceed it.
|
||||
# Chunking uses no secret (the parameters of the FastCDC chunking algorithm
|
||||
# are fixed and public). A full blob at a large limit holds hundreds of
|
||||
# chunks, so its size does not show any one chunk's length. The last blob of
|
||||
# each configured path can hold a single chunk. Lowering the limit toward
|
||||
# chunk_size exposes chunk lengths.
|
||||
# Chunking uses no secret (the FastCDC parameters are fixed and public). At a
|
||||
# large limit a blob holds hundreds of chunks, so individual chunk lengths are
|
||||
# not visible in its size; lowering the limit toward chunk_size exposes them.
|
||||
# A backup needs free temporary space, because each blob is written whole to
|
||||
# a temporary file before it is uploaded (up to about blob_size_limit; an
|
||||
# rclone destination that cannot stream uploads needs about twice that) and
|
||||
@@ -359,14 +329,12 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
# Default: 3
|
||||
compression_level: 5
|
||||
|
||||
# How old a blob that no snapshot lists must be before vaultik prune deletes
|
||||
# it. Each snapshot has a directory under metadata/ at the destination, and
|
||||
# its manifest there lists the blobs it uses. This is also how old every
|
||||
# object in a directory with no manifest must be before vaultik prune deletes
|
||||
# that directory
|
||||
# A backup uploads its blobs and its encrypted database (db.zst.age) before
|
||||
# its manifest, so on a destination shared between hosts or users this must
|
||||
# be longer than the longest backup run
|
||||
# How old a blob that no snapshot lists must be before prune deletes it, and
|
||||
# how old every object in a directory under metadata/ with no manifest must
|
||||
# be before prune deletes that directory
|
||||
# A backup uploads its blobs and its db.zst.age before its manifest, so on a
|
||||
# destination shared between hosts or users this must be longer than the
|
||||
# longest backup run
|
||||
# It does not protect a blob a backup reuses from an earlier run that stopped
|
||||
# before its manifest: that blob keeps the time of its first upload, so a
|
||||
# prune can delete it
|
||||
@@ -377,9 +345,3 @@ compression_level: 5
|
||||
# Hostname to use in backup metadata
|
||||
# Default: system hostname
|
||||
#hostname: myserver
|
||||
|
||||
# Environment variables that vaultik sets in its own process when it loads
|
||||
# the config. Its keys are variable names
|
||||
# Default: none
|
||||
#env:
|
||||
# NAME: value
|
||||
+435
-367
@@ -1,418 +1,486 @@
|
||||
# vaultik architecture
|
||||
# Vaultik Architecture
|
||||
|
||||
This document describes how the `vaultik` code is organised. It covers three
|
||||
things:
|
||||
This document describes the internal architecture of Vaultik, focusing on the
|
||||
data model, type instantiation, and the relationships between core modules.
|
||||
|
||||
- the packages
|
||||
- how they are wired together
|
||||
- what happens during a backup, a restore, a verify and a prune
|
||||
## Overview
|
||||
|
||||
Two other documents cover the stored data:
|
||||
Vaultik is a backup system that uses content-defined chunking for deduplication
|
||||
and packs chunks into large, compressed, encrypted blobs for efficient cloud
|
||||
storage. The system is built around dependency injection using
|
||||
[uber-go/fx](https://github.com/uber-go/fx).
|
||||
|
||||
- [`DATAMODEL.md`](DATAMODEL.md) describes the local index.
|
||||
- [`REPOSTRUCTURE.md`](REPOSTRUCTURE.md) describes the layout of the
|
||||
destination.
|
||||
|
||||
## Terms
|
||||
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Snapshot ID**: the name of a snapshot, in the form
|
||||
`<hostname>_<snapshot-name>_<timestamp>`.
|
||||
- **Chunk**: a piece of a file's content. A chunk is named by the SHA-256 hash
|
||||
of its content.
|
||||
- **Blob**: one object at the destination. It holds many chunks, compressed and
|
||||
then encrypted.
|
||||
- **Local index**: the SQLite database on the backed-up machine.
|
||||
- **Destination**: the place the backups are stored.
|
||||
- **Marker file**: `vaultik-destination.txt` at the destination. See
|
||||
[The marker file](REPOSTRUCTURE.md#the-marker-file).
|
||||
- **Manifest**: the unencrypted list of the blobs one snapshot uses.
|
||||
- **Remote key**: a hash of the snapshot ID. It names the snapshot's directory
|
||||
at the destination.
|
||||
|
||||
## How a backup moves data
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
source files
|
||||
Source Files
|
||||
│
|
||||
▼
|
||||
Scanner walks the paths and finds the files that changed
|
||||
│
|
||||
▼
|
||||
Chunker cuts each changed file into chunks
|
||||
│
|
||||
▼
|
||||
Packer packs new chunks into a blob, compressed and encrypted
|
||||
│
|
||||
▼
|
||||
Storer uploads the blob to the destination
|
||||
┌─────────────────┐
|
||||
│ Scanner │ Walks directories, detects changed files
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Chunker │ Splits files into variable-size chunks (FastCDC)
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Packer │ Accumulates chunks, compresses (zstd), encrypts (age)
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ S3 Client │ Uploads blobs to remote storage
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## Packages
|
||||
## Data Model
|
||||
|
||||
- `cmd/vaultik`: the `main` function. It starts optional profiling and calls
|
||||
`cli.Entry`.
|
||||
- `internal/cli`: the commands, their flags and their help text. It builds the
|
||||
application, takes the lock and handles interrupts.
|
||||
- `internal/vaultik`: the code behind each command. `Vaultik` holds every
|
||||
dependency a command uses.
|
||||
- `internal/snapshot`: the `Scanner`, which backs up one path, and the
|
||||
`SnapshotManager`, which creates snapshot records and uploads their metadata.
|
||||
- `internal/chunker`: cuts a stream into chunks with the FastCDC algorithm.
|
||||
- `internal/blob`: the `Packer`, which collects chunks into blobs.
|
||||
- `internal/blobgen`: the writer that hashes, compresses and encrypts a blob,
|
||||
and the reader that reverses it.
|
||||
- `internal/database`: the local index. It holds the schema and the repository
|
||||
types that read and write the tables. [`DATAMODEL.md`](DATAMODEL.md#code) says
|
||||
which type serves which table.
|
||||
- `internal/storage`: the `Storer` interface and its three implementations, for
|
||||
S3, a local directory and [rclone](https://rclone.org/).
|
||||
- `internal/s3`: the S3 client that the S3 `Storer` uses.
|
||||
- `internal/config`: loads and checks the config file.
|
||||
- `internal/ui`: formats the status lines, warnings and errors a user sees.
|
||||
- `internal/log`: the logger. It writes to stderr.
|
||||
- `internal/pidlock`: the lock that `RunWithApp` takes for a command that
|
||||
changes data, so that only one runs at a time.
|
||||
- `internal/globals`: the version, commit and commit date, set when the binary
|
||||
is built.
|
||||
- `internal/types`: distinct types for IDs, hashes and paths.
|
||||
### Core Entities
|
||||
|
||||
## Wiring
|
||||
The database tracks five primary entities and their relationships:
|
||||
|
||||
The application is assembled with [uber-go/fx](https://github.com/uber-go/fx), a
|
||||
dependency injection library. `cli.NewApp` lists the providers:
|
||||
```
|
||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||
│ Snapshot │────▶│ File │────▶│ Chunk │
|
||||
└──────────────┘ └──────────────┘ └──────────────┘
|
||||
│ │
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐
|
||||
│ Blob │◀─────────────────────────│ BlobChunk │
|
||||
└──────────────┘ └──────────────┘
|
||||
```
|
||||
|
||||
### Entity Descriptions
|
||||
|
||||
#### File (`database.File`)
|
||||
|
||||
Represents a file, directory, or symlink in the backup system. Stores metadata
|
||||
needed for restoration:
|
||||
|
||||
- Path, mtime
|
||||
- Size, mode, ownership (uid, gid)
|
||||
- Symlink target (if applicable)
|
||||
|
||||
It also stores `source_path`, the source directory the scan found it under, made
|
||||
absolute and with symlinks resolved. Restore does not read it.
|
||||
|
||||
#### Chunk (`database.Chunk`)
|
||||
|
||||
A content-addressed unit of data. Files are split into variable-size chunks
|
||||
using the FastCDC algorithm:
|
||||
|
||||
- `ChunkHash`: SHA256 hash of chunk content (primary key)
|
||||
- `Size`: Chunk size in bytes
|
||||
|
||||
Chunk sizes vary between `avgChunkSize/4` and `avgChunkSize*4` (2.5MB-40MB for
|
||||
the 10MB default average).
|
||||
|
||||
#### FileChunk (`database.FileChunk`)
|
||||
|
||||
Maps files to their constituent chunks:
|
||||
|
||||
- `FileID`: Reference to the file
|
||||
- `Idx`: Position of this chunk within the file (0-indexed)
|
||||
- `ChunkHash`: Reference to the chunk
|
||||
|
||||
#### Blob (`database.Blob`)
|
||||
|
||||
The final storage unit uploaded to S3. Contains many compressed and encrypted
|
||||
chunks:
|
||||
|
||||
- `ID`: UUID assigned at creation
|
||||
- `Hash`: `hex(SHA256(SHA256(uncompressed blob contents)))`, computed before
|
||||
compression and encryption (see
|
||||
[docs/REPOSTRUCTURE.md](REPOSTRUCTURE.md#blobs-directory-blobs))
|
||||
- `UncompressedSize`: Total raw chunk data before compression
|
||||
- `CompressedSize`: Size after zstd compression and age encryption
|
||||
- `CreatedTS`, `FinishedTS`, `UploadedTS`: Lifecycle timestamps
|
||||
|
||||
Blob creation process:
|
||||
|
||||
1. Chunks are accumulated (up to MaxBlobSize, typically 10GB)
|
||||
2. As each chunk is added, its uncompressed bytes are fed to a running SHA-256
|
||||
3. Concurrently, the same bytes are compressed with zstd, then encrypted with
|
||||
age (recipients configured in config), and written to a temporary file
|
||||
4. On finalize, the blob's name is the double SHA-256 of the uncompressed
|
||||
contents — `hex(SHA256(SHA256(...)))` — not a hash of the compressed,
|
||||
encrypted bytes
|
||||
5. The finished file is uploaded to `blobs/{hash[0:2]}/{hash[2:4]}/{hash}` and
|
||||
then deleted
|
||||
|
||||
A backup needs free temporary space, because each blob is written whole to a
|
||||
temporary file before it is uploaded (up to about `blob_size_limit`; an rclone
|
||||
destination that cannot stream uploads needs about twice that) and the metadata
|
||||
export writes copies of the local index. Temporary files go to `$TMPDIR`
|
||||
(default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to
|
||||
`/var/tmp`.
|
||||
|
||||
#### BlobChunk (`database.BlobChunk`)
|
||||
|
||||
Maps chunks to their position within blobs:
|
||||
|
||||
- `BlobID`: Reference to the blob
|
||||
- `ChunkHash`: Reference to the chunk
|
||||
- `Offset`: Byte offset within the uncompressed blob
|
||||
- `Length`: Chunk size
|
||||
|
||||
#### Snapshot (`database.Snapshot`)
|
||||
|
||||
Represents a point-in-time backup:
|
||||
|
||||
- `ID`: Format is `{hostname}_{snapshot-name}_{RFC3339}` (e.g.
|
||||
`server1_home_2025-06-01T12:00:00Z`)
|
||||
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
||||
- `CompletedAt`: Null until snapshot finishes successfully
|
||||
|
||||
#### SnapshotFile / SnapshotBlob
|
||||
|
||||
Join tables linking snapshots to their files and blobs.
|
||||
|
||||
### Relationship Summary
|
||||
|
||||
```
|
||||
Snapshot 1──────────▶ N SnapshotFile N ◀────────── 1 File
|
||||
Snapshot 1──────────▶ N SnapshotBlob N ◀────────── 1 Blob
|
||||
File 1──────────▶ N FileChunk N ◀────────── 1 Chunk
|
||||
Blob 1──────────▶ N BlobChunk N ◀────────── 1 Chunk
|
||||
```
|
||||
|
||||
## Type Instantiation
|
||||
|
||||
### Application Startup
|
||||
|
||||
The CLI uses fx for dependency injection. Here's the instantiation order:
|
||||
|
||||
```go
|
||||
fx.Supply(config.Path(opts.ConfigPath)),
|
||||
fx.Supply(opts.LogOptions),
|
||||
fx.Provide(globals.New),
|
||||
fx.Provide(log.New),
|
||||
config.Module,
|
||||
database.Module,
|
||||
log.Module,
|
||||
storage.Module,
|
||||
snapshot.Module,
|
||||
fx.Provide(vaultik.New),
|
||||
fx.Invoke(setupGlobals),
|
||||
fx.NopLogger,
|
||||
// cli/app.go: NewApp()
|
||||
fx.New(
|
||||
fx.Supply(config.Path(opts.ConfigPath)), // 1. Config path
|
||||
fx.Supply(opts.LogOptions), // 2. Log options
|
||||
fx.Provide(globals.New), // 3. Globals
|
||||
fx.Provide(log.New), // 4. Logger config
|
||||
config.Module, // 5. Config
|
||||
database.Module, // 6. Database + Repositories
|
||||
log.Module, // 7. Logger initialization
|
||||
storage.Module, // 8. Storage backend (S3/file/rclone)
|
||||
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
||||
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
||||
)
|
||||
```
|
||||
|
||||
Each module provides these:
|
||||
### Key Type Instantiation Points
|
||||
|
||||
#### 1. Config (`config.Config`)
|
||||
|
||||
- **Created by**: `config.Module` via `config.LoadConfig()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All configuration from YAML file (S3 credentials, encryption
|
||||
keys, paths, etc.)
|
||||
|
||||
#### 2. Database (`database.DB`)
|
||||
|
||||
- `config.Module` provides `*config.Config`, from `config.Load`.
|
||||
- `database.Module` provides `*database.DB` and `*database.Repositories`. It
|
||||
creates the index file with mode `0600` when it does not exist.
|
||||
- `storage.Module` provides the `storage.Storer` that the config selects.
|
||||
- `snapshot.Module` provides the `snapshot.ScannerFactory` and the
|
||||
`*snapshot.SnapshotManager`.
|
||||
- `vaultik.New` provides `*vaultik.Vaultik`.
|
||||
|
||||
A command runs through `cli.RunOperation`. It works like this:
|
||||
|
||||
- `RunWithApp` takes the lock when the command changes data, and then starts the
|
||||
application.
|
||||
- The command's function runs in its own goroutine.
|
||||
- When the function returns, it asks the application to shut down.
|
||||
- On `SIGINT` or `SIGTERM`, the stop hook cancels the `Vaultik` context. It then
|
||||
waits for the function to return, so that its temporary files are removed.
|
||||
- The wait ends after 30 seconds at most.
|
||||
|
||||
These commands do not build the application:
|
||||
- **Created by**: `database.Module` via `database.New()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: SQLite connection, path reference
|
||||
|
||||
- the `config` commands
|
||||
- `database delete`
|
||||
- `completion`
|
||||
- `version`
|
||||
#### 3. Repositories (`database.Repositories`)
|
||||
|
||||
- **Created by**: `database.Module` via `database.NewRepositories()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All repository interfaces (Files, Chunks, Blobs, Snapshots,
|
||||
etc.)
|
||||
|
||||
## Backup
|
||||
#### 4. Vaultik (`vaultik.Vaultik`)
|
||||
|
||||
`Vaultik.CreateSnapshot` runs a backup. It does these steps in order:
|
||||
- **Created by**: `vaultik.New(VaultikParams)`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All dependencies for backup operations
|
||||
|
||||
- `markDestination` checks that the local index is not bound to another
|
||||
destination. It then checks the marker file, writes it, and binds the local
|
||||
index to the configured destination.
|
||||
- `PruneDatabase` removes unfinished snapshots and unused records from the local
|
||||
index.
|
||||
- `createNamedSnapshot` runs once for each snapshot name.
|
||||
- With `--prune`, `runPostBackupPrune` removes old snapshots and unused data.
|
||||
```go
|
||||
type Vaultik struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
DB *database.DB
|
||||
Repositories *database.Repositories
|
||||
Storage storage.Storer
|
||||
ScannerFactory snapshot.ScannerFactory
|
||||
SnapshotManager *snapshot.SnapshotManager
|
||||
Shutdowner fx.Shutdowner
|
||||
Fs afero.Fs
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
```
|
||||
|
||||
`CreateSnapshot` calls `skipPostBackupPrune` instead of `runPostBackupPrune` in
|
||||
these cases:
|
||||
#### 5. SnapshotManager (`snapshot.SnapshotManager`)
|
||||
|
||||
- `--skip-errors` skipped a file or directory that could not be read.
|
||||
- A configured path held nothing.
|
||||
- An earlier snapshot of the same name found a mount point that this backup did
|
||||
not find.
|
||||
- **Created by**: `snapshot.Module` via `snapshot.NewSnapshotManager()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Responsibility**: Creates/completes snapshots, exports metadata to S3
|
||||
|
||||
`PruneDatabase` must succeed before anything is scanned. An unfinished snapshot
|
||||
left in the index would make the backup skip files that were never fully stored.
|
||||
#### 6. Scanner (`snapshot.Scanner`)
|
||||
|
||||
### One snapshot
|
||||
|
||||
`createNamedSnapshot` does these steps:
|
||||
- **Created by**: `ScannerFactory(ScannerParams)`
|
||||
- **When**: Each `CreateSnapshot()` call
|
||||
- **Contains**: Chunker, Packer, progress reporter
|
||||
|
||||
- `resolveSnapshotPaths` makes each configured path absolute and resolves
|
||||
symlinks.
|
||||
- `SnapshotManager.CreateSnapshotWithName` creates the snapshot record. The
|
||||
record is not yet marked complete.
|
||||
- `Scanner.Scan` backs up each path in turn.
|
||||
- `finalizeSnapshotMetadata` uploads the snapshot's metadata and marks the
|
||||
snapshot complete.
|
||||
```go
|
||||
// vaultik/snapshot.go: CreateSnapshot()
|
||||
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
Fs: v.Fs,
|
||||
})
|
||||
```
|
||||
|
||||
A new `Scanner` is created for each snapshot, from `ScannerFactory`. It owns a
|
||||
`Chunker` and a `Packer`.
|
||||
#### 7. Chunker (`chunker.Chunker`)
|
||||
|
||||
### Scanning one path
|
||||
- **Created by**: `chunker.NewChunker(avgChunkSize)`
|
||||
- **When**: Inside `snapshot.NewScanner()`
|
||||
- **Configuration**:
|
||||
- `avgChunkSize`: From config (default 10MB)
|
||||
- `minChunkSize`: avgChunkSize / 4
|
||||
- `maxChunkSize`: avgChunkSize \* 4
|
||||
|
||||
`Scanner.Scan` works in these steps:
|
||||
#### 8. Packer (`blob.Packer`)
|
||||
|
||||
- `repairInterruptedBlobs` deletes the records of blobs whose upload never
|
||||
finished. Their data is then stored again.
|
||||
- `loadDatabaseState` loads the known files and chunks into memory.
|
||||
- `scanPhase` walks the path. It sorts each regular file and symlink into
|
||||
changed or unchanged.
|
||||
- `detectDeletedFiles` counts the files that the previous snapshot of this name
|
||||
had and that are now gone.
|
||||
- `batchAddFilesToSnapshot` records the unchanged files in the new snapshot.
|
||||
- `recordDirectories` records every directory, and the mount points found.
|
||||
- `processPhase` reads each changed regular file and stores it. It records each
|
||||
changed symlink.
|
||||
|
||||
The walk handles each kind of entry differently:
|
||||
|
||||
- A regular file or a symlink is compared with its record in the index.
|
||||
- A directory is recorded with its metadata. It has no content to store.
|
||||
- A device node, a named pipe or a socket is skipped.
|
||||
- An entry that matches an exclude pattern is skipped. For a directory, so is
|
||||
everything in it.
|
||||
- The local index, its SQLite side files and the lock file `vaultik.pid` are
|
||||
skipped, whatever the exclude patterns say.
|
||||
- **Created by**: `blob.NewPacker(PackerConfig)`
|
||||
- **When**: Inside `snapshot.NewScanner()`
|
||||
- **Configuration**:
|
||||
- `MaxBlobSize`: Maximum blob size before finalization (typically 10GB)
|
||||
- `CompressionLevel`: zstd level (1-19)
|
||||
- `Recipients`: age public keys for encryption
|
||||
|
||||
### Deduplication
|
||||
|
||||
Data is stored once at two levels.
|
||||
|
||||
- **Files.** A file is not read again when its record in the index still matches
|
||||
it.
|
||||
- **Chunks.** A chunk is named by the SHA-256 hash of its content. A chunk that
|
||||
an uploaded blob already holds, according to the index, is not stored again.
|
||||
|
||||
A file's record matches when all of these are unchanged:
|
||||
|
||||
- the size
|
||||
- the modification time
|
||||
- the mode
|
||||
- the owner and group
|
||||
- the link target
|
||||
|
||||
One case overrides the file check. A file is read again when it lists a chunk
|
||||
that no uploaded blob holds. This happens when the only snapshot that used the
|
||||
blob holding that chunk is removed. `loadFilesToRechunk` finds these files.
|
||||
|
||||
### Chunks
|
||||
|
||||
`chunker.Chunker` cuts a file with the FastCDC algorithm. The cut points depend
|
||||
on the content, so an insertion early in a file moves only nearby chunks.
|
||||
|
||||
`chunk_size` sets the average chunk size. No chunk is larger than four times
|
||||
this size. Only a file's last chunk can be smaller than a quarter of it. With
|
||||
the default of 10 MB, those limits are 40 MB and 2.5 MB.
|
||||
|
||||
`ChunkReaderStreaming` passes each chunk to a callback as it is found. The
|
||||
chunker buffers at most twice the largest chunk size.
|
||||
|
||||
### Blobs
|
||||
|
||||
`blob.Packer` collects new chunks into a blob. For each blob it does this:
|
||||
|
||||
- It gives the blob a UUID and creates a temporary file.
|
||||
- It adds each chunk's bytes to a running SHA-256 hash.
|
||||
- In the same pass it compresses the bytes with
|
||||
[zstd](https://github.com/facebook/zstd) and encrypts them with
|
||||
[age](https://age-encryption.org/). The result goes to the temporary file.
|
||||
- It records each chunk's offset and length within the blob.
|
||||
- It closes the blob when the next chunk would pass `blob_size_limit`, and at
|
||||
the end of each path.
|
||||
```go
|
||||
// snapshot/scanner.go: NewScanner()
|
||||
packerCfg := blob.PackerConfig{
|
||||
MaxBlobSize: cfg.MaxBlobSize,
|
||||
CompressionLevel: cfg.CompressionLevel,
|
||||
Recipients: cfg.AgeRecipients,
|
||||
Repositories: cfg.Repositories,
|
||||
Fs: cfg.FS,
|
||||
}
|
||||
packer, err := blob.NewPacker(packerCfg)
|
||||
```
|
||||
|
||||
The blob's name is `hex(SHA256(SHA256(uncompressed contents)))`. It is computed
|
||||
from the chunk data before compression and encryption.
|
||||
[`REPOSTRUCTURE.md`](REPOSTRUCTURE.md#blobs) says what follows from that.
|
||||
## Module Responsibilities
|
||||
|
||||
When a blob is closed, `Scanner.handleBlobReady` stores it:
|
||||
### `internal/cli`
|
||||
|
||||
- It looks the blob up at the destination. A blob that is already there is not
|
||||
uploaded again.
|
||||
- Otherwise it uploads the temporary file to
|
||||
`blobs/<hash[0:2]>/<hash[2:4]>/<hash>`.
|
||||
- It records the upload in the index and links the blob to the snapshot.
|
||||
- It deletes the temporary file.
|
||||
|
||||
A file's record is written to the index only after every blob that holds its
|
||||
chunks is uploaded.
|
||||
|
||||
### Snapshot metadata
|
||||
|
||||
`finalizeSnapshotMetadata` does these steps in order:
|
||||
|
||||
- `PopulateSnapshotBlobs` records every blob the snapshot uses.
|
||||
- `UpdateSnapshotStatsExtended` stores the snapshot's totals.
|
||||
- `ExportSnapshotMetadata` uploads the metadata.
|
||||
- `MarkSnapshotComplete` marks the snapshot complete.
|
||||
Entry point for fx application. Combines all modules and handles signal
|
||||
interrupts.
|
||||
|
||||
Completion is recorded last on purpose. A backup that stops during the upload
|
||||
leaves the snapshot unfinished, and the next backup removes it and backs up its
|
||||
files again.
|
||||
|
||||
`ExportSnapshotMetadata` works on a copy of the index, never on the index
|
||||
itself:
|
||||
|
||||
- It copies the index into a temporary directory with `VACUUM INTO`.
|
||||
- It deletes the rows of every other snapshot from the copy. It also deletes the
|
||||
files and blobs that this snapshot does not use. It then deletes each chunk
|
||||
that no remaining file and no remaining blob uses.
|
||||
- It keeps the `local_meta` and `schema_migrations` tables as they are. The
|
||||
uploaded database therefore names the destination, and lists the migrations
|
||||
that have run.
|
||||
- It runs `VACUUM`, so that no deleted row is left in the file.
|
||||
- It compresses the copy with zstd and encrypts it with age.
|
||||
- It builds the manifest from the copy.
|
||||
- It uploads `metadata/<remote-key>/db.zst.age`, and then
|
||||
`metadata/<remote-key>/manifest.json.zst`.
|
||||
Key functions:
|
||||
|
||||
The manifest is uploaded last. A directory under `metadata/` with no manifest
|
||||
therefore marks a backup that did not finish.
|
||||
|
||||
## Restore
|
||||
- `NewApp(AppOptions)` → Creates fx.App with all modules
|
||||
- `RunApp(ctx, app)` → Starts app, handles graceful shutdown
|
||||
- `RunWithApp(ctx, opts)` → Convenience wrapper
|
||||
|
||||
`Vaultik.Restore` does these steps:
|
||||
### `internal/vaultik`
|
||||
|
||||
- `restoreIdentities` parses the age private key. A missing or malformed key
|
||||
stops the restore before anything is downloaded.
|
||||
- `downloadSnapshotDB` downloads and decrypts the snapshot's database into a
|
||||
temporary directory.
|
||||
- `getFilesToRestore` selects the files, from the paths given.
|
||||
- `restoreAllFiles` downloads blobs and writes the files.
|
||||
- With `--verify`, `verifyRestoredFiles` reads each restored regular file and
|
||||
checks its chunk hashes.
|
||||
Main orchestrator containing all dependencies and command implementations.
|
||||
|
||||
Given a snapshot ID, the restore looks it up in the local index. When it is not
|
||||
there, the restore checks the marker file and lists the snapshot's directory.
|
||||
Given a remote key, it reads the local index only to check the marker file. It
|
||||
does not read the manifest. It finds each file's chunks and blobs through the
|
||||
downloaded database.
|
||||
Key methods:
|
||||
|
||||
### Checks on the downloaded database
|
||||
- `New(VaultikParams)` → Constructor (fx DI)
|
||||
- `CreateSnapshot(opts)` → Main backup operation
|
||||
- `ListSnapshots(jsonOutput)` → List available snapshots
|
||||
- `VerifySnapshot(id, deep)` → Verify snapshot integrity
|
||||
- `PurgeSnapshots(...)` → Remove old snapshots
|
||||
|
||||
Decryption shows that a database is readable. It does not show that the database
|
||||
is honest. The restore therefore checks it:
|
||||
### `internal/chunker`
|
||||
|
||||
- `database.OpenReadOnly` opens the file read-only. It refuses a file that has a
|
||||
trigger, a view or a virtual table, or that lacks an expected table.
|
||||
- `verifySnapshotDBIdentity` checks that the snapshot in the database hashes to
|
||||
the remote key that was asked for.
|
||||
- `containedRestorePath` refuses a stored path that would leave the target
|
||||
directory. It also refuses to write through a symlink below the target.
|
||||
- A blob is not decompressed beyond the size that the database gives for it.
|
||||
- Each downloaded blob is hashed. The hash must match the blob's name.
|
||||
|
||||
### Order of work
|
||||
|
||||
`restorePlan` orders the files so that few blobs are on disk at once:
|
||||
|
||||
- Downloaded blobs are kept decrypted in a cache directory.
|
||||
- A file is restored once every blob it needs is in the cache.
|
||||
- When no file is ready, the restore downloads the blobs for the file that needs
|
||||
the fewest.
|
||||
- `restoreSweeper` deletes a cached blob once no remaining file needs it.
|
||||
|
||||
A regular file is created with mode `0600` and gets its stored mode after its
|
||||
content is written. A directory gets its stored mode and modification time after
|
||||
the whole restore, and its owner when the command runs as root.
|
||||
|
||||
## Verify
|
||||
|
||||
Without `--deep`, `Vaultik.VerifySnapshotWithOptions` checks a snapshot without
|
||||
the private key:
|
||||
Content-defined chunking using FastCDC algorithm.
|
||||
|
||||
- It downloads the manifest.
|
||||
- It checks that the snapshot's encrypted database is present.
|
||||
- It checks that each listed blob is present with the listed size.
|
||||
Key types:
|
||||
|
||||
`Vaultik.RunDeepVerify` needs the private key. It does these steps:
|
||||
|
||||
- It downloads the manifest and the database. It checks the database as a
|
||||
restore does.
|
||||
- `verifyManifestAgainstDatabase` checks that both list the same blobs with the
|
||||
same sizes.
|
||||
- `verifyBlobExistenceFromDB` checks that every blob is present with the
|
||||
recorded size.
|
||||
- `verifyBlob` downloads each blob, checks the hash of each chunk in it, and
|
||||
checks the hash that names the blob.
|
||||
|
||||
## Prune
|
||||
|
||||
`Vaultik.Prune` calls `checkStorageBinding` and then does three steps:
|
||||
|
||||
- `CleanupLocalSnapshots` drops from the local index each snapshot whose
|
||||
manifest is gone from the destination.
|
||||
- `PruneDatabase` removes unfinished snapshots and unused records from the local
|
||||
index.
|
||||
- `PruneBlobs` deletes unused data at the destination.
|
||||
|
||||
`PruneBlobs` reads every manifest at the destination. It then deletes two kinds
|
||||
of object, once they are older than `prune_grace_period`:
|
||||
|
||||
- a blob that no manifest lists
|
||||
- a directory under `metadata/` that has no manifest, and whose snapshot is not
|
||||
in the local index
|
||||
|
||||
It deletes nothing when any manifest cannot be read.
|
||||
[`REPOSTRUCTURE.md`](REPOSTRUCTURE.md#pruning) explains why this is safe, and
|
||||
where it is not.
|
||||
|
||||
## The destination
|
||||
|
||||
`storage.Storer` is the only way the code reaches the destination. It has these
|
||||
methods:
|
||||
- `Chunk` → Hash, Data, Offset, Size
|
||||
- `Chunker` → avgChunkSize, minChunkSize, maxChunkSize
|
||||
|
||||
- `Put` and `PutWithProgress` store an object.
|
||||
- `Get` reads an object.
|
||||
- `Stat` reports an object's size.
|
||||
- `Delete` removes an object.
|
||||
- `List` and `ListStream` list the objects under a prefix.
|
||||
- `DeletePartialUploads` removes what an interrupted upload left behind.
|
||||
- `Info` describes the destination.
|
||||
Key methods:
|
||||
|
||||
`storage.NewStorer` picks the implementation from `storage_url`:
|
||||
|
||||
- `s3://` gives the S3 implementation.
|
||||
- `file://` gives the local directory implementation.
|
||||
- `rclone://` gives the rclone implementation.
|
||||
|
||||
With no `storage_url`, it builds the S3 implementation from the `s3` settings.
|
||||
- `NewChunker(avgChunkSize)` → Constructor
|
||||
- `ChunkReaderStreaming(reader, callback)` → Stream chunks with callback
|
||||
(preferred)
|
||||
- `ChunkReader(reader)` → Return all chunks at once (memory-intensive)
|
||||
|
||||
## The marker file and the destination binding
|
||||
### `internal/blob`
|
||||
|
||||
These checks keep the local index and the destination in step.
|
||||
Blob packing: accumulates chunks, compresses, encrypts, tracks metadata.
|
||||
|
||||
- `checkDestinationMarker` looks for `vaultik-destination.txt`. It fails when
|
||||
the file is reported missing while the local index holds snapshots. It skips
|
||||
the check when the local index belongs to another destination.
|
||||
- `checkStorageBinding` fails when the local index is bound to a different
|
||||
destination from the configured one. It records nothing.
|
||||
- `EnsureStorageBinding` binds an unbound local index to the configured
|
||||
destination. Only `markDestination` calls it, after the marker file is
|
||||
written, so a backup that writes nothing leaves the local index unbound.
|
||||
Key types:
|
||||
|
||||
Every command that takes the lock calls `checkStorageBinding` first.
|
||||
|
||||
## Concurrency
|
||||
|
||||
- Each open database uses one connection, so its queries run one at a time.
|
||||
- The index runs in SQLite's write-ahead log (WAL) mode. A second process can
|
||||
therefore read it while a backup writes to it.
|
||||
- `Packer` guards its state with a mutex.
|
||||
- `Scanner` holds `packerMu` while a chunk is added to the packer and while a
|
||||
blob is closed.
|
||||
- `Repositories.WithTx` runs a function in one transaction. It commits when the
|
||||
function succeeds and rolls back when it fails.
|
||||
- `Packer` → Thread-safe blob accumulator
|
||||
- `ChunkRef` → Hash + Data for adding to packer
|
||||
- `FinishedBlob` → Completed blob ready for upload
|
||||
- `BlobWithReader` → FinishedBlob + io.Reader for streaming upload
|
||||
|
||||
Key methods:
|
||||
|
||||
- `NewPacker(PackerConfig)` → Constructor
|
||||
- `AddChunk(ChunkRef)` → Add chunk to current blob
|
||||
- `FinalizeBlob()` → Compress, encrypt, hash current blob
|
||||
- `Flush()` → Finalize any in-progress blob
|
||||
- `SetBlobHandler(func)` → Set callback for upload
|
||||
|
||||
### `internal/snapshot`
|
||||
|
||||
#### Scanner
|
||||
|
||||
Orchestrates the backup process for a directory.
|
||||
|
||||
Key methods:
|
||||
|
||||
- `NewScanner(ScannerConfig)` → Constructor (creates Chunker + Packer)
|
||||
- `Scan(ctx, path, snapshotID)` → Main scan operation
|
||||
|
||||
Scan phases:
|
||||
|
||||
1. **Phase 1**: Walk directory, identify files needing processing
|
||||
2. **Phase 1b**: Count as deleted the files that the previous snapshot of the
|
||||
same name lists, the walk did not find, and are gone from disk; a file that
|
||||
is excluded, or inside an excluded directory, is not counted
|
||||
3. **Phase 2**: Process files (chunk → pack → upload)
|
||||
|
||||
#### SnapshotManager
|
||||
|
||||
Manages snapshot lifecycle and metadata export.
|
||||
|
||||
Key methods:
|
||||
|
||||
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
|
||||
- `PopulateSnapshotBlobs(ctx, snapshotID)` → Record every blob the snapshot
|
||||
references
|
||||
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
|
||||
- `MarkSnapshotComplete(ctx, snapshotID)` → Record completion, only after a
|
||||
successful export
|
||||
- `CompleteSnapshot(ctx, snapshotID)` → Convenience: populate blobs, then mark
|
||||
complete (no export between)
|
||||
|
||||
### `internal/database`
|
||||
|
||||
SQLite database for local index. Single-writer mode for thread safety.
|
||||
|
||||
Key types:
|
||||
|
||||
- `DB` → Database connection wrapper
|
||||
- `Repositories` → Collection of all repository interfaces
|
||||
|
||||
Repository interfaces:
|
||||
|
||||
- `FilesRepository` → CRUD for File records
|
||||
- `ChunksRepository` → CRUD for Chunk records
|
||||
- `BlobsRepository` → CRUD for Blob records
|
||||
- `SnapshotsRepository` → CRUD for Snapshot records
|
||||
- Plus join table repositories (FileChunks, BlobChunks, etc.)
|
||||
|
||||
## Snapshot Creation Flow
|
||||
|
||||
```
|
||||
CreateSnapshot(opts)
|
||||
│
|
||||
├─► markDestination() // Check the marker (see Storage Layout in S3), then write it
|
||||
│
|
||||
├─► PruneDatabase() // Critical: avoid dedup errors
|
||||
│
|
||||
├─► SnapshotManager.CreateSnapshot() // Create DB record
|
||||
│
|
||||
├─► For each source directory:
|
||||
│ │
|
||||
│ ├─► scanner.Scan(ctx, path, snapshotID)
|
||||
│ │ │
|
||||
│ │ ├─► Phase 1: scanPhase()
|
||||
│ │ │ Walk directory
|
||||
│ │ │ Check file metadata changes
|
||||
│ │ │ Build list of files to process
|
||||
│ │ │
|
||||
│ │ ├─► Phase 1b: detectDeletedFiles()
|
||||
│ │ │
|
||||
│ │ └─► Phase 2: processPhase()
|
||||
│ │ For each file:
|
||||
│ │ chunker.ChunkReaderStreaming()
|
||||
│ │ For each chunk:
|
||||
│ │ packer.AddChunk()
|
||||
│ │ If blob full → FinalizeBlob()
|
||||
│ │ → handleBlobReady()
|
||||
│ │ → s3Client.PutObjectWithProgress()
|
||||
│ │ packer.Flush() // Final blob
|
||||
│ │
|
||||
│ └─► Accumulate statistics
|
||||
│
|
||||
├─► SnapshotManager.PopulateSnapshotBlobs() // record referenced blobs
|
||||
│
|
||||
├─► SnapshotManager.UpdateSnapshotStatsExtended()
|
||||
│
|
||||
├─► SnapshotManager.ExportSnapshotMetadata()
|
||||
│ │
|
||||
│ ├─► Copy database to temp file
|
||||
│ ├─► Clean to only current snapshot data (VACUUM)
|
||||
│ ├─► Compress binary SQLite with zstd
|
||||
│ ├─► Encrypt with age
|
||||
│ ├─► Upload db.zst.age to storage
|
||||
│ └─► Upload manifest.json.zst to storage
|
||||
│
|
||||
└─► SnapshotManager.MarkSnapshotComplete() // only after the export succeeds
|
||||
```
|
||||
|
||||
## Deduplication Strategy
|
||||
|
||||
1. **File-level**: Files and symlinks unchanged since last backup are skipped
|
||||
(metadata comparison: size, mtime, mode, uid, gid, and a symlink's target),
|
||||
unless the file lists a chunk that no uploaded blob holds; such a file is
|
||||
re-chunked
|
||||
|
||||
2. **Chunk-level**: Chunks are content-addressed by SHA256 hash. If a chunk hash
|
||||
already exists in the database, the chunk data is not re-uploaded.
|
||||
|
||||
3. **Blob-level**: Blobs contain only unique chunks. Duplicate chunks within a
|
||||
blob are skipped.
|
||||
|
||||
## Storage Layout in S3
|
||||
|
||||
```
|
||||
bucket/
|
||||
├── vaultik-destination.txt # Marker written by every backup
|
||||
│
|
||||
├── blobs/
|
||||
│ └── {hash[0:2]}/
|
||||
│ └── {hash[2:4]}/
|
||||
│ └── {full-hash} # Compressed+encrypted blob
|
||||
│
|
||||
└── metadata/
|
||||
└── {remote-key}/
|
||||
├── db.zst.age # Encrypted binary SQLite database
|
||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
||||
```
|
||||
|
||||
A destination that answers a look-up of `vaultik-destination.txt` with not found
|
||||
is treated as not present while the local index holds snapshots stored there
|
||||
(`checkDestinationMarker` in `internal/vaultik/destination_marker.go`):
|
||||
`snapshot create` refuses to write to it, `prune`, `snapshot purge` and
|
||||
`remote nuke` fail, and `snapshot list` and `snapshot remove` behave as for a
|
||||
destination that cannot be listed. This keeps those commands from taking an
|
||||
unplugged volume's empty mount point for a destination whose snapshots are all
|
||||
gone. When the look-up fails for any reason other than not found, for example
|
||||
when S3 refuses it to credentials that allow only PUT, the check is skipped, so
|
||||
a host with such credentials does not notice an emptied destination; see
|
||||
[docs/REPOSTRUCTURE.md](REPOSTRUCTURE.md#marker-file-vaultik-destinationtxt).
|
||||
The `file://` backend never creates its destination directory, so a write to a
|
||||
missing one fails.
|
||||
|
||||
The `{remote-key}` directory name is a one-way double SHA-256 hash of the human
|
||||
snapshot ID, so the human ID (hostname, snapshot name, timestamp) is never
|
||||
written to the store as a directory name. See
|
||||
[docs/REPOSTRUCTURE.md](REPOSTRUCTURE.md#remote-key-derivation) for the
|
||||
derivation and a worked example.
|
||||
|
||||
## Thread Safety
|
||||
|
||||
- `Packer`: Thread-safe via mutex. Multiple goroutines can call `AddChunk()`.
|
||||
- `Scanner`: Uses `packerMu` mutex to coordinate blob finalization.
|
||||
- `Database`: Single-writer mode (`MaxOpenConns=1`) ensures SQLite thread
|
||||
safety.
|
||||
- `Repositories.WithTx()`: Handles transaction lifecycle automatically.
|
||||
+301
-362
@@ -1,450 +1,389 @@
|
||||
# vaultik data model
|
||||
# Vaultik Data Model
|
||||
|
||||
`vaultik` keeps a SQLite database on the backed-up machine, called the local
|
||||
index. This document describes its tables, and how a backup, a restore and
|
||||
`prune` use them.
|
||||
## Overview
|
||||
|
||||
The local index has two jobs:
|
||||
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and
|
||||
blob associations during the backup process. This database serves as an index
|
||||
for incremental backups and enables efficient deduplication.
|
||||
|
||||
- It records which files, chunks and blobs are already stored. The next backup
|
||||
then does not store them again.
|
||||
- It is the source of each snapshot's own database. A backup uploads a trimmed,
|
||||
encrypted copy of the index with every snapshot.
|
||||
**Important Notes:**
|
||||
|
||||
A restore reads no file, chunk or blob rows from the local index. It reads the
|
||||
copy that was uploaded with the snapshot.
|
||||
- **Version Compatibility**: In rare cases, you may need to use the same version
|
||||
of Vaultik to restore a backup as was used to create it. This ensures
|
||||
compatibility with the metadata format stored in S3.
|
||||
|
||||
## Terms
|
||||
## Schema Migrations
|
||||
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Snapshot ID**: the name of a snapshot, in the form
|
||||
`<hostname>_<snapshot-name>_<timestamp>`.
|
||||
- **Chunk**: a piece of a file's content, named by the SHA-256 hash of that
|
||||
content.
|
||||
- **Blob**: one object at the destination. It holds many chunks, compressed and
|
||||
then encrypted.
|
||||
- **Destination**: the place the backups are stored.
|
||||
- **Marker file**: `vaultik-destination.txt` at the destination. See
|
||||
[The marker file](REPOSTRUCTURE.md#the-marker-file).
|
||||
- **Manifest**: the unencrypted list of the blobs one snapshot uses.
|
||||
- **Remote key**: a hash of the snapshot ID. It names the snapshot's directory
|
||||
at the destination.
|
||||
Vaultik supports schema migrations. They are the numbered SQL files in
|
||||
`internal/database/schema/`, embedded in the binary: `000.sql` creates the
|
||||
`schema_migrations` table, which records each migration that has run, and
|
||||
`001.sql` creates the application tables. `database.New` opens a database and
|
||||
applies, in order, every migration that database has not yet recorded.
|
||||
|
||||
## Where the index is
|
||||
**Before 1.0** no migrations are added, because nothing is installed anywhere
|
||||
yet. A schema change edits `001.sql` (and the code that uses the affected
|
||||
tables) directly. A local database created before the change has already
|
||||
recorded `001.sql` as run, so it keeps the old schema and can become unusable;
|
||||
`vaultik database delete` followed by a full backup rebuilds it.
|
||||
|
||||
The index is the file `index.sqlite` in the platform data directory. By default
|
||||
that is `~/.local/share/vaultik/` on Linux and
|
||||
`~/Library/Application Support/vaultik/` on macOS. The `index_path` setting and
|
||||
the `VAULTIK_INDEX_PATH` environment variable change the path. So does
|
||||
`XDG_DATA_HOME`.
|
||||
**After 1.0** each schema change is a new numbered file, so an existing local
|
||||
database is migrated the first time an updated vaultik opens it. A file that has
|
||||
shipped in a release is never edited.
|
||||
|
||||
The file is readable only by its owner, because it lists every backed-up path.
|
||||
`vaultik` sets that mode each time it opens the index.
|
||||
## Database Tables
|
||||
|
||||
## Schema migrations
|
||||
### 1. `files`
|
||||
|
||||
`vaultik` supports schema migrations. They are the numbered SQL files in
|
||||
`internal/database/schema/`, which are built into the binary.
|
||||
Stores metadata about files in the filesystem being backed up.
|
||||
|
||||
- `000.sql` creates the `schema_migrations` table. That table records each
|
||||
migration that has run.
|
||||
- `001.sql` creates every other table.
|
||||
**Columns:**
|
||||
|
||||
`database.New` opens the index. It then applies, in order, every migration that
|
||||
the index has not recorded.
|
||||
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
||||
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
||||
- `mtime` (INTEGER NOT NULL) - Modification time, whole seconds since the Unix
|
||||
epoch
|
||||
- `mtime_nsec` (INTEGER NOT NULL) - Nanoseconds within that second, 0 to
|
||||
999999999
|
||||
- `size` (INTEGER NOT NULL) - File size in bytes
|
||||
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
||||
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
||||
- `gid` (INTEGER NOT NULL) - Group ID of file owner
|
||||
- `link_target` (TEXT) - Symlink target path (NULL for regular files)
|
||||
|
||||
Before version 1.0, no migration is added, because nothing is installed anywhere
|
||||
yet. A schema change edits `001.sql` directly, with the code that uses the
|
||||
changed tables.
|
||||
**Indexes:**
|
||||
|
||||
This has one consequence. An index created before such a change has already
|
||||
recorded `001.sql` as run, so it keeps the old schema and can become unusable.
|
||||
To rebuild it, run `vaultik database delete` and then a full backup.
|
||||
- `idx_files_path` on `path` for efficient lookups
|
||||
|
||||
After version 1.0, each schema change is a new numbered file. An existing index
|
||||
is then migrated the first time an updated `vaultik` opens it. A file that was
|
||||
part of a release is never edited.
|
||||
**Purpose:** Tracks file metadata to detect changes between backup runs. Used
|
||||
for incremental backup decisions. The UUID primary key provides stable
|
||||
references that don't change if files are moved.
|
||||
|
||||
Migrations apply to the local index only. A snapshot's database at the
|
||||
destination is read as it was written. Each snapshot records the `vaultik`
|
||||
version that wrote it.
|
||||
### 2. `chunks`
|
||||
|
||||
## Tables
|
||||
Stores information about content-defined chunks created from files.
|
||||
|
||||
Times are whole seconds since the Unix epoch, in UTC. The two exceptions are
|
||||
noted at their columns.
|
||||
**Columns:**
|
||||
|
||||
### `files`
|
||||
- `chunk_hash` (TEXT PRIMARY KEY) - SHA256 hash of chunk content
|
||||
- `size` (INTEGER NOT NULL) - Chunk size in bytes
|
||||
|
||||
One row for each backed-up path: a regular file, a symlink or a directory.
|
||||
**Purpose:** Enables deduplication by tracking unique chunks across all files.
|
||||
|
||||
- `id`: a UUID. Primary key.
|
||||
- `path`: the absolute path. Unique.
|
||||
- `source_path`: the configured path under which the scan found this entry, made
|
||||
absolute with symlinks resolved. A restore does not read it.
|
||||
- `mtime`: the modification time.
|
||||
- `mtime_nsec`: the nanoseconds within that second.
|
||||
- `size`: the size in bytes.
|
||||
- `mode`: the permission bits and the file type.
|
||||
- `uid`: the owner's user ID.
|
||||
- `gid`: the owner's group ID.
|
||||
- `link_target`: the target of a symlink. It is empty for other entries.
|
||||
### 3. `file_chunks`
|
||||
|
||||
`idx_files_path` indexes `path`.
|
||||
Maps files to their constituent chunks in order.
|
||||
|
||||
There is one row for each path, and it is updated when the file changes. The row
|
||||
is shared by every snapshot in the local index that lists the path. An older
|
||||
snapshot's own view of the file is in that snapshot's database at the
|
||||
destination.
|
||||
**Columns:**
|
||||
|
||||
### `chunks`
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- `idx` (INTEGER) - Chunk index within file (0-based)
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- PRIMARY KEY (`file_id`, `idx`)
|
||||
|
||||
One row for each distinct chunk.
|
||||
**Purpose:** Allows reconstruction of files from chunks during restore.
|
||||
|
||||
- `chunk_hash`: the SHA-256 hash of the chunk's content, in hex. Primary key.
|
||||
- `size`: the size in bytes.
|
||||
### 4. `chunk_files`
|
||||
|
||||
### `file_chunks`
|
||||
Reverse mapping showing which files contain each chunk.
|
||||
|
||||
The chunks that make up each file, in order.
|
||||
**Columns:**
|
||||
|
||||
- `file_id`: the file. References `files.id`.
|
||||
- `idx`: the position of the chunk within the file, counted from 0.
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- `file_offset` (INTEGER) - Byte offset of chunk within file
|
||||
- `length` (INTEGER) - Length of chunk in bytes
|
||||
- PRIMARY KEY (`chunk_hash`, `file_id`)
|
||||
|
||||
The primary key is (`file_id`, `idx`). `idx_file_chunks_chunk_hash` indexes
|
||||
`chunk_hash`.
|
||||
**Purpose:** Supports efficient queries for chunk usage and deduplication
|
||||
statistics.
|
||||
|
||||
### `chunk_files`
|
||||
### 5. `blobs`
|
||||
|
||||
The reverse of `file_chunks`: the files that contain each chunk.
|
||||
Stores information about packed, compressed, and encrypted blob files.
|
||||
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
- `file_id`: the file. References `files.id`.
|
||||
- `file_offset`: the byte offset of the chunk within the file.
|
||||
- `length`: the length of the chunk in bytes.
|
||||
**Columns:**
|
||||
|
||||
The primary key is (`chunk_hash`, `file_id`). `idx_chunk_files_file_id` indexes
|
||||
`file_id`.
|
||||
- `id` (TEXT PRIMARY KEY) - UUID assigned when blob creation starts
|
||||
- `blob_hash` (TEXT UNIQUE) - `hex(SHA256(SHA256(uncompressed blob contents)))`,
|
||||
computed before compression and encryption (NULL until finalized); see
|
||||
[REPOSTRUCTURE.md](REPOSTRUCTURE.md#blobs-directory-blobs)
|
||||
- `created_ts` (INTEGER NOT NULL) - Creation timestamp
|
||||
- `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
|
||||
- `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before
|
||||
compression
|
||||
- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and
|
||||
encryption. Once uploaded, the size of the object in storage: when the blob
|
||||
was already stored, under another compression level or other recipients, this
|
||||
can differ from the size of the local copy
|
||||
- `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
|
||||
|
||||
### `blobs`
|
||||
**Purpose:** Tracks blob lifecycle from creation through upload. The UUID
|
||||
primary key allows immediate association of chunks with blobs.
|
||||
|
||||
One row for each blob.
|
||||
### 6. `blob_chunks`
|
||||
|
||||
- `id`: a UUID, given when packing of the blob starts. Primary key.
|
||||
- `blob_hash`: the name of the blob at the destination. Unique. Until the blob
|
||||
is closed, it holds a placeholder that starts with `temp-placeholder-`.
|
||||
- `created_ts`: when packing started.
|
||||
- `finished_ts`: when the blob was closed. It is `NULL` before that.
|
||||
- `uncompressed_size`: the total size of the blob's chunks.
|
||||
- `compressed_size`: the size after compression and encryption.
|
||||
- `uploaded_ts`: when the upload finished. It is `NULL` before that.
|
||||
Maps chunks to the blobs that contain them.
|
||||
|
||||
`blob_hash` is `hex(SHA256(SHA256(uncompressed contents)))`. See
|
||||
[`REPOSTRUCTURE.md`](REPOSTRUCTURE.md#blobs).
|
||||
**Columns:**
|
||||
|
||||
Once a blob is uploaded, `compressed_size` is the size of the object at the
|
||||
destination. A blob that was already there can have a different size from the
|
||||
local copy. That happens when it was written with another compression level or
|
||||
other recipients.
|
||||
- `blob_id` (TEXT) - Blob ID (FK to blobs.id)
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- `offset` (INTEGER) - Byte offset of chunk within blob (before compression)
|
||||
- `length` (INTEGER) - Length of chunk in bytes
|
||||
- PRIMARY KEY (`blob_id`, `chunk_hash`)
|
||||
|
||||
### `blob_chunks`
|
||||
**Purpose:** Enables chunk retrieval from blobs during restore operations.
|
||||
|
||||
The chunks that each blob holds.
|
||||
### 7. `snapshots`
|
||||
|
||||
- `blob_id`: the blob. References `blobs.id`.
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
- `offset`: the byte offset of the chunk within the uncompressed blob.
|
||||
- `length`: the length of the chunk in bytes.
|
||||
Tracks backup snapshots.
|
||||
|
||||
The primary key is (`blob_id`, `chunk_hash`). `idx_blob_chunks_chunk_hash`
|
||||
indexes `chunk_hash`.
|
||||
**Columns:**
|
||||
|
||||
A restore finds the blob for each chunk through this table.
|
||||
- `id` (TEXT PRIMARY KEY) - Snapshot ID (format: `hostname_name_timestamp`, e.g.
|
||||
`server1_home_2025-06-01T12:00:00Z`: the hostname up to its first `.`, the
|
||||
snapshot name, and an RFC 3339 UTC timestamp)
|
||||
- `hostname` (TEXT) - Hostname where backup was created
|
||||
- `vaultik_version` (TEXT) - Version of Vaultik used
|
||||
- `vaultik_git_revision` (TEXT) - Git revision of Vaultik used
|
||||
- `started_at` (INTEGER) - Start timestamp
|
||||
- `completed_at` (INTEGER) - Completion timestamp (NULL if in progress)
|
||||
- `file_count` (INTEGER) - Number of files in snapshot
|
||||
- `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not
|
||||
stored before
|
||||
- `new_chunk_size` (INTEGER) - Total size of those chunks before compression;
|
||||
`snapshot list` shows it as NEW CHUNK SIZE
|
||||
- `blob_count` (INTEGER) - Number of blobs this snapshot created
|
||||
- `total_size` (INTEGER) - Total size of all files
|
||||
- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the
|
||||
distinct chunks the snapshot's files reference, recorded when its backup ran;
|
||||
`snapshot list` shows it as UNCOMPRESSED SIZE
|
||||
- `blob_size` (INTEGER) - Total compressed size of all referenced blobs
|
||||
- `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced
|
||||
blobs
|
||||
- `compression_ratio` (REAL) - Compression ratio achieved
|
||||
- `compression_level` (INTEGER) - Compression level used for this snapshot
|
||||
- `upload_bytes` (INTEGER) - Total bytes uploaded during this snapshot
|
||||
- `upload_duration_ms` (INTEGER) - Total milliseconds spent uploading to S3
|
||||
|
||||
### `snapshots`
|
||||
**Purpose:** Provides snapshot metadata and statistics including version
|
||||
tracking for compatibility.
|
||||
|
||||
One row for each snapshot.
|
||||
### 8. `snapshot_files`
|
||||
|
||||
- `id`: the snapshot ID. Primary key.
|
||||
- `hostname`: the `hostname` setting of the config. When that setting is empty,
|
||||
it is the machine's hostname.
|
||||
- `vaultik_version`: the `vaultik` version that made it.
|
||||
- `vaultik_git_revision`: the git commit of that version.
|
||||
- `started_at`: when the backup started.
|
||||
- `completed_at`: when the backup finished. It is `NULL` until then.
|
||||
- `file_count`: the number of files and symlinks in the snapshot. Directories
|
||||
are not counted.
|
||||
- `chunk_count`: the number of chunks this snapshot stored that were not stored
|
||||
before.
|
||||
- `new_chunk_size`: the total size of those chunks before compression.
|
||||
- `blob_count`: the number of blobs this snapshot created.
|
||||
- `total_size`: the total size of all files in the snapshot.
|
||||
- `uncompressed_chunk_size`: the total size, before compression, of the distinct
|
||||
chunks the snapshot's files use.
|
||||
- `blob_size`: the total compressed size of the blobs the snapshot uses.
|
||||
- `blob_uncompressed_size`: the total uncompressed size of those blobs.
|
||||
- `compression_ratio`: the compression ratio of those blobs.
|
||||
- `compression_level`: the compression level the backup used.
|
||||
- `upload_bytes`: the bytes uploaded during the backup.
|
||||
- `upload_duration_ms`: the time spent uploading, in milliseconds.
|
||||
Maps snapshots to the files they contain.
|
||||
|
||||
The snapshot ID has the form `<hostname>_<snapshot-name>_<timestamp>`. An
|
||||
example is `server1_home_2025-06-01T12:00:00Z`. It has three parts:
|
||||
**Columns:**
|
||||
|
||||
- the hostname, up to its first `.`
|
||||
- the snapshot name
|
||||
- the start time, in UTC
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- PRIMARY KEY (`snapshot_id`, `file_id`)
|
||||
|
||||
The `snapshot list` table shows `uncompressed_chunk_size` as `UNCOMPRESSED SIZE`
|
||||
and `new_chunk_size` as `NEW CHUNK SIZE`.
|
||||
**Purpose:** Records which files are included in each snapshot.
|
||||
|
||||
A snapshot with `completed_at` set to `NULL` is unfinished. `snapshot list`
|
||||
shows it only when its manifest is at the destination, and then as remote only.
|
||||
The next backup deletes it, and so does `prune`.
|
||||
### 9. `snapshot_blobs`
|
||||
|
||||
### `snapshot_files`
|
||||
Maps snapshots to the blobs they reference.
|
||||
|
||||
The files in each snapshot.
|
||||
**Columns:**
|
||||
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `file_id`: the file. References `files.id`.
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `blob_id` (TEXT) - Blob ID (FK to blobs.id)
|
||||
- `blob_hash` (TEXT) - Denormalized blob hash for manifest generation
|
||||
- PRIMARY KEY (`snapshot_id`, `blob_id`)
|
||||
|
||||
The primary key is (`snapshot_id`, `file_id`). `idx_snapshot_files_file_id`
|
||||
indexes `file_id`.
|
||||
**Purpose:** Tracks blob dependencies for snapshots and enables manifest
|
||||
generation.
|
||||
|
||||
### `snapshot_blobs`
|
||||
### 10. `snapshot_mount_points`
|
||||
|
||||
The blobs that each snapshot uses.
|
||||
The directories at or below a configured path, the configured path itself
|
||||
included, that a snapshot's backup found an entry in and, once it had listed
|
||||
them, found on a different filesystem from their parent directory (a different
|
||||
device number), as a mount point is. An automount point whose volume cannot be
|
||||
mounted is not recorded, because the backup finds no entry in it.
|
||||
|
||||
**Columns:**
|
||||
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `path` (TEXT) - Absolute path of the directory
|
||||
- PRIMARY KEY (`snapshot_id`, `path`)
|
||||
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `blob_id`: the blob. References `blobs.id`.
|
||||
- `blob_hash`: a copy of the blob's hash.
|
||||
**Purpose:** Lets `snapshot create --prune` skip the prune when a backup does
|
||||
not find a mount point that an earlier snapshot of the same name found, as when
|
||||
the volume is not mounted.
|
||||
|
||||
The primary key is (`snapshot_id`, `blob_id`). `idx_snapshot_blobs_blob_id`
|
||||
indexes `blob_id`.
|
||||
### 11. `uploads`
|
||||
|
||||
The manifest is built from this table.
|
||||
Tracks blob upload metrics.
|
||||
|
||||
### `snapshot_mount_points`
|
||||
**Columns:**
|
||||
|
||||
The mount points that a snapshot's backup found.
|
||||
- `blob_hash` (TEXT PRIMARY KEY) - Hash of uploaded blob
|
||||
- `snapshot_id` (TEXT NOT NULL) - The snapshot that triggered this upload (FK to
|
||||
snapshots.id)
|
||||
- `uploaded_at` (INTEGER) - Upload timestamp
|
||||
- `size` (INTEGER) - Size of uploaded blob
|
||||
- `duration_ms` (INTEGER) - Upload duration in milliseconds
|
||||
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `path`: the absolute path of the directory.
|
||||
**Purpose:** Performance monitoring and tracking which blobs were newly created
|
||||
(uploaded) during each snapshot.
|
||||
|
||||
The primary key is (`snapshot_id`, `path`).
|
||||
## Data Flow and Operations
|
||||
|
||||
A directory is recorded here when all of these hold:
|
||||
### 1. Backup Process
|
||||
|
||||
- It is at or below a configured path.
|
||||
- The backup found an entry in it that is not excluded.
|
||||
- It is on a different filesystem from its parent directory.
|
||||
1. **File Scanning**
|
||||
- `INSERT OR REPLACE INTO files` - Update file metadata
|
||||
- `SELECT * FROM files WHERE path = ?` - Check if file has changed
|
||||
- `INSERT INTO snapshot_files` - Add file to current snapshot
|
||||
- `INSERT OR IGNORE INTO snapshot_mount_points` - Record each directory on a
|
||||
different filesystem from its parent that the walk found an entry in
|
||||
|
||||
2. **Chunking** (for changed files)
|
||||
- `INSERT OR IGNORE INTO chunks` - Store new chunks
|
||||
- `INSERT INTO file_chunks` - Map chunks to file
|
||||
- `INSERT INTO chunk_files` - Create reverse mapping
|
||||
|
||||
3. **Blob Packing**
|
||||
- `INSERT INTO blobs` - Create blob record with UUID (blob_hash NULL)
|
||||
- `INSERT INTO blob_chunks` - Associate chunks with blob immediately
|
||||
- `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after
|
||||
packing
|
||||
|
||||
The device number is read after the directory has been listed. An automount
|
||||
point is a directory whose volume is mounted when it is first listed, so the
|
||||
number is that of the volume. An automount point whose volume cannot be mounted
|
||||
is not recorded, because the backup finds no entry in it.
|
||||
4. **Upload**
|
||||
- `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as
|
||||
uploaded, with the size of the object in storage (the upload is skipped
|
||||
when the blob is already stored)
|
||||
- `INSERT INTO uploads` - Record upload metrics with snapshot_id
|
||||
- `INSERT INTO snapshot_blobs` - Associate blob with snapshot
|
||||
|
||||
`snapshot create --prune` reads this table. It skips the prune when an earlier
|
||||
snapshot of the same name recorded a mount point that the new backup did not
|
||||
find. That happens when the volume is not mounted.
|
||||
5. **Snapshot Completion**
|
||||
- `UPDATE snapshots SET completed_at = ?, stats...` - Finalize snapshot
|
||||
- Generate and upload blob manifest from `snapshot_blobs`
|
||||
|
||||
### `uploads`
|
||||
### 2. Incremental Backup
|
||||
|
||||
One row for each blob that a backup uploaded.
|
||||
1. **Change Detection**
|
||||
- `SELECT * FROM files WHERE path = ?` - Get previous file metadata
|
||||
- Compare mtime, size, mode to detect changes
|
||||
- Re-chunk a file that lists a chunk no uploaded blob holds, even when its
|
||||
metadata is unchanged
|
||||
- Skip unchanged files but still add to `snapshot_files`
|
||||
|
||||
- `blob_hash`: the blob. References `blobs.blob_hash`. Primary key.
|
||||
- `snapshot_id`: the snapshot whose backup uploaded it. References
|
||||
`snapshots.id`.
|
||||
- `uploaded_at`: when the upload started. It is written as a date and time, not
|
||||
as seconds.
|
||||
- `size`: the size of the uploaded blob.
|
||||
- `duration_ms`: the duration of the upload, in milliseconds.
|
||||
2. **Chunk Reuse**
|
||||
- `SELECT * FROM blob_chunks WHERE chunk_hash = ?` - Find existing chunks
|
||||
- `INSERT INTO snapshot_blobs` - Reference existing blobs for unchanged
|
||||
files
|
||||
|
||||
`idx_uploads_snapshot_id` indexes `snapshot_id`.
|
||||
### 3. Snapshot Metadata Export
|
||||
|
||||
A blob that was already at the destination gets no row here.
|
||||
|
||||
### `local_meta`
|
||||
|
||||
Settings that belong to this copy of the index.
|
||||
After a snapshot is completed:
|
||||
|
||||
- `key`: the name of the setting. Primary key.
|
||||
- `value`: its value.
|
||||
1. Copy database to temporary file
|
||||
2. Clean temporary database to contain only current snapshot data
|
||||
3. VACUUM the trimmed database so deleted rows leave no pages behind
|
||||
4. Compress with zstd and encrypt with age
|
||||
5. Upload to S3 as `metadata/{remote-key}/db.zst.age`
|
||||
6. Generate blob manifest and upload as
|
||||
`metadata/{remote-key}/manifest.json.zst`
|
||||
|
||||
The only key is `storage_url`. A backup sets it once it has written the marker
|
||||
file at the destination. No other command sets it. These commands refuse to run
|
||||
when the config names a different destination:
|
||||
The `{remote-key}` directory name is a one-way hash of the human snapshot ID, so
|
||||
the ID is never written to the store in plaintext. The hash uses no secret, so a
|
||||
guessed hostname and snapshot name can still be confirmed against a listing; see
|
||||
[REPOSTRUCTURE.md](REPOSTRUCTURE.md#remote-key-derivation) and its
|
||||
[Accepted Risks](REPOSTRUCTURE.md#accepted-risks).
|
||||
|
||||
- `snapshot create`
|
||||
- `snapshot purge`
|
||||
- `snapshot remove`
|
||||
- `prune`
|
||||
- `remote nuke`
|
||||
### 4. Restore Process
|
||||
|
||||
### `schema_migrations`
|
||||
The restore process doesn't use the local database. Instead:
|
||||
|
||||
1. Downloads and decrypts the snapshot's metadata database (`db.zst.age`) from
|
||||
S3
|
||||
2. Downloads the blobs holding the chunks of the files being restored, found
|
||||
through that database's `blob_chunks` table; the manifest is not read
|
||||
3. Reconstructs files from decrypted and decompressed chunks
|
||||
|
||||
The migrations that have run.
|
||||
### 5. Pruning
|
||||
|
||||
1. **Identify Unreferenced Blobs**
|
||||
- Query blobs not referenced by any remaining snapshot
|
||||
- Delete from S3, once older than `prune_grace_period`, and from the local
|
||||
database
|
||||
2. **Delete Metadata Directories With No Manifest**
|
||||
- Delete each directory under `metadata/` that has no `manifest.json.zst`
|
||||
and no local snapshot record, once every object in it is older than
|
||||
`prune_grace_period`
|
||||
|
||||
- `version`: the number of the migration. Primary key.
|
||||
- `applied_at`: when it ran. It is written as a date and time, not as seconds.
|
||||
|
||||
## How commands use the tables
|
||||
|
||||
### Backup
|
||||
|
||||
A backup works on the tables in this order:
|
||||
|
||||
- It saves the destination in `local_meta` when the index has none yet. It does
|
||||
so once it has written the marker file at the destination.
|
||||
- It deletes unfinished snapshots, and the rows that no snapshot uses.
|
||||
- It creates a row in `snapshots`, with `completed_at` set to `NULL`.
|
||||
- It deletes the `blobs` rows that were never uploaded.
|
||||
- It walks each path. It compares each regular file and symlink with its row in
|
||||
`files`.
|
||||
- It links each unchanged file to the snapshot in `snapshot_files`.
|
||||
- It writes a `files` row and a `snapshot_files` row for each directory.
|
||||
- It records each mount point in `snapshot_mount_points`.
|
||||
- It reads each changed regular file and stores it. See below. A changed symlink
|
||||
is recorded without being read.
|
||||
- It fills `snapshot_blobs` with every blob the snapshot uses.
|
||||
- It stores the snapshot's totals in `snapshots`.
|
||||
- It uploads the snapshot's metadata.
|
||||
- It sets `completed_at`.
|
||||
|
||||
A file counts as changed when its row in `files` differs from the file on disk
|
||||
in any of these:
|
||||
|
||||
- size
|
||||
- modification time
|
||||
- mode
|
||||
- owner or group
|
||||
- link target
|
||||
|
||||
A file is also read again when it lists a chunk that no uploaded blob holds. A
|
||||
regular file with no row in `files` is read too.
|
||||
|
||||
Storing a changed file writes these rows:
|
||||
|
||||
- A row in `blobs`, when packing of a blob starts.
|
||||
- When the blob is closed, in one transaction:
|
||||
- a row in `chunks` for each chunk queued since the last blob was closed.
|
||||
When a blob is full, that includes the first chunk of the next blob.
|
||||
- the blob's rows in `blob_chunks`
|
||||
- the blob's `blob_hash`, sizes and `finished_ts`
|
||||
- When the blob is stored at the destination, in one transaction:
|
||||
- `uploaded_ts` and `compressed_size` in `blobs`
|
||||
- a row in `snapshot_blobs`
|
||||
- a row in `uploads`, unless the blob was already at the destination
|
||||
- The row in `files`, and its rows in these tables:
|
||||
- `file_chunks`
|
||||
- `chunk_files`
|
||||
- `snapshot_files`
|
||||
|
||||
The row in `files` is written last. It is written only after every blob that
|
||||
holds the file's new chunks is stored.
|
||||
|
||||
### Snapshot metadata
|
||||
|
||||
The metadata of a snapshot is made from a copy of the index:
|
||||
|
||||
- The index is copied into a temporary directory with `VACUUM INTO`.
|
||||
- The rows of every other snapshot are deleted from the copy. So are the files
|
||||
and blobs that this snapshot does not use. A chunk is then deleted when no
|
||||
remaining file and no remaining blob uses it.
|
||||
- The `local_meta` and `schema_migrations` tables are kept as they are. The
|
||||
uploaded database therefore names the destination, and lists the migrations
|
||||
that have run.
|
||||
- `VACUUM` rewrites the copy, so that no deleted row is left in the file.
|
||||
- The copy is compressed with [zstd](https://github.com/facebook/zstd) and
|
||||
encrypted with [age](https://age-encryption.org/).
|
||||
- The manifest is built from the copy's `snapshot_blobs` table.
|
||||
- The copy is uploaded as `metadata/<remote-key>/db.zst.age`.
|
||||
- The manifest is uploaded as `metadata/<remote-key>/manifest.json.zst`.
|
||||
|
||||
The copy is made through SQLite, not by copying the file. The index runs in
|
||||
SQLite's write-ahead log (WAL) mode, so rows that are already committed can
|
||||
still be in the `-wal` file beside it.
|
||||
|
||||
### Restore
|
||||
|
||||
A restore reads no file, chunk or blob rows from the local index. It reads
|
||||
`snapshots` and `local_meta` there, to look up a snapshot ID and to check the
|
||||
marker file.
|
||||
|
||||
- It downloads and decrypts `db.zst.age` into a temporary directory.
|
||||
- It opens that file read-only. It applies no migration to it.
|
||||
- It reads the snapshot's row in `snapshots`, to check that this is the right
|
||||
database.
|
||||
- It reads the files to restore from `files`.
|
||||
- It finds each file's chunks in `file_chunks`.
|
||||
- It finds the blob and offset of each chunk in `blob_chunks`.
|
||||
- It finds the name of each blob in `blobs`.
|
||||
- With `--verify`, it reads each chunk's size from `chunks`.
|
||||
|
||||
It does not read the manifest.
|
||||
|
||||
### Prune
|
||||
|
||||
`prune` saves nothing in `local_meta`. It refuses to run when `local_meta` holds
|
||||
a different destination from the configured one. It changes the local index in
|
||||
two ways:
|
||||
|
||||
- It deletes each snapshot whose manifest is gone from the destination.
|
||||
- It deletes unfinished snapshots, and the rows that no snapshot uses.
|
||||
|
||||
The second step removes unused rows in this order:
|
||||
|
||||
- files that are in no snapshot
|
||||
- blobs that are in no snapshot
|
||||
- `blob_chunks` rows whose chunk is gone
|
||||
- chunks that no file and no blob uses
|
||||
|
||||
`prune` then deletes unused data at the destination. That step reads the
|
||||
manifests, not the local index. It reads the local index only to keep a
|
||||
directory under `metadata/` whose snapshot is still recorded there.
|
||||
|
||||
## Code
|
||||
|
||||
Each repository type in `internal/database` serves one or more tables:
|
||||
|
||||
- `FileRepository`: `files`
|
||||
- `ChunkRepository`: `chunks`
|
||||
- `FileChunkRepository`: `file_chunks`
|
||||
- `ChunkFileRepository`: `chunk_files`
|
||||
- `BlobRepository`: `blobs`
|
||||
- `BlobChunkRepository`: `blob_chunks`
|
||||
- `SnapshotRepository`:
|
||||
- `snapshots`
|
||||
- `snapshot_files`
|
||||
- `snapshot_blobs`
|
||||
- `snapshot_mount_points`
|
||||
- `uploads`, where it deletes a snapshot's rows
|
||||
- `UploadRepository`: `uploads`
|
||||
- `LocalMetaRepository`: `local_meta`
|
||||
|
||||
No repository type serves `schema_migrations`. The code that applies the
|
||||
migrations reads and writes it.
|
||||
|
||||
`Repositories` holds one of each type.
|
||||
|
||||
`Repositories.WithTx` runs a function in one transaction:
|
||||
### 6. Incomplete Snapshot Cleanup
|
||||
|
||||
Before each backup:
|
||||
|
||||
1. Query incomplete snapshots (where `completed_at IS NULL`)
|
||||
2. Delete each one and all its associations, without checking S3 for its
|
||||
metadata
|
||||
3. Clean up orphaned files, chunks, and blobs
|
||||
|
||||
## Repository Pattern
|
||||
|
||||
Vaultik uses a repository pattern for database access:
|
||||
|
||||
- `FileRepository` - CRUD operations for files and file metadata
|
||||
- `ChunkRepository` - CRUD operations for content chunks
|
||||
- `FileChunkRepository` - Manage file-to-chunk mappings
|
||||
- `ChunkFileRepository` - Manage chunk-to-file reverse mappings
|
||||
- `BlobRepository` - Manage blob lifecycle (creation, finalization, upload)
|
||||
- `BlobChunkRepository` - Manage blob-to-chunk associations
|
||||
- `SnapshotRepository` - Manage snapshots and their relationships
|
||||
- `UploadRepository` - Track blob upload metrics
|
||||
|
||||
Each repository provides methods like:
|
||||
|
||||
- `Create()` - Insert new record
|
||||
- `GetByID()` / `GetByPath()` / `GetByHash()` - Retrieve records
|
||||
- `Update()` - Update existing records
|
||||
- `Delete()` - Remove records
|
||||
- Specialized queries for each entity type (e.g., `DeleteOrphaned()`,
|
||||
`GetIncompleteByHostname()`)
|
||||
|
||||
## Transaction Management
|
||||
|
||||
All database operations that modify multiple tables are wrapped in transactions:
|
||||
|
||||
```go
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
// several repository calls that use tx
|
||||
// Multiple repository operations using tx
|
||||
})
|
||||
```
|
||||
|
||||
The transaction commits when the function returns no error. Otherwise it is
|
||||
rolled back.
|
||||
This ensures consistency, especially important for operations like:
|
||||
|
||||
## SQLite settings
|
||||
- Creating file-chunk mappings
|
||||
- Associating chunks with blobs
|
||||
- Updating snapshot statistics
|
||||
|
||||
- The index runs in WAL mode. A command that only reads, such as
|
||||
`snapshot list`, can read the index while a backup writes to it.
|
||||
- A connection waits up to 10 seconds for a lock before it fails.
|
||||
- Foreign keys are enforced.
|
||||
- Each open database uses one connection, so its queries run one at a time.
|
||||
- Queries pass their values as parameters.
|
||||
## Performance Considerations
|
||||
|
||||
1. **Indexes**:
|
||||
- Primary keys are automatically indexed
|
||||
- `idx_files_path` on `files(path)` for efficient file lookups
|
||||
|
||||
2. **Prepared Statements**: All queries use prepared statements for performance
|
||||
and security
|
||||
|
||||
3. **Batch Operations**: Where possible, operations are batched within
|
||||
transactions
|
||||
|
||||
4. **Write-Ahead Logging**: The local index runs in SQLite WAL mode with a
|
||||
10-second busy timeout, so a read-only command such as `snapshot list` can
|
||||
read it while a backup writes to it. Committed rows can sit in the `-wal`
|
||||
file beside the index until a checkpoint, so the metadata export copies the
|
||||
index through SQLite (`VACUUM INTO`), not as a file
|
||||
|
||||
## Data Integrity
|
||||
|
||||
1. **Foreign Keys**: Enforced through CASCADE DELETE and application-level
|
||||
repository methods
|
||||
2. **Unique Constraints**: Chunk hashes, file paths, and blob hashes are unique
|
||||
3. **Null Handling**: Nullable fields clearly indicate in-progress operations
|
||||
4. **Timestamp Tracking**: All major operations record timestamps for auditing
|
||||
+289
-382
@@ -1,181 +1,158 @@
|
||||
# vaultik destination layout
|
||||
# Vaultik S3 Repository Structure
|
||||
|
||||
This document describes what `vaultik` stores at the destination. It covers the
|
||||
layout, what the layout protects, and the risks it accepts.
|
||||
This document describes the structure and organization of data stored in the S3
|
||||
bucket by Vaultik.
|
||||
|
||||
The destination is the place the backups are stored. It is an S3 bucket and
|
||||
prefix, a local directory or an [rclone](https://rclone.org/) remote. A remote
|
||||
is a storage service set up in rclone. The layout is the same on all three.
|
||||
## Overview
|
||||
|
||||
## Terms
|
||||
Vaultik stores all backup data in an S3-compatible object store. The repository
|
||||
consists of two main components, plus a marker file:
|
||||
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Snapshot ID**: the name of a snapshot, in the form
|
||||
`<hostname>_<snapshot-name>_<timestamp>`.
|
||||
- **Remote key**: a hash of the snapshot ID. It names the snapshot's directory
|
||||
under `metadata/`. See [Remote keys](#remote-keys).
|
||||
- **Chunk**: a piece of a file's content, named by the SHA-256 hash of that
|
||||
content.
|
||||
- **Blob**: one object under `blobs/`. It holds many chunks.
|
||||
- **Local index**: the SQLite database on the backed-up machine.
|
||||
- **Recipient**: an [age](https://age-encryption.org/) public key that backups
|
||||
are encrypted to.
|
||||
1. **Blobs** - The actual backup data (content-addressed, encrypted)
|
||||
2. **Metadata** - Snapshot information and manifests (partially encrypted)
|
||||
|
||||
## Layout
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
<destination>/
|
||||
<bucket>/<prefix>/
|
||||
├── vaultik-destination.txt
|
||||
├── blobs/
|
||||
│ └── <hash[0:2]>/
|
||||
│ └── <hash[2:4]>/
|
||||
│ └── <hash>
|
||||
│ └── <full-hash>
|
||||
└── metadata/
|
||||
└── <remote-key>/
|
||||
├── db.zst.age
|
||||
└── manifest.json.zst
|
||||
```
|
||||
|
||||
The destination holds three things:
|
||||
The metadata subdirectory is named with the **remote key**, a one-way hash of
|
||||
the snapshot ID, not with the human-readable snapshot ID itself. See
|
||||
[Remote Key Derivation](#remote-key-derivation).
|
||||
|
||||
- `vaultik-destination.txt`, the marker file
|
||||
- `blobs/`, which holds the file contents
|
||||
- `metadata/`, which holds one directory for each snapshot
|
||||
|
||||
## The marker file
|
||||
|
||||
Every backup writes `vaultik-destination.txt` before it uploads anything else.
|
||||
The file holds one fixed line of text. It names no host, path or time.
|
||||
|
||||
The marker tells an emptied or missing destination apart from a new one. An
|
||||
unplugged volume leaves an empty directory at its mount point. Without the
|
||||
marker, that directory would look like a destination whose snapshots are all
|
||||
gone.
|
||||
|
||||
The check applies while the local index holds snapshots and belongs to the
|
||||
configured destination. When the destination then answers that the marker does
|
||||
not exist, commands treat the destination as not present:
|
||||
|
||||
- `snapshot create` refuses to write to it.
|
||||
- These commands fail:
|
||||
- `prune`
|
||||
- `snapshot purge`
|
||||
- `remote nuke`
|
||||
- `snapshot verify` and `snapshot restore`, when given a remote key, or a
|
||||
snapshot ID that is not in the local index
|
||||
- `snapshot list` shows the local index alone.
|
||||
- `snapshot remove` deletes nothing at the destination.
|
||||
|
||||
The check is skipped when the look-up fails for any other reason. S3 refuses the
|
||||
look-up to credentials that can only write. A machine with such credentials
|
||||
therefore does not notice an emptied destination. Its next backup would not
|
||||
upload data that its local index says is stored. After a destination is emptied,
|
||||
run `vaultik database delete` on such a machine before it backs up again.
|
||||
## Marker File (`vaultik-destination.txt`)
|
||||
|
||||
Every backup writes this file at the root before it uploads anything. It holds
|
||||
one fixed line of plain text and names no host, path or time. While the local
|
||||
index holds snapshots stored there, a destination that answers a look-up of the
|
||||
marker with not found is treated as not present: `snapshot create` refuses to
|
||||
write to it, and `prune`, `snapshot purge`, `snapshot remove`, `snapshot list`
|
||||
and `remote nuke` behave as for a destination that cannot be listed. An
|
||||
unplugged volume leaves an empty directory at its mount point, and without the
|
||||
marker that directory would read as a destination whose snapshots are all gone.
|
||||
`remote nuke` deletes the marker last.
|
||||
|
||||
## Blobs
|
||||
When the look-up of the marker fails for any reason other than not found, the
|
||||
check is skipped and the command goes on. S3 refuses it on every run to
|
||||
credentials that allow only PUT, so a host with such credentials does not notice
|
||||
a destination emptied since its last backup, and its next backup would leave out
|
||||
the data its local index records as already stored. After the destination is
|
||||
emptied, run `vaultik database delete` on such a host before it backs up again.
|
||||
|
||||
A blob holds the content of many chunks, one after another.
|
||||
## Blobs Directory (`blobs/`)
|
||||
|
||||
- **Path**: `blobs/<first 2 characters>/<next 2 characters>/<full hash>`.
|
||||
### Structure
|
||||
|
||||
- **Path format**: `blobs/<first-2-chars>/<next-2-chars>/<full-hash>`
|
||||
- **Example**:
|
||||
`blobs/ca/fe/cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678`.
|
||||
- **Format**: the chunk data, compressed with
|
||||
[zstd](https://github.com/facebook/zstd) and then encrypted with age.
|
||||
- **Recipients**: the public keys in the `age_recipients` setting when the blob
|
||||
was written.
|
||||
`blobs/ca/fe/cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678`
|
||||
- **Sharding**: The two-level directory structure (using the first 4 characters
|
||||
of the hash) prevents any single directory from containing too many objects
|
||||
|
||||
The two directory levels keep any one directory from holding too many objects.
|
||||
### Content
|
||||
|
||||
### Blob names
|
||||
- **What it contains**: Packed collections of content-defined chunks from files
|
||||
- **Format**: Zstandard compressed, then Age encrypted
|
||||
- **Encryption**: Always encrypted with Age using the configured recipients
|
||||
- **Naming**: Content-addressed. The blob's name is
|
||||
`hex(SHA256(SHA256(uncompressed blob contents)))` — the double SHA-256 of the
|
||||
concatenated chunk data, computed before compression and encryption, not a
|
||||
hash of the stored (compressed, encrypted) bytes. One consequence: only a
|
||||
holder of the age private key can check a stored blob's integrity, because
|
||||
matching a blob to its name means decrypting and decompressing it first —
|
||||
which is what `restore` and `verify --deep` do. Implemented in
|
||||
`internal/blobgen` (`DoubleSHA256`). This is the canonical description of blob
|
||||
naming; other documents and comments point here.
|
||||
|
||||
A blob's name is `hex(SHA256(SHA256(uncompressed contents)))`. The contents are
|
||||
the chunk data, joined in order, before compression and encryption. The name is
|
||||
not a hash of the stored bytes.
|
||||
### Why Encrypted
|
||||
|
||||
This has three consequences:
|
||||
Blobs contain the actual file data from backups and must be encrypted for
|
||||
security. The content-addressing ensures deduplication while the encryption
|
||||
ensures privacy.
|
||||
|
||||
- Checking a blob against its name needs the private key. The blob must be
|
||||
decrypted and decompressed first. `snapshot restore` and
|
||||
`snapshot verify --deep` do this.
|
||||
- Without the key, only a blob's presence and size can be checked.
|
||||
`snapshot verify` does this.
|
||||
- Someone who has the whole contents of a blob can compute its name, and so see
|
||||
whether it is stored. This works for a file they have that fills blobs on its
|
||||
own.
|
||||
## Metadata Directory (`metadata/`)
|
||||
|
||||
`blobgen.DoubleSHA256` in `internal/blobgen` computes the name.
|
||||
Each snapshot has its own subdirectory. The directory is **not** named with the
|
||||
human-readable snapshot ID; it is named with the remote key — a one-way hash of
|
||||
that ID. The human ID is never written to the destination store as a directory
|
||||
name (see [Remote Key Derivation](#remote-key-derivation)).
|
||||
|
||||
### Uploads
|
||||
### Snapshot ID Format
|
||||
|
||||
A backup looks a blob up by name before it uploads it. It does not upload a blob
|
||||
that is already there.
|
||||
The human-readable snapshot ID is used in CLI arguments, log lines, and the
|
||||
local database. It is not written to the destination store.
|
||||
|
||||
On S3, when the destination refuses the look-up, the backup uploads the blob. An
|
||||
object with that name is then replaced by one that holds the same chunk data.
|
||||
- **Format**: `<hostname>_<snapshot-name>_<RFC3339>` (or `<hostname>_<RFC3339>`
|
||||
if no name was specified)
|
||||
- **Example**: `laptop_home_2024-01-15T14:30:52Z`
|
||||
- **Components**:
|
||||
- Short hostname (everything before the first dot is stripped from the FQDN)
|
||||
- Snapshot name from the configured `snapshots:` map (optional)
|
||||
- RFC3339 UTC timestamp
|
||||
|
||||
## Metadata
|
||||
This ID reveals the hostname, the configured snapshot name, and the backup time,
|
||||
so it is never used as the on-disk directory name — the remote key is used
|
||||
instead.
|
||||
|
||||
Each snapshot has one directory under `metadata/`. The directory is named with
|
||||
the snapshot's remote key, not with its snapshot ID.
|
||||
### Remote Key Derivation
|
||||
|
||||
### Snapshot IDs
|
||||
The remote key is `hex(SHA256(SHA256("vaultik|" + snapshot-id)))`: a double
|
||||
SHA-256 over the snapshot ID, with a `vaultik|` domain-separation prefix. The
|
||||
result is a 64-character hex string. The hash is not reversible, but it uses no
|
||||
secret: an observer who guesses a candidate hostname and snapshot name can hash
|
||||
it the same way and confirm whether that snapshot is present. The remote key
|
||||
keeps names out of a plain listing; it does not hide them from a guess.
|
||||
Implemented in `internal/snapshot/remotekey.go`.
|
||||
|
||||
A snapshot ID has the form `<hostname>_<snapshot-name>_<timestamp>`. An example
|
||||
is `laptop_home_2024-01-15T14:30:52Z`. It has three parts:
|
||||
|
||||
- the hostname, up to its first `.`
|
||||
- the snapshot name from the config
|
||||
- the time the backup started, in UTC
|
||||
|
||||
The snapshot ID is used on the command line, in log records and in the local
|
||||
index. It reveals the hostname, the snapshot name and the backup time. It is
|
||||
therefore never written to the destination without encryption.
|
||||
|
||||
### Remote keys
|
||||
|
||||
The remote key is `hex(SHA256(SHA256("vaultik|" + snapshot ID)))`. It is 64 hex
|
||||
characters long. `internal/snapshot/remotekey.go` computes it.
|
||||
|
||||
For example:
|
||||
Worked example:
|
||||
|
||||
- Snapshot ID: `server1_home_2025-06-01T12:00:00Z`
|
||||
- Remote key: `17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa`
|
||||
- Directory:
|
||||
`metadata/17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa/`
|
||||
|
||||
A listing of the destination therefore shows no hostname and no snapshot name.
|
||||
A plain listing of the destination store therefore shows only these hashes, not
|
||||
the hostname or snapshot name of any backup — but because the hash uses no
|
||||
secret, a guessed hostname and snapshot name can be hashed and confirmed against
|
||||
the listing. The same remote key is stored in the manifest's `snapshot_id`
|
||||
field.
|
||||
|
||||
The hash cannot be reversed, but it uses no secret. Someone who guesses a
|
||||
snapshot ID can hash it the same way and see whether that snapshot exists. The
|
||||
remote key keeps names out of a listing. It does not hide them from a guess.
|
||||
### Files in Each Snapshot Directory
|
||||
|
||||
### `db.zst.age`
|
||||
#### `db.zst.age` - Encrypted Database
|
||||
|
||||
The snapshot's database. A restore needs it.
|
||||
- **What it contains**: Pruned binary SQLite database for this snapshot
|
||||
- **Format**: Binary SQLite → Zstandard compressed → Age encrypted
|
||||
- **Encryption**: Encrypted with Age
|
||||
- **Purpose**: Contains full file metadata, chunk mappings, and all
|
||||
relationships
|
||||
- **Why encrypted**: Contains sensitive metadata like file paths, permissions,
|
||||
and ownership
|
||||
|
||||
- **Contents**: a SQLite database. It is a copy of the local index without the
|
||||
rows of other snapshots. It lists every file in the snapshot with its
|
||||
metadata, its chunks, and the blob that holds each chunk.
|
||||
- **Format**: the database file, compressed with zstd and then encrypted with
|
||||
age.
|
||||
#### `manifest.json.zst` - Unencrypted Blob Manifest
|
||||
|
||||
It is encrypted because it holds file paths, permissions and owners.
|
||||
[`DATAMODEL.md`](DATAMODEL.md) describes its tables.
|
||||
|
||||
### `manifest.json.zst`
|
||||
|
||||
The list of the blobs the snapshot uses.
|
||||
|
||||
- **Format**: JSON, compressed with zstd. It is not encrypted.
|
||||
- **What it contains**: JSON list of all blob hashes referenced by this snapshot
|
||||
- **Format**: JSON → Zstandard compressed (NOT encrypted)
|
||||
- **Encryption**: NOT encrypted
|
||||
- **Purpose**: Enables pruning operations without requiring decryption keys
|
||||
- **Structure**:
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot_id": "17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa",
|
||||
"timestamp": "2025-06-01T12:04:31Z",
|
||||
"blob_count": 2,
|
||||
"total_compressed_size": 57344,
|
||||
"timestamp": "2025-06-01T12:00:00Z",
|
||||
"blob_count": 42,
|
||||
"total_compressed_size": 1048576,
|
||||
"blobs": [
|
||||
{
|
||||
"hash": "cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
@@ -189,259 +166,189 @@ The list of the blobs the snapshot uses.
|
||||
}
|
||||
```
|
||||
|
||||
- `snapshot_id`: the remote key, not the snapshot ID.
|
||||
- `timestamp`: when the manifest was written, at the end of the backup.
|
||||
- `blob_count`: the number of blobs.
|
||||
- `total_compressed_size`: the total size of those blobs, in bytes.
|
||||
- `blobs`: the hash and the size of each blob.
|
||||
|
||||
The manifest holds no file name and no path.
|
||||
|
||||
The manifest is not encrypted so that these commands work without the private
|
||||
key:
|
||||
|
||||
- `prune`, which must know which blobs are still in use
|
||||
- `snapshot verify`, which checks that each blob is present
|
||||
- `snapshot list` and `remote info`, which report what the destination holds
|
||||
|
||||
## What is protected
|
||||
|
||||
These are encrypted:
|
||||
|
||||
- file contents, in the blobs
|
||||
- in `db.zst.age`:
|
||||
- file names and paths
|
||||
- permissions and owners
|
||||
- modification times
|
||||
- which chunks make up which file
|
||||
|
||||
These are not encrypted:
|
||||
|
||||
- the remote key of each snapshot
|
||||
- the time each backup finished, in the manifest
|
||||
- the hash and the compressed size of each blob, in the manifest
|
||||
- the number of blobs and their total size for each snapshot, in the manifest
|
||||
|
||||
Someone who can read the destination therefore learns these:
|
||||
|
||||
- when each backup finished
|
||||
- how many blobs each snapshot uses, and how large they are
|
||||
- which blobs two snapshots share
|
||||
- whether a blob holds data they already have in full
|
||||
- whether a snapshot ID they guess exists
|
||||
|
||||
They do not learn these:
|
||||
|
||||
- any hostname or snapshot name, unless they guess it
|
||||
- file names or paths
|
||||
- file contents
|
||||
- permissions or owners
|
||||
- the directory structure
|
||||
- which chunks belong to which file
|
||||
|
||||
The times and sizes could not be hidden by encrypting the manifest. The
|
||||
destination itself shows when each object was written and how large it is.
|
||||
|
||||
## Accepted risks
|
||||
|
||||
These are known properties of the format. They are written down so that you can
|
||||
weigh them.
|
||||
|
||||
### No proof of who wrote the data
|
||||
|
||||
A restore and `snapshot verify --deep` prove two things. The data decrypts with
|
||||
your private key, and it matches its hashes. They do not prove who wrote it.
|
||||
|
||||
The hashes use no secret. Someone who knows a recipient's public key and can
|
||||
replace objects at the destination can put a snapshot of their own there. The
|
||||
public key is not stored at the destination, but a backed-up machine that is
|
||||
broken into has it.
|
||||
|
||||
The defences are on the destination's side:
|
||||
|
||||
- object versioning or object lock on the bucket
|
||||
- credentials for the backed-up machine that cannot delete or overwrite
|
||||
- running `prune` from a trusted machine
|
||||
|
||||
An S3 `PutObject` replaces an existing object. Permission to write alone
|
||||
therefore does not stop overwrites.
|
||||
|
||||
### Sizes are visible
|
||||
|
||||
An object's size follows exactly from the size of its compressed contents.
|
||||
|
||||
All the new chunks in one blob share one zstd stream. The stream looks back 8
|
||||
MiB, or 4 MiB at compression levels 1 and 2. A blob is closed at the
|
||||
`blob_size_limit` setting and at the end of each configured path. A chunk that
|
||||
the local index records as uploaded is never packed again.
|
||||
|
||||
Data that an outsider writes compresses smaller when it matches a secret near it
|
||||
in the same blob, so the blob gets smaller. The blob's size therefore shows the
|
||||
outsider whether their data matched the secret.
|
||||
|
||||
This works only when all of these hold:
|
||||
|
||||
- The outsider can write into a backed-up file.
|
||||
- The outsider can see the blob sizes at the destination.
|
||||
- The outsider's data and the secret are in the same blob, no further apart than
|
||||
the stream looks back.
|
||||
- The file keeps changing.
|
||||
|
||||
Back up a directory that outsiders can write to as a snapshot of its own.
|
||||
|
||||
### Chunk boundaries use no secret
|
||||
|
||||
`vaultik` finds chunk boundaries with the FastCDC algorithm. Its parameters are
|
||||
fixed and public. `chunk_size` sets the average chunk size. No chunk is larger
|
||||
than four times this size. Only a file's last chunk can be smaller than a
|
||||
quarter of it. With the default of 10 MB, those limits are 40 MB and 2.5 MB. A
|
||||
file of 2.5 MB or less is one chunk. An empty file has no chunk.
|
||||
|
||||
At the default `blob_size_limit` of 10 GB, a full blob holds hundreds of chunks.
|
||||
The size of the blob then does not show the size of any one chunk. A limit close
|
||||
to `chunk_size` begins to show chunk sizes.
|
||||
|
||||
### Decrypted data on the local disk
|
||||
|
||||
These commands write unencrypted data to the temporary directory:
|
||||
|
||||
- `snapshot restore` writes decrypted blobs under `vaultik-blobcache-*/`. There
|
||||
is no limit on their total size.
|
||||
- `snapshot restore` writes the decrypted database to
|
||||
`vaultik-restore-*/snapshot.db`.
|
||||
- `snapshot verify --deep` writes the decrypted database to
|
||||
`vaultik-verify-*/snapshot.db`.
|
||||
- `snapshot create` writes a copy of the local index to
|
||||
`vaultik-snapshot-*/snapshot.db`.
|
||||
|
||||
These files are readable only by their owner, and are removed when the command
|
||||
ends. A `kill -9` or a power loss leaves them behind. Delete any leftover
|
||||
`vaultik-*` entry in the temporary directory yourself.
|
||||
|
||||
Trust the temporary directory as much as the directory you restore into.
|
||||
|
||||
### Commands need different permissions
|
||||
|
||||
`snapshot create` without `--prune` writes objects. On S3 it needs no other
|
||||
permission. Other commands need more:
|
||||
|
||||
- These commands list and read:
|
||||
- `snapshot list`
|
||||
- `snapshot verify`
|
||||
- `snapshot restore`
|
||||
- `remote info`
|
||||
- These commands list and delete:
|
||||
- `snapshot purge`
|
||||
- `snapshot remove`
|
||||
- `remote nuke`
|
||||
- `prune` lists, reads and deletes.
|
||||
|
||||
An rclone remote that can move objects also needs permission to read and delete.
|
||||
An upload there looks up the object it replaces and deletes it. It stops when
|
||||
the look-up is refused.
|
||||
|
||||
`snapshot verify` and `snapshot restore` list only when they are given a remote
|
||||
key, or a snapshot ID that is not in the local index.
|
||||
`snapshot remove --local-only` does not reach the destination.
|
||||
`snapshot remove --dry-run` deletes nothing.
|
||||
|
||||
`snapshot create --prune` runs a prune on the backed-up machine. That machine
|
||||
then holds credentials that can delete its own backups. To keep it to write-only
|
||||
credentials, run `prune` from a separate, trusted machine. `prune` there deletes
|
||||
only data that no snapshot uses. It does not remove old snapshots.
|
||||
|
||||
### New recipients cannot read old data
|
||||
|
||||
Changing `age_recipients` does not re-encrypt what is already stored. A chunk
|
||||
that the local index records as uploaded is not written again, so it stays
|
||||
encrypted to the earlier recipients.
|
||||
|
||||
A new snapshot that uses such chunks cannot be restored with a new recipient's
|
||||
key alone. To make everything readable by a new key, run
|
||||
`vaultik database delete` and take a full backup to a new destination.
|
||||
|
||||
### X25519 recipients only
|
||||
|
||||
`vaultik` accepts age X25519 public keys only. It rejects age SSH and plugin
|
||||
recipients.
|
||||
|
||||
The encrypted data is held by a third party for a long time. There is no second
|
||||
kind of recipient to fall back on if X25519 is broken, and no kind that resists
|
||||
a quantum computer.
|
||||
|
||||
## Order of writes
|
||||
|
||||
A backup writes in a fixed order:
|
||||
|
||||
- the marker file
|
||||
- the blobs
|
||||
- `db.zst.age`
|
||||
- `manifest.json.zst`
|
||||
|
||||
A snapshot's metadata is written only after all its blobs are uploaded. The
|
||||
manifest is written last. A directory under `metadata/` that has no manifest
|
||||
therefore belongs to a backup that did not finish.
|
||||
|
||||
The local index marks a snapshot complete only after both metadata files are
|
||||
uploaded. A backup that stops during that upload leaves the snapshot unfinished.
|
||||
The next backup removes it from the local index and backs up its files again.
|
||||
|
||||
One short gap remains. A backup can stop after the upload and before the
|
||||
snapshot is marked complete. The destination then holds a snapshot that can be
|
||||
restored, and the local index marks it unfinished. `snapshot list` shows it as
|
||||
remote only.
|
||||
|
||||
An upload that is cut off leaves no partial object under the final name on S3 or
|
||||
on a local directory. The same holds for an rclone remote that can move objects.
|
||||
An rclone remote that cannot move objects can keep a partial object.
|
||||
|
||||
## Pruning
|
||||
|
||||
`prune` deletes two kinds of object at the destination:
|
||||
|
||||
- a blob that no manifest lists
|
||||
- a directory under `metadata/` that has no manifest
|
||||
|
||||
It reads the manifests, which are not encrypted, so it needs no private key.
|
||||
|
||||
These rules keep it from deleting data that is in use:
|
||||
|
||||
- It keeps every blob that any manifest lists.
|
||||
- It deletes nothing when any manifest cannot be downloaded or decoded.
|
||||
- It deletes a blob only once the blob is older than `prune_grace_period`.
|
||||
- It deletes a directory only once every object in it is that old.
|
||||
- It keeps a directory whose snapshot is in the local index.
|
||||
- It leaves alone a directory whose name is not a remote key.
|
||||
|
||||
`prune_grace_period` defaults to `24h`. It protects a backup that is still
|
||||
running. Such a backup has uploaded blobs and perhaps `db.zst.age`, but not yet
|
||||
its manifest. A prune from this machine or another keeps those objects, as long
|
||||
as no backup takes longer than the grace period.
|
||||
|
||||
The grace period does not protect one case. A backup does not upload a blob that
|
||||
is already at the destination, so that blob keeps the time of its first upload.
|
||||
The blob can be left over from an earlier backup that stopped before its
|
||||
manifest. A prune that runs before the new manifest is written can then delete
|
||||
that blob. The new snapshot cannot be restored afterwards.
|
||||
|
||||
To avoid this case, give each machine and each user its own destination.
|
||||
|
||||
`remote nuke` applies none of these rules.
|
||||
|
||||
## What a restore needs
|
||||
|
||||
- the age private key
|
||||
- the snapshot's `db.zst.age`
|
||||
- the blobs that hold the files to restore
|
||||
|
||||
A restore does not read the manifest, or the file, chunk and blob rows of the
|
||||
local index.
|
||||
|
||||
It works in these steps:
|
||||
|
||||
- It downloads and decrypts `db.zst.age`.
|
||||
- It checks that the database is the one for the snapshot that was asked for.
|
||||
- It downloads and decrypts the blobs it needs. It checks each blob against its
|
||||
name.
|
||||
- It writes each file from its chunks.
|
||||
- It sets each file's permissions and times, and its owner when it runs as root.
|
||||
`snapshot_id` is the remote key (a hash), not the human ID; `timestamp` is
|
||||
written in the clear.
|
||||
|
||||
### Why Manifest is Unencrypted
|
||||
|
||||
The manifest must be readable without the private key to enable:
|
||||
|
||||
1. **Pruning operations** - Identifying unreferenced blobs for deletion
|
||||
2. **Storage analysis** - Understanding space usage without decryption
|
||||
3. **Verification** - Checking blob existence without decryption
|
||||
4. **Cross-snapshot deduplication analysis** - Finding shared blobs between
|
||||
snapshots
|
||||
|
||||
The manifest contains the remote key, the backup timestamp, the blob count and
|
||||
total compressed size, and each blob's hash and compressed size. It contains no
|
||||
file names, paths, or other decrypted metadata.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### What's Encrypted
|
||||
|
||||
- **All file content** (in blobs)
|
||||
- **All file metadata** (paths, permissions, timestamps, ownership in
|
||||
db.zst.age)
|
||||
- **File-to-chunk mappings** (in db.zst.age)
|
||||
|
||||
### What's Not Encrypted
|
||||
|
||||
- **The remote key** — directory names and the manifest `snapshot_id`, a one-way
|
||||
hash of the snapshot ID (see [Remote Key Derivation](#remote-key-derivation))
|
||||
- **The backup timestamp** (in manifest.json.zst)
|
||||
- **Blob hashes and their compressed sizes** (in manifest.json.zst)
|
||||
- **Blob count and total compressed size per snapshot** (in manifest.json.zst)
|
||||
|
||||
### Privacy Implications
|
||||
|
||||
From the unencrypted data, an observer of the destination store can determine:
|
||||
|
||||
- **When each backup was taken** — not from the directory name, which is a
|
||||
one-way hash, but from the plaintext `timestamp` field in manifest.json.zst,
|
||||
which is published in the clear
|
||||
- How many blobs each snapshot references, and the total compressed size
|
||||
- The compressed size of each blob, and which blobs are shared between snapshots
|
||||
(deduplication patterns)
|
||||
- **Whether a guessed hostname and snapshot name are present** — the remote key
|
||||
is an unkeyed hash, so an observer holding candidate names can hash each one
|
||||
and match it against the directory listing. The human ID is never published,
|
||||
so it cannot be read off directly, but it can be confirmed by guessing.
|
||||
|
||||
Together these give an observer a timing-and-size profile of every snapshot.
|
||||
This is an accepted, documented property of the format, not a defect: the
|
||||
manifest is unencrypted so that pruning can run without the private key, and the
|
||||
timing channel could not be closed by encrypting it anyway — object creation
|
||||
times and per-object sizes stay visible at the storage layer on both `s3://` and
|
||||
`file://` destinations regardless.
|
||||
|
||||
An observer cannot determine:
|
||||
|
||||
- The hostname or snapshot name of any backup by reading it off the store — the
|
||||
directory name and the manifest `snapshot_id` are unkeyed hashes of the human
|
||||
ID, so the text is never published (though a guessed name can be confirmed, as
|
||||
above)
|
||||
- File names or paths
|
||||
- File contents
|
||||
- File permissions or ownership
|
||||
- Directory structure
|
||||
- Which chunks belong to which files
|
||||
|
||||
### Accepted Risks
|
||||
|
||||
These are known, deliberate properties of the format and the tooling, recorded
|
||||
so an operator can weigh them rather than discover them.
|
||||
|
||||
1. **No proof of authorship.** Restore and `verify --deep` prove that data
|
||||
decrypts with the age private key and matches its unkeyed content hashes.
|
||||
They do not prove who wrote it: anyone who knows a recipient public key and
|
||||
can replace objects on the destination can substitute a snapshot they built.
|
||||
The recipient string is not stored at the destination, but a compromised
|
||||
backed-up host has it. Defences live on the destination side — bucket
|
||||
versioning or object lock, credentials for the source host that cannot delete
|
||||
or overwrite existing versions, and pruning from a trusted host. Note that S3
|
||||
`PutObject` overwrites an existing key, so PUT permission alone is not
|
||||
append-only.
|
||||
2. **Compression reveals sizes.** Blobs and `db.zst.age` are zstd-compressed
|
||||
then age-encrypted; the manifest is compressed only. age does not pad, so an
|
||||
object's size is the exact compressed length of its contents. All new chunks
|
||||
packed into one blob share a single zstd stream (8 MiB window, 4 MiB at
|
||||
compression levels 1-2), and a blob is closed at `blob_size_limit` and at the
|
||||
end of each configured path. Because a stored chunk is never packed again,
|
||||
someone who can write into a backed-up file and watch blob sizes learns
|
||||
something only when their controlled data and a secret land in the same chunk
|
||||
of a file that keeps changing. Advice: back up any outsider-writable
|
||||
directory as its own snapshot.
|
||||
3. **Chunking uses no secret.** The FastCDC parameters are fixed and public. The
|
||||
default 10 MB average yields chunks between 2.5 MB and 40 MB, and any file of
|
||||
2.5 MB or less is a single chunk. At the default 10 GB `blob_size_limit` a
|
||||
blob holds hundreds of chunks, so individual chunk lengths are not visible in
|
||||
the blob's size; lowering the limit toward the chunk size begins to expose
|
||||
them.
|
||||
4. **Decrypted data on local disk.** Several commands stage plaintext under
|
||||
`$TMPDIR`: `snapshot restore` writes decrypted blobs under
|
||||
`vaultik-blobcache-*/` (no size cap) and the decrypted metadata database at
|
||||
`vaultik-restore-*/snapshot.db`; `verify --deep` writes that database at
|
||||
`vaultik-verify-*/snapshot.db`; `snapshot create` keeps a plaintext copy of
|
||||
the index at `vaultik-snapshot-*/snapshot.db`. These files are created `0600`
|
||||
and removed on success, but a `kill -9` or a power loss leaves them behind —
|
||||
delete any leftover `vaultik-*` directory under `$TMPDIR` by hand. `$TMPDIR`
|
||||
should be trusted to the same degree as the restore target.
|
||||
5. **Store permissions differ per command.** The backed-up host needs only PUT
|
||||
to run `snapshot create`: it writes blobs and metadata and neither reads nor
|
||||
deletes them. Other commands need more — `snapshot verify`,
|
||||
`snapshot restore`, and `prune` list and read; `prune`, `snapshot purge`,
|
||||
`snapshot remove`, and `remote nuke` also delete. The recommended cron line
|
||||
uses `--prune`, which runs `prune` on the backed-up host, so granting that
|
||||
host `--prune` gives it credentials that can delete its own backups. To keep
|
||||
the source host to PUT only, prune from a separate trusted host instead.
|
||||
6. **Changing recipients does not re-encrypt existing data.** Deduplicated
|
||||
chunks and same-named blobs already on the destination stay encrypted to the
|
||||
recipients in force when they were written. A new snapshot that reuses them
|
||||
cannot be restored with a newly added recipient's key alone, because those
|
||||
reused objects were never encrypted to it. To make everything readable by a
|
||||
new key, run `vaultik database delete` and take a full backup to a fresh
|
||||
destination or prefix.
|
||||
7. **X25519 recipients only.** vaultik rejects age ssh and plugin recipients.
|
||||
Long-lived ciphertext held by a third party (the destination operator) has no
|
||||
fallback if X25519 is ever broken: there is no second recipient type and no
|
||||
post-quantum option.
|
||||
|
||||
## Consistency Guarantees
|
||||
|
||||
1. **Blobs are immutable** - Once written, a blob is never modified
|
||||
2. **Blobs are written before metadata** - A snapshot's metadata is only written
|
||||
after all its blobs are successfully uploaded
|
||||
3. **Metadata is written atomically** - Both db.zst.age and manifest.json.zst
|
||||
are written as complete files
|
||||
4. **A snapshot is marked complete in the local DB only after its metadata is
|
||||
uploaded** - `finalizeSnapshotMetadata` runs `ExportSnapshotMetadata` first
|
||||
and records completion (`MarkSnapshotComplete`) only once the export succeeds
|
||||
(see the backup data flow in [ARCHITECTURE.md](ARCHITECTURE.md)). A crash
|
||||
during the export therefore leaves the snapshot incomplete, so the next
|
||||
backup's `PruneDatabase` drops it and re-backs-up its data, rather than
|
||||
leaving a completed-looking row in the local index with no matching metadata
|
||||
on the destination store. (A crash in the brief moment after the export
|
||||
succeeds but before completion is recorded leaves a fully-restorable snapshot
|
||||
on the destination that the local index drops as incomplete on the next run;
|
||||
`snapshot list` then reports it honestly as remote-only, which is the safe
|
||||
direction: the destination copy stays restorable.)
|
||||
|
||||
## Pruning Safety
|
||||
|
||||
The prune operation is safe because:
|
||||
|
||||
1. It keeps every blob listed in any snapshot's manifest and deletes only blobs
|
||||
that no manifest references. Under `metadata/` it deletes only a directory
|
||||
that has no `manifest.json.zst`, whose name is a remote key, and that no
|
||||
local snapshot record claims
|
||||
2. Manifests are unencrypted and can be read without keys
|
||||
3. If any manifest cannot be downloaded or decoded, prune deletes nothing and
|
||||
exits with an error, rather than treating that snapshot's blobs as
|
||||
unreferenced
|
||||
4. It deletes an unreferenced blob only once the blob is older than
|
||||
`prune_grace_period` (default `24h`), and a directory under `metadata/` with
|
||||
no manifest only once every object in it is. A backup uploads its blobs and
|
||||
its `db.zst.age` before its manifest, so a prune running meanwhile, on this
|
||||
host or another sharing the destination, leaves them as long as no backup run
|
||||
takes longer than the grace period. The grace period does not protect a blob
|
||||
a backup reuses from an earlier run that stopped before its manifest: the
|
||||
backup skips the upload of a blob already on the destination, so that blob
|
||||
keeps the time of its first upload, and a prune before the backup's manifest
|
||||
is written can delete it; the new snapshot then cannot be restored.
|
||||
`remote nuke` has no such check
|
||||
|
||||
## Restoration Requirements
|
||||
|
||||
To restore from a backup, you need:
|
||||
|
||||
1. **The Age private key** - To decrypt blobs and database
|
||||
2. **The snapshot metadata** - Both files from the snapshot's metadata directory
|
||||
3. **All referenced blobs** - As listed in the manifest
|
||||
|
||||
The restoration process:
|
||||
|
||||
1. Download and decrypt the database dump to understand file structure
|
||||
2. Download and decrypt the required blobs
|
||||
3. Reconstruct files from their chunks
|
||||
4. Restore file metadata (permissions, timestamps, etc.)
|
||||
+1
-59
@@ -19,62 +19,4 @@ into `main` is the owner's.
|
||||
|
||||
# Future Steps
|
||||
|
||||
Work planned after 1.0, loosely ordered by priority.
|
||||
|
||||
## Correctness and operability
|
||||
|
||||
- **Outside security audit.** Before 1.0 the encryption and blob-generation code
|
||||
was reviewed. Every bug the review found was fixed. The risks it accepted are
|
||||
listed in [Accepted risks](REPOSTRUCTURE.md#accepted-risks). No outside audit
|
||||
has been done. age, zstd and content-defined chunking are mostly off-the-shelf
|
||||
pieces. The code that joins them needs an outside read:
|
||||
- key handling
|
||||
- recipient parsing
|
||||
- manifest trust boundary
|
||||
- restore-time identity validation
|
||||
- **Verify restored content end-to-end in CI.** The current integration test
|
||||
does this for a small synthetic snapshot but not at scale. A nightly job
|
||||
against a multi-GB representative snapshot would catch silent regressions in
|
||||
the chunker, packer, or restore planner.
|
||||
|
||||
## Performance
|
||||
|
||||
- **Parallel blob downloads during restore.** A restore downloads one blob at a
|
||||
time now. With a fast S3 endpoint and a multi-core machine, restore speed is
|
||||
bound by the fetch, decryption and decompression of each blob. Fetching,
|
||||
decrypting and decompressing several blobs at once would make such a restore
|
||||
faster. The change must work with `restorePlan`, which orders the files, and
|
||||
with `restoreSweeper`, which deletes a cached blob once no remaining file
|
||||
needs it.
|
||||
- **Bandwidth limiting (`--bwlimit`).** For both upload and download. It is
|
||||
useful for backing up over a shared link. It is tricky to make work correctly
|
||||
with parallel downloads.
|
||||
- **Restart of interrupted restore.** Today a restore can be started again, and
|
||||
it then overwrites the partial output. It does not resume from where it
|
||||
stopped, and it does not skip files that are already present. A `--resume`
|
||||
mode that checks targets before fetching blobs would matter for very large
|
||||
restores.
|
||||
- **Daemon mode.** A long-running mode that watches for file changes, so that
|
||||
frequent backups, such as hourly ones, skip the full scan. It adds little for
|
||||
the usual backups from cron every 12 to 36 hours. See
|
||||
[issue #204](https://git.eeqj.de/sneak/vaultik/issues/204).
|
||||
|
||||
## Usability
|
||||
|
||||
- **Man pages and richer `--help` examples.** Cobra generates basic help. Man
|
||||
pages would be a separate target.
|
||||
- **`vaultik snapshot diff <a> <b>`.** Show which files changed between two
|
||||
snapshots, without restoring either.
|
||||
- **Status reporting hook for `--cron`.** When a backup fails silently in cron,
|
||||
the user has no idea. A configurable hook on completion, for both success and
|
||||
failure, would close the loop. It could be a webhook, an email or
|
||||
`notify-send`.
|
||||
|
||||
## Infrastructure
|
||||
|
||||
- **Schema migrations after 1.0.** Migrations are supported, but none are added
|
||||
before 1.0 because there is no installed base yet. After 1.0, each schema
|
||||
change is a new migration, so an existing local index is migrated when
|
||||
`vaultik` is updated. See [`DATAMODEL.md`](DATAMODEL.md#schema-migrations).
|
||||
- **Storage backend coverage tests.** S3, `file://` and `rclone://` all share
|
||||
the `Storer` interface, but the rclone path is the least exercised in CI.
|
||||
Work planned after 1.0 is listed in the README [roadmap](../README.md#roadmap).
|
||||
@@ -24,7 +24,6 @@ require (
|
||||
github.com/stretchr/testify v1.11.1
|
||||
go.uber.org/dig v1.19.0
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/mod v0.29.0
|
||||
golang.org/x/sync v0.18.0
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/term v0.37.0
|
||||
|
||||
@@ -2,10 +2,8 @@ package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -122,38 +120,6 @@ func TestNewReaderTrailingAndGarbage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestNewReaderGarbageErrorQuotesNoBytes checks that lines of text in
|
||||
// place of a blob fail with ErrNotAgeStream, and that the error carries
|
||||
// none of their bytes. The lines end in a newline because age's own error
|
||||
// quotes the data's first line only when it reads a whole line.
|
||||
func TestNewReaderGarbageErrorQuotesNoBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, _ := makeIdentity(t)
|
||||
garbage := bytes.Repeat([]byte("garbage-bytes-\n"), 100)
|
||||
|
||||
_, err := blobgen.NewReader(bytes.NewReader(garbage), id)
|
||||
|
||||
require.ErrorIs(t, err, blobgen.ErrNotAgeStream)
|
||||
assert.NotContains(t, err.Error(), "garbage-bytes-")
|
||||
}
|
||||
|
||||
// errConnectionReset stands for a storage read that fails.
|
||||
var errConnectionReset = errors.New("connection reset")
|
||||
|
||||
// TestNewReaderReadFailure checks that a failed read of the age header is
|
||||
// returned as that read error, not as ErrNotAgeStream.
|
||||
func TestNewReaderReadFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, _ := makeIdentity(t)
|
||||
|
||||
_, err := blobgen.NewReader(iotest.ErrReader(errConnectionReset), id)
|
||||
|
||||
require.ErrorIs(t, err, errConnectionReset)
|
||||
assert.NotErrorIs(t, err, blobgen.ErrNotAgeStream)
|
||||
}
|
||||
|
||||
// TestNewWriterInvalidLevel covers the rejected end of issue case 9: an
|
||||
// out-of-range compression level errors and writes nothing to the destination.
|
||||
func TestNewWriterInvalidLevel(t *testing.T) {
|
||||
|
||||
@@ -11,11 +11,6 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// ErrNotAgeStream is returned by NewReader when the data cannot be opened
|
||||
// as an age stream, for example because its header is malformed. It
|
||||
// replaces age's own error, which can quote the data's first bytes.
|
||||
var ErrNotAgeStream = errors.New("not a valid age stream")
|
||||
|
||||
// Reader wraps decompression and decryption with SHA256 verification
|
||||
type Reader struct {
|
||||
reader io.Reader
|
||||
@@ -29,25 +24,11 @@ type Reader struct {
|
||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies
|
||||
// data. Every supplied identity is offered to age.Decrypt, so a blob
|
||||
// encrypted to any one of them can be read.
|
||||
//
|
||||
// A failure to read r is returned as that error, and a blob encrypted to
|
||||
// none of the identities as age.NoIdentityMatchError. Any other failure
|
||||
// to open the age stream is ErrNotAgeStream.
|
||||
func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||
src := &readErrorRecorder{reader: r}
|
||||
|
||||
decReader, err := age.Decrypt(src, identities...)
|
||||
// Create decryption reader
|
||||
decReader, err := age.Decrypt(r, identities...)
|
||||
if err != nil {
|
||||
var noMatch *age.NoIdentityMatchError
|
||||
|
||||
switch {
|
||||
case src.err != nil:
|
||||
return nil, fmt.Errorf("reading age header: %w", src.err)
|
||||
case errors.As(err, &noMatch):
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
default:
|
||||
return nil, ErrNotAgeStream
|
||||
}
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
}
|
||||
|
||||
// Create decompression reader
|
||||
@@ -71,23 +52,6 @@ func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readErrorRecorder passes reads through to reader and keeps the first
|
||||
// error other than io.EOF, so NewReader can tell a failed read from bad
|
||||
// data.
|
||||
type readErrorRecorder struct {
|
||||
reader io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *readErrorRecorder) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if err != nil && !errors.Is(err, io.EOF) && r.err == nil {
|
||||
r.err = err
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.teeReader.Read(p)
|
||||
|
||||
+5
-2
@@ -7,9 +7,11 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/dig"
|
||||
"go.uber.org/fx"
|
||||
@@ -337,7 +339,7 @@ func runVaultikApp(
|
||||
},
|
||||
Mode: mode,
|
||||
}, op, func(err error) {
|
||||
log.Info(failMsg, "error", err)
|
||||
log.Error(failMsg, "error", err)
|
||||
ReportErrorf("%s: %v", failMsg, err)
|
||||
})
|
||||
}
|
||||
@@ -349,7 +351,8 @@ func runVaultikApp(
|
||||
// only one runs at a time; a read-only command runs without it and is not
|
||||
// blocked while a mutator holds the lock (opts.Mode).
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
release, err := acquireLockIfMutating(opts.Mode, pidlock.Dir())
|
||||
release, err := acquireLockIfMutating(opts.Mode,
|
||||
filepath.Join(xdg.DataHome, "vaultik"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+48
-48
@@ -49,14 +49,8 @@ var (
|
||||
errWouldNotLoad = errors.New(
|
||||
"vaultik would not load the config after this change, " +
|
||||
"so the file was not changed")
|
||||
errSetS3KeysFirst = errors.New(
|
||||
"set s3.access_key_id and s3.secret_access_key first, " +
|
||||
"then storage_url")
|
||||
)
|
||||
|
||||
// storageURLKey is the config key that selects the storage backend.
|
||||
const storageURLKey = "storage_url"
|
||||
|
||||
// macOS is the value of runtime.GOOS on macOS.
|
||||
const macOS = "darwin"
|
||||
|
||||
@@ -282,8 +276,7 @@ storage_url: ""
|
||||
# Hostname used in snapshot IDs. Default: system hostname.
|
||||
# hostname: myserver
|
||||
|
||||
# Path to the local SQLite index database. It must be an absolute path;
|
||||
# ~ is expanded.
|
||||
# Path to the local SQLite index database.
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
@@ -318,14 +311,12 @@ func newConfigInitCommand() *cobra.Command {
|
||||
Use: "init",
|
||||
Short: "Write a default config file",
|
||||
Long: `Creates a default configuration file with commented explanations
|
||||
for the settings it lists. If a config file already exists at the target
|
||||
path, the command refuses to overwrite it. The file is created with mode
|
||||
0600, because it can hold credentials.
|
||||
for every setting. If a config file already exists at the target path,
|
||||
the command refuses to overwrite it.
|
||||
|
||||
The config is written to the path from --config, $VAULTIK_CONFIG, or
|
||||
config.yml in the platform default config directory
|
||||
(~/Library/Application Support/vaultik/ on macOS, ~/.config/vaultik/ on
|
||||
Linux, /etc/vaultik/ as root).`,
|
||||
the platform default config directory (e.g. ~/Library/Application Support/
|
||||
on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
path := configPathForInit()
|
||||
@@ -364,7 +355,7 @@ Linux, /etc/vaultik/ as root).`,
|
||||
func newConfigEditCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "edit",
|
||||
Short: "Open the config file in $EDITOR (vi when it is not set)",
|
||||
Short: "Open the config file in $EDITOR",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
@@ -446,16 +437,17 @@ func newConfigSetCommand() *cobra.Command {
|
||||
Use: "set <key> <value>",
|
||||
Short: "Set a config value by dotted path (e.g. compression_level 5)",
|
||||
Long: `Sets a scalar config value addressed by dotted YAML path and writes
|
||||
the file back. Comments in the file are kept. Blank lines can be
|
||||
removed. Missing keys along the path are created: as a list when the
|
||||
next path element is a number, otherwise as a map.
|
||||
the file back, preserving comments and formatting. Missing keys along
|
||||
the path are created: as a list when the next path element is a
|
||||
number, otherwise as a map.
|
||||
|
||||
If the config loads, a change after which it would not, such as
|
||||
compression_level 99, is refused and the file is left unchanged. A
|
||||
config that does not load is checked one setting at a time. Examples
|
||||
are the file config init writes before storage_url is set, and one
|
||||
with a ${ENV:...} reference to a variable not set in this shell. In
|
||||
such a config, an unknown key or a value of the wrong type is refused.
|
||||
config that does not load, such as the one config init writes before
|
||||
storage_url is set or one with a ${ENV:...} reference to a variable
|
||||
not set here, is checked one setting at a time: only a change after
|
||||
which the changed setting cannot be read, such as one giving it the
|
||||
wrong type, is refused.
|
||||
|
||||
Examples:
|
||||
vaultik config set storage_url "file:///mnt/backups"
|
||||
@@ -492,13 +484,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
|
||||
|
||||
keys := strings.Split(key, ".")
|
||||
|
||||
if keys[0] != config.EnvSection {
|
||||
_, err = config.SettingType(keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, keys, value)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -521,7 +506,7 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
|
||||
|
||||
err = checkConfigChange(oldData, data, settingData)
|
||||
if err != nil {
|
||||
return wouldNotLoadError(key, err)
|
||||
return fmt.Errorf("%w: %w", errWouldNotLoad, err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, data, configFileMode)
|
||||
@@ -545,21 +530,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// wouldNotLoadError returns the refusal of a change to key that vaultik
|
||||
// would not load for reason. When storage_url moves to s3:// before the S3
|
||||
// credentials are set, it says to set them first: they can be set while
|
||||
// the old storage_url is still in place.
|
||||
func wouldNotLoadError(key string, reason error) error {
|
||||
if key == storageURLKey &&
|
||||
(errors.Is(reason, config.ErrS3KeyIDRequired) ||
|
||||
errors.Is(reason, config.ErrS3SecretRequired)) {
|
||||
return fmt.Errorf("%w: %w; %w", errWouldNotLoad, reason,
|
||||
errSetS3KeysFirst)
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: %w", errWouldNotLoad, reason)
|
||||
}
|
||||
|
||||
// checkConfigChange returns the reason vaultik would not load newData, the
|
||||
// config after a change, if the change brings that failure in. If oldData,
|
||||
// the config before it, loads, the new one must load too. A config that
|
||||
@@ -769,11 +739,41 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
}
|
||||
|
||||
// configKeyIsString reports whether the dotted key names a string in
|
||||
// config.Config, as s3.access_key_id and snapshots.home.exclude.0 do.
|
||||
// config.Config, following the fields' yaml tags, as s3.access_key_id and
|
||||
// snapshots.home.exclude.0 do.
|
||||
func configKeyIsString(keys []string) bool {
|
||||
typ, err := config.SettingType(keys)
|
||||
typ := reflect.TypeFor[config.Config]()
|
||||
|
||||
return err == nil && typ.Kind() == reflect.String
|
||||
for _, key := range keys {
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
|
||||
// The key is a snapshot name or a list index.
|
||||
typ = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
typ = field.Type
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return typ.Kind() == reflect.String
|
||||
}
|
||||
|
||||
// yamlField returns the field of struct type typ whose yaml tag names key.
|
||||
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
|
||||
for field := range typ.Fields() {
|
||||
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
|
||||
if name == key {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
|
||||
return reflect.StructField{}, false
|
||||
}
|
||||
|
||||
// yamlSetInMapping resolves (creating if needed) the value node for the
|
||||
|
||||
+4
-139
@@ -2,7 +2,6 @@ package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -96,8 +95,8 @@ func TestDefaultConfigOnLinux(t *testing.T) {
|
||||
// testRecipient is a valid age public key.
|
||||
const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
|
||||
|
||||
// TestConfigSetRecipientOnFreshConfig follows the README's getting started
|
||||
// steps: on the file `config init` writes, `config set age_recipients.0` and
|
||||
// TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
|
||||
// file `config init` writes, `config set age_recipients.0` and
|
||||
// `config set storage_url` give a config that loads with that recipient.
|
||||
// Neither is refused, although the config does not load until both are set;
|
||||
// once it loads, a recipient that is not a valid key is.
|
||||
@@ -153,21 +152,6 @@ snapshots:
|
||||
- "~"
|
||||
`
|
||||
|
||||
// testS3URL is an S3 storage_url, which needs credentials to load.
|
||||
const testS3URL = "s3://bucket/prefix?endpoint=https://s3.example.com"
|
||||
|
||||
// configWithS3Keys loads: its storage_url is testS3URL and both S3 keys are
|
||||
// set.
|
||||
const configWithS3Keys = storageURLKey + ": " + testS3URL + `
|
||||
s3:
|
||||
access_key_id: AKIAEXAMPLE
|
||||
secret_access_key: example-secret
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
`
|
||||
|
||||
// configWithUnsetReference loads only where VAULTIK_TEST_UNSET_SECRET is
|
||||
// set, as a backup job's environment might set it. No test sets it.
|
||||
const configWithUnsetReference = configWithoutRecipients + `s3:
|
||||
@@ -241,7 +225,7 @@ func TestConfigSetIndexUnderMissingKey(t *testing.T) {
|
||||
// change after which the config would not load, with an error giving the
|
||||
// reason, and leaves the file as it was. On the file `config init` writes,
|
||||
// which does not load until storage_url is set, and on one that cannot be
|
||||
// read here, only an unknown key or a value of the wrong type is refused.
|
||||
// read here, only a value of the wrong type is refused.
|
||||
func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -253,14 +237,6 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
value string
|
||||
reason string
|
||||
}{
|
||||
// Keys that no setting has, on a config that loads and on one
|
||||
// that does not.
|
||||
{configWithoutRecipients, "nosuchkey", "1",
|
||||
`unknown config key "nosuchkey"`},
|
||||
{configWithoutRecipients, "snapshots.home.exlude", "/tmp",
|
||||
`unknown config key "snapshots.home.exlude"`},
|
||||
{defaultConfigText(macOS), "compression_lvl", "9",
|
||||
`unknown config key "compression_lvl"`},
|
||||
// Index 1 of the empty list the missing key gives.
|
||||
{configWithoutRecipients, "age_recipients.1", testRecipient,
|
||||
"index out of range"},
|
||||
@@ -268,20 +244,12 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
{configWithoutRecipients, "snapshots.home.paths", "/srv",
|
||||
"into []string"},
|
||||
{configWithoutRecipients, "chunk_size", "fast",
|
||||
`chunk_size: "fast" is not a size`},
|
||||
"invalid size format"},
|
||||
// Values of the right type that fail validation.
|
||||
{configWithoutRecipients, "compression_level", "99",
|
||||
"compression_level must be between 1 and 19"},
|
||||
{configWithoutRecipients, "chunk_size", "1KB",
|
||||
"chunk_size must be at least 1MB"},
|
||||
// An s3:// URL before its credentials names them.
|
||||
{configWithoutRecipients, storageURLKey, testS3URL,
|
||||
"s3.access_key_id is required for s3:// URLs; " +
|
||||
"set s3.access_key_id and s3.secret_access_key first"},
|
||||
{configWithoutRecipients + "s3:\n access_key_id: AKIAEXAMPLE\n",
|
||||
storageURLKey, testS3URL,
|
||||
"s3.secret_access_key is required for s3:// URLs; " +
|
||||
"set s3.access_key_id and s3.secret_access_key first"},
|
||||
// With its reference filled in, the config loads.
|
||||
{levelReference, "blob_size_limit", "1MB",
|
||||
"blob_size_limit must be at least"},
|
||||
@@ -367,109 +335,6 @@ func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetS3KeysBeforeStorageURL checks that the order the refusal
|
||||
// gives works on a config that loads with a file:// storage_url.
|
||||
func TestConfigSetS3KeysBeforeStorageURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
for _, set := range [][2]string{
|
||||
{"s3.access_key_id", "AKIAEXAMPLE"},
|
||||
{"s3.secret_access_key", "example-secret"},
|
||||
{storageURLKey, testS3URL},
|
||||
} {
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, set[0], set[1])
|
||||
if err != nil {
|
||||
t.Fatalf("config set %s: %v", set[0], err)
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(data), storageURLKey+": "+testS3URL) {
|
||||
t.Errorf("config set storage_url wrote:\n%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetRefusesClearingS3Key checks that clearing s3.access_key_id on
|
||||
// a config with an s3:// storage_url is refused without saying to set the
|
||||
// S3 keys first. That advice is only for a change to storage_url.
|
||||
func TestConfigSetRefusesClearingS3Key(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const key = "s3.access_key_id"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configWithS3Keys), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false), path, key, "")
|
||||
if !errors.Is(err, errWouldNotLoad) {
|
||||
t.Errorf("clearing %s gives error %v, want one saying %q",
|
||||
key, err, errWouldNotLoad)
|
||||
}
|
||||
|
||||
if errors.Is(err, errSetS3KeysFirst) {
|
||||
t.Errorf("clearing %s gives error %v, want one not saying %q",
|
||||
key, err, errSetS3KeysFirst)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != configWithS3Keys {
|
||||
t.Errorf("clearing %s changed the file to:\n%s", key, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetWritesEnvSection checks that config set writes a key under
|
||||
// the top-level env section, whose keys are not settings, on a config that
|
||||
// loads.
|
||||
func TestConfigSetWritesEnvSection(t *testing.T) {
|
||||
const name = "VAULTIK_TEST_CONFIG_SET_ENV"
|
||||
|
||||
// Loading the changed config sets the variable; t.Setenv restores it
|
||||
// after the test.
|
||||
t.Setenv(name, "")
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, "env."+name, "value")
|
||||
if err != nil {
|
||||
t.Fatalf("config set env.%s: %v", name, err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(data), "env:\n "+name+": value\n") {
|
||||
t.Errorf("config set env.%s wrote:\n%s", name, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetStringLooksLikeNumber sets string settings to values that
|
||||
// YAML reads as numbers or booleans when they are unquoted, and checks that
|
||||
// config.Load returns each one unchanged.
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
@@ -101,16 +100,18 @@ Use --force to skip the confirmation prompt.`,
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the database file and its side files. A side file
|
||||
// may not exist, so only failing to delete the database file
|
||||
// is an error.
|
||||
for _, indexFile := range database.IndexFiles(dbPath) {
|
||||
err = os.Remove(indexFile)
|
||||
if err != nil && indexFile == dbPath {
|
||||
return fmt.Errorf("failed to delete database: %w", err)
|
||||
}
|
||||
// Delete the database file
|
||||
err = os.Remove(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete database: %w", err)
|
||||
}
|
||||
|
||||
// Also delete WAL and SHM files if they exist
|
||||
walPath := dbPath + "-wal"
|
||||
shmPath := dbPath + "-shm"
|
||||
_ = os.Remove(walPath) // Ignore errors - files may not exist
|
||||
_ = os.Remove(shmPath)
|
||||
|
||||
out.Infof("Local state database deleted: %s.", dbPath)
|
||||
log.Info("Local state database deleted", "path", dbPath)
|
||||
|
||||
|
||||
@@ -5,11 +5,9 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
@@ -48,28 +46,3 @@ func TestDatabaseDeleteDeclinedIsError(t *testing.T) {
|
||||
out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestDatabaseDeleteRemovesSideFiles checks that the side files SQLite
|
||||
// keeps beside the database file are deleted with it.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestDatabaseDeleteRemovesSideFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath, cfgPath := seedDatabaseDeleteConfig(t, dir)
|
||||
|
||||
indexFiles := database.IndexFiles(dbPath)
|
||||
for _, indexFile := range indexFiles {
|
||||
seedFile(t, dir, filepath.Base(indexFile), "data")
|
||||
}
|
||||
|
||||
setRootFlags(t, RootFlags{ConfigPath: cfgPath})
|
||||
|
||||
mustExecute(t, newDatabaseDeleteCommand(), "--force")
|
||||
|
||||
for _, indexFile := range indexFiles {
|
||||
_, err := os.Stat(indexFile)
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("database delete left %s: stat err = %v", indexFile, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,6 @@ const shortCommitLen = 12
|
||||
const exitCodeInterrupted = 130
|
||||
|
||||
// Entry is the main entry point for the CLI application.
|
||||
// It first calls globals.FillUnsetFromBuildInfo, which fills in the
|
||||
// version, commit and commit date that the linker flags did not set.
|
||||
// It prints the startup banner to stderr (unless a banner-suppressing
|
||||
// flag is present in os.Args — see bannerSuppressedInArgs), executes the
|
||||
// root cobra command, and routes any returned error through the
|
||||
@@ -35,7 +33,6 @@ const exitCodeInterrupted = 130
|
||||
// deferred profile writers run before the process ends. See run in
|
||||
// cmd/vaultik/main.go.
|
||||
func Entry() int {
|
||||
globals.FillUnsetFromBuildInfo()
|
||||
emitStartupBanner(os.Args[1:], os.Stderr)
|
||||
|
||||
rootCmd := NewRootCommand()
|
||||
@@ -85,10 +82,6 @@ func emitStartupBanner(args []string, w io.Writer) {
|
||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
||||
// must surface a failure outside the normal RunE return path.
|
||||
//
|
||||
// This line is the one report of the failure. A caller that also logs
|
||||
// the failure logs it at info level, so the record appears only under
|
||||
// --verbose or --debug and the error is not printed twice.
|
||||
func ReportErrorf(format string, args ...any) {
|
||||
ui.New(os.Stderr).Errorf(format, args...)
|
||||
}
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
package cli //nolint:testpackage // shares the fixtures and capture helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
const (
|
||||
// errorLinePrefix starts the final line that reports a failed command.
|
||||
errorLinePrefix = "ERROR: "
|
||||
|
||||
flagVerbose = "--verbose"
|
||||
cmdRestore = "restore"
|
||||
|
||||
// blobsDir is the directory of a destination store that holds the
|
||||
// blobs.
|
||||
blobsDir = "blobs"
|
||||
)
|
||||
|
||||
// TestEntryFailureIsPrintedOnce runs commands that fail after startup
|
||||
// against a destination they cannot use. `info` is not among them: it
|
||||
// cannot fail after startup.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryFailureIsPrintedOnce(t *testing.T) {
|
||||
for _, command := range [][]string{
|
||||
{cmdSnapshot, cmdCreate},
|
||||
{cmdSnapshot, cmdRemove, someSnapshotID},
|
||||
{cmdSnapshot, cmdVerify, someSnapshotID},
|
||||
{cmdRemote, cmdInfo},
|
||||
{cmdPrune, flagJSON},
|
||||
} {
|
||||
for _, verbose := range []bool{false, true} {
|
||||
name := fmt.Sprintf("%s verbose=%t", strings.Join(command, " "), verbose)
|
||||
|
||||
t.Run(name, func(t *testing.T) {
|
||||
requireErrorPrintedOnce(t, verbose, append([]string{flagConfig,
|
||||
writeUnusableDestinationConfig(t)}, command...)...)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntryBlobUploadFailureIsPrintedOnce backs up to a destination
|
||||
// whose blobs directory is a file, so the blob upload fails.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryBlobUploadFailureIsPrintedOnce(t *testing.T) {
|
||||
for _, verbose := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(storeDir, blobsDir), nil, 0o600))
|
||||
|
||||
requireErrorPrintedOnce(t, verbose,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntryBlobDownloadFailureIsPrintedOnce backs up, deletes the blobs
|
||||
// from the destination, and restores, so the blob download fails.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryBlobDownloadFailureIsPrintedOnce(t *testing.T) {
|
||||
for _, verbose := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfigWithPrivateKey(t)
|
||||
|
||||
code, _, _ := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
require.Equal(t, 0, code)
|
||||
|
||||
require.NoError(t, os.RemoveAll(filepath.Join(storeDir, blobsDir)))
|
||||
|
||||
requireErrorPrintedOnce(t, verbose, flagConfig, configPath,
|
||||
cmdSnapshot, cmdRestore, onlySnapshotID(t, configPath), t.TempDir())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// requireErrorPrintedOnce runs Entry with args, adding --verbose when
|
||||
// verbose is set. The command must fail, and stderr must carry its error
|
||||
// once, as the final ERROR: line. A log record carrying the same error
|
||||
// appears only under --verbose.
|
||||
func requireErrorPrintedOnce(t *testing.T, verbose bool, args ...string) {
|
||||
t.Helper()
|
||||
|
||||
if verbose {
|
||||
args = append(args, flagVerbose)
|
||||
}
|
||||
|
||||
code, _, stderr := runEntry(t, args...)
|
||||
require.Equal(t, 1, code)
|
||||
|
||||
errorLine, records := failureOutput(t, stderr)
|
||||
|
||||
if !verbose {
|
||||
assert.Zero(t, records,
|
||||
"without --verbose no log record repeats the error:\n%s", stderr)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
assert.NotZero(t, records,
|
||||
"--verbose must keep the log record of %q", errorLine)
|
||||
}
|
||||
|
||||
// failureOutput requires stderr to hold exactly one ERROR: line and
|
||||
// returns it, with the number of log records whose error attribute is
|
||||
// the error that line reports. stderr is a pipe here, so every log
|
||||
// record is a JSON object on its own line.
|
||||
func failureOutput(t *testing.T, stderr string) (string, int) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
errorLines []string
|
||||
records []map[string]any
|
||||
)
|
||||
|
||||
for line := range strings.Lines(stderr) {
|
||||
var record map[string]any
|
||||
if json.Unmarshal([]byte(line), &record) == nil {
|
||||
records = append(records, record)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(line, errorLinePrefix) {
|
||||
errorLines = append(errorLines, strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
|
||||
require.Len(t, errorLines, 1,
|
||||
"the error must be printed once as an ERROR: line:\n%s", stderr)
|
||||
|
||||
// An error can run over several lines; the ERROR: line holds the
|
||||
// first of them.
|
||||
repeats := 0
|
||||
|
||||
for _, record := range records {
|
||||
failure, ok := record["error"].(string)
|
||||
firstLine, _, _ := strings.Cut(failure, "\n")
|
||||
|
||||
if ok && strings.Contains(errorLines[0], firstLine) {
|
||||
repeats++
|
||||
}
|
||||
}
|
||||
|
||||
return errorLines[0], repeats
|
||||
}
|
||||
|
||||
// writeBackupConfigWithPrivateKey builds what writeBackupConfig builds,
|
||||
// with a generated key pair in place of its recipient. The config also
|
||||
// holds the private key, so that restore can decrypt the backup. Returns
|
||||
// the config path and the store directory.
|
||||
func writeBackupConfigWithPrivateKey(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
out := ui.NewWithColor(io.Discard, false)
|
||||
|
||||
require.NoError(t, writeConfigSet(out, configPath,
|
||||
"age_recipients.0", identity.Recipient().String()))
|
||||
require.NoError(t, writeConfigSet(out, configPath,
|
||||
"age_secret_key", identity.String()))
|
||||
|
||||
return configPath, storeDir
|
||||
}
|
||||
|
||||
// onlySnapshotID returns the ID of the one snapshot in the local index
|
||||
// that writeBackupConfig puts beside the config file.
|
||||
func onlySnapshotID(t *testing.T, configPath string) string {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx,
|
||||
filepath.Join(filepath.Dir(configPath), "index.sqlite"))
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { require.NoError(t, db.Close()) }()
|
||||
|
||||
backedUp, err := database.NewRepositories(db).Snapshots.GetOnlySnapshot(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
return backedUp.ID.String()
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotCreateSkipsIndexAndLockFiles backs up a directory
|
||||
// that holds vaultik's index and PID file beside one other file, with no
|
||||
// exclude pattern set. The summary counts only the other file. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/421.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout, os.Stderr and the xdg
|
||||
// globals.
|
||||
func TestEntrySnapshotCreateSkipsIndexAndLockFiles(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
// linked sets index_path to a symlink outside the backed-up
|
||||
// directory that points to the index inside it.
|
||||
linked bool
|
||||
}{
|
||||
{name: "index in the backed-up directory"},
|
||||
{name: "index_path links into the backed-up directory", linked: true},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
sourceDir := filepath.Join(dir, "source")
|
||||
storeDir := filepath.Join(dir, "store")
|
||||
|
||||
indexPath := filepath.Join(sourceDir, "index.sqlite")
|
||||
|
||||
if testCase.linked {
|
||||
link := filepath.Join(dir, "index.sqlite")
|
||||
require.NoError(t, os.Symlink(indexPath, link))
|
||||
|
||||
indexPath = link
|
||||
}
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
sourceDir, storeDir, indexPath)
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
require.NoError(t, os.Mkdir(sourceDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(sourceDir, "file.txt"), []byte("hello"), 0o600))
|
||||
|
||||
// The PID file goes in a directory under XDG_DATA_HOME, which
|
||||
// xdg reads at package init.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(sourceDir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
code, stdout, _ := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
|
||||
require.Equal(t, 0, code)
|
||||
assert.Contains(t, stdout,
|
||||
"Files: 1 examined, 1 backed up, 0 unchanged.")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -61,8 +61,8 @@ func TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup(t *testing.T) {
|
||||
|
||||
// TestEntrySnapshotCreateReportsPostBackupPruneFailure makes the
|
||||
// post-backup prune fail after a good backup, by planting a manifest
|
||||
// that cannot be decoded, and checks that the failure is reported once,
|
||||
// as the prune's and not as a failed snapshot creation.
|
||||
// that cannot be decoded, and checks that the failure is reported as the
|
||||
// prune's and not as a failed snapshot creation.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||
@@ -83,9 +83,6 @@ func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||
assert.Contains(t, stderr, "Backup completed, but post-backup prune failed")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
_, repeats := failureOutput(t, stderr)
|
||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2,
|
||||
@@ -96,8 +93,7 @@ func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||
// empties the source directory, as a volume that is not mounted leaves
|
||||
// its mount point, and runs `snapshot create --prune`. The command exits
|
||||
// non-zero and reports the backup as completed, not as a failed snapshot
|
||||
// creation. The error is printed once. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||
// creation. See https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||
@@ -118,9 +114,6 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||
"Backup completed, but 1 source path(s) were empty")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
_, repeats := failureOutput(t, stderr)
|
||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
@@ -131,8 +124,7 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||
// directory as a mount point, and runs `snapshot create --prune`, which
|
||||
// does not find it as one. The command exits non-zero, reports the
|
||||
// backup as completed, not as a failed snapshot creation, and keeps the
|
||||
// older snapshot. The error is printed once. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
// older snapshot. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
||||
@@ -167,9 +159,6 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
||||
"an earlier snapshot recorded were not found")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
_, repeats := failureOutput(t, stderr)
|
||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
|
||||
@@ -39,7 +39,7 @@ func NewInfoCommand() *cobra.Command {
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.ShowInfo()
|
||||
}, func(err error) {
|
||||
log.Info("Failed to show info", "error", err)
|
||||
log.Error("Failed to show info", "error", err)
|
||||
ReportErrorf("Failed to show info: %v", err)
|
||||
})
|
||||
},
|
||||
|
||||
@@ -17,11 +17,7 @@ func NewPruneCommand() *cobra.Command {
|
||||
unreferenced blobs and directories under metadata/ with no manifest
|
||||
from the backup destination store.
|
||||
|
||||
Needs permission to list, read and delete objects on the destination
|
||||
store. It does not need the age private key.
|
||||
|
||||
Local cleanup drops snapshots whose manifest is gone from the
|
||||
destination store, incomplete snapshots, and any files, chunks, or
|
||||
Local cleanup drops incomplete snapshots and any files, chunks, or
|
||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||
scans every snapshot manifest in the destination store, builds the
|
||||
set of still-referenced blob hashes, and deletes any blob not in that
|
||||
@@ -64,7 +60,7 @@ referenced.`,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.Prune(opts)
|
||||
}, func(err error) {
|
||||
log.Info("Prune operation failed", "error", err)
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
ReportErrorf("Prune failed: %v", err)
|
||||
})
|
||||
},
|
||||
|
||||
+6
-11
@@ -40,13 +40,11 @@ func newRemoteNukeCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "nuke",
|
||||
Short: "Delete ALL snapshot metadata and blobs from the backup destination store",
|
||||
Long: `Removes every snapshot's metadata, every blob and the destination
|
||||
marker from the backup destination store, and each removed snapshot from
|
||||
the local index. After this command completes successfully the backup
|
||||
destination store is empty and the next backup starts from scratch.
|
||||
Long: `Removes every snapshot's metadata and every blob from remote
|
||||
storage. After this command completes successfully the bucket prefix is
|
||||
empty and the next backup starts from scratch.
|
||||
|
||||
This is destructive and irreversible. Requires --force, and permission
|
||||
to list and delete objects on the destination store.`,
|
||||
This is destructive and irreversible. Requires --force.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if !force {
|
||||
@@ -77,10 +75,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
- Size of all snapshot metadata (per snapshot and total)
|
||||
- Count and total size of all blobs
|
||||
- Count and size of referenced blobs (from all manifests)
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)
|
||||
|
||||
Needs permission to list and read the destination store. It does not
|
||||
need the age private key.`,
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
@@ -103,7 +98,7 @@ need the age private key.`,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.RemoteInfo(jsonOutput)
|
||||
}, func(err error) {
|
||||
log.Info("Failed to get remote info", "error", err)
|
||||
log.Error("Failed to get remote info", "error", err)
|
||||
ReportErrorf("Failed to get remote info: %v", err)
|
||||
})
|
||||
},
|
||||
|
||||
@@ -35,10 +35,9 @@ func NewRootCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "vaultik",
|
||||
Short: "Secure incremental backup tool with asymmetric encryption",
|
||||
Long: `vaultik is an incremental backup tool that encrypts data to age public
|
||||
keys and uploads it to an S3-compatible object store, a local directory or
|
||||
an rclone remote. The backed-up machine needs no private key: only
|
||||
'snapshot restore' and 'snapshot verify --deep' need one.`,
|
||||
Long: `vaultik is a secure incremental backup tool that encrypts data using age
|
||||
public keys and uploads to S3-compatible storage. No private keys are needed
|
||||
on the source system.`,
|
||||
SilenceUsage: true,
|
||||
// Bare 'vaultik' (no subcommand): print help. The banner is
|
||||
// printed once at process startup by Entry, before cobra
|
||||
@@ -51,16 +50,13 @@ an rclone remote. The backed-up machine needs no private key: only
|
||||
|
||||
// Add global flags
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
|
||||
"Path to config file (default: $VAULTIK_CONFIG, then the platform "+
|
||||
"config directory, then /etc/vaultik/config.yml)")
|
||||
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false,
|
||||
"Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false,
|
||||
"Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false,
|
||||
"Leave out the banner, status lines, tables and reports (warnings, "+
|
||||
"errors, JSON documents, 'config get' values and confirmation "+
|
||||
"prompts are still printed)")
|
||||
"Suppress non-error output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
|
||||
"Skip files that cannot be read when creating a snapshot, or "+
|
||||
"that cannot be restored when restoring, instead of aborting "+
|
||||
|
||||
+31
-77
@@ -60,22 +60,6 @@ func NewSnapshotCommand() *cobra.Command {
|
||||
return cmd
|
||||
}
|
||||
|
||||
// snapshotCreateLong is the long help text of 'snapshot create'.
|
||||
const snapshotCreateLong = `Creates new snapshots of the configured directories.
|
||||
|
||||
If snapshot names are provided, only those snapshots are created.
|
||||
If no names are provided, all configured snapshots are created.
|
||||
|
||||
Needs at least one public key in age_recipients, and permission to write
|
||||
objects at the backup destination store. It does not need the age private
|
||||
key. With --prune it also needs permission to list, read and delete objects
|
||||
there. An rclone remote that can move objects also needs permission to
|
||||
read and delete.
|
||||
|
||||
The config is read from the path given by --config or VAULTIK_CONFIG;
|
||||
otherwise from the platform config directory (~/.config/vaultik/config.yml
|
||||
on Linux), then /etc/vaultik/config.yml.`
|
||||
|
||||
// newSnapshotCreateCommand creates the 'snapshot create' subcommand
|
||||
func newSnapshotCreateCommand() *cobra.Command {
|
||||
opts := &vaultik.SnapshotCreateOptions{}
|
||||
@@ -83,8 +67,15 @@ func newSnapshotCreateCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "create [snapshot-names...]",
|
||||
Short: "Create new snapshots",
|
||||
Long: snapshotCreateLong,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
Long: `Creates new snapshots of the configured directories.
|
||||
|
||||
If snapshot names are provided, only those snapshots are created.
|
||||
If no names are provided, all configured snapshots are created.
|
||||
|
||||
The config is read from the path given by --config or VAULTIK_CONFIG;
|
||||
otherwise from the platform config directory (~/.config/vaultik/config.yml
|
||||
on Linux), then /etc/vaultik/config.yml.`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
err := opts.Validate()
|
||||
if err != nil {
|
||||
@@ -118,7 +109,7 @@ func newSnapshotCreateCommand() *cobra.Command {
|
||||
return v.CreateSnapshot(opts)
|
||||
}, func(err error) {
|
||||
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
||||
log.Info("Post-backup prune failed", "error", err)
|
||||
log.Error("Post-backup prune failed", "error", err)
|
||||
ReportErrorf("Backup completed, but %v", err)
|
||||
|
||||
return
|
||||
@@ -127,13 +118,13 @@ func newSnapshotCreateCommand() *cobra.Command {
|
||||
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
||||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
||||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
|
||||
log.Info("Backup completed with an error", "error", err)
|
||||
log.Error("Backup completed with an error", "error", err)
|
||||
ReportErrorf("Backup completed, but %v", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("Snapshot creation failed", "error", err)
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
ReportErrorf("Snapshot creation failed: %v", err)
|
||||
})
|
||||
},
|
||||
@@ -165,15 +156,8 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
Use: "list",
|
||||
Aliases: []string{"ls"},
|
||||
Short: "List all snapshots",
|
||||
Long: `Lists the snapshots in the local index and on the backup destination
|
||||
store, with their ID, timestamp, and sizes.
|
||||
|
||||
Needs permission to list and read the destination store. It does not need
|
||||
the age private key. A snapshot found only on the destination store is
|
||||
listed by its remote key, because its ID is stored only in the local index
|
||||
and in the snapshot's encrypted database. If the destination store cannot
|
||||
be listed, the command warns and lists the local index alone.`,
|
||||
Args: cobra.NoArgs,
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return runVaultikApp(cmd, readOnly, false,
|
||||
"Failed to list snapshots",
|
||||
@@ -195,19 +179,11 @@ func newSnapshotPurgeCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: `Removes snapshots based on age or count criteria. Exactly one of
|
||||
--keep-latest and --older-than is required.
|
||||
Long: `Removes snapshots based on age or count criteria.
|
||||
|
||||
Retention is per-snapshot-name: --keep-latest keeps the latest snapshot of
|
||||
each name, not the latest globally. Use --snapshot to restrict the
|
||||
operation to specific snapshot names.
|
||||
|
||||
Only snapshots in the local index are removed. Each one is removed from
|
||||
the local index and its metadata is deleted from the backup destination
|
||||
store. Blobs are not touched: run 'vaultik prune' afterwards.
|
||||
|
||||
Needs permission to list and delete objects on the destination store. If
|
||||
the destination store cannot be listed, nothing is removed.`,
|
||||
Retention is per-snapshot-name: --keep-latest keeps the latest of each
|
||||
configured snapshot name, not the latest globally. Use --snapshot to
|
||||
restrict the operation to specific snapshot names.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Validate flags
|
||||
@@ -246,30 +222,14 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Check a snapshot's blobs are present with the listed size",
|
||||
Long: `Checks that every blob the snapshot's manifest lists is present
|
||||
in storage with the size the manifest records, and that the
|
||||
snapshot's encrypted database is present. It does not read blob
|
||||
contents and does not need the age private key.
|
||||
|
||||
Needs permission to read the backup destination store, and to list it
|
||||
when given a remote key, or a snapshot ID that is not in the local
|
||||
index.
|
||||
|
||||
Use --deep to download, decrypt, and re-hash every blob to detect
|
||||
corruption -- integrity, not who wrote it.
|
||||
|
||||
--deep requires the age private key, in the VAULTIK_AGE_SECRET_KEY
|
||||
environment variable or as age_secret_key in the config file; the
|
||||
environment variable wins. The variable may hold the whole age-keygen
|
||||
file, with its comments and all of its keys. Read the key from the
|
||||
file rather than typing it, so that it does not land in your shell
|
||||
history:
|
||||
|
||||
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||
|
||||
The snapshot may be named by its ID or, on a host with no local
|
||||
index, by the remote key that 'snapshot list' prints for a
|
||||
remote-only snapshot (an unambiguous leading part is enough).`,
|
||||
Long: "Checks that every blob the snapshot's manifest lists is present\n" +
|
||||
"in storage with the size the manifest records, and that the\n" +
|
||||
"snapshot's encrypted database is present. It does not read blob\n" +
|
||||
"contents; use --deep to download, decrypt, and re-hash every blob\n" +
|
||||
"to detect corruption -- integrity, not who wrote it.\n\n" +
|
||||
"The snapshot may be named by its ID or, on a host with no local\n" +
|
||||
"index, by the remote key that 'snapshot list' prints for a\n" +
|
||||
"remote-only snapshot (an unambiguous leading part is enough).",
|
||||
Args: requireSnapshotIDArg,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
@@ -298,16 +258,13 @@ remote-only snapshot (an unambiguous leading part is enough).`,
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("Verification failed", "error", err)
|
||||
log.Error("Verification failed", "error", err)
|
||||
ReportErrorf("Verification failed: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false,
|
||||
"Download, decrypt and re-hash every blob; requires the age private "+
|
||||
"key in VAULTIK_AGE_SECRET_KEY, or as age_secret_key in the config "+
|
||||
"file")
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Download and verify blob hashes")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output verification results as JSON")
|
||||
|
||||
return cmd
|
||||
@@ -331,10 +288,8 @@ know about), which is what 'vaultik prune' does. The command prints the
|
||||
'vaultik prune' invocation to run as a follow-up only when it removed the
|
||||
snapshot's metadata from the destination store, and never under --json.
|
||||
|
||||
The argument must be a snapshot ID: a remote key is not accepted. An ID
|
||||
that names no snapshot in the local index or on the destination store is
|
||||
an error. Needs permission to list and delete objects on the destination
|
||||
store, unless --local-only is given.
|
||||
An ID that names no snapshot in the local index or on the destination
|
||||
store is an error.
|
||||
|
||||
Use --local-only to skip the remote half (e.g. when you want to forget a
|
||||
snapshot locally without touching the destination store); the snapshot
|
||||
@@ -363,8 +318,7 @@ nuke --force' — it is the single supported entry point for that.`,
|
||||
|
||||
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
|
||||
"Show what would be removed without removing; needs no permission "+
|
||||
"to delete")
|
||||
"Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false,
|
||||
"Output result as JSON; skips the confirmation prompt, as --force does")
|
||||
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false,
|
||||
|
||||
@@ -37,22 +37,13 @@ The snapshot may be named by its ID or, when restoring on a host with no
|
||||
local index, by the remote key that 'snapshot list' prints for a
|
||||
remote-only snapshot (an unambiguous leading part is enough).
|
||||
|
||||
Each file is restored under its original absolute path inside the target
|
||||
directory. File owners are restored only when the command runs as root.
|
||||
|
||||
Requires the age private key, in the VAULTIK_AGE_SECRET_KEY environment
|
||||
variable or as age_secret_key in the config file; the environment
|
||||
variable wins. The variable may hold the whole age-keygen file, with its
|
||||
comments and all of its keys. Read the key from the file rather than
|
||||
typing it, so that it does not land in your shell history:
|
||||
Requires the age private key in the VAULTIK_AGE_SECRET_KEY environment
|
||||
variable. The variable may hold the whole age-keygen file (comments and
|
||||
all of its identities are accepted); read it from the file rather than
|
||||
typing the key, so it does not land in your shell history:
|
||||
|
||||
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||
|
||||
Also needs permission to read the backup destination store, and to list
|
||||
it when given a remote key, or a snapshot ID that is not in the local
|
||||
index. It also needs free space in the temporary directory for the
|
||||
decrypted blobs.
|
||||
|
||||
Examples:
|
||||
# Restore entire snapshot
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||
@@ -113,7 +104,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
SkipErrors: rootFlags.SkipErrors,
|
||||
})
|
||||
}, func(err error) {
|
||||
log.Info("Restore operation failed", "error", err)
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
ReportErrorf("Restore failed: %v", err)
|
||||
})
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -70,22 +69,14 @@ var (
|
||||
errFileURLNotAbsolute = errors.New(
|
||||
"a file:// storage_url needs an absolute path, " +
|
||||
"as in file:///path/to/backup")
|
||||
errIndexPathNotAbsolute = errors.New(
|
||||
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
||||
"as in ~/.local/share/vaultik/index.sqlite")
|
||||
errNegativePruneGracePeriod = errors.New(
|
||||
"prune_grace_period must not be negative")
|
||||
errStorageNotConfigured = errors.New(
|
||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||
"s3.bucket + credentials")
|
||||
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
|
||||
)
|
||||
|
||||
// Validate returns these when S3 storage is configured without its
|
||||
// credentials.
|
||||
var (
|
||||
ErrS3KeyIDRequired = errors.New("s3.access_key_id is required")
|
||||
ErrS3SecretRequired = errors.New("s3.secret_access_key is required")
|
||||
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
|
||||
errS3SecretRequired = errors.New("s3.secret_access_key is required")
|
||||
)
|
||||
|
||||
// expandTilde expands ~ at the start of a path to the user's home directory.
|
||||
@@ -295,39 +286,15 @@ func Load(path string) (*Config, error) {
|
||||
}
|
||||
|
||||
// Parse reads YAML config data as Load does, without validating it. It
|
||||
// refuses a key that no setting has and a size setting whose value is not a
|
||||
// size, naming the setting. It applies default values for optional
|
||||
// fields, fills in ${...} references using smartconfig, and expands ~ in
|
||||
// paths.
|
||||
// applies default values for optional fields, fills in ${...} references
|
||||
// using smartconfig, and expands ~ in paths.
|
||||
func Parse(data []byte) (*Config, error) {
|
||||
// smartconfig drops line numbers, so keys are checked in the data as
|
||||
// written.
|
||||
var root yaml.Node
|
||||
|
||||
err := yaml.Unmarshal(data, &root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
err = checkKeys(&root, reflect.TypeFor[Config](), "",
|
||||
map[*yaml.Node]bool{}, map[nodeAndType]bool{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load config using smartconfig for interpolation
|
||||
sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config file: %w", err)
|
||||
}
|
||||
|
||||
// A ${...} reference can bring in keys that are not in the file, and
|
||||
// is the value of a size setting only once it is filled in.
|
||||
err = checkFilledConfig(sc.Data())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
// Set defaults
|
||||
BlobSizeLimit: defaultBlobSizeLimit,
|
||||
@@ -397,8 +364,6 @@ func Parse(data []byte) (*Config, error) {
|
||||
// create needs a recipient and it checks for one itself
|
||||
// - At least one snapshot must be configured with at least one path
|
||||
// - Storage must be configured (either storage_url or s3.* fields)
|
||||
// - The index path must be absolute: a relative one would resolve against
|
||||
// the working directory, so each directory would get its own index
|
||||
// - Chunk size must be at least 1MB
|
||||
// - Blob size limit must be at least the largest chunk the chunker can emit
|
||||
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
|
||||
@@ -432,10 +397,6 @@ func (c *Config) Validate() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if !filepath.IsAbs(c.IndexPath) {
|
||||
return fmt.Errorf("%w: %q", errIndexPathNotAbsolute, c.IndexPath)
|
||||
}
|
||||
|
||||
if c.ChunkSize.Int64() < minChunkSize {
|
||||
return errChunkSizeTooSmall
|
||||
}
|
||||
@@ -529,11 +490,11 @@ func (c *Config) validateStorage() error {
|
||||
}
|
||||
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return ErrS3KeyIDRequired
|
||||
return errS3KeyIDRequired
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return ErrS3SecretRequired
|
||||
return errS3SecretRequired
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -560,11 +521,11 @@ func (c *Config) validateStorageURL() error {
|
||||
case strings.HasPrefix(c.StorageURL, "s3://"):
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -165,7 +165,6 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
||||
AgeRecipients: []string{testSneakAgePublicKey},
|
||||
Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
|
||||
StorageURL: "file:///tmp/vaultik-test-store",
|
||||
IndexPath: "/tmp/vaultik-test-index.sqlite",
|
||||
ChunkSize: chunkSize,
|
||||
BlobSizeLimit: blobLimit,
|
||||
CompressionLevel: 3,
|
||||
@@ -511,65 +510,6 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadIndexPath checks that a relative index_path fails at load, naming
|
||||
// the value. Accepted, it would resolve against the working directory, so a
|
||||
// backup run from another directory would start from an empty index. A path
|
||||
// starting with ~ is expanded and loads.
|
||||
func TestLoadIndexPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
t.Fatalf("home directory: %v", err)
|
||||
}
|
||||
|
||||
load := func(t *testing.T, indexPath string) (*Config, error) {
|
||||
t.Helper()
|
||||
|
||||
data := "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n" +
|
||||
"index_path: " + indexPath + "\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(data), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
return Load(path)
|
||||
}
|
||||
|
||||
t.Run("relative path is rejected", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := load(t, "relative/index.sqlite")
|
||||
if !errors.Is(err, errIndexPathNotAbsolute) {
|
||||
t.Fatalf("Load() error = %v, want errIndexPathNotAbsolute", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), `"relative/index.sqlite"`) {
|
||||
t.Errorf("error %q does not name the value", err.Error())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("path starting with ~ is expanded", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := load(t, "~/vaultik/index.sqlite")
|
||||
if err != nil {
|
||||
t.Fatalf("Load() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
want := filepath.Join(home, "vaultik", "index.sqlite")
|
||||
if cfg.IndexPath != want {
|
||||
t.Errorf("IndexPath = %q, want %q", cfg.IndexPath, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||
@@ -582,7 +522,6 @@ func TestValidateAgeRecipients(t *testing.T) {
|
||||
AgeRecipients: recipients,
|
||||
Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
|
||||
StorageURL: "file:///tmp/vaultik-test-store",
|
||||
IndexPath: "/tmp/vaultik-test-index.sqlite",
|
||||
ChunkSize: Size(10 * 1024 * 1024),
|
||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
|
||||
CompressionLevel: 3,
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// yamlMergeTag is the tag YAML gives a merge key (<<), which adds the keys
|
||||
// of the map it names to the map it sits in.
|
||||
const yamlMergeTag = "!!merge"
|
||||
|
||||
// EnvSection is the top-level key of a map of environment variables that
|
||||
// smartconfig sets while it loads the config. It is not a vaultik setting,
|
||||
// so its keys are not checked.
|
||||
const EnvSection = "env"
|
||||
|
||||
var (
|
||||
errUnknownKey = errors.New("unknown config key")
|
||||
errAnchorContainsItself = errors.New("anchor contains itself")
|
||||
)
|
||||
|
||||
// nodeAndType is a node of a parsed config and the setting type it is
|
||||
// checked against.
|
||||
type nodeAndType struct {
|
||||
node *yaml.Node
|
||||
typ reflect.Type
|
||||
}
|
||||
|
||||
// SettingType returns the type of the setting that the dotted key names in
|
||||
// Config, following the fields' yaml tags, as for s3.bucket or
|
||||
// snapshots.home.exclude.0. A snapshot name or a list index may be any
|
||||
// word. It returns an unknown key error when no setting has that key.
|
||||
func SettingType(keys []string) (reflect.Type, error) {
|
||||
typ := reflect.TypeFor[Config]()
|
||||
|
||||
for _, key := range keys {
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
|
||||
typ = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w %q", errUnknownKey, strings.Join(keys, "."))
|
||||
}
|
||||
|
||||
typ = field.Type
|
||||
default:
|
||||
return nil, fmt.Errorf("%w %q", errUnknownKey, strings.Join(keys, "."))
|
||||
}
|
||||
}
|
||||
|
||||
return typ, nil
|
||||
}
|
||||
|
||||
// yamlField returns the field of struct type typ whose yaml tag names key.
|
||||
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
|
||||
for field := range typ.Fields() {
|
||||
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
|
||||
if name == key && name != "-" {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
|
||||
return reflect.StructField{}, false
|
||||
}
|
||||
|
||||
// checkKeys returns an error naming the first key in node, a parsed config
|
||||
// or part of one at the dotted path, that no setting in typ has. The error
|
||||
// gives the key's line in the file, when it has one. A value of the wrong
|
||||
// type is left for the YAML decoder to report. following holds the aliases
|
||||
// the walk is inside. Meeting one of them again means its anchor contains
|
||||
// itself, which is an error. checked holds each anchored node the walk has
|
||||
// checked completely, with the type it was checked against. An alias to
|
||||
// that node is not followed again for that type.
|
||||
func checkKeys(node *yaml.Node, typ reflect.Type, path string,
|
||||
following map[*yaml.Node]bool, checked map[nodeAndType]bool,
|
||||
) error {
|
||||
switch node.Kind {
|
||||
case yaml.DocumentNode:
|
||||
for _, child := range node.Content {
|
||||
err := checkKeys(child, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.AliasNode:
|
||||
if following[node] {
|
||||
return fmt.Errorf("%w: %q on line %d",
|
||||
errAnchorContainsItself, node.Value, node.Line)
|
||||
}
|
||||
|
||||
target := nodeAndType{node.Alias, typ}
|
||||
if checked[target] {
|
||||
return nil
|
||||
}
|
||||
|
||||
following[node] = true
|
||||
defer delete(following, node)
|
||||
|
||||
err := checkKeys(node.Alias, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
checked[target] = true
|
||||
case yaml.SequenceNode:
|
||||
if typ.Kind() != reflect.Slice {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i, child := range node.Content {
|
||||
err := checkKeys(child, typ.Elem(), joinKey(path, strconv.Itoa(i)),
|
||||
following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.MappingNode:
|
||||
return checkMappingKeys(node, typ, path, following, checked)
|
||||
case yaml.ScalarNode:
|
||||
// A scalar has no keys.
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkFilledConfig checks data, the config after smartconfig has filled in
|
||||
// its ${...} references: its keys with checkKeys, then its size settings
|
||||
// with checkSizes. A key that a reference brings in is not in the file, and
|
||||
// the copy that Encode makes has no line numbers.
|
||||
func checkFilledConfig(data map[string]any) error {
|
||||
var filled yaml.Node
|
||||
|
||||
err := filled.Encode(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal config data: %w", err)
|
||||
}
|
||||
|
||||
err = checkKeys(&filled, reflect.TypeFor[Config](), "",
|
||||
map[*yaml.Node]bool{}, map[nodeAndType]bool{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return checkSizes(&filled, nil)
|
||||
}
|
||||
|
||||
// checkMappingKeys is checkKeys for a map.
|
||||
func checkMappingKeys(node *yaml.Node, typ reflect.Type, path string,
|
||||
following map[*yaml.Node]bool, checked map[nodeAndType]bool,
|
||||
) error {
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
key, value := node.Content[i], node.Content[i+1]
|
||||
|
||||
if key.Tag == yamlMergeTag {
|
||||
// The value is one map or a list of maps.
|
||||
merged := []*yaml.Node{value}
|
||||
if value.Kind == yaml.SequenceNode {
|
||||
merged = value.Content
|
||||
}
|
||||
|
||||
for _, m := range merged {
|
||||
err := checkKeys(m, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
keyPath := joinKey(path, key.Value)
|
||||
if keyPath == EnvSection {
|
||||
continue
|
||||
}
|
||||
|
||||
var valueType reflect.Type
|
||||
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map:
|
||||
valueType = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key.Value)
|
||||
if !ok && key.Line == 0 {
|
||||
// A key that a ${...} reference brought in has no line.
|
||||
return fmt.Errorf("%w %q", errUnknownKey, keyPath)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return fmt.Errorf("%w %q on line %d",
|
||||
errUnknownKey, keyPath, key.Line)
|
||||
}
|
||||
|
||||
valueType = field.Type
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
err := checkKeys(value, valueType, keyPath, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// joinKey appends key to the dotted path.
|
||||
func joinKey(path, key string) string {
|
||||
if path == "" {
|
||||
return key
|
||||
}
|
||||
|
||||
return path + "." + key
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
package config //nolint:testpackage // checks the unexported errUnknownKey
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// keysTestConfig loads; each case below adds one line to it.
|
||||
const keysTestConfig = "storage_url: file:///tmp/vaultik-test-storage\n" +
|
||||
"s3:\n" +
|
||||
" region: us-east-1\n" +
|
||||
"snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n"
|
||||
|
||||
// TestParseRefusesUnknownKey checks that a key no setting has, at any
|
||||
// depth, fails the parse with an error naming the key and its line. A key
|
||||
// that a ${...} reference brings in is not in the file, so its error gives
|
||||
// no line.
|
||||
func TestParseRefusesUnknownKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
snapshotPath := filepath.Join(t.TempDir(), "snapshot.yml")
|
||||
|
||||
err := os.WriteFile(snapshotPath, []byte("other:\n"+
|
||||
" paths: [/tmp/vaultik-test-source]\n"+
|
||||
" exlude: [/tmp/cache]\n"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write snapshot: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{"top level", keysTestConfig + "compression_lvl: 9\n",
|
||||
`unknown config key "compression_lvl" on line 7`},
|
||||
{"in a snapshot", keysTestConfig + " exlude: [/tmp/cache]\n",
|
||||
`unknown config key "snapshots.test.exlude" on line 7`},
|
||||
{"in s3", strings.Replace(keysTestConfig, " region:",
|
||||
" bukcet: b\n region:", 1),
|
||||
`unknown config key "s3.bukcet" on line 3`},
|
||||
{"from a reference",
|
||||
keysTestConfig + " other: ${YAML:" + snapshotPath + ":other}\n",
|
||||
`unknown config key "snapshots.other.exlude"`},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := Parse([]byte(tt.yaml))
|
||||
if !errors.Is(err, errUnknownKey) {
|
||||
t.Fatalf("Parse() error = %v, want errUnknownKey", err)
|
||||
}
|
||||
|
||||
if err.Error() != tt.want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRefusesAnchorThatContainsItself checks that a YAML merge key (<<)
|
||||
// naming the anchor of the map it sits in fails the parse with an error.
|
||||
func TestParseRefusesAnchorThatContainsItself(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := strings.Replace(keysTestConfig, " test:", " test: &test", 1) +
|
||||
" <<: *test\n"
|
||||
|
||||
_, err := Parse([]byte(data))
|
||||
if !errors.Is(err, errAnchorContainsItself) {
|
||||
t.Fatalf("Parse() error = %v, want errAnchorContainsItself", err)
|
||||
}
|
||||
|
||||
want := `anchor contains itself: "test" on line 7`
|
||||
if err.Error() != want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRefusesExcessiveAliasing checks that Parse returns an error for
|
||||
// a file whose merge keys (<<) fan out. Each of eight snapshots merges the
|
||||
// one before it ten times. A key check that followed every alias would walk
|
||||
// the first snapshot over 10^8 times and run past the package's test
|
||||
// timeout.
|
||||
func TestParseRefusesExcessiveAliasing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const snapshots, merges = 8, 10
|
||||
|
||||
var data strings.Builder
|
||||
|
||||
data.WriteString(strings.Replace(keysTestConfig, " test:", " test: &s0", 1))
|
||||
|
||||
for i := 1; i <= snapshots; i++ {
|
||||
aliases := slices.Repeat([]string{"*s" + strconv.Itoa(i-1)}, merges)
|
||||
fmt.Fprintf(&data, " s%d: &s%d\n <<: [%s]\n",
|
||||
i, i, strings.Join(aliases, ", "))
|
||||
}
|
||||
|
||||
_, err := Parse([]byte(data.String()))
|
||||
if err == nil {
|
||||
t.Fatal("Parse() returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAcceptsKnownKeys checks that a config with a ${...} reference
|
||||
// and a YAML merge key (<<) still parses.
|
||||
func TestParseAcceptsKnownKeys(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
levelPath := filepath.Join(t.TempDir(), "level")
|
||||
|
||||
err := os.WriteFile(levelPath, []byte("9"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write level: %v", err)
|
||||
}
|
||||
|
||||
data := strings.Replace(keysTestConfig, " test:", " test: &test", 1) +
|
||||
" other:\n" +
|
||||
" <<: *test\n" +
|
||||
" exclude: [/tmp/vaultik-test-source/cache]\n" +
|
||||
"compression_level: ${FILE:" + levelPath + "}\n"
|
||||
|
||||
cfg, err := Parse([]byte(data))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.CompressionLevel != 9 {
|
||||
t.Errorf("compression_level = %d, want 9", cfg.CompressionLevel)
|
||||
}
|
||||
|
||||
other := cfg.Snapshots["other"]
|
||||
if len(other.Paths) != 1 || len(other.Exclude) != 1 {
|
||||
t.Errorf("snapshot other = %+v, want the paths of test and one exclude",
|
||||
other)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAcceptsEnvSection checks that a config with a top-level env
|
||||
// section still parses. smartconfig sets the environment variables it
|
||||
// lists, so its keys are not settings.
|
||||
func TestParseAcceptsEnvSection(t *testing.T) {
|
||||
const name = "VAULTIK_TEST_ENV_SECTION"
|
||||
|
||||
// Parse sets the variable; t.Setenv restores it after the test.
|
||||
t.Setenv(name, "")
|
||||
|
||||
_, err := Parse([]byte(keysTestConfig + "env:\n " + name + ": set\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
+11
-77
@@ -4,20 +4,13 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
errSizeType = errors.New("size must be a number or string")
|
||||
errSizeTooLarge = errors.New("size exceeds maximum supported value")
|
||||
errNotASize = errors.New(
|
||||
"is not a size; use a number of bytes or a unit such as 10MB")
|
||||
)
|
||||
|
||||
// Size represents a byte size that can be specified in configuration files.
|
||||
@@ -39,15 +32,6 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// A number too large for an int64 decodes as a float64. Its text can be
|
||||
// 1e+20, which is not a size string.
|
||||
var floatVal float64
|
||||
|
||||
err = unmarshal(&floatVal)
|
||||
if err == nil && floatVal > math.MaxInt64 {
|
||||
return errSizeTooLarge
|
||||
}
|
||||
|
||||
// Try to unmarshal as string
|
||||
var strVal string
|
||||
|
||||
@@ -56,66 +40,17 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
return errSizeType
|
||||
}
|
||||
|
||||
size, err := ParseSize(strVal)
|
||||
// Parse the string using go-humanize
|
||||
bytes, err := humanize.ParseBytes(strVal)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
*s = size
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkSizes returns an error naming the first size setting in node whose
|
||||
// value is not a size. node is the config, or the part of it at the dotted
|
||||
// keys, after smartconfig has filled in its ${...} references.
|
||||
func checkSizes(node *yaml.Node, keys []string) error {
|
||||
typ, err := SettingType(keys)
|
||||
if err != nil {
|
||||
// Unknown keys are refused before this check, so keys is in one of
|
||||
// two places. The env section holds no settings. A value of the
|
||||
// wrong shape, such as a map given for hostname or inside exclude,
|
||||
// is reported later by the YAML decoder.
|
||||
return nil //nolint:nilerr // not an error here
|
||||
if bytes > math.MaxInt64 {
|
||||
return fmt.Errorf("%w: %s", errSizeTooLarge, strVal)
|
||||
}
|
||||
|
||||
if typ == reflect.TypeFor[Size]() {
|
||||
var size Size
|
||||
|
||||
err = node.Decode(&size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", strings.Join(keys, "."), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.DocumentNode:
|
||||
for _, child := range node.Content {
|
||||
err = checkSizes(child, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.MappingNode:
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
err = checkSizes(node.Content[i+1],
|
||||
slices.Concat(keys, []string{node.Content[i].Value}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
for i, child := range node.Content {
|
||||
err = checkSizes(child, slices.Concat(keys, []string{strconv.Itoa(i)}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.ScalarNode, yaml.AliasNode:
|
||||
// A scalar holds no settings, and the filled config has no aliases.
|
||||
}
|
||||
*s = Size(bytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -135,17 +70,16 @@ func (s Size) String() string {
|
||||
return humanize.Bytes(uint64(s))
|
||||
}
|
||||
|
||||
// ParseSize parses a size string, such as "10MB", into a Size value. An
|
||||
// error quotes s.
|
||||
// ParseSize parses a size string into a Size value
|
||||
func ParseSize(s string) (Size, error) {
|
||||
bytes, err := humanize.ParseBigBytes(s)
|
||||
bytes, err := humanize.ParseBytes(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%q %w", s, errNotASize)
|
||||
return 0, fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
if !bytes.IsInt64() {
|
||||
if bytes > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s)
|
||||
}
|
||||
|
||||
return Size(bytes.Int64()), nil
|
||||
return Size(bytes), nil
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package config //nolint:testpackage // checks errNotASize and errSizeTooLarge
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestParseRefusesValueThatIsNotASize checks that a size setting whose
|
||||
// value is not a size fails the parse with an error naming the setting and
|
||||
// quoting the value, also when a ${...} reference fills the value in.
|
||||
func TestParseRefusesValueThatIsNotASize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sizePath := filepath.Join(t.TempDir(), "size")
|
||||
|
||||
err := os.WriteFile(sizePath, []byte("lots"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write size: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{"chunk_size", keysTestConfig + "chunk_size: banana\n",
|
||||
`chunk_size: "banana" is not a size; ` +
|
||||
"use a number of bytes or a unit such as 10MB"},
|
||||
{"blob_size_limit", keysTestConfig + "blob_size_limit: 10XB\n",
|
||||
`blob_size_limit: "10XB" is not a size; ` +
|
||||
"use a number of bytes or a unit such as 10MB"},
|
||||
{"s3.part_size", strings.Replace(keysTestConfig, " region:",
|
||||
" part_size: big\n region:", 1),
|
||||
`s3.part_size: "big" is not a size; ` +
|
||||
"use a number of bytes or a unit such as 10MB"},
|
||||
{"from a reference",
|
||||
keysTestConfig + "chunk_size: ${FILE:" + sizePath + "}\n",
|
||||
`chunk_size: "lots" is not a size; ` +
|
||||
"use a number of bytes or a unit such as 10MB"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := Parse([]byte(tt.yaml))
|
||||
if !errors.Is(err, errNotASize) {
|
||||
t.Fatalf("Parse() error = %v, want errNotASize", err)
|
||||
}
|
||||
|
||||
if err.Error() != tt.want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRefusesSizeTooLarge checks that a size setting whose value is
|
||||
// too large for a size fails the parse with an error naming the setting.
|
||||
// YAML reads the second value as a float.
|
||||
func TestParseRefusesSizeTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
value string
|
||||
want string
|
||||
}{
|
||||
{"100EB", "chunk_size: size exceeds maximum supported value: 100EB"},
|
||||
{"99999999999999999999",
|
||||
"chunk_size: size exceeds maximum supported value"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := Parse([]byte(keysTestConfig + "chunk_size: " + tt.value + "\n"))
|
||||
if !errors.Is(err, errSizeTooLarge) {
|
||||
t.Fatalf("Parse() error = %v, want errSizeTooLarge", err)
|
||||
}
|
||||
|
||||
if err.Error() != tt.want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAcceptsSizes checks that size settings given as a number of
|
||||
// bytes, as a unit, and through a ${...} reference still parse.
|
||||
func TestParseAcceptsSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sizePath := filepath.Join(t.TempDir(), "size")
|
||||
|
||||
err := os.WriteFile(sizePath, []byte("20MB"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write size: %v", err)
|
||||
}
|
||||
|
||||
data := strings.Replace(keysTestConfig, " region:",
|
||||
" part_size: 6000000\n region:", 1) +
|
||||
"chunk_size: ${FILE:" + sizePath + "}\n" +
|
||||
"blob_size_limit: 1GB\n"
|
||||
|
||||
cfg, err := Parse([]byte(data))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.ChunkSize != 20_000_000 || cfg.BlobSizeLimit != 1_000_000_000 ||
|
||||
cfg.S3.PartSize != 6_000_000 {
|
||||
t.Errorf("chunk_size, blob_size_limit, s3.part_size = %d, %d, %d, "+
|
||||
"want 20000000, 1000000000, 6000000",
|
||||
cfg.ChunkSize, cfg.BlobSizeLimit, cfg.S3.PartSize)
|
||||
}
|
||||
}
|
||||
@@ -20,17 +20,6 @@ const indexDirPerm = 0o700
|
||||
// backed-up path and chunk hash and must stay private.
|
||||
const indexFilePerm = 0o600
|
||||
|
||||
// IndexFiles returns the path of the index at indexPath and the paths of
|
||||
// the side files SQLite keeps beside it while it writes.
|
||||
func IndexFiles(indexPath string) []string {
|
||||
return []string{
|
||||
indexPath,
|
||||
indexPath + "-wal",
|
||||
indexPath + "-shm",
|
||||
indexPath + "-journal",
|
||||
}
|
||||
}
|
||||
|
||||
// ensureIndexFileMode makes the index file owner-only before the SQLite
|
||||
// driver opens it: it creates the file 0600 if absent, or chmods an existing
|
||||
// one to 0600. Doing this first matters because SQLite creates its -wal and
|
||||
|
||||
@@ -1,24 +1,19 @@
|
||||
// Package globals holds application-wide metadata (name, version,
|
||||
// commit) that is populated at build time via linker flags, or from the
|
||||
// build info Go records in the binary when the linker flags set none.
|
||||
// commit) that is populated at build time via linker flags.
|
||||
package globals
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
)
|
||||
|
||||
// Appname is the application name, populated from main().
|
||||
var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// DevVersion is the version a binary reports when it was built without
|
||||
// git metadata: script/version emits it outside a git checkout, and a
|
||||
// binary keeps it when neither its linker flags nor its Go build info
|
||||
// name a version. goreleaser's snapshot template stamps
|
||||
// git metadata: script/version emits it outside a git checkout, and an
|
||||
// unstamped `go build` keeps it. goreleaser's snapshot template stamps
|
||||
// it followed by "-" and the commit it was built from. It is
|
||||
// deliberately not a number.
|
||||
const DevVersion = "dev"
|
||||
@@ -37,54 +32,6 @@ var Commit = Unknown //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
// CommitDate is the ISO-8601 date of the commit, populated from main().
|
||||
var CommitDate = Unknown //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// develVersion is the main module version Go records when it cannot
|
||||
// tell which version it built.
|
||||
const develVersion = "(devel)"
|
||||
|
||||
// FillUnsetFromBuildInfo applies FillUnset to Version, Commit and
|
||||
// CommitDate, with the build info of the running binary.
|
||||
func FillUnsetFromBuildInfo() {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
Version, Commit, CommitDate = FillUnset(
|
||||
info, Version, Commit, CommitDate)
|
||||
}
|
||||
|
||||
// FillUnset returns version, commit and commitDate. Each one that
|
||||
// still holds its default ("dev" or empty for the version, "unknown"
|
||||
// for the others) is replaced by the value Go recorded in info. A
|
||||
// `go install` of a tag sets no linker flags, but Go records the tag
|
||||
// as the main module's version. Go records the commit and its time
|
||||
// only when it builds from a git checkout. The time is cut to a date,
|
||||
// as the Makefile stamps it.
|
||||
func FillUnset(
|
||||
info *debug.BuildInfo, version, commit, commitDate string,
|
||||
) (string, string, string) {
|
||||
if (version == "" || version == DevVersion) &&
|
||||
info.Main.Version != "" && info.Main.Version != develVersion {
|
||||
version = info.Main.Version
|
||||
}
|
||||
|
||||
for _, s := range info.Settings {
|
||||
switch s.Key {
|
||||
case "vcs.revision":
|
||||
if commit == Unknown {
|
||||
commit = s.Value
|
||||
}
|
||||
case "vcs.time":
|
||||
t, err := time.Parse(time.RFC3339, s.Value)
|
||||
if commitDate == Unknown && err == nil {
|
||||
commitDate = t.UTC().Format(time.DateOnly)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return version, commit, commitDate
|
||||
}
|
||||
|
||||
// Author identifies the upstream author of vaultik.
|
||||
const Author = "Jeffrey Paul <sneak@sneak.berlin>"
|
||||
|
||||
@@ -121,10 +68,8 @@ func New() (*Globals, error) {
|
||||
// a release. "dev" and goreleaser's snapshot "dev-<sha>" count, and so
|
||||
// does what `git describe --tags --always --dirty` gives a make or
|
||||
// docker build of an untagged commit: the bare short commit, or
|
||||
// tag-N-gHASH on a commit after a tag. A Go pseudo-version counts: Go
|
||||
// records one for a build of an untagged commit without linker flags.
|
||||
// Any version ending in "-dirty" or "+dirty" counts, a modified
|
||||
// checkout of a tag ("v1.0.0-dirty", "v1.0.0+dirty") included.
|
||||
// tag-N-gHASH on a commit after a tag. Any version ending in "-dirty"
|
||||
// counts, a modified checkout of a tag ("v1.0.0-dirty") included.
|
||||
// A plain tag such as "v1.0.0" or "1.0.0" is a release.
|
||||
//
|
||||
// The empty string counts too. Nothing that knows its version reports
|
||||
@@ -137,8 +82,7 @@ func New() (*Globals, error) {
|
||||
func IsDevVersion(v string) bool {
|
||||
if v == "" || v == Unknown || v == DevVersion ||
|
||||
strings.HasPrefix(v, DevVersion+"-") ||
|
||||
strings.HasSuffix(v, "-dirty") || strings.HasSuffix(v, "+dirty") ||
|
||||
module.IsPseudoVersion(v) {
|
||||
strings.HasSuffix(v, "-dirty") {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package globals_test
|
||||
|
||||
import (
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
@@ -60,20 +59,12 @@ func TestIsDevVersion(t *testing.T) {
|
||||
{"1.0.0-rc.1-12-g877eb2f", true},
|
||||
// A tagged commit with uncommitted changes is not that tag.
|
||||
{"v1.0.0-dirty", true},
|
||||
// What Go records for a build without linker flags: a
|
||||
// pseudo-version for an untagged commit, "+dirty" for a
|
||||
// modified checkout.
|
||||
{"v1.0.0-rc3.0.20261111001226-abcdefabcdef", true},
|
||||
{"v0.0.0-20261111001226-abcdefabcdef", true},
|
||||
{"v1.0.1-0.20261111001226-abcdefabcdef", true},
|
||||
{"v1.0.0+dirty", true},
|
||||
// What a tagged build produces (goreleaser's .Version strips
|
||||
// the leading "v"; script/version keeps it).
|
||||
{"1.0.0", false},
|
||||
{"0.1.0", false},
|
||||
{"1.0.0-rc.1", false},
|
||||
{"v1.0.0", false},
|
||||
{"v1.0.0-rc3", false},
|
||||
// A release must not be mistaken for a dev build just because
|
||||
// the string happens to contain "dev".
|
||||
{"1.0.0-dev", false},
|
||||
@@ -95,9 +86,10 @@ func TestIsDevVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultVersionIsDev pins the default: a binary whose linker flags
|
||||
// and build info name no version must report itself as a development
|
||||
// build rather than as some default release number.
|
||||
// TestDefaultVersionIsDev pins the linker-flag contract: an unstamped
|
||||
// binary (no -ldflags at all, which is what `go build ./...` and `go
|
||||
// install` produce) must report itself as a development build rather
|
||||
// than as some default release number.
|
||||
func TestDefaultVersionIsDev(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -106,104 +98,3 @@ func TestDefaultVersionIsDev(t *testing.T) {
|
||||
globals.DevVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// buildInfo returns build info with the given main module version and,
|
||||
// when revision is not empty, the commit settings Go records for a
|
||||
// build from a git checkout.
|
||||
func buildInfo(version, revision string) *debug.BuildInfo {
|
||||
info := &debug.BuildInfo{
|
||||
Main: debug.Module{Path: "sneak.berlin/go/vaultik", Version: version},
|
||||
}
|
||||
|
||||
if revision != "" {
|
||||
info.Settings = []debug.BuildSetting{
|
||||
{Key: "vcs.revision", Value: revision},
|
||||
{Key: "vcs.time", Value: "2026-11-10T23:12:26Z"},
|
||||
}
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
// TestFillUnset covers a binary from `go install` of a tag, which
|
||||
// carries no linker flags: it must report the tag. Values the linker
|
||||
// flags set are kept.
|
||||
func TestFillUnset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
tag = "v1.0.0-rc3"
|
||||
rev = "0123456789abcdef0123456789abcdef01234567"
|
||||
)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
info *debug.BuildInfo
|
||||
version, commit, date string
|
||||
wantVersion, wantCommit string
|
||||
wantDate string
|
||||
}{
|
||||
{
|
||||
name: "go install of a tag",
|
||||
info: buildInfo(tag, ""),
|
||||
version: globals.DevVersion,
|
||||
commit: globals.Unknown,
|
||||
date: globals.Unknown,
|
||||
wantVersion: tag,
|
||||
wantCommit: globals.Unknown,
|
||||
wantDate: globals.Unknown,
|
||||
},
|
||||
{
|
||||
name: "empty version",
|
||||
info: buildInfo(tag, ""),
|
||||
version: "",
|
||||
commit: globals.Unknown,
|
||||
date: globals.Unknown,
|
||||
wantVersion: tag,
|
||||
wantCommit: globals.Unknown,
|
||||
wantDate: globals.Unknown,
|
||||
},
|
||||
{
|
||||
name: "go build in a git checkout",
|
||||
info: buildInfo(tag, rev),
|
||||
version: globals.DevVersion,
|
||||
commit: globals.Unknown,
|
||||
date: globals.Unknown,
|
||||
wantVersion: tag,
|
||||
wantCommit: rev,
|
||||
wantDate: "2026-11-10",
|
||||
},
|
||||
{
|
||||
name: "Go could not tell the version",
|
||||
info: buildInfo("(devel)", ""),
|
||||
version: globals.DevVersion,
|
||||
commit: globals.Unknown,
|
||||
date: globals.Unknown,
|
||||
wantVersion: globals.DevVersion,
|
||||
wantCommit: globals.Unknown,
|
||||
wantDate: globals.Unknown,
|
||||
},
|
||||
{
|
||||
name: "linker flags set",
|
||||
info: buildInfo(tag, rev),
|
||||
version: "v1.0.0-rc2-4-gabcdef1",
|
||||
commit: "abcdef1",
|
||||
date: "2026-11-01",
|
||||
wantVersion: "v1.0.0-rc2-4-gabcdef1",
|
||||
wantCommit: "abcdef1",
|
||||
wantDate: "2026-11-01",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
version, commit, date := globals.FillUnset(
|
||||
tc.info, tc.version, tc.commit, tc.date)
|
||||
|
||||
if version != tc.wantVersion || commit != tc.wantCommit ||
|
||||
date != tc.wantDate {
|
||||
t.Errorf("%s: got %q, %q, %q; want %q, %q, %q", tc.name,
|
||||
version, commit, date,
|
||||
tc.wantVersion, tc.wantCommit, tc.wantDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
@@ -30,20 +29,8 @@ const (
|
||||
lockDirPerm = 0o700
|
||||
// pidFilePerm is the mode for the PID file (owner-only).
|
||||
pidFilePerm = 0o600
|
||||
// pidFileName is the name of the PID file in the lock directory.
|
||||
pidFileName = "vaultik.pid"
|
||||
)
|
||||
|
||||
// Dir returns the directory that holds vaultik's PID file.
|
||||
func Dir() string {
|
||||
return filepath.Join(xdg.DataHome, "vaultik")
|
||||
}
|
||||
|
||||
// Path returns the path of the PID file that Acquire uses in lockDir.
|
||||
func Path(lockDir string) string {
|
||||
return filepath.Join(lockDir, pidFileName)
|
||||
}
|
||||
|
||||
// Acquire attempts to acquire a PID lock in the specified directory.
|
||||
// If another process holds the lock, it returns ErrAlreadyRunning with
|
||||
// that process's PID. On success, it writes the current PID to the lock
|
||||
@@ -55,7 +42,7 @@ func Acquire(lockDir string) (*Lock, error) {
|
||||
return nil, fmt.Errorf("creating lock directory: %w", err)
|
||||
}
|
||||
|
||||
lockPath := Path(lockDir)
|
||||
lockPath := filepath.Join(lockDir, "vaultik.pid")
|
||||
|
||||
// No O_TRUNC: the file may hold the PID of the process that has the
|
||||
// lock, which the error below reports.
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package snapshot_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// A backup of the directory that holds vaultik's index and PID file
|
||||
// stores neither, with no exclude pattern set, and still stores the
|
||||
// files beside them. See https://git.eeqj.de/sneak/vaultik/issues/421.
|
||||
func TestScanSkipsIndexAndLockFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const dataDir = "/home/user/.local/share/vaultik"
|
||||
|
||||
indexAndLockFiles := append(database.IndexFiles(dataDir+"/index.sqlite"),
|
||||
dataDir+"/vaultik.pid")
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
for _, p := range append(indexAndLockFiles, dataDir+"/notes.txt") {
|
||||
require.NoError(t, afero.WriteFile(fs, p, []byte("data"), 0o600))
|
||||
}
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
||||
FS: fs,
|
||||
ChunkSize: 64 * 1024,
|
||||
Repositories: repos,
|
||||
MaxBlobSize: 1024 * 1024,
|
||||
CompressionLevel: 3,
|
||||
AgeRecipients: []string{
|
||||
"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
|
||||
IndexAndLockFiles: indexAndLockFiles,
|
||||
})
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||
|
||||
result, err := scanner.Scan(ctx, "/home/user", "test-snapshot")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, 1, result.FilesScanned)
|
||||
|
||||
stored, err := repos.Files.GetByPath(ctx, dataDir+"/notes.txt")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, stored, "a file beside the index is stored")
|
||||
|
||||
for _, p := range indexAndLockFiles {
|
||||
f, err := repos.Files.GetByPath(ctx, p)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, f, "%s is not stored", p)
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,11 @@ package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
@@ -64,52 +62,9 @@ func provideScannerFactory(
|
||||
EnableProgress: params.EnableProgress,
|
||||
UI: params.UI,
|
||||
Exclude: excludes,
|
||||
IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()),
|
||||
SkipErrors: params.SkipErrors,
|
||||
PreviousSnapshotID: params.PreviousSnapshotID,
|
||||
DeviceNumber: params.DeviceNumber,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// indexAndLockFiles returns the paths of the index at indexPath, its side
|
||||
// files and the PID file in lockDir. Symlinks in their directories are
|
||||
// resolved, as they are in the snapshot's source paths, so the scan finds
|
||||
// them under the same names. If indexPath is a symlink, the file it points
|
||||
// to and that file's side files are returned too.
|
||||
func indexAndLockFiles(indexPath, lockDir string) []string {
|
||||
paths := append(database.IndexFiles(indexPath), pidlock.Path(lockDir))
|
||||
for i, p := range paths {
|
||||
paths[i] = resolveParentDir(p)
|
||||
}
|
||||
|
||||
// SQLite writes the file a symlinked index points to and keeps the
|
||||
// side files beside that file. The database package creates the index
|
||||
// before the scanner is built, so the link resolves.
|
||||
index := resolveParentDir(indexPath)
|
||||
|
||||
target, err := filepath.EvalSymlinks(index)
|
||||
if err == nil && target != index {
|
||||
paths = append(paths, database.IndexFiles(target)...)
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// resolveParentDir returns the absolute form of path with the symlinks in
|
||||
// its parent directory resolved. If the parent directory cannot be
|
||||
// resolved, it returns the absolute form unresolved. If path has no
|
||||
// absolute form, it returns path.
|
||||
func resolveParentDir(path string) string {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return path
|
||||
}
|
||||
|
||||
dir, err := filepath.EvalSymlinks(filepath.Dir(abs))
|
||||
if err != nil {
|
||||
return abs
|
||||
}
|
||||
|
||||
return filepath.Join(dir, filepath.Base(abs))
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
//nolint:testpackage // exercises the unexported indexAndLockFiles helper
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The scan walks source paths with symlinks resolved, so vaultik's own
|
||||
// files are named the same way, or a symlinked home would back them up.
|
||||
func TestIndexAndLockFilesResolvesSymlinkedDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
realDir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
link := filepath.Join(t.TempDir(), "link")
|
||||
require.NoError(t, os.Symlink(realDir, link))
|
||||
|
||||
paths := indexAndLockFiles(filepath.Join(link, "index.sqlite"), link)
|
||||
|
||||
require.Equal(t, []string{
|
||||
filepath.Join(realDir, "index.sqlite"),
|
||||
filepath.Join(realDir, "index.sqlite-wal"),
|
||||
filepath.Join(realDir, "index.sqlite-shm"),
|
||||
filepath.Join(realDir, "index.sqlite-journal"),
|
||||
filepath.Join(realDir, "vaultik.pid"),
|
||||
}, paths)
|
||||
}
|
||||
@@ -67,9 +67,6 @@ type Scanner struct {
|
||||
currentSourcePath string
|
||||
exclude []string // Glob patterns for files/directories to exclude
|
||||
compiledExclude []compiledPattern // Compiled glob patterns
|
||||
// indexAndLockFiles holds the paths of vaultik's own working files,
|
||||
// which the scan treats as excluded (ScannerConfig.IndexAndLockFiles).
|
||||
indexAndLockFiles map[string]struct{}
|
||||
progress *ProgressReporter
|
||||
// skipErrors skips files that cannot be opened or read (logged loudly);
|
||||
// packer, database, encryption, and upload errors still abort the run.
|
||||
@@ -136,11 +133,6 @@ type ScannerConfig struct {
|
||||
EnableProgress bool // Enable the live progress reporter (ETAs, throughput)
|
||||
UI *ui.Writer // Where user-facing scanner messages go; nil = discard
|
||||
Exclude []string // Glob patterns for files/directories to exclude
|
||||
// IndexAndLockFiles are the absolute paths of vaultik's own working
|
||||
// files: the index, its side files and the PID file. The scan treats
|
||||
// them as excluded, whatever Exclude says, since a copy taken while
|
||||
// they are being written is torn.
|
||||
IndexAndLockFiles []string
|
||||
// SkipErrors skips files that cannot be opened or read (log loudly but
|
||||
// continue); packer, database, encryption, and upload errors still abort.
|
||||
SkipErrors bool
|
||||
@@ -234,11 +226,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
deviceNumber = deviceNumberFromStat
|
||||
}
|
||||
|
||||
indexAndLockFiles := make(map[string]struct{}, len(cfg.IndexAndLockFiles))
|
||||
for _, p := range cfg.IndexAndLockFiles {
|
||||
indexAndLockFiles[filepath.Clean(p)] = struct{}{}
|
||||
}
|
||||
|
||||
return &Scanner{
|
||||
fs: cfg.FS,
|
||||
chunker: chunker.NewChunker(cfg.ChunkSize),
|
||||
@@ -250,7 +237,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
ageRecipient: strings.Join(cfg.AgeRecipients, ","),
|
||||
exclude: cfg.Exclude,
|
||||
compiledExclude: compiledExclude,
|
||||
indexAndLockFiles: indexAndLockFiles,
|
||||
progress: progress,
|
||||
skipErrors: cfg.SkipErrors,
|
||||
previousSnapshotID: cfg.PreviousSnapshotID,
|
||||
@@ -1875,13 +1861,7 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
|
||||
finishedBlob.Compressed, progressCallback)
|
||||
if err != nil {
|
||||
// The record is at info level: the returned error ends the backup
|
||||
// and is printed as its ERROR: line. Only an interrupt cancels
|
||||
// ctx, and the user is told about it, so an upload it stopped is
|
||||
// not logged as a storage failure.
|
||||
if ctx.Err() == nil {
|
||||
log.Info("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||
}
|
||||
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||
|
||||
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||
}
|
||||
@@ -2268,9 +2248,8 @@ func (s *Scanner) detectDeletedFiles(
|
||||
|
||||
// excludedOrInExcludedDir reports whether filePath, which is at or
|
||||
// beneath rootPath, or any directory between rootPath and it, matches an
|
||||
// exclude pattern or is one of vaultik's own files. A pattern such as
|
||||
// */.local/share/Trash matches the directory but not the files inside it,
|
||||
// which the walk skips too.
|
||||
// exclude pattern. A pattern such as */.local/share/Trash matches the
|
||||
// directory but not the files inside it, which the walk skips too.
|
||||
func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
|
||||
for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
|
||||
if s.shouldExclude(p, rootPath) {
|
||||
@@ -2394,14 +2373,9 @@ func doubleStarVariants(pattern string) []string {
|
||||
}
|
||||
|
||||
// shouldExclude checks if a path should be excluded based on exclude patterns
|
||||
// or because it is one of vaultik's own working files.
|
||||
// filePath is the full path to the file
|
||||
// rootPath is the root of the backup source directory
|
||||
func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
|
||||
if _, ok := s.indexAndLockFiles[filePath]; ok {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(s.compiledExclude) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
//nolint:testpackage // exercises the unexported uploadBlobIfNeeded
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/blob"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
const uploadFailedLogMessage = "Failed to upload blob"
|
||||
|
||||
// uploadTestBlob uploads a small blob through store.
|
||||
func uploadTestBlob(ctx context.Context, store storage.Storer) error {
|
||||
data := []byte("blob data")
|
||||
s := &Scanner{storage: store, ui: ui.NewWithColor(io.Discard, false)}
|
||||
|
||||
_, _, err := s.uploadBlobIfNeeded(ctx, "blobs/test", &blob.WithReader{
|
||||
FinishedBlob: &blob.FinishedBlob{
|
||||
Hash: "test",
|
||||
Compressed: int64(len(data)),
|
||||
},
|
||||
Reader: bytes.NewReader(data),
|
||||
}, time.Now(), &ScanResult{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// captureLog runs fn and returns what the logger wrote to stderr
|
||||
// meanwhile, at the --verbose level. The logger writes to the os.Stderr
|
||||
// it finds when it is initialized, so os.Stderr is pointed at a file
|
||||
// first. Not parallel-safe: os.Stderr and the logger are process-global.
|
||||
func captureLog(t *testing.T, fn func()) string {
|
||||
t.Helper()
|
||||
|
||||
stderrPath := filepath.Join(t.TempDir(), "stderr")
|
||||
|
||||
stderrFile, err := os.Create(stderrPath) //nolint:gosec // G304: test temp path
|
||||
if err != nil {
|
||||
t.Fatalf("creating stderr file: %v", err)
|
||||
}
|
||||
|
||||
previous := os.Stderr
|
||||
os.Stderr = stderrFile
|
||||
|
||||
log.Initialize(log.Config{Verbose: true})
|
||||
|
||||
fn()
|
||||
|
||||
os.Stderr = previous
|
||||
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
_ = stderrFile.Close()
|
||||
|
||||
captured, err := os.ReadFile(stderrPath) //nolint:gosec // G304: test temp path
|
||||
if err != nil {
|
||||
t.Fatalf("reading stderr file: %v", err)
|
||||
}
|
||||
|
||||
return string(captured)
|
||||
}
|
||||
|
||||
// An upload stopped by Ctrl-C or SIGTERM is not logged as an error; the
|
||||
// interrupted message is all the user sees. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/434.
|
||||
//
|
||||
//nolint:paralleltest // captureLog replaces os.Stderr
|
||||
func TestInterruptedUploadIsNotLoggedAsError(t *testing.T) {
|
||||
inner, err := storage.NewFileStorer(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
// The user interrupts mid-upload. The injected error does not wrap
|
||||
// context.Canceled, so only ctx shows why the upload failed.
|
||||
store := faultstore.New(inner)
|
||||
store.OnPut = func(string) faultstore.PutAction {
|
||||
cancel()
|
||||
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
var uploadErr error
|
||||
|
||||
stderr := captureLog(t, func() { uploadErr = uploadTestBlob(ctx, store) })
|
||||
|
||||
if uploadErr == nil {
|
||||
t.Fatal("expected the upload to fail")
|
||||
}
|
||||
|
||||
if strings.Contains(stderr, uploadFailedLogMessage) {
|
||||
t.Errorf("interrupted upload logged as a failure: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// An upload that fails for any other reason is still logged.
|
||||
//
|
||||
//nolint:paralleltest // captureLog replaces os.Stderr
|
||||
func TestFailedUploadIsLogged(t *testing.T) {
|
||||
inner, err := storage.NewFileStorer(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := faultstore.New(inner)
|
||||
store.OnPut = func(string) faultstore.PutAction { return faultstore.PutFail }
|
||||
|
||||
var uploadErr error
|
||||
|
||||
stderr := captureLog(t, func() {
|
||||
uploadErr = uploadTestBlob(context.Background(), store)
|
||||
})
|
||||
|
||||
if uploadErr == nil {
|
||||
t.Fatal("expected the upload to fail")
|
||||
}
|
||||
|
||||
if !strings.Contains(stderr, uploadFailedLogMessage) {
|
||||
t.Errorf("failed upload not logged: %q", stderr)
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func TestS3ListingCutShortIsAnError(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
|
||||
// writes: the layout in docs/REPOSTRUCTURE.md, with the prefix joined to
|
||||
// writes: the README's remote storage layout, with the prefix joined to
|
||||
// each key by one "/". s3://b/p and s3://b/p/ must be the same
|
||||
// destination, or a host that writes the URL the other way finds no
|
||||
// snapshots. The listed object is put straight into the bucket, as
|
||||
|
||||
@@ -22,10 +22,6 @@ var errBlobHashMismatch = errors.New("blob hash mismatch")
|
||||
var errBlobNotFullyRead = errors.New(
|
||||
"blob closed before fully read; hash not verified")
|
||||
|
||||
// errWrongPrivateKeyForBlob is returned when the configured key parses
|
||||
// but is not one of the keys a blob was encrypted to.
|
||||
var errWrongPrivateKeyForBlob = errors.New("cannot decrypt this blob")
|
||||
|
||||
// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash
|
||||
// of decrypted plaintext when Close is called. It reuses the hash that
|
||||
// blobgen.Reader already computes internally via its TeeReader, avoiding
|
||||
@@ -91,14 +87,10 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader, err := v.newBlobReader(rc, identities)
|
||||
reader, err := blobgen.NewReader(rc, identities...)
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
|
||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("creating blob reader: %w", err)
|
||||
}
|
||||
|
||||
@@ -110,28 +102,6 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newBlobReader opens the decrypting reader for a blob. When none of the
|
||||
// identities can decrypt it, the error says in plain words that the
|
||||
// configured private key is not one of the keys the blob was encrypted
|
||||
// to and names where the key was configured.
|
||||
func (v *Vaultik) newBlobReader(
|
||||
r io.Reader, identities []age.Identity,
|
||||
) (*blobgen.Reader, error) {
|
||||
blobReader, err := blobgen.NewReader(r, identities...)
|
||||
|
||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: the private key in %s is not one of the keys the blob was encrypted to",
|
||||
errWrongPrivateKeyForBlob, v.Config.AgeSecretKeySourceName())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blobReader, nil
|
||||
}
|
||||
|
||||
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
||||
// The reader is closed as soon as ctx is cancelled, which ends a read
|
||||
// blocked inside it. A storage error is returned as is: restore and deep
|
||||
|
||||
@@ -51,7 +51,7 @@ func (v *Vaultik) checkDestinationMarker() error {
|
||||
}
|
||||
|
||||
// Config is nil only in tests, which then record no binding (see
|
||||
// checkStorageBinding).
|
||||
// EnsureStorageBinding).
|
||||
if v.Config != nil {
|
||||
stored, configured, err := v.storageBinding()
|
||||
if err != nil {
|
||||
@@ -83,19 +83,11 @@ func (v *Vaultik) checkDestinationMarker() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// markDestination refuses a local index bound to a different
|
||||
// destination, checks the backup destination store for its marker,
|
||||
// writes the marker, and then records the binding. A first backup marks
|
||||
// its destination and a later one writes the same text again. The
|
||||
// binding is recorded last, so a first backup that cannot write to the
|
||||
// destination leaves the local index unbound.
|
||||
// markDestination checks the backup destination store with
|
||||
// checkDestinationMarker and then writes the marker, so a first backup
|
||||
// marks its destination and a later one writes the same text again.
|
||||
func (v *Vaultik) markDestination() error {
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = v.checkDestinationMarker()
|
||||
err := v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -107,5 +99,5 @@ func (v *Vaultik) markDestination() error {
|
||||
destinationMarkerKey, err)
|
||||
}
|
||||
|
||||
return v.EnsureStorageBinding()
|
||||
return nil
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func (v *Vaultik) NukeRemote(force bool) error {
|
||||
// the same way, so the advice below must not follow these errors.
|
||||
// The listing also fails on a destination that answers that it has
|
||||
// no marker while the local index holds snapshots.
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -192,7 +192,7 @@ type PruneBlobsResult struct {
|
||||
// PruneDatabase or PruneBlobs individually unless it specifically wants
|
||||
// one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+28
-99
@@ -40,9 +40,6 @@ var (
|
||||
// which is secret, so the message is safe to print and log.
|
||||
errInvalidAgeSecretKey = errors.New(
|
||||
"configured age secret key holds no usable age identity")
|
||||
// errWrongPrivateKey is returned when the configured key parses but
|
||||
// is not one of the keys the snapshot was encrypted to.
|
||||
errWrongPrivateKey = errors.New("cannot decrypt this snapshot")
|
||||
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
||||
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
||||
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
||||
@@ -109,12 +106,7 @@ type RestoreResult struct {
|
||||
BytesDownloaded int64
|
||||
Duration time.Duration
|
||||
FilesFailed int
|
||||
FailedFiles []string // Paths of files that failed to restore for their own reason
|
||||
// Blobs that could not be downloaded, and the paths of the files
|
||||
// skipped because they need one of them. Those files are counted in
|
||||
// FilesFailed but not listed in the summary.
|
||||
BlobsFailed int
|
||||
FilesNeedingFailedBlob []string
|
||||
FailedFiles []string // Paths of files that failed to restore
|
||||
// Verification results (only populated if Verify option is set)
|
||||
FilesVerified int
|
||||
BytesVerified int64
|
||||
@@ -136,19 +128,10 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
"paths", opts.Paths,
|
||||
)
|
||||
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(opts.SnapshotID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Step 1: Download and decrypt the snapshot metadata database
|
||||
log.Info("Downloading snapshot metadata...")
|
||||
|
||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, remoteKey, identities)
|
||||
if errors.Is(err, errWrongPrivateKey) {
|
||||
return err
|
||||
}
|
||||
|
||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities)
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||
}
|
||||
@@ -243,11 +226,6 @@ func (v *Vaultik) finishRestore(
|
||||
if result.FilesFailed > 0 {
|
||||
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
|
||||
|
||||
if result.BlobsFailed > 0 {
|
||||
v.UI.Detailf("%d of them need a blob that could not be read (%d blob(s) in all).",
|
||||
len(result.FilesNeedingFailedBlob), result.BlobsFailed)
|
||||
}
|
||||
|
||||
for _, path := range result.FailedFiles {
|
||||
v.UI.Detailf("%s", v.UI.Path(path))
|
||||
}
|
||||
@@ -345,28 +323,6 @@ func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
||||
return identities, nil
|
||||
}
|
||||
|
||||
// newSnapshotDBReader opens the decrypting reader for a snapshot
|
||||
// database. When none of the identities can decrypt it, the error says
|
||||
// in plain words that the configured private key is the wrong one and
|
||||
// names where it was configured.
|
||||
func (v *Vaultik) newSnapshotDBReader(
|
||||
r io.Reader, identities []age.Identity,
|
||||
) (*blobgen.Reader, error) {
|
||||
blobReader, err := blobgen.NewReader(r, identities...)
|
||||
|
||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: the private key in %s is not one of the keys the backup was encrypted to",
|
||||
errWrongPrivateKey, v.Config.AgeSecretKeySourceName())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blobReader, nil
|
||||
}
|
||||
|
||||
// restoreAllFiles processes files in blob-locality order: drain every
|
||||
// file whose blob set is on disk, download the missing blobs for the
|
||||
// pending file with the smallest uncached count, repeat. This keeps
|
||||
@@ -538,8 +494,8 @@ func (v *Vaultik) runRestoreLoop(
|
||||
// Returns false when nothing is pending download (the caller stops).
|
||||
//
|
||||
// A blob that cannot be downloaded is reported through
|
||||
// handleRestoreBlobError, so it aborts the restore unless --skip-errors
|
||||
// is set.
|
||||
// handleRestoreFileError for every pending file that references it, so
|
||||
// it aborts the restore unless --skip-errors is set.
|
||||
func (s *restoreSession) downloadNextBlobSet(
|
||||
plan *restorePlan, filesByID map[types.FileID]*database.File,
|
||||
) (bool, error) {
|
||||
@@ -572,11 +528,12 @@ func (s *restoreSession) downloadNextBlobSet(
|
||||
return false, err
|
||||
}
|
||||
|
||||
blobErr := s.v.handleRestoreBlobError(
|
||||
plan, s.opts, s.result, filesByID,
|
||||
plan.filesReferencingBlob(hash), err)
|
||||
if blobErr != nil {
|
||||
return false, blobErr
|
||||
for _, fileID := range plan.filesReferencingBlob(hash) {
|
||||
fileErr := s.v.handleRestoreFileError(
|
||||
plan, s.opts, s.result, filesByID[fileID], fileID, err)
|
||||
if fileErr != nil {
|
||||
return false, fileErr
|
||||
}
|
||||
}
|
||||
|
||||
// next is among the failed files, so the rest of its blob set
|
||||
@@ -667,19 +624,19 @@ func (v *Vaultik) restoreProgressTick(
|
||||
|
||||
// handleRestoreFileError records a per-file restore failure: fatal unless
|
||||
// --skip-errors is set, in which case the file is counted as failed and
|
||||
// dropped from the plan. A fatal failure is not logged here: the returned
|
||||
// error is printed as the command's ERROR: line.
|
||||
// dropped from the plan.
|
||||
func (v *Vaultik) handleRestoreFileError(
|
||||
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
|
||||
file *database.File, fileID types.FileID, err error,
|
||||
) error {
|
||||
log.Error("Failed to restore file", "path", file.Path, "error", err)
|
||||
|
||||
if !opts.SkipErrors {
|
||||
return fmt.Errorf(
|
||||
"restoring %s: %w (pass --skip-errors to continue past "+
|
||||
"restore failures)", file.Path, err)
|
||||
}
|
||||
|
||||
log.Error("Failed to restore file", "path", file.Path, "error", err)
|
||||
v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).",
|
||||
v.UI.Path(file.Path.String()), err)
|
||||
|
||||
@@ -691,41 +648,6 @@ func (v *Vaultik) handleRestoreFileError(
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleRestoreBlobError records a blob that could not be downloaded.
|
||||
// fileIDs are the pending files that need it, never empty. Without
|
||||
// --skip-errors the first of them fails the restore. With it, the blob is
|
||||
// reported once, and its files are counted as failed and dropped from
|
||||
// the plan without being reported one by one.
|
||||
func (v *Vaultik) handleRestoreBlobError(
|
||||
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
|
||||
filesByID map[types.FileID]*database.File, fileIDs []types.FileID,
|
||||
err error,
|
||||
) error {
|
||||
if !opts.SkipErrors {
|
||||
return v.handleRestoreFileError(
|
||||
plan, opts, result, filesByID[fileIDs[0]], fileIDs[0], err)
|
||||
}
|
||||
|
||||
log.Error("Failed to restore files", "files", len(fileIDs), "error", err)
|
||||
v.UI.Errorf("Failed to restore %d file(s): %v. Skipping them (--skip-errors).",
|
||||
len(fileIDs), err)
|
||||
|
||||
result.BlobsFailed++
|
||||
|
||||
for _, fileID := range fileIDs {
|
||||
path := filesByID[fileID].Path.String()
|
||||
|
||||
log.Debug("Skipped file that needs a failed blob", "path", path)
|
||||
|
||||
result.FilesFailed++
|
||||
result.FilesNeedingFailedBlob = append(result.FilesNeedingFailedBlob, path)
|
||||
|
||||
plan.finishFile(fileID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// printRestoreProgress emits a periodic restore-phase status line via
|
||||
// the UI writer, mirroring scanner.printProcessingProgress so the two
|
||||
// long-running commands have the same on-screen rhythm.
|
||||
@@ -797,8 +719,8 @@ func (v *Vaultik) handleRestoreVerification(
|
||||
) error {
|
||||
// A file that failed to restore was not written and is already
|
||||
// reported as such, so it is not verified.
|
||||
failedToRestore := make(map[string]bool, result.FilesFailed)
|
||||
for _, path := range slices.Concat(result.FailedFiles, result.FilesNeedingFailedBlob) {
|
||||
failedToRestore := make(map[string]bool, len(result.FailedFiles))
|
||||
for _, path := range result.FailedFiles {
|
||||
failedToRestore[path] = true
|
||||
}
|
||||
|
||||
@@ -832,12 +754,19 @@ func (v *Vaultik) handleRestoreVerification(
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadSnapshotDB downloads and decrypts the metadata database of the
|
||||
// snapshot at remoteKey, which resolveSnapshotRemoteKey returned for
|
||||
// snapshotID.
|
||||
// downloadSnapshotDB downloads and decrypts the snapshot metadata
|
||||
// database. The identifier is resolved to the snapshot's remote key: a
|
||||
// human ID is hashed, and a remote key (or its abbreviation, as printed
|
||||
// for a remote-only snapshot) is used as-is, so a host with no local
|
||||
// index can restore the snapshots it can only see on the store.
|
||||
func (v *Vaultik) downloadSnapshotDB(
|
||||
snapshotID, remoteKey string, identities []age.Identity,
|
||||
snapshotID string, identities []age.Identity,
|
||||
) (*database.DB, string, error) {
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Download encrypted database from storage
|
||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||
|
||||
@@ -854,9 +783,9 @@ func (v *Vaultik) downloadSnapshotDB(
|
||||
// the plaintext to a temp file. Neither the encrypted bytes nor the
|
||||
// decrypted database is ever held whole in memory; a snapshot database
|
||||
// can be large.
|
||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
|
||||
@@ -51,72 +51,27 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
backup := backupThenDeleteOneBlob(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
assertLostFilesCountedNotListed(t, backup, out)
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsReportsUnreadableBlobOnce restores with SkipErrors
|
||||
// after one blob of a two-blob snapshot was overwritten with other bytes
|
||||
// of the same size. The blob must be reported once, by its hash, as not a
|
||||
// valid age stream, without quoting its bytes, and its files must be
|
||||
// counted, not listed.
|
||||
func TestRestoreSkipErrorsReportsUnreadableBlobOnce(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
backup, lostBlob := backupThenOverwriteOneBlob(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
assert.Equal(t, 1, strings.Count(out, "not a valid age stream"), out)
|
||||
assert.Contains(t, out, lostBlob[:16])
|
||||
assert.NotContains(t, out, overwriteBlobPattern)
|
||||
assertLostFilesCountedNotListed(t, backup, out)
|
||||
assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
|
||||
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
|
||||
// earlier backup defects left behind. Each such file fails for its own
|
||||
// reason, so each is listed.
|
||||
// earlier backup defects left behind.
|
||||
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
backup := backupThenDropOneBlobFromMetadata(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.Containsf(t, out, path, "%s must be reported", path)
|
||||
}
|
||||
assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
|
||||
}
|
||||
|
||||
// assertLostFilesCountedNotListed checks that the restore output out
|
||||
// counts the files of the lost blob and names none of them.
|
||||
func assertLostFilesCountedNotListed(
|
||||
t *testing.T, backup *missingBlobBackup, out string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
assert.Contains(t, out, fmt.Sprintf(
|
||||
"%d of them need a blob that could not be read (1 blob(s) in all)",
|
||||
len(backup.lost)))
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.NotContainsf(t, out, path,
|
||||
"%s needs the lost blob and must be counted, not listed", path)
|
||||
}
|
||||
}
|
||||
|
||||
// assertRestoreSkipsLostFiles restores backup with SkipErrors and returns
|
||||
// what it printed. Every file stored in the lost blob must be left
|
||||
// absent, every other file must be restored intact, and Restore must
|
||||
// still return an error.
|
||||
// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
|
||||
// stored in the lost blob must be reported as failed and left absent,
|
||||
// every other file must be restored intact, and Restore must still return
|
||||
// an error.
|
||||
func assertRestoreSkipsLostFiles(
|
||||
ctx context.Context, t *testing.T, backup *missingBlobBackup,
|
||||
) string {
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var out bytes.Buffer
|
||||
@@ -138,6 +93,8 @@ func assertRestoreSkipsLostFiles(
|
||||
restored := filepath.Join(backup.restoreDir, path)
|
||||
|
||||
if backup.lost[path] {
|
||||
assert.Containsf(t, out.String(), path,
|
||||
"%s needs the lost blob and must be reported", path)
|
||||
assert.NoFileExists(t, restored)
|
||||
|
||||
continue
|
||||
@@ -147,8 +104,6 @@ func assertRestoreSkipsLostFiles(
|
||||
require.NoErrorf(t, err, "%s does not need the lost blob", path)
|
||||
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
||||
}
|
||||
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with
|
||||
@@ -189,9 +144,9 @@ func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) {
|
||||
// TestRestoreSkipErrorsVerifyReportsFailuresApart restores with
|
||||
// SkipErrors and Verify after one blob of a two-blob snapshot was
|
||||
// deleted, while one file that does restore is written one byte too long.
|
||||
// The restore warning must count the files of the deleted blob without
|
||||
// listing them, the verification summary must list only the long file,
|
||||
// once, and Restore must return an error.
|
||||
// The restore warning must list only the files of the deleted blob, the
|
||||
// verification summary only the long file, each path once, and Restore
|
||||
// must return an error.
|
||||
func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
@@ -231,9 +186,9 @@ func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
|
||||
"Verification failed: 1 files did not match")
|
||||
require.Truef(t, found, "no verification summary in:\n%s", out.String())
|
||||
|
||||
assertLostFilesCountedNotListed(t, backup, warning)
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.Equalf(t, 1, strings.Count(warning, path),
|
||||
"%s must be listed once as failed to restore", path)
|
||||
assert.NotContainsf(t, summary, path,
|
||||
"%s was not restored and must not fail verification", path)
|
||||
}
|
||||
@@ -337,35 +292,6 @@ func backupThenDeleteOneBlob(
|
||||
return backup
|
||||
}
|
||||
|
||||
// overwriteBlobPattern repeats, each time followed by a newline, through
|
||||
// the bytes that backupThenOverwriteOneBlob writes over a blob. age's own
|
||||
// error quotes the data's first line only when it reads a whole line.
|
||||
const overwriteBlobPattern = "overwritten-blob-bytes-"
|
||||
|
||||
// backupThenOverwriteOneBlob backs up missingBlobFileCount single-chunk
|
||||
// files, then overwrites the blob holding the first of them with as many
|
||||
// bytes of overwriteBlobPattern lines. It also returns that blob's hash.
|
||||
func backupThenOverwriteOneBlob(
|
||||
ctx context.Context, t *testing.T,
|
||||
) (*missingBlobBackup, string) {
|
||||
t.Helper()
|
||||
|
||||
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
key := fmt.Sprintf("blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)
|
||||
|
||||
info, err := backup.storer.Stat(ctx, key)
|
||||
require.NoError(t, err)
|
||||
|
||||
line := overwriteBlobPattern + "\n"
|
||||
pattern := bytes.Repeat([]byte(line), int(info.Size)/len(line)+1)
|
||||
require.NoError(t, backup.storer.Put(
|
||||
ctx, key, bytes.NewReader(pattern[:info.Size])))
|
||||
|
||||
return backup, lostBlob
|
||||
}
|
||||
|
||||
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
|
||||
// single-chunk files, then removes the blob holding the first of them from
|
||||
// the snapshot and exports the snapshot metadata again. The exported
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestWrongPrivateKeyError checks that restore and deep verify, given a
|
||||
// valid private key the backup was not encrypted to, fail with one plain
|
||||
// sentence that names where the key was configured.
|
||||
func TestWrongPrivateKeyError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
store := NewMockStorer()
|
||||
|
||||
const snapshotID = "test-host_2026-01-01T00:00:00Z"
|
||||
|
||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
||||
|
||||
manifest, err := snapshot.EncodeManifest(
|
||||
&snapshot.Manifest{SnapshotID: remoteKey}, 3)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, store.Put(ctx,
|
||||
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(manifest)))
|
||||
|
||||
// The database is encrypted to the test recipient only.
|
||||
var db bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(&db, 1, []string{testAgePublicKey})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write([]byte("snapshot database"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
require.NoError(t, store.Put(ctx,
|
||||
"metadata/"+remoteKey+"/db.zst.age", &db))
|
||||
|
||||
otherKey, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: &config.Config{AgeSecretKey: otherKey.String()},
|
||||
Storage: store,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
// A Config built directly has no recorded source, so the error names
|
||||
// the config-file field.
|
||||
const want = "cannot decrypt this snapshot: the private key in " +
|
||||
"age_secret_key is not one of the keys the backup was encrypted to"
|
||||
|
||||
err = v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
require.EqualError(t, err, want)
|
||||
|
||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
||||
require.EqualError(t, err, want)
|
||||
}
|
||||
|
||||
// TestWrongPrivateKeyForBlobError backs up a file, then backs it up again
|
||||
// from an empty local index with a second recipient added. The second
|
||||
// snapshot reuses the stored blob, which stays encrypted to the first
|
||||
// recipient only. With only the added key configured, restore and deep
|
||||
// verify read the database but fail on the blob with one plain sentence
|
||||
// that names where the key was configured.
|
||||
func TestWrongPrivateKeyForBlobError(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
dataPath := filepath.Join(dataDir, "data.bin")
|
||||
storeDir := t.TempDir()
|
||||
secondDBPath := filepath.Join(tempDir, "second.sqlite")
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
maxBlobSize := int64(512 * 1024)
|
||||
|
||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, dataPath,
|
||||
bytesPattern("key-", int(chunkSize)), 0o644))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
|
||||
filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
|
||||
|
||||
env := setupDedupBackupEnv(
|
||||
ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
|
||||
|
||||
defer func() { _ = env.db.Close() }()
|
||||
|
||||
addedKey, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
|
||||
addedKey.Recipient().String())
|
||||
|
||||
snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
|
||||
env.cfg.Hostname, "second", dataDir, secondDBPath)
|
||||
require.Zero(t, result.BlobsUploaded,
|
||||
"the second backup should reuse the stored blob")
|
||||
|
||||
require.NoError(t, env.db.Close())
|
||||
|
||||
blobKeys, err := env.storer.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, blobKeys, 1)
|
||||
|
||||
blobHash := path.Base(blobKeys[0])
|
||||
|
||||
env.cfg.AgeSecretKey = addedKey.String()
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: env.cfg,
|
||||
Storage: env.storer,
|
||||
Fs: fs,
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
const sentence = "cannot decrypt this blob: the private key in " +
|
||||
"age_secret_key is not one of the keys the blob was encrypted to"
|
||||
|
||||
// Restore names the blob by the first 16 characters of its hash.
|
||||
err = v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
require.EqualError(t, err, "restoring "+dataPath+": downloading blob "+
|
||||
blobHash[:16]+": "+sentence+
|
||||
" (pass --skip-errors to continue past restore failures)")
|
||||
|
||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
||||
require.EqualError(t, err,
|
||||
"blob "+blobHash+" verification failed: "+sentence)
|
||||
}
|
||||
@@ -115,9 +115,14 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
"index_path", v.Config.IndexPath,
|
||||
)
|
||||
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Before PruneDatabase: it drops incomplete snapshots, which could
|
||||
// leave the local index without one and so skip the marker check.
|
||||
err := v.markDestination()
|
||||
err = v.markDestination()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -744,7 +749,7 @@ type SnapshotPurgeOptions struct {
|
||||
// snapshot name, not the latest globally. This prevents `home` and `system`
|
||||
// snapshots from cannibalizing each other.
|
||||
func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -959,22 +964,25 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
||||
Mode: "shallow",
|
||||
}
|
||||
|
||||
// The key is kept so we can also check for the snapshot's encrypted
|
||||
// database below.
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
// remote-only snapshot) is used as-is, so a host with no local index
|
||||
// can verify a snapshot it can only see on the store. The key is kept
|
||||
// so we can also check for the snapshot's encrypted database below.
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
result.Status = verifyStatusFailed
|
||||
result.ErrorMessage = err.Error()
|
||||
result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err)
|
||||
|
||||
return v.outputVerifyJSON(result)
|
||||
}
|
||||
|
||||
return err
|
||||
return fmt.Errorf("resolving snapshot identifier: %w", err)
|
||||
}
|
||||
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
@@ -1272,7 +1280,7 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
|
||||
// jq` fail (issue #108). The narration is duplicated as log records,
|
||||
// which go to stderr and so cannot corrupt the document.
|
||||
func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1462,12 +1470,12 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
SnapshotID: snapshotID,
|
||||
}
|
||||
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly)
|
||||
inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -1530,20 +1538,17 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// checkSnapshotExists returns errSnapshotNotFound when the local index
|
||||
// has no row for snapshotID and, unless localOnly is set, the
|
||||
// destination store has no metadata for it either. inLocalIndex reports
|
||||
// whether the local index has that row. The destination store is
|
||||
// consulted only when the local index has no row, so a snapshot in the
|
||||
// local index can still be removed from it while the destination store
|
||||
// is unreachable, and only after checkDestinationMarker passes, so an
|
||||
// unplugged volume's empty mount point is not taken for a store without
|
||||
// the snapshot.
|
||||
//
|
||||
//nolint:nonamedreturns // the result name says what the bool means
|
||||
func (v *Vaultik) checkSnapshotExists(
|
||||
snapshotID string, localOnly bool,
|
||||
) (inLocalIndex bool, err error) {
|
||||
// findSnapshotToRemove reports whether the local index has a row for
|
||||
// snapshotID, and returns errSnapshotNotFound when there is nothing to
|
||||
// remove: no row and, unless LocalOnly is set, no metadata on the
|
||||
// destination store either. The destination store is consulted only
|
||||
// when the local index has no row, so a snapshot in the local index can
|
||||
// still be removed from it while the destination store is unreachable,
|
||||
// and only after checkDestinationMarker passes, so an unplugged volume's
|
||||
// empty mount point is not taken for a store without the snapshot.
|
||||
func (v *Vaultik) findSnapshotToRemove(
|
||||
snapshotID string, opts *RemoveOptions,
|
||||
) (bool, error) {
|
||||
if v.Repositories != nil {
|
||||
snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID)
|
||||
if err != nil {
|
||||
@@ -1555,12 +1560,12 @@ func (v *Vaultik) checkSnapshotExists(
|
||||
}
|
||||
}
|
||||
|
||||
if localOnly {
|
||||
if opts.LocalOnly {
|
||||
return false, fmt.Errorf("%w in the local index: %s",
|
||||
errSnapshotNotFound, snapshotID)
|
||||
}
|
||||
|
||||
err = v.checkDestinationMarker()
|
||||
err := v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -1702,7 +1707,7 @@ func (v *Vaultik) removeSnapshotRemote(
|
||||
// before it removes anything when the destination store cannot be
|
||||
// listed or is not present by checkDestinationMarker.
|
||||
func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) {
|
||||
err := v.checkStorageBinding()
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -28,9 +28,8 @@ var (
|
||||
// Two forms are accepted, matching the two things a host can know:
|
||||
//
|
||||
// - A human snapshot ID (hostname_name_timestamp), which a host holding
|
||||
// the local index has. It is hashed to its remote key. An ID that
|
||||
// names no snapshot in the local index or on the destination store
|
||||
// is an error (see checkSnapshotExists).
|
||||
// the local index has. It is hashed to its remote key; the store is
|
||||
// not consulted.
|
||||
// - A remote key, or the leading part of one, which is all a host with
|
||||
// no local index can know — it is exactly what `snapshot list` prints
|
||||
// for a remote-only snapshot (see formatRemoteOnlyID). It is resolved
|
||||
@@ -42,11 +41,6 @@ var (
|
||||
// RFC3339 timestamp).
|
||||
func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) {
|
||||
if !isRemoteKeyOrPrefix(identifier) {
|
||||
_, err := v.checkSnapshotExists(identifier, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return snapshot.RemoteSnapshotKey(identifier), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,30 +13,35 @@ import (
|
||||
var errStorageBindingMismatch = errors.New(
|
||||
"local index is bound to a different backup destination")
|
||||
|
||||
// checkStorageBinding refuses when the local index database is bound to
|
||||
// a different destination than the configured one, with an error that
|
||||
// tells the user how to recover (revert the config, or run `vaultik
|
||||
// database delete`). Every mutating command calls it before touching
|
||||
// either the local index or the destination store, because the two live
|
||||
// in lockstep: the local index records which chunks/blobs already exist
|
||||
// at the destination, and a mismatched pair produces silent corruption
|
||||
// (the scanner sees "known" chunks and skips uploads, then writes
|
||||
// snapshots whose manifests reference blobs that aren't on the new
|
||||
// destination).
|
||||
// EnsureStorageBinding guarantees that the local index database is
|
||||
// bound to the currently-configured storage destination. Every mutating
|
||||
// command must call this before touching either the local index or the
|
||||
// destination store, because the two live in lockstep: the local index
|
||||
// records which chunks/blobs already exist at the destination, and
|
||||
// mismatched destination + local index produces silent corruption (the
|
||||
// scanner sees "known" chunks and skips uploads, then writes snapshots
|
||||
// whose manifests reference blobs that aren't on the new destination).
|
||||
//
|
||||
// It records nothing. Only a backup binds the local index, through
|
||||
// EnsureStorageBinding, and it does so before adding anything to it. An
|
||||
// unbound local index therefore holds nothing from any destination.
|
||||
// Behaviour:
|
||||
// - On first use (empty stored value), record the configured
|
||||
// destination (storage_url, or the s3.* endpoint, bucket and
|
||||
// prefix) and log the binding.
|
||||
// - When the stored value matches the configured destination, do
|
||||
// nothing and return nil.
|
||||
// - When the two differ, refuse with an error that tells the user
|
||||
// how to recover (revert the config, or run `vaultik database
|
||||
// delete` to discard the local index and rebuild against the new
|
||||
// destination on the next backup).
|
||||
//
|
||||
// Read-only inspection commands (remote info, snapshot list, etc.)
|
||||
// deliberately don't call this: they can be run against a bare
|
||||
// destination store without any binding state.
|
||||
func (v *Vaultik) checkStorageBinding() error {
|
||||
func (v *Vaultik) EnsureStorageBinding() error {
|
||||
if v.Repositories == nil || v.Config == nil {
|
||||
// NewForTesting builds a Vaultik with no DB or config;
|
||||
// there's no binding to check. Callers exercising the bind
|
||||
// path use a real DB and populate Config explicitly (see
|
||||
// storage_bind_test.go).
|
||||
// there's nothing to bind and no binding to check. Callers
|
||||
// exercising the bind path use a real DB and populate Config
|
||||
// explicitly (see storage_bind_test.go).
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -45,24 +50,6 @@ func (v *Vaultik) checkStorageBinding() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if stored == "" || stored == configured {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
}
|
||||
|
||||
// EnsureStorageBinding records the configured destination (storage_url,
|
||||
// or the s3.* endpoint, bucket and prefix) as the binding of an unbound
|
||||
// local index. It refuses as checkStorageBinding does when the local
|
||||
// index is bound to a different destination.
|
||||
func (v *Vaultik) EnsureStorageBinding() error {
|
||||
stored, configured, err := v.storageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if stored == "" {
|
||||
err = v.Repositories.LocalMeta.Set(
|
||||
v.ctx, database.LocalMetaKeyStorageURL, configured)
|
||||
@@ -75,12 +62,12 @@ func (v *Vaultik) EnsureStorageBinding() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if stored != configured {
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
if stored == configured {
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
}
|
||||
|
||||
// storageBinding returns the destination the local index is bound to
|
||||
|
||||
@@ -2,7 +2,6 @@ package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -10,126 +9,9 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// storageBindingOf returns the destination the local index is bound to,
|
||||
// or "" when it is not bound.
|
||||
func storageBindingOf(
|
||||
ctx context.Context, t *testing.T, repos *database.Repositories,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
// TestFailedFirstBackupLeavesIndexUnbound checks that a first backup
|
||||
// the destination refuses, as a mistyped bucket would, leaves the local
|
||||
// index unbound, so a backup with the corrected destination succeeds.
|
||||
func TestFailedFirstBackupLeavesIndexUnbound(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := t.TempDir()
|
||||
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||
working := v.Storage
|
||||
|
||||
refusing := faultstore.New(working)
|
||||
refusing.OnPut = func(string) faultstore.PutAction { return faultstore.PutFail }
|
||||
refusing.OnStat = func(string) bool { return true }
|
||||
v.Storage = refusing
|
||||
v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "typo")
|
||||
|
||||
require.ErrorIs(t, backUp(v, "first"), faultstore.ErrInjectedUpload)
|
||||
assert.Empty(t, storageBindingOf(ctx, t, repos))
|
||||
|
||||
v.Storage = working
|
||||
v.Config.StorageURL = "file://" + storeDir
|
||||
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
assert.Equal(t, "file://"+storeDir, storageBindingOf(ctx, t, repos))
|
||||
}
|
||||
|
||||
// TestBackupToOtherDestinationWritesNoMarker checks that a backup
|
||||
// refuses a local index bound to a different destination before it
|
||||
// writes the marker to the configured one.
|
||||
func TestBackupToOtherDestinationWritesNoMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
firstDir := t.TempDir()
|
||||
v, _ := newFileDestinationVaultik(ctx, t, firstDir)
|
||||
v.Config.StorageURL = "file://" + firstDir
|
||||
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
|
||||
secondDir := t.TempDir()
|
||||
secondStore, err := storage.NewFileStorer(secondDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
v.Storage = secondStore
|
||||
v.Config.StorageURL = "file://" + secondDir
|
||||
|
||||
require.ErrorContains(t, backUp(v, "first"),
|
||||
"local index is bound to a different backup destination")
|
||||
assert.NoFileExists(t, filepath.Join(secondDir, markerFile))
|
||||
}
|
||||
|
||||
// TestCommandsOtherThanBackupLeaveIndexUnbound checks that commands
|
||||
// other than a backup do not bind an unbound local index, whether or not
|
||||
// they fail. Each runs on a fresh local index.
|
||||
func TestCommandsOtherThanBackupLeaveIndexUnbound(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(v *vaultik.Vaultik) error
|
||||
}{
|
||||
{"prune", func(v *vaultik.Vaultik) error {
|
||||
return v.Prune(&vaultik.PruneOptions{Force: true})
|
||||
}},
|
||||
{"snapshot purge", func(v *vaultik.Vaultik) error {
|
||||
return v.PurgeSnapshotsWithOptions(
|
||||
&vaultik.SnapshotPurgeOptions{KeepLatest: true, Force: true})
|
||||
}},
|
||||
{"snapshot remove", func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveSnapshot("host_first_2026-01-02T03:04:05Z",
|
||||
&vaultik.RemoveOptions{Force: true})
|
||||
|
||||
return err
|
||||
}},
|
||||
{"snapshot remove --all", func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveAllSnapshots(&vaultik.RemoveOptions{Force: true})
|
||||
|
||||
return err
|
||||
}},
|
||||
{"remote nuke", func(v *vaultik.Vaultik) error {
|
||||
return v.NukeRemote(true)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos := newFileDestinationVaultik(ctx, t, t.TempDir())
|
||||
|
||||
_ = tt.run(v)
|
||||
|
||||
assert.Empty(t, storageBindingOf(ctx, t, repos))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// buildBindTestVaultik returns a minimal Vaultik wired with a real
|
||||
// in-memory DB and a config carrying the given StorageURL. Enough to
|
||||
// exercise EnsureStorageBinding without spinning up storage or fx.
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestUnknownSnapshotID checks that restore and verify, given an ID that
|
||||
// names no snapshot in the local index or on the destination store, say
|
||||
// so in the words `snapshot remove` uses and print nothing else.
|
||||
func TestUnknownSnapshotID(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
const unknownID = "host1_home_2026-01-01T00:00:00Z"
|
||||
|
||||
commands := map[string]func(v *vaultik.Vaultik) error{
|
||||
"restore": func(v *vaultik.Vaultik) error {
|
||||
return v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: unknownID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
},
|
||||
"verify": func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(unknownID, &vaultik.VerifyOptions{})
|
||||
},
|
||||
"verify --deep": func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(unknownID,
|
||||
&vaultik.VerifyOptions{Deep: true})
|
||||
},
|
||||
}
|
||||
|
||||
for name, run := range commands {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", nil)
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Config = &config.Config{AgeSecretKey: testAgeSecretKey}
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
|
||||
err := run(tv.Vaultik)
|
||||
|
||||
require.EqualError(t, err, "snapshot not found in the local "+
|
||||
"index or on the destination store: "+unknownID)
|
||||
assert.Empty(t, tv.Stdout.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
+16
-21
@@ -120,17 +120,12 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
|
||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
|
||||
if !opts.JSON && !v.UI.Quiet() {
|
||||
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
|
||||
}
|
||||
|
||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
||||
snapshotID, remoteKey, opts, result, identities)
|
||||
snapshotID, opts, result, identities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -175,13 +170,21 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadVerificationData downloads manifest, database, and blob list for
|
||||
// verification of the snapshot at remoteKey, which
|
||||
// resolveSnapshotRemoteKey returned for snapshotID.
|
||||
// loadVerificationData downloads manifest, database, and blob list for verification
|
||||
func (v *Vaultik) loadVerificationData(
|
||||
snapshotID, remoteKey string, opts *VerifyOptions, result *VerifyResult,
|
||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||
identities []age.Identity,
|
||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
// remote-only snapshot) is used as-is, so a host with no local index
|
||||
// can verify a snapshot it can only see on the store.
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("resolving snapshot identifier: %v", err), err)
|
||||
}
|
||||
|
||||
// Download manifest. downloadManifestByKey is the single reader for
|
||||
// remote manifests; see its doc comment.
|
||||
log.Info("Downloading manifest", "remote_key", remoteKey)
|
||||
@@ -264,10 +267,6 @@ func (v *Vaultik) downloadVerifiedSnapshotDB(
|
||||
defer func() { _ = dbReader.Close() }()
|
||||
|
||||
tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
|
||||
if errors.Is(err, errWrongPrivateKey) {
|
||||
return nil, v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||
@@ -352,9 +351,9 @@ func (v *Vaultik) decryptAndLoadDatabase(
|
||||
reader io.ReadCloser, identities []age.Identity,
|
||||
) (*tempDB, error) {
|
||||
// Decrypt and decompress through the shared blobgen reader.
|
||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
@@ -427,11 +426,7 @@ func (v *Vaultik) verifyBlob(
|
||||
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||
// encrypted bytes.
|
||||
blobReader, err := v.newBlobReader(reader, identities)
|
||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
||||
return err
|
||||
}
|
||||
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||
}
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@
|
||||
# when no tag is reachable, each with a "-dirty" suffix when tracked
|
||||
# files have uncommitted changes. Untracked files are ignored: a stray
|
||||
# scratch file does not change what was compiled. Outside a git checkout
|
||||
# (such as an unpacked release tarball), or in one with no commits, it
|
||||
# is "dev".
|
||||
# (release tarball, `go install`), or in one with no commits, it is
|
||||
# "dev".
|
||||
#
|
||||
# Nothing here ever invents a version number.
|
||||
set -eu
|
||||
|
||||
Reference in new issue
Block a user