Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b43bc3a7bf |
+3
-3
@@ -63,7 +63,7 @@ A content-addressed unit of data. Files are split into variable-size chunks usin
|
||||
- `ChunkHash`: SHA256 hash of chunk content (primary key)
|
||||
- `Size`: Chunk size in bytes
|
||||
|
||||
Chunk sizes vary between `avgChunkSize/4` and `avgChunkSize*4` (typically 16KB-256KB for 64KB average).
|
||||
Chunk sizes vary between `avgChunkSize/4` and `avgChunkSize*4` (2.5MB-40MB for the 10MB default average).
|
||||
|
||||
#### FileChunk (`database.FileChunk`)
|
||||
Maps files to their constituent chunks:
|
||||
@@ -120,7 +120,7 @@ The CLI uses fx for dependency injection. Here's the instantiation order:
|
||||
```go
|
||||
// cli/app.go: NewApp()
|
||||
fx.New(
|
||||
fx.Supply(config.ConfigPath(opts.ConfigPath)), // 1. Config path
|
||||
fx.Supply(config.Path(opts.ConfigPath)), // 1. Config path
|
||||
fx.Supply(opts.LogOptions), // 2. Log options
|
||||
fx.Provide(globals.New), // 3. Globals
|
||||
fx.Provide(log.New), // 4. Logger config
|
||||
@@ -193,7 +193,7 @@ scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||
- **Created by**: `chunker.NewChunker(avgChunkSize)`
|
||||
- **When**: Inside `snapshot.NewScanner()`
|
||||
- **Configuration**:
|
||||
- `avgChunkSize`: From config (typically 64KB)
|
||||
- `avgChunkSize`: From config (default 10MB)
|
||||
- `minChunkSize`: avgChunkSize / 4
|
||||
- `maxChunkSize`: avgChunkSize * 4
|
||||
|
||||
|
||||
@@ -147,10 +147,10 @@ vaultik [--config <path>] config edit
|
||||
vaultik [--config <path>] config get <key>
|
||||
vaultik [--config <path>] config set <key> <value>
|
||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
|
||||
vaultik [--config <path>] snapshot list [--json]
|
||||
vaultik [--config <path>] snapshot list [--json] # alias: ls
|
||||
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
||||
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--snapshot <name>...] [--force]
|
||||
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force] [--local-only] [--json]
|
||||
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force] [--local-only] [--json] # alias: rm
|
||||
vaultik [--config <path>] snapshot restore <snapshot-id> <target-dir> [paths...] [--verify]
|
||||
vaultik [--config <path>] prune [--force] [--json]
|
||||
vaultik [--config <path>] info
|
||||
@@ -169,6 +169,21 @@ vaultik version
|
||||
* `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
|
||||
* `--skip-errors`: Continue past per-file errors instead of aborting (applies to `snapshot create` and `restore`)
|
||||
|
||||
### locking
|
||||
|
||||
Every command that opens the local index — `snapshot create`, `snapshot
|
||||
list`, `snapshot verify`, `snapshot purge`, `snapshot remove`, `snapshot
|
||||
restore`, `prune`, `info`, and `remote info`/`remote nuke` — takes a
|
||||
process-wide lock at `$XDG_DATA_HOME/vaultik/vaultik.pid`
|
||||
(`~/.local/share/vaultik/vaultik.pid` on Linux) for the whole run. Only
|
||||
one such command runs at a time: a second one exits immediately with an
|
||||
"already running" error rather than waiting. The lock is not scoped to
|
||||
mutating commands, so read-only commands are affected too — `vaultik
|
||||
snapshot list` fails while a backup is in progress; scoping it so
|
||||
read-only commands run during a backup is tracked in
|
||||
[issue #150](https://git.eeqj.de/sneak/vaultik/issues/150). `config`,
|
||||
`database delete`, `completion`, and `version` do not take the lock.
|
||||
|
||||
### stdout and stderr
|
||||
|
||||
Log output — everything from `--verbose` and `--debug`, and every
|
||||
@@ -203,6 +218,8 @@ and `vaultik prune --json | jq .` both work as written.
|
||||
* `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for `snapshot restore` and `snapshot verify --deep`)
|
||||
* `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
||||
* `VAULTIK_INDEX_PATH`: Override local SQLite index path
|
||||
* `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the run (development/debugging)
|
||||
* `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits (development/debugging)
|
||||
|
||||
### shell completion
|
||||
|
||||
@@ -395,6 +412,10 @@ both are set.
|
||||
|
||||
## architecture
|
||||
|
||||
For an implementation-level view of the internals — the data model, the
|
||||
`fx` dependency-injection wiring, and the scanner — see
|
||||
[`ARCHITECTURE.md`](ARCHITECTURE.md).
|
||||
|
||||
### remote storage layout
|
||||
|
||||
```
|
||||
@@ -472,19 +493,24 @@ derivation.
|
||||
|
||||
### compression
|
||||
|
||||
* zstd compression at configurable level (1-19, default 3)
|
||||
* zstd compression at configurable level (1-19, default 3). The level is
|
||||
accepted as 1-19 but maps onto zstd's four internal speed presets:
|
||||
1-2 fastest, 3-5 default, 6-9 better, 10-19 best. Levels within the
|
||||
same band compress identically.
|
||||
* Applied before encryption at the blob level
|
||||
|
||||
---
|
||||
|
||||
## configuration reference
|
||||
|
||||
Run `vaultik config init` to generate a fully commented config file.
|
||||
Key fields:
|
||||
Run `vaultik config init` to generate a fully commented config file; a
|
||||
complete annotated example also lives in
|
||||
[`config.example.yml`](config.example.yml). Key fields:
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `age_recipients` | (required) | Age public keys for encryption |
|
||||
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
|
||||
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
||||
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
||||
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
|
||||
@@ -600,9 +626,17 @@ priority.
|
||||
|
||||
## output style
|
||||
|
||||
All user-facing output goes through helpers in `internal/ui` and conforms
|
||||
to a uniform style. Color is enabled when stdout is a TTY and the
|
||||
`NO_COLOR` environment variable is unset (https://no-color.org/).
|
||||
The operational narration of the long-running commands — the Begin,
|
||||
Complete, Progress, and status lines of `snapshot create`, `prune`,
|
||||
`snapshot restore`, and the like — goes through helpers in `internal/ui`
|
||||
and conforms to the uniform style below. Some commands instead write
|
||||
plain text straight to stdout (`version`, `info`, `config`, the
|
||||
`database delete` prompt, and the `snapshot list` table); that output is
|
||||
unstyled and does not honor `--quiet`. Routing it through `internal/ui`
|
||||
is tracked in
|
||||
[issue #149](https://git.eeqj.de/sneak/vaultik/issues/149). Color is
|
||||
enabled when stdout is a TTY and the `NO_COLOR` environment variable is
|
||||
unset (https://no-color.org/).
|
||||
|
||||
`internal/ui` writes to stdout; it is the output the user asked for.
|
||||
Structured log records are a different thing and go through
|
||||
|
||||
@@ -1,209 +0,0 @@
|
||||
// Package faultstore provides a storage.Storer wrapper that injects
|
||||
// faults on demand, so tests can reproduce the failure modes a real
|
||||
// backend exhibits: an upload that fails partway, a backend that reports
|
||||
// success while storing nothing, and reads that return corrupt or
|
||||
// truncated bytes. It is the seam called for by the fault-injection
|
||||
// tests (sneak/vaultik issue 72) and is meant to be reused by future
|
||||
// tests rather than re-implemented per case.
|
||||
//
|
||||
// The wrapper delegates every method to the inner Storer. Two hooks
|
||||
// change that: OnPut decides the fate of each write, and OnGet decides
|
||||
// how each read's bytes are returned. Both are keyed by the object key,
|
||||
// so a test can fault only blobs, only metadata, or a single object.
|
||||
package faultstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
|
||||
var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
|
||||
|
||||
// PutAction is the disposition OnPut assigns to a write.
|
||||
type PutAction int
|
||||
|
||||
const (
|
||||
// PutNormal writes through to the inner Storer.
|
||||
PutNormal PutAction = iota
|
||||
// PutFail reads part of the stream, then fails without storing the
|
||||
// object — a network upload that dies partway through.
|
||||
PutFail
|
||||
// PutSwallow reports success but stores nothing — a backend that
|
||||
// lies about durability.
|
||||
PutSwallow
|
||||
)
|
||||
|
||||
// GetFault is how OnGet chooses to damage a read.
|
||||
type GetFault int
|
||||
|
||||
const (
|
||||
// GetNormal returns the stored bytes unchanged.
|
||||
GetNormal GetFault = iota
|
||||
// GetCorrupt flips a byte so the returned object no longer matches
|
||||
// what was stored.
|
||||
GetCorrupt
|
||||
// GetTruncate returns a short read: the object's bytes cut off
|
||||
// before the end.
|
||||
GetTruncate
|
||||
)
|
||||
|
||||
// Storer wraps an inner storage.Storer with fault-injection hooks. A
|
||||
// zero-valued hook means "no fault": construct with New and set only the
|
||||
// hook a test needs.
|
||||
type Storer struct {
|
||||
inner storage.Storer
|
||||
|
||||
// OnPut, when set, is consulted before every Put and
|
||||
// PutWithProgress with the object key.
|
||||
OnPut func(key string) PutAction
|
||||
|
||||
// OnGet, when set, is consulted for every Get with the object key
|
||||
// and damages the returned bytes accordingly.
|
||||
OnGet func(key string) GetFault
|
||||
}
|
||||
|
||||
// New wraps inner. inner must be non-nil.
|
||||
func New(inner storage.Storer) *Storer {
|
||||
return &Storer{inner: inner}
|
||||
}
|
||||
|
||||
// midStreamBytes is how far a PutFail reads before failing, enough to be
|
||||
// past the start of any real blob without depending on the blob's size.
|
||||
const midStreamBytes = 512
|
||||
|
||||
// Put stores data unless OnPut faults the write.
|
||||
func (f *Storer) Put(ctx context.Context, key string, data io.Reader) error {
|
||||
handled, err := f.injectPut(key, data)
|
||||
if handled {
|
||||
return err
|
||||
}
|
||||
|
||||
return f.inner.Put(ctx, key, data)
|
||||
}
|
||||
|
||||
// PutWithProgress stores data unless OnPut faults the write.
|
||||
func (f *Storer) PutWithProgress(
|
||||
ctx context.Context, key string, data io.Reader,
|
||||
size int64, progress storage.ProgressCallback,
|
||||
) error {
|
||||
handled, err := f.injectPut(key, data)
|
||||
if handled {
|
||||
return err
|
||||
}
|
||||
|
||||
return f.inner.PutWithProgress(ctx, key, data, size, progress)
|
||||
}
|
||||
|
||||
// Get retrieves data, damaging it if OnGet faults the read.
|
||||
func (f *Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
rc, err := f.inner.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fault := GetNormal
|
||||
if f.OnGet != nil {
|
||||
fault = f.OnGet(key)
|
||||
}
|
||||
|
||||
if fault == GetNormal {
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(damage(fault, data))), nil
|
||||
}
|
||||
|
||||
// damage returns a faulted copy of the stored bytes. GetCorrupt flips a
|
||||
// byte in the middle so decryption authentication fails; GetTruncate
|
||||
// drops the final byte so the read ends short. Both are no-ops on empty
|
||||
// input, which cannot be damaged into something distinguishable.
|
||||
func damage(fault GetFault, data []byte) []byte {
|
||||
out := make([]byte, len(data))
|
||||
copy(out, data)
|
||||
|
||||
if len(out) == 0 {
|
||||
return out
|
||||
}
|
||||
|
||||
switch fault {
|
||||
case GetCorrupt:
|
||||
out[len(out)/2] ^= 0xff
|
||||
case GetTruncate:
|
||||
out = out[:len(out)-1]
|
||||
case GetNormal:
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Stat delegates unchanged.
|
||||
func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
||||
return f.inner.Stat(ctx, key)
|
||||
}
|
||||
|
||||
// Delete delegates unchanged.
|
||||
func (f *Storer) Delete(ctx context.Context, key string) error {
|
||||
return f.inner.Delete(ctx, key)
|
||||
}
|
||||
|
||||
// List delegates unchanged.
|
||||
func (f *Storer) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
return f.inner.List(ctx, prefix)
|
||||
}
|
||||
|
||||
// ListStream delegates unchanged.
|
||||
func (f *Storer) ListStream(
|
||||
ctx context.Context, prefix string,
|
||||
) <-chan storage.ObjectInfo {
|
||||
return f.inner.ListStream(ctx, prefix)
|
||||
}
|
||||
|
||||
// Info delegates unchanged.
|
||||
func (f *Storer) Info() storage.Info {
|
||||
return f.inner.Info()
|
||||
}
|
||||
|
||||
func (f *Storer) putAction(key string) PutAction {
|
||||
if f.OnPut == nil {
|
||||
return PutNormal
|
||||
}
|
||||
|
||||
return f.OnPut(key)
|
||||
}
|
||||
|
||||
// injectPut handles the non-normal write dispositions. It reports
|
||||
// whether it handled the write and, if so, with what error.
|
||||
func (f *Storer) injectPut(key string, data io.Reader) (bool, error) {
|
||||
switch f.putAction(key) {
|
||||
case PutFail:
|
||||
// Consume part of the stream so the failure lands mid-transfer,
|
||||
// the way a dropped connection would, then error without
|
||||
// storing anything.
|
||||
_, _ = io.CopyN(io.Discard, data, midStreamBytes)
|
||||
|
||||
return true, fmt.Errorf("%w for %q", ErrInjectedUpload, key)
|
||||
case PutSwallow:
|
||||
// A lying backend still drains the request body, then keeps
|
||||
// nothing.
|
||||
_, _ = io.Copy(io.Discard, data)
|
||||
|
||||
return true, nil
|
||||
case PutNormal:
|
||||
return false, nil
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
@@ -1,634 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// These tests cover the failure modes a backup tool must survive:
|
||||
// interrupted uploads, an interrupted metadata export, corrupt and
|
||||
// truncated reads, a full restore disk, and a backend that reports
|
||||
// success while storing nothing. Faults are injected through the
|
||||
// storage.Storer seam (internal/storage/faultstore), never by patching
|
||||
// production code. Each test asserts on the observable end state — what
|
||||
// is in the index, what is at the destination, what the user is told —
|
||||
// not merely that an error was returned. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/72.
|
||||
//
|
||||
// Object-level write atomicity (no partial blob object left behind) is
|
||||
// covered by the file:// backend's atomic-write work
|
||||
// (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested
|
||||
// here; these tests target the layers above the backend.
|
||||
//
|
||||
// The tests run serially, not with t.Parallel: each calls
|
||||
// log.Initialize, which replaces the package-global logger, and a
|
||||
// backup or restore running concurrently reads that same logger. Under
|
||||
// -race the two collide. Running one at a time is the same choice
|
||||
// prune_count_test.go already makes for the same reason.
|
||||
|
||||
const (
|
||||
faultChunkSize = int64(64 * 1024)
|
||||
faultMaxBlobSize = int64(256 * 1024)
|
||||
)
|
||||
|
||||
// faultTestConfig returns the config shared by the fault-injection
|
||||
// tests: a real recipient/secret keypair so blobs are genuinely
|
||||
// encrypted, and a blob size limit the restore sweeper can divide.
|
||||
func faultTestConfig() *config.Config {
|
||||
return &config.Config{
|
||||
AgeRecipients: []string{testAgePublicKey},
|
||||
AgeSecretKey: testAgeSecretKey,
|
||||
CompressionLevel: 3,
|
||||
Hostname: testHostname,
|
||||
BlobSizeLimit: config.Size(faultMaxBlobSize),
|
||||
}
|
||||
}
|
||||
|
||||
// writeFaultSourceTree writes a spread of file sizes that forces several
|
||||
// chunks across more than one blob, so a fault landing on a single blob
|
||||
// still leaves other data intact. Returns the expected content by path.
|
||||
func writeFaultSourceTree(
|
||||
t *testing.T, fs afero.Fs, dataDir string,
|
||||
) map[string][]byte {
|
||||
t.Helper()
|
||||
|
||||
files := map[string][]byte{
|
||||
filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"),
|
||||
filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(faultChunkSize*3)),
|
||||
filepath.Join(dataDir, "sub", "b.bin"): bytesPattern("b-", int(faultChunkSize*3)),
|
||||
filepath.Join(dataDir, "sub", "c.bin"): bytesPattern("c-", int(faultChunkSize*2)),
|
||||
}
|
||||
|
||||
for path, content := range files {
|
||||
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// newFaultScanner builds a scanner writing through the given storer.
|
||||
func newFaultScanner(
|
||||
fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
) *snapshot.Scanner {
|
||||
return snapshot.NewScanner(snapshot.ScannerConfig{
|
||||
FS: fs,
|
||||
Storage: storer,
|
||||
ChunkSize: faultChunkSize,
|
||||
MaxBlobSize: faultMaxBlobSize,
|
||||
CompressionLevel: cfg.CompressionLevel,
|
||||
AgeRecipients: cfg.AgeRecipients,
|
||||
Repositories: repos,
|
||||
})
|
||||
}
|
||||
|
||||
// newFaultSnapshotManager builds a snapshot manager writing through the
|
||||
// given storer.
|
||||
func newFaultSnapshotManager(
|
||||
fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
) *snapshot.SnapshotManager {
|
||||
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
||||
Repos: repos,
|
||||
Storage: storer,
|
||||
Config: cfg,
|
||||
})
|
||||
sm.SetFilesystem(fs)
|
||||
|
||||
return sm
|
||||
}
|
||||
|
||||
// fullFaultBackup runs a complete backup (create, scan, complete,
|
||||
// export) through storer and returns the snapshot ID.
|
||||
func fullFaultBackup(
|
||||
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
dataDir, dbPath, name string,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
sm := newFaultSnapshotManager(fs, storer, cfg, repos)
|
||||
scanner := newFaultScanner(fs, storer, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, name, "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id))
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
// newReaderVaultik builds a Vaultik that reads (restore/verify) through
|
||||
// storer, with the given repositories (nil is fine for restore/verify,
|
||||
// which read metadata from storage).
|
||||
func newReaderVaultik(
|
||||
ctx context.Context, cfg *config.Config, storer storage.Storer,
|
||||
repos *database.Repositories, fs afero.Fs,
|
||||
) *vaultik.Vaultik {
|
||||
v := &vaultik.Vaultik{
|
||||
Config: cfg,
|
||||
Storage: storer,
|
||||
Repositories: repos,
|
||||
Fs: fs,
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// Scenario 3: a stored blob's bytes are flipped before restore reads
|
||||
// them. Restore must fail loudly, and no file must be left on the
|
||||
// restore target holding corrupt content.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreRejectsCorruptBlob(t *testing.T) {
|
||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt")
|
||||
}
|
||||
|
||||
// Scenario 4: a stored blob is truncated before restore reads it. Same
|
||||
// contract as the corrupt case.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreRejectsTruncatedBlob(t *testing.T) {
|
||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated")
|
||||
}
|
||||
|
||||
// assertRestoreRejectsDamagedBlob backs up the source tree, then restores
|
||||
// through a store that damages every blob read with the given fault, and
|
||||
// asserts restore fails naming a blob and leaves no file on the target
|
||||
// holding wrong bytes. Metadata reads are returned intact so the failure
|
||||
// is isolated to the blob.
|
||||
func assertRestoreRejectsDamagedBlob(
|
||||
t *testing.T, fault faultstore.GetFault, name string,
|
||||
) {
|
||||
t.Helper()
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, name)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
faultStore := faultstore.New(inner)
|
||||
faultStore.OnGet = func(key string) faultstore.GetFault {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return fault
|
||||
}
|
||||
|
||||
return faultstore.GetNormal
|
||||
}
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, faultStore, nil, fs)
|
||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
||||
|
||||
require.Error(t, err, "restore must fail on a damaged blob")
|
||||
assert.Contains(t, err.Error(), "blob",
|
||||
"error should name the blob that failed")
|
||||
assertNoCorruptFiles(t, fs, restoreDir, testFiles)
|
||||
}
|
||||
|
||||
// Scenario 6: the backend accepts blob uploads and reports success but
|
||||
// stores nothing. verify --deep must catch it.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Blob uploads are swallowed; metadata uploads land, so verify can
|
||||
// download the manifest and database and then discover the blobs are
|
||||
// absent.
|
||||
lying := faultstore.New(inner)
|
||||
lying.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutSwallow
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, fs, lying, cfg, repos, dataDir, dbPath, "lying")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// No blob objects were actually written.
|
||||
blobKeys, err := inner.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, blobKeys, "lying backend should have stored no blobs")
|
||||
|
||||
// Read back through the honest underlying store.
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
||||
err = v.VerifySnapshotWithOptions(id, &vaultik.VerifyOptions{Deep: true})
|
||||
require.Error(t, err, "deep verify must catch a backend that stored nothing")
|
||||
}
|
||||
|
||||
// Scenario 1a: a blob upload fails partway through. The interrupted run
|
||||
// must not record the blob as uploaded, must not reference it from the
|
||||
// snapshot, and must leave no blob object at the destination.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Every blob upload fails partway through. The scan must surface it.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.Error(t, err, "scan must fail when a blob upload fails")
|
||||
|
||||
// No blob may claim to be uploaded.
|
||||
blobs, err := repos.Blobs.GetAll(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, b := range blobs {
|
||||
assert.Nilf(t, b.UploadedTS,
|
||||
"blob %s marked uploaded after a failed upload", b.Hash)
|
||||
}
|
||||
|
||||
// The snapshot may reference no blobs, and the destination holds none.
|
||||
hashes, err := repos.Snapshots.GetBlobHashes(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, hashes, "interrupted snapshot must reference no blobs")
|
||||
|
||||
blobKeys, err := inner.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, blobKeys, "no blob object may survive at the destination")
|
||||
}
|
||||
|
||||
// Scenario 1b: after an interrupted upload, a retry on the same local
|
||||
// index must produce a restorable snapshot. It does not: the interrupted
|
||||
// run's chunk rows persist, the retry deduplicates against them, and the
|
||||
// backup silently emits a snapshot referencing data never stored. Skipped
|
||||
// pending the fix. See https://git.eeqj.de/sneak/vaultik/issues/148.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
|
||||
t.Skip("blocked on https://git.eeqj.de/sneak/vaultik/issues/148: " +
|
||||
"retry after an interrupted upload silently produces an " +
|
||||
"unrestorable snapshot")
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Attempt 1: every blob upload fails.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
||||
|
||||
id1, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id1)
|
||||
require.Error(t, err)
|
||||
|
||||
// Retry on the same local index with a working backend.
|
||||
id2 := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, "retry")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
||||
require.NoError(t, v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: id2,
|
||||
TargetDir: restoreDir,
|
||||
Verify: true,
|
||||
}), "retry after an interrupted upload must produce a restorable snapshot")
|
||||
|
||||
assertRestoredTree(t, fs, restoreDir, testFiles)
|
||||
}
|
||||
|
||||
// Scenario 2: the process dies during the metadata export, after the
|
||||
// database is uploaded but before the manifest. The destination is left
|
||||
// with blobs and a database but no manifest. verify and snapshot list
|
||||
// must report the damage honestly rather than crashing or passing.
|
||||
// Automatic detection and repair of this partial state on the next run
|
||||
// is tracked in https://git.eeqj.de/sneak/vaultik/issues/148 and is not
|
||||
// asserted here.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Back up and complete with a working backend.
|
||||
sm := newFaultSnapshotManager(fs, inner, cfg, repos)
|
||||
scanner := newFaultScanner(fs, inner, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "export", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
||||
|
||||
// Export through a backend that fails only the manifest upload. The
|
||||
// database uploads first and lands; the manifest does not.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasSuffix(key, "manifest.json.zst") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
smFault := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
|
||||
err = smFault.ExportSnapshotMetadata(ctx, dbPath, id)
|
||||
require.Error(t, err, "export must fail when the manifest upload fails")
|
||||
|
||||
// The destination is in the partial state the scenario describes.
|
||||
key := snapshot.RemoteSnapshotKey(id)
|
||||
|
||||
_, err = inner.Stat(ctx, "metadata/"+key+"/db.zst.age")
|
||||
require.NoError(t, err, "database should have been uploaded before the manifest")
|
||||
|
||||
_, err = inner.Stat(ctx, "metadata/"+key+"/manifest.json.zst")
|
||||
require.ErrorIs(t, err, storage.ErrNotFound, "manifest upload should not have landed")
|
||||
|
||||
// verify must fail loudly for this snapshot, in both modes.
|
||||
reader := newReaderVaultik(ctx, cfg, inner, repos, fs)
|
||||
|
||||
deepOpts := &vaultik.VerifyOptions{Deep: true}
|
||||
require.Error(t, reader.VerifySnapshotWithOptions(id, deepOpts),
|
||||
"deep verify must report the missing manifest")
|
||||
|
||||
shallowOpts := &vaultik.VerifyOptions{Deep: false}
|
||||
require.Error(t, reader.VerifySnapshotWithOptions(id, shallowOpts),
|
||||
"shallow verify must report the missing manifest")
|
||||
|
||||
// snapshot list must not crash on the partial snapshot.
|
||||
require.NoError(t, reader.ListSnapshots(false),
|
||||
"snapshot list must tolerate a partially-exported snapshot")
|
||||
}
|
||||
|
||||
// Scenario 5: the restore target runs out of space mid-file. Restore
|
||||
// must fail with a clear, file-naming error rather than reporting
|
||||
// success over a truncated file.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreReportsDiskFull(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
osFS := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, osFS, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, osFS, inner, cfg, repos, dataDir, dbPath, "diskfull")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Restore onto a filesystem that allows only a few bytes of file
|
||||
// content: enough to create files, far too little to hold them.
|
||||
budget := int64(8)
|
||||
quota := "aFS{Fs: osFS, remaining: &budget}
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, quota)
|
||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
||||
|
||||
require.Error(t, err, "restore must fail when the target disk is full")
|
||||
assert.Contains(t, err.Error(), errNoSpace.Error(),
|
||||
"restore error should surface the out-of-space cause")
|
||||
}
|
||||
|
||||
// assertRestoredTree byte-compares every restored file against the
|
||||
// original.
|
||||
func assertRestoredTree(
|
||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for origPath, expected := range testFiles {
|
||||
restoredPath := filepath.Join(restoreDir, origPath)
|
||||
got, err := afero.ReadFile(fs, restoredPath)
|
||||
require.NoErrorf(t, err, "restored file missing: %s", origPath)
|
||||
require.Equalf(t, expected, got, "restored content mismatch for %s", origPath)
|
||||
}
|
||||
}
|
||||
|
||||
// errNoSpace is the out-of-space error quotaFS returns once its byte
|
||||
// budget is exhausted, mirroring a real ENOSPC.
|
||||
var errNoSpace = errors.New("no space left on device")
|
||||
|
||||
// quotaFS is an afero.Fs whose files may write only a fixed total number
|
||||
// of content bytes before failing, simulating a full restore target. It
|
||||
// wraps the interface so every method except Create delegates to the
|
||||
// real filesystem; only file writes are capped.
|
||||
type quotaFS struct {
|
||||
afero.Fs
|
||||
|
||||
remaining *int64
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Create's signature requires returning afero.File.
|
||||
func (q *quotaFS) Create(name string) (afero.File, error) {
|
||||
f, err := q.Fs.Create(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return "aFile{File: f, remaining: q.remaining}, nil
|
||||
}
|
||||
|
||||
// quotaFile fails writes once the shared byte budget is exhausted.
|
||||
type quotaFile struct {
|
||||
afero.File
|
||||
|
||||
remaining *int64
|
||||
}
|
||||
|
||||
func (q *quotaFile) Write(p []byte) (int, error) {
|
||||
if *q.remaining <= 0 {
|
||||
return 0, errNoSpace
|
||||
}
|
||||
|
||||
allowed := min(int64(len(p)), *q.remaining)
|
||||
|
||||
n, err := q.File.Write(p[:allowed])
|
||||
*q.remaining -= int64(n)
|
||||
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if int64(n) < int64(len(p)) {
|
||||
return n, errNoSpace
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// assertNoCorruptFiles fails if any file that made it to the restore
|
||||
// target holds content that differs from the original: a failed restore
|
||||
// may leave a file absent, but must never leave wrong bytes presenting
|
||||
// as the real file.
|
||||
func assertNoCorruptFiles(
|
||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for origPath, expected := range testFiles {
|
||||
restoredPath := filepath.Join(restoreDir, origPath)
|
||||
|
||||
got, err := afero.ReadFile(fs, restoredPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equalf(t, expected, got,
|
||||
"restored file %s holds corrupt content", origPath)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user