Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7fcb78013c |
@@ -25,6 +25,17 @@ release" is exactly the contradiction
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-09-21: Stopped `--json` from silencing stderr diagnostics
|
||||||
|
([issue #112](https://git.eeqj.de/sneak/vaultik/issues/112)). `--json`
|
||||||
|
used to be folded into `Quiet`, which pinned the log level to `WARN`,
|
||||||
|
so `prune --json` gave a machine consumer no record of the local index
|
||||||
|
rows it deleted even under `--verbose`. `--json` now quiets only the
|
||||||
|
stdout UI (the JSON document must stay clean, per
|
||||||
|
[issue #108](https://git.eeqj.de/sneak/vaultik/issues/108)); the stderr
|
||||||
|
log level follows `--verbose`/`--debug` again. The coupling was
|
||||||
|
removed the same way for `snapshot verify`, `snapshot remove`, and
|
||||||
|
`remote info`, which carried it for the same outdated reason.
|
||||||
|
|
||||||
- 2026-09-21: Stopped `prune` from reporting a failed row count as 0
|
- 2026-09-21: Stopped `prune` from reporting a failed row count as 0
|
||||||
([issue #96](https://git.eeqj.de/sneak/vaultik/issues/96)). The seven
|
([issue #96](https://git.eeqj.de/sneak/vaultik/issues/96)). The seven
|
||||||
`getTableCount` reads in `PruneDatabase` discarded their error, so a
|
`getTableCount` reads in `PruneDatabase` discarded their error, so a
|
||||||
|
|||||||
+12
-5
@@ -49,6 +49,11 @@ type AppOptions struct {
|
|||||||
// silenced — per the documented convention that --quiet suppresses
|
// silenced — per the documented convention that --quiet suppresses
|
||||||
// non-error output only. The startup banner is printed by Entry
|
// non-error output only. The startup banner is printed by Entry
|
||||||
// before cobra parses arguments, gated by the same arg-level check.
|
// before cobra parses arguments, gated by the same arg-level check.
|
||||||
|
//
|
||||||
|
// --json quiets the UI here too, because stdout then carries a JSON
|
||||||
|
// document and human narration would corrupt it. Unlike Quiet it does
|
||||||
|
// not lower the stderr log level (issue #112), so --verbose/--debug
|
||||||
|
// still surface diagnostics alongside the document.
|
||||||
func setupGlobals(
|
func setupGlobals(
|
||||||
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
|
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
|
||||||
) {
|
) {
|
||||||
@@ -56,7 +61,7 @@ func setupGlobals(
|
|||||||
OnStart: func(_ context.Context) error {
|
OnStart: func(_ context.Context) error {
|
||||||
g.StartTime = time.Now().UTC()
|
g.StartTime = time.Now().UTC()
|
||||||
|
|
||||||
if opts.Cron || opts.Quiet {
|
if opts.Cron || opts.Quiet || opts.JSON {
|
||||||
v.UI.SetQuiet(true)
|
v.UI.SetQuiet(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,10 +284,11 @@ func RunOperation(
|
|||||||
// shared by the list/purge/verify/remove/remote-info subcommands:
|
// shared by the list/purge/verify/remove/remote-info subcommands:
|
||||||
// resolve the config, then run op against the Vaultik instance through
|
// resolve the config, then run op against the Vaultik instance through
|
||||||
// RunOperation, reporting a failure prefixed with failMsg (suppressed
|
// RunOperation, reporting a failure prefixed with failMsg (suppressed
|
||||||
// while suppressErrors is true, e.g. under --json). extraQuiet is OR-ed
|
// while suppressErrors is true, e.g. under --json). jsonOutput marks a
|
||||||
// into LogOptions.Quiet (e.g. --json output modes).
|
// command whose stdout is a JSON document: it quiets the UI but, unlike
|
||||||
|
// Quiet, leaves the stderr log level alone.
|
||||||
func runVaultikApp(
|
func runVaultikApp(
|
||||||
cmd *cobra.Command, extraQuiet, suppressErrors bool,
|
cmd *cobra.Command, jsonOutput, suppressErrors bool,
|
||||||
failMsg string, op func(v *vaultik.Vaultik) error,
|
failMsg string, op func(v *vaultik.Vaultik) error,
|
||||||
) error {
|
) error {
|
||||||
configPath, err := ResolveConfigPath()
|
configPath, err := ResolveConfigPath()
|
||||||
@@ -297,7 +303,8 @@ func runVaultikApp(
|
|||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || extraQuiet,
|
Quiet: rootFlags.Quiet,
|
||||||
|
JSON: jsonOutput,
|
||||||
},
|
},
|
||||||
}, op, func(err error) {
|
}, op, func(err error) {
|
||||||
if suppressErrors {
|
if suppressErrors {
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
package cli //nolint:testpackage // shares the prune fixtures and capture helpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// staleRecordLogMessage is the local-cleanup audit line CleanupLocalSnapshots
|
||||||
|
// logs for each stale record. It is exactly the signal issue #112 says a
|
||||||
|
// machine consumer lost under --json: gated off stdout, and pinned below
|
||||||
|
// the log level on stderr because --json used to force Quiet.
|
||||||
|
const staleRecordLogMessage = "Removing stale local snapshot record"
|
||||||
|
|
||||||
|
// TestEntryPruneJSONStderrHonoursVerbosity is the end-to-end regression
|
||||||
|
// guard for issue #112. Under --json the log level must still follow
|
||||||
|
// --verbose/--debug rather than being pinned to WARN, so the
|
||||||
|
// local-cleanup records reach stderr under --verbose while stdout stays
|
||||||
|
// exactly one JSON document; without --verbose they stay below the
|
||||||
|
// level, as they do without --json.
|
||||||
|
//
|
||||||
|
// Both halves are asserted together on the same run, because the fix has
|
||||||
|
// to keep the document clean (issue #108) while freeing stderr.
|
||||||
|
//
|
||||||
|
// Not parallel: it replaces os.Args, os.Stdout, os.Stderr and the xdg
|
||||||
|
// globals.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntryPruneJSONStderrHonoursVerbosity(t *testing.T) {
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
name string
|
||||||
|
verbose bool
|
||||||
|
wantOnStderr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "verbose json surfaces the cleanup record on stderr",
|
||||||
|
verbose: true,
|
||||||
|
wantOnStderr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "json alone keeps the cleanup record below the level",
|
||||||
|
verbose: false,
|
||||||
|
wantOnStderr: false,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
configPath := writeHermeticPruneConfig(t, true)
|
||||||
|
|
||||||
|
previousArgs := os.Args
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
os.Args = previousArgs
|
||||||
|
rootFlags = RootFlags{}
|
||||||
|
})
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||||
|
}
|
||||||
|
if testCase.verbose {
|
||||||
|
args = append(args, "--verbose")
|
||||||
|
}
|
||||||
|
|
||||||
|
os.Args = args
|
||||||
|
|
||||||
|
stdout, stderr := captureProcessStdoutAndStderr(t,
|
||||||
|
func() { _ = Entry() })
|
||||||
|
|
||||||
|
// The document stays clean in both cases: freeing stderr must
|
||||||
|
// not regress issue #108.
|
||||||
|
requireExactlyOneJSONDocument(t, stdout)
|
||||||
|
|
||||||
|
if testCase.wantOnStderr {
|
||||||
|
assert.Contains(t, stderr, staleRecordLogMessage,
|
||||||
|
"--verbose --json must emit the cleanup record on stderr")
|
||||||
|
assert.Contains(t, stderr, stalePruneSnapshotID,
|
||||||
|
"the record must name the snapshot it removed")
|
||||||
|
} else {
|
||||||
|
assert.NotContains(t, stderr, staleRecordLogMessage,
|
||||||
|
"without --verbose the record stays below the log level")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// captureProcessStdoutAndStderr redirects both of the process's own
|
||||||
|
// standard streams to pipes for the duration of fn and returns what was
|
||||||
|
// written to each. The redirection is at the file-descriptor level
|
||||||
|
// because the logger binds os.Stderr when it initializes inside fn, and
|
||||||
|
// the JSON document reaches os.Stdout independently; the point is to see
|
||||||
|
// where each actually lands.
|
||||||
|
//
|
||||||
|
// Not parallel-safe: os.Stdout and os.Stderr are process-global.
|
||||||
|
func captureProcessStdoutAndStderr(t *testing.T, fn func()) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
outReader, outWriter, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
errReader, errWriter, err := os.Pipe()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
previousOut, previousErr := os.Stdout, os.Stderr
|
||||||
|
os.Stdout, os.Stderr = outWriter, errWriter
|
||||||
|
|
||||||
|
capturedOut := drain(outReader)
|
||||||
|
capturedErr := drain(errReader)
|
||||||
|
|
||||||
|
fn()
|
||||||
|
|
||||||
|
os.Stdout, os.Stderr = previousOut, previousErr
|
||||||
|
|
||||||
|
require.NoError(t, outWriter.Close())
|
||||||
|
require.NoError(t, errWriter.Close())
|
||||||
|
|
||||||
|
out, errOut := <-capturedOut, <-capturedErr
|
||||||
|
|
||||||
|
require.NoError(t, outReader.Close())
|
||||||
|
require.NoError(t, errReader.Close())
|
||||||
|
|
||||||
|
return out, errOut
|
||||||
|
}
|
||||||
|
|
||||||
|
// drain copies a reader to a string on a goroutine and delivers the
|
||||||
|
// result once the writer end is closed.
|
||||||
|
func drain(reader io.Reader) <-chan string {
|
||||||
|
captured := make(chan string, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
_, _ = io.Copy(&buf, reader)
|
||||||
|
captured <- buf.String()
|
||||||
|
}()
|
||||||
|
|
||||||
|
return captured
|
||||||
|
}
|
||||||
@@ -41,7 +41,8 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
|||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || opts.JSON,
|
Quiet: rootFlags.Quiet,
|
||||||
|
JSON: opts.JSON,
|
||||||
},
|
},
|
||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.Prune(opts)
|
return v.Prune(opts)
|
||||||
|
|||||||
@@ -85,7 +85,8 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || jsonOutput,
|
Quiet: rootFlags.Quiet,
|
||||||
|
JSON: jsonOutput,
|
||||||
},
|
},
|
||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.RemoteInfo(jsonOutput)
|
return v.RemoteInfo(jsonOutput)
|
||||||
|
|||||||
@@ -209,7 +209,8 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || opts.JSON,
|
Quiet: rootFlags.Quiet,
|
||||||
|
JSON: opts.JSON,
|
||||||
},
|
},
|
||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.VerifySnapshotWithOptions(snapshotID, opts)
|
return v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||||
|
|||||||
+15
-1
@@ -14,8 +14,18 @@ var Module = fx.Module("log",
|
|||||||
)
|
)
|
||||||
|
|
||||||
// New creates a new logger configuration from provided options.
|
// New creates a new logger configuration from provided options.
|
||||||
|
//
|
||||||
|
// JSON is intentionally not carried into Config: a command emitting a
|
||||||
|
// JSON document on stdout must keep its stderr log level under
|
||||||
|
// --verbose/--debug, so --json must not lower it (issue #112). JSON
|
||||||
|
// silences the stdout UI in setupGlobals instead.
|
||||||
func New(opts Options) Config {
|
func New(opts Options) Config {
|
||||||
return Config(opts)
|
return Config{
|
||||||
|
Verbose: opts.Verbose,
|
||||||
|
Debug: opts.Debug,
|
||||||
|
Cron: opts.Cron,
|
||||||
|
Quiet: opts.Quiet,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Options are provided by the CLI.
|
// Options are provided by the CLI.
|
||||||
@@ -24,4 +34,8 @@ type Options struct {
|
|||||||
Debug bool
|
Debug bool
|
||||||
Cron bool
|
Cron bool
|
||||||
Quiet bool
|
Quiet bool
|
||||||
|
// JSON marks a command whose stdout carries a machine-readable
|
||||||
|
// document. It silences the human UI on stdout (see setupGlobals),
|
||||||
|
// but unlike Quiet it leaves the stderr log level alone.
|
||||||
|
JSON bool
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,209 +0,0 @@
|
|||||||
// Package faultstore provides a storage.Storer wrapper that injects
|
|
||||||
// faults on demand, so tests can reproduce the failure modes a real
|
|
||||||
// backend exhibits: an upload that fails partway, a backend that reports
|
|
||||||
// success while storing nothing, and reads that return corrupt or
|
|
||||||
// truncated bytes. It is the seam called for by the fault-injection
|
|
||||||
// tests (sneak/vaultik issue 72) and is meant to be reused by future
|
|
||||||
// tests rather than re-implemented per case.
|
|
||||||
//
|
|
||||||
// The wrapper delegates every method to the inner Storer. Two hooks
|
|
||||||
// change that: OnPut decides the fate of each write, and OnGet decides
|
|
||||||
// how each read's bytes are returned. Both are keyed by the object key,
|
|
||||||
// so a test can fault only blobs, only metadata, or a single object.
|
|
||||||
package faultstore
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
|
|
||||||
var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
|
|
||||||
|
|
||||||
// PutAction is the disposition OnPut assigns to a write.
|
|
||||||
type PutAction int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// PutNormal writes through to the inner Storer.
|
|
||||||
PutNormal PutAction = iota
|
|
||||||
// PutFail reads part of the stream, then fails without storing the
|
|
||||||
// object — a network upload that dies partway through.
|
|
||||||
PutFail
|
|
||||||
// PutSwallow reports success but stores nothing — a backend that
|
|
||||||
// lies about durability.
|
|
||||||
PutSwallow
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetFault is how OnGet chooses to damage a read.
|
|
||||||
type GetFault int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// GetNormal returns the stored bytes unchanged.
|
|
||||||
GetNormal GetFault = iota
|
|
||||||
// GetCorrupt flips a byte so the returned object no longer matches
|
|
||||||
// what was stored.
|
|
||||||
GetCorrupt
|
|
||||||
// GetTruncate returns a short read: the object's bytes cut off
|
|
||||||
// before the end.
|
|
||||||
GetTruncate
|
|
||||||
)
|
|
||||||
|
|
||||||
// Storer wraps an inner storage.Storer with fault-injection hooks. A
|
|
||||||
// zero-valued hook means "no fault": construct with New and set only the
|
|
||||||
// hook a test needs.
|
|
||||||
type Storer struct {
|
|
||||||
inner storage.Storer
|
|
||||||
|
|
||||||
// OnPut, when set, is consulted before every Put and
|
|
||||||
// PutWithProgress with the object key.
|
|
||||||
OnPut func(key string) PutAction
|
|
||||||
|
|
||||||
// OnGet, when set, is consulted for every Get with the object key
|
|
||||||
// and damages the returned bytes accordingly.
|
|
||||||
OnGet func(key string) GetFault
|
|
||||||
}
|
|
||||||
|
|
||||||
// New wraps inner. inner must be non-nil.
|
|
||||||
func New(inner storage.Storer) *Storer {
|
|
||||||
return &Storer{inner: inner}
|
|
||||||
}
|
|
||||||
|
|
||||||
// midStreamBytes is how far a PutFail reads before failing, enough to be
|
|
||||||
// past the start of any real blob without depending on the blob's size.
|
|
||||||
const midStreamBytes = 512
|
|
||||||
|
|
||||||
// Put stores data unless OnPut faults the write.
|
|
||||||
func (f *Storer) Put(ctx context.Context, key string, data io.Reader) error {
|
|
||||||
handled, err := f.injectPut(key, data)
|
|
||||||
if handled {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return f.inner.Put(ctx, key, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutWithProgress stores data unless OnPut faults the write.
|
|
||||||
func (f *Storer) PutWithProgress(
|
|
||||||
ctx context.Context, key string, data io.Reader,
|
|
||||||
size int64, progress storage.ProgressCallback,
|
|
||||||
) error {
|
|
||||||
handled, err := f.injectPut(key, data)
|
|
||||||
if handled {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return f.inner.PutWithProgress(ctx, key, data, size, progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get retrieves data, damaging it if OnGet faults the read.
|
|
||||||
func (f *Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
|
||||||
rc, err := f.inner.Get(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fault := GetNormal
|
|
||||||
if f.OnGet != nil {
|
|
||||||
fault = f.OnGet(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fault == GetNormal {
|
|
||||||
return rc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := io.ReadAll(rc)
|
|
||||||
_ = rc.Close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return io.NopCloser(bytes.NewReader(damage(fault, data))), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// damage returns a faulted copy of the stored bytes. GetCorrupt flips a
|
|
||||||
// byte in the middle so decryption authentication fails; GetTruncate
|
|
||||||
// drops the final byte so the read ends short. Both are no-ops on empty
|
|
||||||
// input, which cannot be damaged into something distinguishable.
|
|
||||||
func damage(fault GetFault, data []byte) []byte {
|
|
||||||
out := make([]byte, len(data))
|
|
||||||
copy(out, data)
|
|
||||||
|
|
||||||
if len(out) == 0 {
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
switch fault {
|
|
||||||
case GetCorrupt:
|
|
||||||
out[len(out)/2] ^= 0xff
|
|
||||||
case GetTruncate:
|
|
||||||
out = out[:len(out)-1]
|
|
||||||
case GetNormal:
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat delegates unchanged.
|
|
||||||
func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
|
||||||
return f.inner.Stat(ctx, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete delegates unchanged.
|
|
||||||
func (f *Storer) Delete(ctx context.Context, key string) error {
|
|
||||||
return f.inner.Delete(ctx, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List delegates unchanged.
|
|
||||||
func (f *Storer) List(ctx context.Context, prefix string) ([]string, error) {
|
|
||||||
return f.inner.List(ctx, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListStream delegates unchanged.
|
|
||||||
func (f *Storer) ListStream(
|
|
||||||
ctx context.Context, prefix string,
|
|
||||||
) <-chan storage.ObjectInfo {
|
|
||||||
return f.inner.ListStream(ctx, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Info delegates unchanged.
|
|
||||||
func (f *Storer) Info() storage.Info {
|
|
||||||
return f.inner.Info()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Storer) putAction(key string) PutAction {
|
|
||||||
if f.OnPut == nil {
|
|
||||||
return PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
return f.OnPut(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// injectPut handles the non-normal write dispositions. It reports
|
|
||||||
// whether it handled the write and, if so, with what error.
|
|
||||||
func (f *Storer) injectPut(key string, data io.Reader) (bool, error) {
|
|
||||||
switch f.putAction(key) {
|
|
||||||
case PutFail:
|
|
||||||
// Consume part of the stream so the failure lands mid-transfer,
|
|
||||||
// the way a dropped connection would, then error without
|
|
||||||
// storing anything.
|
|
||||||
_, _ = io.CopyN(io.Discard, data, midStreamBytes)
|
|
||||||
|
|
||||||
return true, fmt.Errorf("%w for %q", ErrInjectedUpload, key)
|
|
||||||
case PutSwallow:
|
|
||||||
// A lying backend still drains the request body, then keeps
|
|
||||||
// nothing.
|
|
||||||
_, _ = io.Copy(io.Discard, data)
|
|
||||||
|
|
||||||
return true, nil
|
|
||||||
case PutNormal:
|
|
||||||
return false, nil
|
|
||||||
default:
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,634 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// These tests cover the failure modes a backup tool must survive:
|
|
||||||
// interrupted uploads, an interrupted metadata export, corrupt and
|
|
||||||
// truncated reads, a full restore disk, and a backend that reports
|
|
||||||
// success while storing nothing. Faults are injected through the
|
|
||||||
// storage.Storer seam (internal/storage/faultstore), never by patching
|
|
||||||
// production code. Each test asserts on the observable end state — what
|
|
||||||
// is in the index, what is at the destination, what the user is told —
|
|
||||||
// not merely that an error was returned. See
|
|
||||||
// https://git.eeqj.de/sneak/vaultik/issues/72.
|
|
||||||
//
|
|
||||||
// Object-level write atomicity (no partial blob object left behind) is
|
|
||||||
// covered by the file:// backend's atomic-write work
|
|
||||||
// (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested
|
|
||||||
// here; these tests target the layers above the backend.
|
|
||||||
//
|
|
||||||
// The tests run serially, not with t.Parallel: each calls
|
|
||||||
// log.Initialize, which replaces the package-global logger, and a
|
|
||||||
// backup or restore running concurrently reads that same logger. Under
|
|
||||||
// -race the two collide. Running one at a time is the same choice
|
|
||||||
// prune_count_test.go already makes for the same reason.
|
|
||||||
|
|
||||||
const (
|
|
||||||
faultChunkSize = int64(64 * 1024)
|
|
||||||
faultMaxBlobSize = int64(256 * 1024)
|
|
||||||
)
|
|
||||||
|
|
||||||
// faultTestConfig returns the config shared by the fault-injection
|
|
||||||
// tests: a real recipient/secret keypair so blobs are genuinely
|
|
||||||
// encrypted, and a blob size limit the restore sweeper can divide.
|
|
||||||
func faultTestConfig() *config.Config {
|
|
||||||
return &config.Config{
|
|
||||||
AgeRecipients: []string{testAgePublicKey},
|
|
||||||
AgeSecretKey: testAgeSecretKey,
|
|
||||||
CompressionLevel: 3,
|
|
||||||
Hostname: testHostname,
|
|
||||||
BlobSizeLimit: config.Size(faultMaxBlobSize),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFaultSourceTree writes a spread of file sizes that forces several
|
|
||||||
// chunks across more than one blob, so a fault landing on a single blob
|
|
||||||
// still leaves other data intact. Returns the expected content by path.
|
|
||||||
func writeFaultSourceTree(
|
|
||||||
t *testing.T, fs afero.Fs, dataDir string,
|
|
||||||
) map[string][]byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
files := map[string][]byte{
|
|
||||||
filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"),
|
|
||||||
filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(faultChunkSize*3)),
|
|
||||||
filepath.Join(dataDir, "sub", "b.bin"): bytesPattern("b-", int(faultChunkSize*3)),
|
|
||||||
filepath.Join(dataDir, "sub", "c.bin"): bytesPattern("c-", int(faultChunkSize*2)),
|
|
||||||
}
|
|
||||||
|
|
||||||
for path, content := range files {
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFaultScanner builds a scanner writing through the given storer.
|
|
||||||
func newFaultScanner(
|
|
||||||
fs afero.Fs, storer storage.Storer,
|
|
||||||
cfg *config.Config, repos *database.Repositories,
|
|
||||||
) *snapshot.Scanner {
|
|
||||||
return snapshot.NewScanner(snapshot.ScannerConfig{
|
|
||||||
FS: fs,
|
|
||||||
Storage: storer,
|
|
||||||
ChunkSize: faultChunkSize,
|
|
||||||
MaxBlobSize: faultMaxBlobSize,
|
|
||||||
CompressionLevel: cfg.CompressionLevel,
|
|
||||||
AgeRecipients: cfg.AgeRecipients,
|
|
||||||
Repositories: repos,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFaultSnapshotManager builds a snapshot manager writing through the
|
|
||||||
// given storer.
|
|
||||||
func newFaultSnapshotManager(
|
|
||||||
fs afero.Fs, storer storage.Storer,
|
|
||||||
cfg *config.Config, repos *database.Repositories,
|
|
||||||
) *snapshot.SnapshotManager {
|
|
||||||
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
|
||||||
Repos: repos,
|
|
||||||
Storage: storer,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
sm.SetFilesystem(fs)
|
|
||||||
|
|
||||||
return sm
|
|
||||||
}
|
|
||||||
|
|
||||||
// fullFaultBackup runs a complete backup (create, scan, complete,
|
|
||||||
// export) through storer and returns the snapshot ID.
|
|
||||||
func fullFaultBackup(
|
|
||||||
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
|
||||||
cfg *config.Config, repos *database.Repositories,
|
|
||||||
dataDir, dbPath, name string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sm := newFaultSnapshotManager(fs, storer, cfg, repos)
|
|
||||||
scanner := newFaultScanner(fs, storer, cfg, repos)
|
|
||||||
|
|
||||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, name, "v", "g")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = scanner.Scan(ctx, dataDir, id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
|
||||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id))
|
|
||||||
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
// newReaderVaultik builds a Vaultik that reads (restore/verify) through
|
|
||||||
// storer, with the given repositories (nil is fine for restore/verify,
|
|
||||||
// which read metadata from storage).
|
|
||||||
func newReaderVaultik(
|
|
||||||
ctx context.Context, cfg *config.Config, storer storage.Storer,
|
|
||||||
repos *database.Repositories, fs afero.Fs,
|
|
||||||
) *vaultik.Vaultik {
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: cfg,
|
|
||||||
Storage: storer,
|
|
||||||
Repositories: repos,
|
|
||||||
Fs: fs,
|
|
||||||
Stdout: io.Discard,
|
|
||||||
Stderr: io.Discard,
|
|
||||||
UI: ui.NewWithColor(io.Discard, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 3: a stored blob's bytes are flipped before restore reads
|
|
||||||
// them. Restore must fail loudly, and no file must be left on the
|
|
||||||
// restore target holding corrupt content.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestRestoreRejectsCorruptBlob(t *testing.T) {
|
|
||||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 4: a stored blob is truncated before restore reads it. Same
|
|
||||||
// contract as the corrupt case.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestRestoreRejectsTruncatedBlob(t *testing.T) {
|
|
||||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated")
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRestoreRejectsDamagedBlob backs up the source tree, then restores
|
|
||||||
// through a store that damages every blob read with the given fault, and
|
|
||||||
// asserts restore fails naming a blob and leaves no file on the target
|
|
||||||
// holding wrong bytes. Metadata reads are returned intact so the failure
|
|
||||||
// is isolated to the blob.
|
|
||||||
func assertRestoreRejectsDamagedBlob(
|
|
||||||
t *testing.T, fault faultstore.GetFault, name string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
restoreDir := filepath.Join(tempDir, "restored")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
id := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, name)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
faultStore := faultstore.New(inner)
|
|
||||||
faultStore.OnGet = func(key string) faultstore.GetFault {
|
|
||||||
if strings.HasPrefix(key, "blobs/") {
|
|
||||||
return fault
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.GetNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
v := newReaderVaultik(ctx, cfg, faultStore, nil, fs)
|
|
||||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
|
||||||
|
|
||||||
require.Error(t, err, "restore must fail on a damaged blob")
|
|
||||||
assert.Contains(t, err.Error(), "blob",
|
|
||||||
"error should name the blob that failed")
|
|
||||||
assertNoCorruptFiles(t, fs, restoreDir, testFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 6: the backend accepts blob uploads and reports success but
|
|
||||||
// stores nothing. verify --deep must catch it.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
|
|
||||||
writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Blob uploads are swallowed; metadata uploads land, so verify can
|
|
||||||
// download the manifest and database and then discover the blobs are
|
|
||||||
// absent.
|
|
||||||
lying := faultstore.New(inner)
|
|
||||||
lying.OnPut = func(key string) faultstore.PutAction {
|
|
||||||
if strings.HasPrefix(key, "blobs/") {
|
|
||||||
return faultstore.PutSwallow
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
id := fullFaultBackup(ctx, t, fs, lying, cfg, repos, dataDir, dbPath, "lying")
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
// No blob objects were actually written.
|
|
||||||
blobKeys, err := inner.List(ctx, "blobs/")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, blobKeys, "lying backend should have stored no blobs")
|
|
||||||
|
|
||||||
// Read back through the honest underlying store.
|
|
||||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
|
||||||
err = v.VerifySnapshotWithOptions(id, &vaultik.VerifyOptions{Deep: true})
|
|
||||||
require.Error(t, err, "deep verify must catch a backend that stored nothing")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 1a: a blob upload fails partway through. The interrupted run
|
|
||||||
// must not record the blob as uploaded, must not reference it from the
|
|
||||||
// snapshot, and must leave no blob object at the destination.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
|
|
||||||
writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
// Every blob upload fails partway through. The scan must surface it.
|
|
||||||
fault := faultstore.New(inner)
|
|
||||||
fault.OnPut = func(key string) faultstore.PutAction {
|
|
||||||
if strings.HasPrefix(key, "blobs/") {
|
|
||||||
return faultstore.PutFail
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
|
||||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
|
||||||
|
|
||||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = scanner.Scan(ctx, dataDir, id)
|
|
||||||
require.Error(t, err, "scan must fail when a blob upload fails")
|
|
||||||
|
|
||||||
// No blob may claim to be uploaded.
|
|
||||||
blobs, err := repos.Blobs.GetAll(ctx)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, b := range blobs {
|
|
||||||
assert.Nilf(t, b.UploadedTS,
|
|
||||||
"blob %s marked uploaded after a failed upload", b.Hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The snapshot may reference no blobs, and the destination holds none.
|
|
||||||
hashes, err := repos.Snapshots.GetBlobHashes(ctx, id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, hashes, "interrupted snapshot must reference no blobs")
|
|
||||||
|
|
||||||
blobKeys, err := inner.List(ctx, "blobs/")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, blobKeys, "no blob object may survive at the destination")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 1b: after an interrupted upload, a retry on the same local
|
|
||||||
// index must produce a restorable snapshot. It does not: the interrupted
|
|
||||||
// run's chunk rows persist, the retry deduplicates against them, and the
|
|
||||||
// backup silently emits a snapshot referencing data never stored. Skipped
|
|
||||||
// pending the fix. See https://git.eeqj.de/sneak/vaultik/issues/148.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
|
|
||||||
t.Skip("blocked on https://git.eeqj.de/sneak/vaultik/issues/148: " +
|
|
||||||
"retry after an interrupted upload silently produces an " +
|
|
||||||
"unrestorable snapshot")
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
restoreDir := filepath.Join(tempDir, "restored")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
// Attempt 1: every blob upload fails.
|
|
||||||
fault := faultstore.New(inner)
|
|
||||||
fault.OnPut = func(key string) faultstore.PutAction {
|
|
||||||
if strings.HasPrefix(key, "blobs/") {
|
|
||||||
return faultstore.PutFail
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
|
||||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
|
||||||
|
|
||||||
id1, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = scanner.Scan(ctx, dataDir, id1)
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
// Retry on the same local index with a working backend.
|
|
||||||
id2 := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, "retry")
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
|
||||||
require.NoError(t, v.Restore(&vaultik.RestoreOptions{
|
|
||||||
SnapshotID: id2,
|
|
||||||
TargetDir: restoreDir,
|
|
||||||
Verify: true,
|
|
||||||
}), "retry after an interrupted upload must produce a restorable snapshot")
|
|
||||||
|
|
||||||
assertRestoredTree(t, fs, restoreDir, testFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 2: the process dies during the metadata export, after the
|
|
||||||
// database is uploaded but before the manifest. The destination is left
|
|
||||||
// with blobs and a database but no manifest. verify and snapshot list
|
|
||||||
// must report the damage honestly rather than crashing or passing.
|
|
||||||
// Automatic detection and repair of this partial state on the next run
|
|
||||||
// is tracked in https://git.eeqj.de/sneak/vaultik/issues/148 and is not
|
|
||||||
// asserted here.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
|
|
||||||
writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
// Back up and complete with a working backend.
|
|
||||||
sm := newFaultSnapshotManager(fs, inner, cfg, repos)
|
|
||||||
scanner := newFaultScanner(fs, inner, cfg, repos)
|
|
||||||
|
|
||||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "export", "v", "g")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = scanner.Scan(ctx, dataDir, id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
|
||||||
|
|
||||||
// Export through a backend that fails only the manifest upload. The
|
|
||||||
// database uploads first and lands; the manifest does not.
|
|
||||||
fault := faultstore.New(inner)
|
|
||||||
fault.OnPut = func(key string) faultstore.PutAction {
|
|
||||||
if strings.HasSuffix(key, "manifest.json.zst") {
|
|
||||||
return faultstore.PutFail
|
|
||||||
}
|
|
||||||
|
|
||||||
return faultstore.PutNormal
|
|
||||||
}
|
|
||||||
|
|
||||||
smFault := newFaultSnapshotManager(fs, fault, cfg, repos)
|
|
||||||
|
|
||||||
err = smFault.ExportSnapshotMetadata(ctx, dbPath, id)
|
|
||||||
require.Error(t, err, "export must fail when the manifest upload fails")
|
|
||||||
|
|
||||||
// The destination is in the partial state the scenario describes.
|
|
||||||
key := snapshot.RemoteSnapshotKey(id)
|
|
||||||
|
|
||||||
_, err = inner.Stat(ctx, "metadata/"+key+"/db.zst.age")
|
|
||||||
require.NoError(t, err, "database should have been uploaded before the manifest")
|
|
||||||
|
|
||||||
_, err = inner.Stat(ctx, "metadata/"+key+"/manifest.json.zst")
|
|
||||||
require.ErrorIs(t, err, storage.ErrNotFound, "manifest upload should not have landed")
|
|
||||||
|
|
||||||
// verify must fail loudly for this snapshot, in both modes.
|
|
||||||
reader := newReaderVaultik(ctx, cfg, inner, repos, fs)
|
|
||||||
|
|
||||||
deepOpts := &vaultik.VerifyOptions{Deep: true}
|
|
||||||
require.Error(t, reader.VerifySnapshotWithOptions(id, deepOpts),
|
|
||||||
"deep verify must report the missing manifest")
|
|
||||||
|
|
||||||
shallowOpts := &vaultik.VerifyOptions{Deep: false}
|
|
||||||
require.Error(t, reader.VerifySnapshotWithOptions(id, shallowOpts),
|
|
||||||
"shallow verify must report the missing manifest")
|
|
||||||
|
|
||||||
// snapshot list must not crash on the partial snapshot.
|
|
||||||
require.NoError(t, reader.ListSnapshots(false),
|
|
||||||
"snapshot list must tolerate a partially-exported snapshot")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scenario 5: the restore target runs out of space mid-file. Restore
|
|
||||||
// must fail with a clear, file-naming error rather than reporting
|
|
||||||
// success over a truncated file.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
|
||||||
func TestRestoreReportsDiskFull(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
|
|
||||||
osFS := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
restoreDir := filepath.Join(tempDir, "restored")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
|
|
||||||
writeFaultSourceTree(t, osFS, dataDir)
|
|
||||||
|
|
||||||
inner, err := storage.NewFileStorer(storeDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
id := fullFaultBackup(ctx, t, osFS, inner, cfg, repos, dataDir, dbPath, "diskfull")
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
// Restore onto a filesystem that allows only a few bytes of file
|
|
||||||
// content: enough to create files, far too little to hold them.
|
|
||||||
budget := int64(8)
|
|
||||||
quota := "aFS{Fs: osFS, remaining: &budget}
|
|
||||||
|
|
||||||
v := newReaderVaultik(ctx, cfg, inner, nil, quota)
|
|
||||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
|
||||||
|
|
||||||
require.Error(t, err, "restore must fail when the target disk is full")
|
|
||||||
assert.Contains(t, err.Error(), errNoSpace.Error(),
|
|
||||||
"restore error should surface the out-of-space cause")
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRestoredTree byte-compares every restored file against the
|
|
||||||
// original.
|
|
||||||
func assertRestoredTree(
|
|
||||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for origPath, expected := range testFiles {
|
|
||||||
restoredPath := filepath.Join(restoreDir, origPath)
|
|
||||||
got, err := afero.ReadFile(fs, restoredPath)
|
|
||||||
require.NoErrorf(t, err, "restored file missing: %s", origPath)
|
|
||||||
require.Equalf(t, expected, got, "restored content mismatch for %s", origPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errNoSpace is the out-of-space error quotaFS returns once its byte
|
|
||||||
// budget is exhausted, mirroring a real ENOSPC.
|
|
||||||
var errNoSpace = errors.New("no space left on device")
|
|
||||||
|
|
||||||
// quotaFS is an afero.Fs whose files may write only a fixed total number
|
|
||||||
// of content bytes before failing, simulating a full restore target. It
|
|
||||||
// wraps the interface so every method except Create delegates to the
|
|
||||||
// real filesystem; only file writes are capped.
|
|
||||||
type quotaFS struct {
|
|
||||||
afero.Fs
|
|
||||||
|
|
||||||
remaining *int64
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:ireturn // afero.Fs.Create's signature requires returning afero.File.
|
|
||||||
func (q *quotaFS) Create(name string) (afero.File, error) {
|
|
||||||
f, err := q.Fs.Create(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return "aFile{File: f, remaining: q.remaining}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// quotaFile fails writes once the shared byte budget is exhausted.
|
|
||||||
type quotaFile struct {
|
|
||||||
afero.File
|
|
||||||
|
|
||||||
remaining *int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *quotaFile) Write(p []byte) (int, error) {
|
|
||||||
if *q.remaining <= 0 {
|
|
||||||
return 0, errNoSpace
|
|
||||||
}
|
|
||||||
|
|
||||||
allowed := min(int64(len(p)), *q.remaining)
|
|
||||||
|
|
||||||
n, err := q.File.Write(p[:allowed])
|
|
||||||
*q.remaining -= int64(n)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if int64(n) < int64(len(p)) {
|
|
||||||
return n, errNoSpace
|
|
||||||
}
|
|
||||||
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertNoCorruptFiles fails if any file that made it to the restore
|
|
||||||
// target holds content that differs from the original: a failed restore
|
|
||||||
// may leave a file absent, but must never leave wrong bytes presenting
|
|
||||||
// as the real file.
|
|
||||||
func assertNoCorruptFiles(
|
|
||||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for origPath, expected := range testFiles {
|
|
||||||
restoredPath := filepath.Join(restoreDir, origPath)
|
|
||||||
|
|
||||||
got, err := afero.ReadFile(fs, restoredPath)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equalf(t, expected, got,
|
|
||||||
"restored file %s holds corrupt content", origPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user