Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e34f7aef99 |
@@ -25,17 +25,6 @@ release" is exactly the contradiction
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-09-21: Stopped `--json` from silencing stderr diagnostics
|
||||
([issue #112](https://git.eeqj.de/sneak/vaultik/issues/112)). `--json`
|
||||
used to be folded into `Quiet`, which pinned the log level to `WARN`,
|
||||
so `prune --json` gave a machine consumer no record of the local index
|
||||
rows it deleted even under `--verbose`. `--json` now quiets only the
|
||||
stdout UI (the JSON document must stay clean, per
|
||||
[issue #108](https://git.eeqj.de/sneak/vaultik/issues/108)); the stderr
|
||||
log level follows `--verbose`/`--debug` again. The coupling was
|
||||
removed the same way for `snapshot verify`, `snapshot remove`, and
|
||||
`remote info`, which carried it for the same outdated reason.
|
||||
|
||||
- 2026-09-21: Stopped `prune` from reporting a failed row count as 0
|
||||
([issue #96](https://git.eeqj.de/sneak/vaultik/issues/96)). The seven
|
||||
`getTableCount` reads in `PruneDatabase` discarded their error, so a
|
||||
|
||||
+5
-12
@@ -49,11 +49,6 @@ type AppOptions struct {
|
||||
// silenced — per the documented convention that --quiet suppresses
|
||||
// non-error output only. The startup banner is printed by Entry
|
||||
// before cobra parses arguments, gated by the same arg-level check.
|
||||
//
|
||||
// --json quiets the UI here too, because stdout then carries a JSON
|
||||
// document and human narration would corrupt it. Unlike Quiet it does
|
||||
// not lower the stderr log level (issue #112), so --verbose/--debug
|
||||
// still surface diagnostics alongside the document.
|
||||
func setupGlobals(
|
||||
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
|
||||
) {
|
||||
@@ -61,7 +56,7 @@ func setupGlobals(
|
||||
OnStart: func(_ context.Context) error {
|
||||
g.StartTime = time.Now().UTC()
|
||||
|
||||
if opts.Cron || opts.Quiet || opts.JSON {
|
||||
if opts.Cron || opts.Quiet {
|
||||
v.UI.SetQuiet(true)
|
||||
}
|
||||
|
||||
@@ -284,11 +279,10 @@ func RunOperation(
|
||||
// shared by the list/purge/verify/remove/remote-info subcommands:
|
||||
// resolve the config, then run op against the Vaultik instance through
|
||||
// RunOperation, reporting a failure prefixed with failMsg (suppressed
|
||||
// while suppressErrors is true, e.g. under --json). jsonOutput marks a
|
||||
// command whose stdout is a JSON document: it quiets the UI but, unlike
|
||||
// Quiet, leaves the stderr log level alone.
|
||||
// while suppressErrors is true, e.g. under --json). extraQuiet is OR-ed
|
||||
// into LogOptions.Quiet (e.g. --json output modes).
|
||||
func runVaultikApp(
|
||||
cmd *cobra.Command, jsonOutput, suppressErrors bool,
|
||||
cmd *cobra.Command, extraQuiet, suppressErrors bool,
|
||||
failMsg string, op func(v *vaultik.Vaultik) error,
|
||||
) error {
|
||||
configPath, err := ResolveConfigPath()
|
||||
@@ -303,8 +297,7 @@ func runVaultikApp(
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
Quiet: rootFlags.Quiet || extraQuiet,
|
||||
},
|
||||
}, op, func(err error) {
|
||||
if suppressErrors {
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
package cli //nolint:testpackage // shares the prune fixtures and capture helpers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// staleRecordLogMessage is the local-cleanup audit line CleanupLocalSnapshots
|
||||
// logs for each stale record. It is exactly the signal issue #112 says a
|
||||
// machine consumer lost under --json: gated off stdout, and pinned below
|
||||
// the log level on stderr because --json used to force Quiet.
|
||||
const staleRecordLogMessage = "Removing stale local snapshot record"
|
||||
|
||||
// TestEntryPruneJSONStderrHonoursVerbosity is the end-to-end regression
|
||||
// guard for issue #112. Under --json the log level must still follow
|
||||
// --verbose/--debug rather than being pinned to WARN, so the
|
||||
// local-cleanup records reach stderr under --verbose while stdout stays
|
||||
// exactly one JSON document; without --verbose they stay below the
|
||||
// level, as they do without --json.
|
||||
//
|
||||
// Both halves are asserted together on the same run, because the fix has
|
||||
// to keep the document clean (issue #108) while freeing stderr.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout, os.Stderr and the xdg
|
||||
// globals.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryPruneJSONStderrHonoursVerbosity(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
verbose bool
|
||||
wantOnStderr bool
|
||||
}{
|
||||
{
|
||||
name: "verbose json surfaces the cleanup record on stderr",
|
||||
verbose: true,
|
||||
wantOnStderr: true,
|
||||
},
|
||||
{
|
||||
name: "json alone keeps the cleanup record below the level",
|
||||
verbose: false,
|
||||
wantOnStderr: false,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, true)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
args := []string{
|
||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||
}
|
||||
if testCase.verbose {
|
||||
args = append(args, "--verbose")
|
||||
}
|
||||
|
||||
os.Args = args
|
||||
|
||||
stdout, stderr := captureProcessStdoutAndStderr(t,
|
||||
func() { _ = Entry() })
|
||||
|
||||
// The document stays clean in both cases: freeing stderr must
|
||||
// not regress issue #108.
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
if testCase.wantOnStderr {
|
||||
assert.Contains(t, stderr, staleRecordLogMessage,
|
||||
"--verbose --json must emit the cleanup record on stderr")
|
||||
assert.Contains(t, stderr, stalePruneSnapshotID,
|
||||
"the record must name the snapshot it removed")
|
||||
} else {
|
||||
assert.NotContains(t, stderr, staleRecordLogMessage,
|
||||
"without --verbose the record stays below the log level")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// captureProcessStdoutAndStderr redirects both of the process's own
|
||||
// standard streams to pipes for the duration of fn and returns what was
|
||||
// written to each. The redirection is at the file-descriptor level
|
||||
// because the logger binds os.Stderr when it initializes inside fn, and
|
||||
// the JSON document reaches os.Stdout independently; the point is to see
|
||||
// where each actually lands.
|
||||
//
|
||||
// Not parallel-safe: os.Stdout and os.Stderr are process-global.
|
||||
func captureProcessStdoutAndStderr(t *testing.T, fn func()) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
outReader, outWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
errReader, errWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
previousOut, previousErr := os.Stdout, os.Stderr
|
||||
os.Stdout, os.Stderr = outWriter, errWriter
|
||||
|
||||
capturedOut := drain(outReader)
|
||||
capturedErr := drain(errReader)
|
||||
|
||||
fn()
|
||||
|
||||
os.Stdout, os.Stderr = previousOut, previousErr
|
||||
|
||||
require.NoError(t, outWriter.Close())
|
||||
require.NoError(t, errWriter.Close())
|
||||
|
||||
out, errOut := <-capturedOut, <-capturedErr
|
||||
|
||||
require.NoError(t, outReader.Close())
|
||||
require.NoError(t, errReader.Close())
|
||||
|
||||
return out, errOut
|
||||
}
|
||||
|
||||
// drain copies a reader to a string on a goroutine and delivers the
|
||||
// result once the writer end is closed.
|
||||
func drain(reader io.Reader) <-chan string {
|
||||
captured := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, _ = io.Copy(&buf, reader)
|
||||
captured <- buf.String()
|
||||
}()
|
||||
|
||||
return captured
|
||||
}
|
||||
@@ -41,8 +41,7 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.Prune(opts)
|
||||
|
||||
@@ -85,8 +85,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
Quiet: rootFlags.Quiet || jsonOutput,
|
||||
},
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.RemoteInfo(jsonOutput)
|
||||
|
||||
@@ -209,8 +209,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
|
||||
+1
-15
@@ -14,18 +14,8 @@ var Module = fx.Module("log",
|
||||
)
|
||||
|
||||
// New creates a new logger configuration from provided options.
|
||||
//
|
||||
// JSON is intentionally not carried into Config: a command emitting a
|
||||
// JSON document on stdout must keep its stderr log level under
|
||||
// --verbose/--debug, so --json must not lower it (issue #112). JSON
|
||||
// silences the stdout UI in setupGlobals instead.
|
||||
func New(opts Options) Config {
|
||||
return Config{
|
||||
Verbose: opts.Verbose,
|
||||
Debug: opts.Debug,
|
||||
Cron: opts.Cron,
|
||||
Quiet: opts.Quiet,
|
||||
}
|
||||
return Config(opts)
|
||||
}
|
||||
|
||||
// Options are provided by the CLI.
|
||||
@@ -34,8 +24,4 @@ type Options struct {
|
||||
Debug bool
|
||||
Cron bool
|
||||
Quiet bool
|
||||
// JSON marks a command whose stdout carries a machine-readable
|
||||
// document. It silences the human UI on stdout (see setupGlobals),
|
||||
// but unlike Quiet it leaves the stderr log level alone.
|
||||
JSON bool
|
||||
}
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Package faultstore provides a storage.Storer wrapper that injects
|
||||
// faults on demand, so tests can reproduce the failure modes a real
|
||||
// backend exhibits: an upload that fails partway, a backend that reports
|
||||
// success while storing nothing, and reads that return corrupt or
|
||||
// truncated bytes. It is the seam called for by the fault-injection
|
||||
// tests (sneak/vaultik issue 72) and is meant to be reused by future
|
||||
// tests rather than re-implemented per case.
|
||||
//
|
||||
// The wrapper delegates every method to the inner Storer. Two hooks
|
||||
// change that: OnPut decides the fate of each write, and OnGet decides
|
||||
// how each read's bytes are returned. Both are keyed by the object key,
|
||||
// so a test can fault only blobs, only metadata, or a single object.
|
||||
package faultstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
|
||||
var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
|
||||
|
||||
// PutAction is the disposition OnPut assigns to a write.
|
||||
type PutAction int
|
||||
|
||||
const (
|
||||
// PutNormal writes through to the inner Storer.
|
||||
PutNormal PutAction = iota
|
||||
// PutFail reads part of the stream, then fails without storing the
|
||||
// object — a network upload that dies partway through.
|
||||
PutFail
|
||||
// PutSwallow reports success but stores nothing — a backend that
|
||||
// lies about durability.
|
||||
PutSwallow
|
||||
)
|
||||
|
||||
// GetFault is how OnGet chooses to damage a read.
|
||||
type GetFault int
|
||||
|
||||
const (
|
||||
// GetNormal returns the stored bytes unchanged.
|
||||
GetNormal GetFault = iota
|
||||
// GetCorrupt flips a byte so the returned object no longer matches
|
||||
// what was stored.
|
||||
GetCorrupt
|
||||
// GetTruncate returns a short read: the object's bytes cut off
|
||||
// before the end.
|
||||
GetTruncate
|
||||
)
|
||||
|
||||
// Storer wraps an inner storage.Storer with fault-injection hooks. A
|
||||
// zero-valued hook means "no fault": construct with New and set only the
|
||||
// hook a test needs.
|
||||
type Storer struct {
|
||||
inner storage.Storer
|
||||
|
||||
// OnPut, when set, is consulted before every Put and
|
||||
// PutWithProgress with the object key.
|
||||
OnPut func(key string) PutAction
|
||||
|
||||
// OnGet, when set, is consulted for every Get with the object key
|
||||
// and damages the returned bytes accordingly.
|
||||
OnGet func(key string) GetFault
|
||||
}
|
||||
|
||||
// New wraps inner. inner must be non-nil.
|
||||
func New(inner storage.Storer) *Storer {
|
||||
return &Storer{inner: inner}
|
||||
}
|
||||
|
||||
// midStreamBytes is how far a PutFail reads before failing, enough to be
|
||||
// past the start of any real blob without depending on the blob's size.
|
||||
const midStreamBytes = 512
|
||||
|
||||
// Put stores data unless OnPut faults the write.
|
||||
func (f *Storer) Put(ctx context.Context, key string, data io.Reader) error {
|
||||
handled, err := f.injectPut(key, data)
|
||||
if handled {
|
||||
return err
|
||||
}
|
||||
|
||||
return f.inner.Put(ctx, key, data)
|
||||
}
|
||||
|
||||
// PutWithProgress stores data unless OnPut faults the write.
|
||||
func (f *Storer) PutWithProgress(
|
||||
ctx context.Context, key string, data io.Reader,
|
||||
size int64, progress storage.ProgressCallback,
|
||||
) error {
|
||||
handled, err := f.injectPut(key, data)
|
||||
if handled {
|
||||
return err
|
||||
}
|
||||
|
||||
return f.inner.PutWithProgress(ctx, key, data, size, progress)
|
||||
}
|
||||
|
||||
// Get retrieves data, damaging it if OnGet faults the read.
|
||||
func (f *Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
rc, err := f.inner.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fault := GetNormal
|
||||
if f.OnGet != nil {
|
||||
fault = f.OnGet(key)
|
||||
}
|
||||
|
||||
if fault == GetNormal {
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(damage(fault, data))), nil
|
||||
}
|
||||
|
||||
// damage returns a faulted copy of the stored bytes. GetCorrupt flips a
|
||||
// byte in the middle so decryption authentication fails; GetTruncate
|
||||
// drops the final byte so the read ends short. Both are no-ops on empty
|
||||
// input, which cannot be damaged into something distinguishable.
|
||||
func damage(fault GetFault, data []byte) []byte {
|
||||
out := make([]byte, len(data))
|
||||
copy(out, data)
|
||||
|
||||
if len(out) == 0 {
|
||||
return out
|
||||
}
|
||||
|
||||
switch fault {
|
||||
case GetCorrupt:
|
||||
out[len(out)/2] ^= 0xff
|
||||
case GetTruncate:
|
||||
out = out[:len(out)-1]
|
||||
case GetNormal:
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Stat delegates unchanged.
|
||||
func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
||||
return f.inner.Stat(ctx, key)
|
||||
}
|
||||
|
||||
// Delete delegates unchanged.
|
||||
func (f *Storer) Delete(ctx context.Context, key string) error {
|
||||
return f.inner.Delete(ctx, key)
|
||||
}
|
||||
|
||||
// List delegates unchanged.
|
||||
func (f *Storer) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
return f.inner.List(ctx, prefix)
|
||||
}
|
||||
|
||||
// ListStream delegates unchanged.
|
||||
func (f *Storer) ListStream(
|
||||
ctx context.Context, prefix string,
|
||||
) <-chan storage.ObjectInfo {
|
||||
return f.inner.ListStream(ctx, prefix)
|
||||
}
|
||||
|
||||
// Info delegates unchanged.
|
||||
func (f *Storer) Info() storage.Info {
|
||||
return f.inner.Info()
|
||||
}
|
||||
|
||||
func (f *Storer) putAction(key string) PutAction {
|
||||
if f.OnPut == nil {
|
||||
return PutNormal
|
||||
}
|
||||
|
||||
return f.OnPut(key)
|
||||
}
|
||||
|
||||
// injectPut handles the non-normal write dispositions. It reports
|
||||
// whether it handled the write and, if so, with what error.
|
||||
func (f *Storer) injectPut(key string, data io.Reader) (bool, error) {
|
||||
switch f.putAction(key) {
|
||||
case PutFail:
|
||||
// Consume part of the stream so the failure lands mid-transfer,
|
||||
// the way a dropped connection would, then error without
|
||||
// storing anything.
|
||||
_, _ = io.CopyN(io.Discard, data, midStreamBytes)
|
||||
|
||||
return true, fmt.Errorf("%w for %q", ErrInjectedUpload, key)
|
||||
case PutSwallow:
|
||||
// A lying backend still drains the request body, then keeps
|
||||
// nothing.
|
||||
_, _ = io.Copy(io.Discard, data)
|
||||
|
||||
return true, nil
|
||||
case PutNormal:
|
||||
return false, nil
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,634 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// These tests cover the failure modes a backup tool must survive:
|
||||
// interrupted uploads, an interrupted metadata export, corrupt and
|
||||
// truncated reads, a full restore disk, and a backend that reports
|
||||
// success while storing nothing. Faults are injected through the
|
||||
// storage.Storer seam (internal/storage/faultstore), never by patching
|
||||
// production code. Each test asserts on the observable end state — what
|
||||
// is in the index, what is at the destination, what the user is told —
|
||||
// not merely that an error was returned. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/72.
|
||||
//
|
||||
// Object-level write atomicity (no partial blob object left behind) is
|
||||
// covered by the file:// backend's atomic-write work
|
||||
// (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested
|
||||
// here; these tests target the layers above the backend.
|
||||
//
|
||||
// The tests run serially, not with t.Parallel: each calls
|
||||
// log.Initialize, which replaces the package-global logger, and a
|
||||
// backup or restore running concurrently reads that same logger. Under
|
||||
// -race the two collide. Running one at a time is the same choice
|
||||
// prune_count_test.go already makes for the same reason.
|
||||
|
||||
const (
|
||||
faultChunkSize = int64(64 * 1024)
|
||||
faultMaxBlobSize = int64(256 * 1024)
|
||||
)
|
||||
|
||||
// faultTestConfig returns the config shared by the fault-injection
|
||||
// tests: a real recipient/secret keypair so blobs are genuinely
|
||||
// encrypted, and a blob size limit the restore sweeper can divide.
|
||||
func faultTestConfig() *config.Config {
|
||||
return &config.Config{
|
||||
AgeRecipients: []string{testAgePublicKey},
|
||||
AgeSecretKey: testAgeSecretKey,
|
||||
CompressionLevel: 3,
|
||||
Hostname: testHostname,
|
||||
BlobSizeLimit: config.Size(faultMaxBlobSize),
|
||||
}
|
||||
}
|
||||
|
||||
// writeFaultSourceTree writes a spread of file sizes that forces several
|
||||
// chunks across more than one blob, so a fault landing on a single blob
|
||||
// still leaves other data intact. Returns the expected content by path.
|
||||
func writeFaultSourceTree(
|
||||
t *testing.T, fs afero.Fs, dataDir string,
|
||||
) map[string][]byte {
|
||||
t.Helper()
|
||||
|
||||
files := map[string][]byte{
|
||||
filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"),
|
||||
filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(faultChunkSize*3)),
|
||||
filepath.Join(dataDir, "sub", "b.bin"): bytesPattern("b-", int(faultChunkSize*3)),
|
||||
filepath.Join(dataDir, "sub", "c.bin"): bytesPattern("c-", int(faultChunkSize*2)),
|
||||
}
|
||||
|
||||
for path, content := range files {
|
||||
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// newFaultScanner builds a scanner writing through the given storer.
|
||||
func newFaultScanner(
|
||||
fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
) *snapshot.Scanner {
|
||||
return snapshot.NewScanner(snapshot.ScannerConfig{
|
||||
FS: fs,
|
||||
Storage: storer,
|
||||
ChunkSize: faultChunkSize,
|
||||
MaxBlobSize: faultMaxBlobSize,
|
||||
CompressionLevel: cfg.CompressionLevel,
|
||||
AgeRecipients: cfg.AgeRecipients,
|
||||
Repositories: repos,
|
||||
})
|
||||
}
|
||||
|
||||
// newFaultSnapshotManager builds a snapshot manager writing through the
|
||||
// given storer.
|
||||
func newFaultSnapshotManager(
|
||||
fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
) *snapshot.SnapshotManager {
|
||||
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
||||
Repos: repos,
|
||||
Storage: storer,
|
||||
Config: cfg,
|
||||
})
|
||||
sm.SetFilesystem(fs)
|
||||
|
||||
return sm
|
||||
}
|
||||
|
||||
// fullFaultBackup runs a complete backup (create, scan, complete,
|
||||
// export) through storer and returns the snapshot ID.
|
||||
func fullFaultBackup(
|
||||
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
||||
cfg *config.Config, repos *database.Repositories,
|
||||
dataDir, dbPath, name string,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
sm := newFaultSnapshotManager(fs, storer, cfg, repos)
|
||||
scanner := newFaultScanner(fs, storer, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, name, "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id))
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
// newReaderVaultik builds a Vaultik that reads (restore/verify) through
|
||||
// storer, with the given repositories (nil is fine for restore/verify,
|
||||
// which read metadata from storage).
|
||||
func newReaderVaultik(
|
||||
ctx context.Context, cfg *config.Config, storer storage.Storer,
|
||||
repos *database.Repositories, fs afero.Fs,
|
||||
) *vaultik.Vaultik {
|
||||
v := &vaultik.Vaultik{
|
||||
Config: cfg,
|
||||
Storage: storer,
|
||||
Repositories: repos,
|
||||
Fs: fs,
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// Scenario 3: a stored blob's bytes are flipped before restore reads
|
||||
// them. Restore must fail loudly, and no file must be left on the
|
||||
// restore target holding corrupt content.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreRejectsCorruptBlob(t *testing.T) {
|
||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt")
|
||||
}
|
||||
|
||||
// Scenario 4: a stored blob is truncated before restore reads it. Same
|
||||
// contract as the corrupt case.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreRejectsTruncatedBlob(t *testing.T) {
|
||||
assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated")
|
||||
}
|
||||
|
||||
// assertRestoreRejectsDamagedBlob backs up the source tree, then restores
|
||||
// through a store that damages every blob read with the given fault, and
|
||||
// asserts restore fails naming a blob and leaves no file on the target
|
||||
// holding wrong bytes. Metadata reads are returned intact so the failure
|
||||
// is isolated to the blob.
|
||||
func assertRestoreRejectsDamagedBlob(
|
||||
t *testing.T, fault faultstore.GetFault, name string,
|
||||
) {
|
||||
t.Helper()
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, name)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
faultStore := faultstore.New(inner)
|
||||
faultStore.OnGet = func(key string) faultstore.GetFault {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return fault
|
||||
}
|
||||
|
||||
return faultstore.GetNormal
|
||||
}
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, faultStore, nil, fs)
|
||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
||||
|
||||
require.Error(t, err, "restore must fail on a damaged blob")
|
||||
assert.Contains(t, err.Error(), "blob",
|
||||
"error should name the blob that failed")
|
||||
assertNoCorruptFiles(t, fs, restoreDir, testFiles)
|
||||
}
|
||||
|
||||
// Scenario 6: the backend accepts blob uploads and reports success but
|
||||
// stores nothing. verify --deep must catch it.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Blob uploads are swallowed; metadata uploads land, so verify can
|
||||
// download the manifest and database and then discover the blobs are
|
||||
// absent.
|
||||
lying := faultstore.New(inner)
|
||||
lying.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutSwallow
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, fs, lying, cfg, repos, dataDir, dbPath, "lying")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// No blob objects were actually written.
|
||||
blobKeys, err := inner.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, blobKeys, "lying backend should have stored no blobs")
|
||||
|
||||
// Read back through the honest underlying store.
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
||||
err = v.VerifySnapshotWithOptions(id, &vaultik.VerifyOptions{Deep: true})
|
||||
require.Error(t, err, "deep verify must catch a backend that stored nothing")
|
||||
}
|
||||
|
||||
// Scenario 1a: a blob upload fails partway through. The interrupted run
|
||||
// must not record the blob as uploaded, must not reference it from the
|
||||
// snapshot, and must leave no blob object at the destination.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Every blob upload fails partway through. The scan must surface it.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.Error(t, err, "scan must fail when a blob upload fails")
|
||||
|
||||
// No blob may claim to be uploaded.
|
||||
blobs, err := repos.Blobs.GetAll(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, b := range blobs {
|
||||
assert.Nilf(t, b.UploadedTS,
|
||||
"blob %s marked uploaded after a failed upload", b.Hash)
|
||||
}
|
||||
|
||||
// The snapshot may reference no blobs, and the destination holds none.
|
||||
hashes, err := repos.Snapshots.GetBlobHashes(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, hashes, "interrupted snapshot must reference no blobs")
|
||||
|
||||
blobKeys, err := inner.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, blobKeys, "no blob object may survive at the destination")
|
||||
}
|
||||
|
||||
// Scenario 1b: after an interrupted upload, a retry on the same local
|
||||
// index must produce a restorable snapshot. It does not: the interrupted
|
||||
// run's chunk rows persist, the retry deduplicates against them, and the
|
||||
// backup silently emits a snapshot referencing data never stored. Skipped
|
||||
// pending the fix. See https://git.eeqj.de/sneak/vaultik/issues/148.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
|
||||
t.Skip("blocked on https://git.eeqj.de/sneak/vaultik/issues/148: " +
|
||||
"retry after an interrupted upload silently produces an " +
|
||||
"unrestorable snapshot")
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
testFiles := writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Attempt 1: every blob upload fails.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasPrefix(key, "blobs/") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
sm := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
scanner := newFaultScanner(fs, fault, cfg, repos)
|
||||
|
||||
id1, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "interrupted", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id1)
|
||||
require.Error(t, err)
|
||||
|
||||
// Retry on the same local index with a working backend.
|
||||
id2 := fullFaultBackup(ctx, t, fs, inner, cfg, repos, dataDir, dbPath, "retry")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, fs)
|
||||
require.NoError(t, v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: id2,
|
||||
TargetDir: restoreDir,
|
||||
Verify: true,
|
||||
}), "retry after an interrupted upload must produce a restorable snapshot")
|
||||
|
||||
assertRestoredTree(t, fs, restoreDir, testFiles)
|
||||
}
|
||||
|
||||
// Scenario 2: the process dies during the metadata export, after the
|
||||
// database is uploaded but before the manifest. The destination is left
|
||||
// with blobs and a database but no manifest. verify and snapshot list
|
||||
// must report the damage honestly rather than crashing or passing.
|
||||
// Automatic detection and repair of this partial state on the next run
|
||||
// is tracked in https://git.eeqj.de/sneak/vaultik/issues/148 and is not
|
||||
// asserted here.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Back up and complete with a working backend.
|
||||
sm := newFaultSnapshotManager(fs, inner, cfg, repos)
|
||||
scanner := newFaultScanner(fs, inner, cfg, repos)
|
||||
|
||||
id, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "export", "v", "g")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = scanner.Scan(ctx, dataDir, id)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
||||
|
||||
// Export through a backend that fails only the manifest upload. The
|
||||
// database uploads first and lands; the manifest does not.
|
||||
fault := faultstore.New(inner)
|
||||
fault.OnPut = func(key string) faultstore.PutAction {
|
||||
if strings.HasSuffix(key, "manifest.json.zst") {
|
||||
return faultstore.PutFail
|
||||
}
|
||||
|
||||
return faultstore.PutNormal
|
||||
}
|
||||
|
||||
smFault := newFaultSnapshotManager(fs, fault, cfg, repos)
|
||||
|
||||
err = smFault.ExportSnapshotMetadata(ctx, dbPath, id)
|
||||
require.Error(t, err, "export must fail when the manifest upload fails")
|
||||
|
||||
// The destination is in the partial state the scenario describes.
|
||||
key := snapshot.RemoteSnapshotKey(id)
|
||||
|
||||
_, err = inner.Stat(ctx, "metadata/"+key+"/db.zst.age")
|
||||
require.NoError(t, err, "database should have been uploaded before the manifest")
|
||||
|
||||
_, err = inner.Stat(ctx, "metadata/"+key+"/manifest.json.zst")
|
||||
require.ErrorIs(t, err, storage.ErrNotFound, "manifest upload should not have landed")
|
||||
|
||||
// verify must fail loudly for this snapshot, in both modes.
|
||||
reader := newReaderVaultik(ctx, cfg, inner, repos, fs)
|
||||
|
||||
deepOpts := &vaultik.VerifyOptions{Deep: true}
|
||||
require.Error(t, reader.VerifySnapshotWithOptions(id, deepOpts),
|
||||
"deep verify must report the missing manifest")
|
||||
|
||||
shallowOpts := &vaultik.VerifyOptions{Deep: false}
|
||||
require.Error(t, reader.VerifySnapshotWithOptions(id, shallowOpts),
|
||||
"shallow verify must report the missing manifest")
|
||||
|
||||
// snapshot list must not crash on the partial snapshot.
|
||||
require.NoError(t, reader.ListSnapshots(false),
|
||||
"snapshot list must tolerate a partially-exported snapshot")
|
||||
}
|
||||
|
||||
// Scenario 5: the restore target runs out of space mid-file. Restore
|
||||
// must fail with a clear, file-naming error rather than reporting
|
||||
// success over a truncated file.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreReportsDiskFull(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
osFS := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
|
||||
writeFaultSourceTree(t, osFS, dataDir)
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
id := fullFaultBackup(ctx, t, osFS, inner, cfg, repos, dataDir, dbPath, "diskfull")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Restore onto a filesystem that allows only a few bytes of file
|
||||
// content: enough to create files, far too little to hold them.
|
||||
budget := int64(8)
|
||||
quota := "aFS{Fs: osFS, remaining: &budget}
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, quota)
|
||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
||||
|
||||
require.Error(t, err, "restore must fail when the target disk is full")
|
||||
assert.Contains(t, err.Error(), errNoSpace.Error(),
|
||||
"restore error should surface the out-of-space cause")
|
||||
}
|
||||
|
||||
// assertRestoredTree byte-compares every restored file against the
|
||||
// original.
|
||||
func assertRestoredTree(
|
||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for origPath, expected := range testFiles {
|
||||
restoredPath := filepath.Join(restoreDir, origPath)
|
||||
got, err := afero.ReadFile(fs, restoredPath)
|
||||
require.NoErrorf(t, err, "restored file missing: %s", origPath)
|
||||
require.Equalf(t, expected, got, "restored content mismatch for %s", origPath)
|
||||
}
|
||||
}
|
||||
|
||||
// errNoSpace is the out-of-space error quotaFS returns once its byte
|
||||
// budget is exhausted, mirroring a real ENOSPC.
|
||||
var errNoSpace = errors.New("no space left on device")
|
||||
|
||||
// quotaFS is an afero.Fs whose files may write only a fixed total number
|
||||
// of content bytes before failing, simulating a full restore target. It
|
||||
// wraps the interface so every method except Create delegates to the
|
||||
// real filesystem; only file writes are capped.
|
||||
type quotaFS struct {
|
||||
afero.Fs
|
||||
|
||||
remaining *int64
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Create's signature requires returning afero.File.
|
||||
func (q *quotaFS) Create(name string) (afero.File, error) {
|
||||
f, err := q.Fs.Create(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return "aFile{File: f, remaining: q.remaining}, nil
|
||||
}
|
||||
|
||||
// quotaFile fails writes once the shared byte budget is exhausted.
|
||||
type quotaFile struct {
|
||||
afero.File
|
||||
|
||||
remaining *int64
|
||||
}
|
||||
|
||||
func (q *quotaFile) Write(p []byte) (int, error) {
|
||||
if *q.remaining <= 0 {
|
||||
return 0, errNoSpace
|
||||
}
|
||||
|
||||
allowed := min(int64(len(p)), *q.remaining)
|
||||
|
||||
n, err := q.File.Write(p[:allowed])
|
||||
*q.remaining -= int64(n)
|
||||
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if int64(n) < int64(len(p)) {
|
||||
return n, errNoSpace
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// assertNoCorruptFiles fails if any file that made it to the restore
|
||||
// target holds content that differs from the original: a failed restore
|
||||
// may leave a file absent, but must never leave wrong bytes presenting
|
||||
// as the real file.
|
||||
func assertNoCorruptFiles(
|
||||
t *testing.T, fs afero.Fs, restoreDir string, testFiles map[string][]byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for origPath, expected := range testFiles {
|
||||
restoredPath := filepath.Join(restoreDir, origPath)
|
||||
|
||||
got, err := afero.ReadFile(fs, restoredPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equalf(t, expected, got,
|
||||
"restored file %s holds corrupt content", origPath)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user