Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6ed84bf6d8 |
@@ -372,7 +372,9 @@ on the destination in one go, use `vaultik remote nuke --force`.
|
||||
store — its remote key (or an unambiguous leading part of it) as shown
|
||||
by `snapshot list`. See
|
||||
[restoring on another machine](#restoring-on-another-machine).
|
||||
* Optional path arguments to restore specific files/directories (default: all)
|
||||
* Optional path arguments to restore specific files/directories (default: all).
|
||||
A path that matches nothing in the snapshot is named in a warning, and the
|
||||
command exits non-zero after restoring the paths that match
|
||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
||||
symlinks, and empty directories
|
||||
* `--verify`: After restoring, verify every file's chunk hashes match
|
||||
@@ -433,10 +435,7 @@ object store. Credentials are read from `s3.access_key_id` and
|
||||
`s3.secret_access_key` in the config file.
|
||||
|
||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
||||
a local or mounted filesystem. Useful for testing or backing up to a NAS. The
|
||||
path must be absolute; a relative one is refused when the config loads.
|
||||
Percent-escapes in the path are decoded, so `my%20stick` and `my stick` name
|
||||
the same directory.
|
||||
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
||||
|
||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||
providers. Requires rclone to be configured separately (`rclone config`).
|
||||
|
||||
@@ -245,17 +245,6 @@ func (p *Packer) Flush(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
||||
// packed, if any, and drops that blob. A caller that stops before Flush
|
||||
// must call it, or the temp file is left behind. Thread-safe.
|
||||
func (p *Packer) DiscardCurrentBlob() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
}
|
||||
|
||||
// FinalizeBlob finalizes the current blob being assembled.
|
||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||
// the content-addressed hash. The finalized blob is either passed to the
|
||||
|
||||
@@ -30,6 +30,8 @@ This command will restore files from the specified snapshot to the
|
||||
target directory.
|
||||
If no paths are specified, all files are restored.
|
||||
If paths are specified, only matching files/directories are restored.
|
||||
A path that matches nothing in the snapshot is named in a warning, and
|
||||
the command exits non-zero after restoring the paths that match.
|
||||
|
||||
The snapshot may be named by its ID or, when restoring on a host with no
|
||||
local index, by the remote key that 'snapshot list' prints for a
|
||||
|
||||
@@ -61,9 +61,6 @@ var (
|
||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||
errBadStorageScheme = errors.New(
|
||||
"storage_url must start with s3://, file://, or rclone://")
|
||||
errFileURLNotAbsolute = errors.New(
|
||||
"a file:// storage_url needs an absolute path, " +
|
||||
"as in file:///path/to/backup")
|
||||
errStorageNotConfigured = errors.New(
|
||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||
"s3.bucket + credentials")
|
||||
@@ -464,18 +461,11 @@ func (c *Config) validateStorage() error {
|
||||
|
||||
// validateStorageURL validates URL-based storage configuration. File and
|
||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||
// credential fields. A file URL must name an absolute path: a relative one
|
||||
// would resolve against the working directory, so the destination would
|
||||
// move with it.
|
||||
// credential fields.
|
||||
func (c *Config) validateStorageURL() error {
|
||||
switch {
|
||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||
// storage.ParseStorageURL applies the same rule, but config
|
||||
// cannot import storage, which imports config.
|
||||
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||
return errFileURLNotAbsolute
|
||||
}
|
||||
|
||||
// File storage doesn't need S3 credentials
|
||||
return nil
|
||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||
// Rclone storage uses rclone's own config
|
||||
|
||||
@@ -337,35 +337,6 @@ func TestLoadS3PartSize(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||
// relative path fails at load. Accepted, it would resolve against the
|
||||
// working directory, so a backup run from another directory would write
|
||||
// to a different destination and still succeed.
|
||||
func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const relativeConfig = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file://dest\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(relativeConfig), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
_, err = Load(path)
|
||||
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "absolute path") {
|
||||
t.Errorf("error %q does not say to use an absolute path", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||
|
||||
@@ -1295,11 +1295,6 @@ func (s *Scanner) batchAddFilesToSnapshot(
|
||||
func (s *Scanner) processPhase(
|
||||
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
||||
) error {
|
||||
// On an error or cancellation the blob being packed is never
|
||||
// finalized, so its temp file must be removed here. After a
|
||||
// successful flush there is no such blob and this does nothing.
|
||||
defer s.packer.DiscardCurrentBlob()
|
||||
|
||||
// Calculate total bytes to process
|
||||
var totalBytes int64
|
||||
for _, f := range filesToProcess {
|
||||
|
||||
@@ -381,60 +381,3 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
|
||||
// part-way, by a file it cannot read or by a cancellation, removes the
|
||||
// temp file of the blob it was packing.
|
||||
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// a.txt is backed up first and starts a blob; the run stops at b.txt.
|
||||
const target = "/source/b.txt"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
newFs func(cancel context.CancelFunc) afero.Fs
|
||||
}{
|
||||
{
|
||||
name: "file cannot be read",
|
||||
newFs: func(context.CancelFunc) afero.Fs {
|
||||
return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cancelled",
|
||||
newFs: func(cancel context.CancelFunc) afero.Fs {
|
||||
return &cancelOnOpenFs{
|
||||
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
fs := tt.newFs(cancel)
|
||||
writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
|
||||
writeSkipErrorTestFile(t, fs, target, "second file content")
|
||||
|
||||
_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected the scan to stop with an error, got nil")
|
||||
}
|
||||
|
||||
left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
|
||||
if err != nil {
|
||||
t.Fatalf("listing blob temp files: %v", err)
|
||||
}
|
||||
|
||||
if len(left) != 0 {
|
||||
t.Fatalf("expected no blob temp file left, got %v", left)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+12
-26
@@ -17,10 +17,8 @@ const (
|
||||
|
||||
// Sentinel errors for storage URL parsing.
|
||||
var (
|
||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||
ErrRelativeFilePath = errors.New(
|
||||
"file URL needs an absolute path, as in file:///path/to/backup")
|
||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||
ErrUnsupportedScheme = errors.New(
|
||||
@@ -53,8 +51,17 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
||||
return nil, ErrEmptyStorageURL
|
||||
}
|
||||
|
||||
// Handle file:// URLs
|
||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||
return parseFileURL(after)
|
||||
path := after
|
||||
if path == "" {
|
||||
return nil, ErrEmptyFilePath
|
||||
}
|
||||
|
||||
return &URL{
|
||||
Scheme: schemeFile,
|
||||
Prefix: path,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(rawURL, "s3://") {
|
||||
@@ -68,27 +75,6 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
||||
return nil, ErrUnsupportedScheme
|
||||
}
|
||||
|
||||
// parseFileURL parses the path that follows file://. The path must be
|
||||
// absolute: a relative one would resolve against the working directory,
|
||||
// so the destination would move with it. Percent-escapes are decoded, so
|
||||
// my%20stick names the directory "my stick"; a literal space is kept.
|
||||
func parseFileURL(rawPath string) (*URL, error) {
|
||||
if rawPath == "" {
|
||||
return nil, ErrEmptyFilePath
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(rawPath, "/") {
|
||||
return nil, ErrRelativeFilePath
|
||||
}
|
||||
|
||||
path, err := url.PathUnescape(rawPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||
}
|
||||
|
||||
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
||||
}
|
||||
|
||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||
// the userinfo and any query parameter other than endpoint, region and
|
||||
// ssl, so a credential-bearing URL is never stored or echoed.
|
||||
|
||||
@@ -14,10 +14,7 @@ import (
|
||||
func TestParseStorageURLValid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
bucket = "mybucket"
|
||||
fileScheme = "file"
|
||||
)
|
||||
const bucket = "mybucket"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -27,17 +24,7 @@ func TestParseStorageURLValid(t *testing.T) {
|
||||
{
|
||||
name: "file absolute path",
|
||||
raw: "file:///var/backups/vaultik",
|
||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||
},
|
||||
{
|
||||
name: "file percent-escaped path is decoded",
|
||||
raw: "file:///Volumes/my%20stick/backup",
|
||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||
},
|
||||
{
|
||||
name: "file path with a literal space",
|
||||
raw: "file:///Volumes/my stick/backup",
|
||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
|
||||
},
|
||||
{
|
||||
name: "s3 bucket and prefix, ssl defaults on",
|
||||
@@ -103,7 +90,6 @@ func TestParseStorageURLErrors(t *testing.T) {
|
||||
}{
|
||||
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
||||
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||
|
||||
@@ -103,8 +103,6 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
||||
}
|
||||
|
||||
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
||||
// The reader is closed as soon as ctx is cancelled, which ends a read
|
||||
// blocked inside it.
|
||||
func (v *Vaultik) FetchBlob(
|
||||
ctx context.Context, blobHash string,
|
||||
) (io.ReadCloser, error) {
|
||||
@@ -122,47 +120,5 @@ func (v *Vaultik) FetchBlob(
|
||||
return nil, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
|
||||
}
|
||||
|
||||
return newCloseOnCancelReader(ctx, rc), nil
|
||||
}
|
||||
|
||||
// closeOnCancelReader closes the reader it wraps when ctx is cancelled.
|
||||
// Not every backend's reader stops on cancel by itself (the file
|
||||
// backend's does not), and a read stalled on a slow or hung source would
|
||||
// otherwise outlast the shutdown timeout, so the process would exit
|
||||
// before restore or deep verify removed its decrypted temporary files.
|
||||
type closeOnCancelReader struct {
|
||||
io.ReadCloser
|
||||
|
||||
ctx context.Context //nolint:containedctx // Read reports its error
|
||||
stop func() bool
|
||||
}
|
||||
|
||||
func newCloseOnCancelReader(
|
||||
ctx context.Context, rc io.ReadCloser,
|
||||
) *closeOnCancelReader {
|
||||
return &closeOnCancelReader{
|
||||
ReadCloser: rc,
|
||||
ctx: ctx,
|
||||
stop: context.AfterFunc(ctx, func() { _ = rc.Close() }),
|
||||
}
|
||||
}
|
||||
|
||||
// Read returns the context's error, not the closed reader's, once ctx
|
||||
// has been cancelled.
|
||||
func (r *closeOnCancelReader) Read(p []byte) (int, error) {
|
||||
n, err := r.ReadCloser.Read(p)
|
||||
if err != nil && r.ctx.Err() != nil {
|
||||
return n, r.ctx.Err()
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the wrapped reader, unless the cancel already closed it.
|
||||
func (r *closeOnCancelReader) Close() error {
|
||||
if !r.stop() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return r.ReadCloser.Close()
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
+43
-15
@@ -27,6 +27,7 @@ import (
|
||||
// Sentinel errors for restore failures.
|
||||
var (
|
||||
errFilesFailedRestore = errors.New("file(s) failed to restore")
|
||||
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
||||
errFilesFailedVerify = errors.New("files failed verification")
|
||||
errDecryptionKeyRequired = errors.New(
|
||||
"decryption key required for restore\n\n" +
|
||||
@@ -149,7 +150,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
repos := database.NewRepositories(tempDB)
|
||||
|
||||
// Step 2: Get list of files to restore
|
||||
files, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
|
||||
files, unmatched, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting files to restore: %w", err)
|
||||
}
|
||||
@@ -158,7 +159,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
log.Warn("No files found to restore")
|
||||
v.UI.Warningf("No files found to restore.")
|
||||
|
||||
return nil
|
||||
return v.reportUnmatchedPaths(unmatched)
|
||||
}
|
||||
|
||||
log.Info("Found files to restore", "count", len(files))
|
||||
@@ -198,14 +199,16 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
v.UI.Duration(result.Duration),
|
||||
)
|
||||
|
||||
return v.finishRestore(repos, files, opts, result)
|
||||
return v.finishRestore(repos, files, unmatched, opts, result)
|
||||
}
|
||||
|
||||
// finishRestore emits the post-restore warnings, runs optional
|
||||
// verification, and converts any failed-file count into an error.
|
||||
// verification, and converts any failed-file count or requested path
|
||||
// that matched nothing into an error.
|
||||
func (v *Vaultik) finishRestore(
|
||||
repos *database.Repositories,
|
||||
files []*database.File,
|
||||
unmatched []string,
|
||||
opts *RestoreOptions,
|
||||
result *RestoreResult,
|
||||
) error {
|
||||
@@ -223,6 +226,8 @@ func (v *Vaultik) finishRestore(
|
||||
}
|
||||
}
|
||||
|
||||
unmatchedErr := v.reportUnmatchedPaths(unmatched)
|
||||
|
||||
// Run verification if requested
|
||||
if opts.Verify {
|
||||
err := v.handleRestoreVerification(repos, files, opts, result)
|
||||
@@ -235,7 +240,23 @@ func (v *Vaultik) finishRestore(
|
||||
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
|
||||
}
|
||||
|
||||
return nil
|
||||
return unmatchedErr
|
||||
}
|
||||
|
||||
// reportUnmatchedPaths warns, by name, about each requested path that
|
||||
// matched nothing in the snapshot, and returns an error counting them,
|
||||
// or nil when there are none.
|
||||
func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
|
||||
if len(unmatched) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, path := range unmatched {
|
||||
log.Warn("Requested path matches nothing in the snapshot", "path", path)
|
||||
v.UI.Warningf("%s matches nothing in the snapshot.", v.UI.Path(path))
|
||||
}
|
||||
|
||||
return fmt.Errorf("%d %w", len(unmatched), errPathsNotInSnapshot)
|
||||
}
|
||||
|
||||
// restoreIdentities parses the configured age secret key once into every
|
||||
@@ -812,27 +833,34 @@ func (v *Vaultik) materializeSnapshotDB(
|
||||
return db, tempDir, nil
|
||||
}
|
||||
|
||||
// getFilesToRestore returns the list of files to restore based on path filters
|
||||
// getFilesToRestore returns the list of files to restore based on path
|
||||
// filters, and the filters, as given, that matched no file.
|
||||
func (v *Vaultik) getFilesToRestore(
|
||||
ctx context.Context, repos *database.Repositories, pathFilters []string,
|
||||
) ([]*database.File, error) {
|
||||
) ([]*database.File, []string, error) {
|
||||
// If no filters, get all files
|
||||
if len(pathFilters) == 0 {
|
||||
return repos.Files.ListAll(ctx)
|
||||
files, err := repos.Files.ListAll(ctx)
|
||||
|
||||
return files, nil, err
|
||||
}
|
||||
|
||||
// Get files matching the path filters
|
||||
var result []*database.File
|
||||
var (
|
||||
result []*database.File
|
||||
unmatched []string
|
||||
)
|
||||
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, filter := range pathFilters {
|
||||
// Normalize the filter path
|
||||
filter = filepath.Clean(filter)
|
||||
|
||||
files, err := repos.Files.ListUnderPath(ctx, filter)
|
||||
files, err := repos.Files.ListUnderPath(ctx, filepath.Clean(filter))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing files under %s: %w", filter, err)
|
||||
return nil, nil, fmt.Errorf("listing files under %s: %w", filter, err)
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
unmatched = append(unmatched, filter)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
@@ -843,7 +871,7 @@ func (v *Vaultik) getFilesToRestore(
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return result, unmatched, nil
|
||||
}
|
||||
|
||||
// buildChunkToBlobMap creates a mapping from chunk hash to blob information
|
||||
|
||||
@@ -142,14 +142,14 @@ func TestRestoreCleansTempDirOnInterrupt(t *testing.T) {
|
||||
}
|
||||
|
||||
// scratchEntries returns the vaultik blob-cache and snapshot-database
|
||||
// scratch entries (restore's and deep verify's) currently present in dir.
|
||||
// scratch entries currently present in dir.
|
||||
func scratchEntries(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
|
||||
var matches []string
|
||||
|
||||
for _, pattern := range []string{
|
||||
"vaultik-blobcache-*", "vaultik-restore-*", "vaultik-verify-*",
|
||||
"vaultik-blobcache-*", "vaultik-restore-*",
|
||||
} {
|
||||
found, err := filepath.Glob(filepath.Join(dir, pattern))
|
||||
require.NoError(t, err)
|
||||
@@ -206,119 +206,3 @@ func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
|
||||
assert.NotContains(t, out.String(), srcPath,
|
||||
"the cancel was reported as a failed file")
|
||||
}
|
||||
|
||||
// stallingBlobStorer wraps a Storer so that every blob download hands
|
||||
// over the first half of the blob and then stalls, ignoring the context,
|
||||
// as a file:// blob on a hung USB stick or network share does. Only
|
||||
// closing the returned reader ends the stall.
|
||||
type stallingBlobStorer struct {
|
||||
storage.Storer
|
||||
|
||||
once sync.Once
|
||||
stalled chan struct{}
|
||||
}
|
||||
|
||||
func newStallingBlobStorer(inner storage.Storer) *stallingBlobStorer {
|
||||
return &stallingBlobStorer{Storer: inner, stalled: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (s *stallingBlobStorer) Get(
|
||||
ctx context.Context, key string,
|
||||
) (io.ReadCloser, error) {
|
||||
rc, err := s.Storer.Get(ctx, key)
|
||||
if err != nil || !strings.HasPrefix(key, "blobs/") {
|
||||
return rc, err
|
||||
}
|
||||
|
||||
blob, err := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The writer is never closed, so once the first half has been read
|
||||
// the next Read blocks until the reader is closed.
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
_, _ = pw.Write(blob[:len(blob)/2])
|
||||
|
||||
s.once.Do(func() { close(s.stalled) })
|
||||
}()
|
||||
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
// TestInterruptDuringStalledBlobReadCleansTempDir stops a restore and a
|
||||
// deep verify while a blob read is stalled on a source that ignores the
|
||||
// context. Stopping must end that read, so the operation unwinds and
|
||||
// removes its decrypted scratch files well within the shutdown timeout
|
||||
// (issue #325).
|
||||
//
|
||||
// Not parallel: it points TMPDIR at a private directory (via t.Setenv)
|
||||
// so it can assert on exactly the scratch files the operation created.
|
||||
func TestInterruptDuringStalledBlobReadCleansTempDir(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
operations := map[string]func(v *Vaultik, snapshotID, targetDir string) error{
|
||||
"restore": func(v *Vaultik, snapshotID, targetDir string) error {
|
||||
return v.Restore(&RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: targetDir,
|
||||
})
|
||||
},
|
||||
"deep verify": func(v *Vaultik, snapshotID, _ string) error {
|
||||
return v.RunDeepVerify(snapshotID, &VerifyOptions{Deep: true})
|
||||
},
|
||||
}
|
||||
|
||||
for name, op := range operations {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
root := t.TempDir()
|
||||
|
||||
// Random content, so the blob stays large enough to stall
|
||||
// part-way through its compressed, encrypted body.
|
||||
cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
|
||||
t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
|
||||
|
||||
scratch := filepath.Join(root, "scratch")
|
||||
require.NoError(t, fs.MkdirAll(scratch, 0o755))
|
||||
t.Setenv("TMPDIR", scratch)
|
||||
|
||||
stalling := newStallingBlobStorer(storer)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
v := newRestoreVaultik(ctx, cfg, stalling, fs)
|
||||
v.cancel = cancel
|
||||
|
||||
var opErr error
|
||||
|
||||
stop := v.StartOperation(func() {
|
||||
opErr = op(v, snapshotID, filepath.Join(root, "restored"))
|
||||
})
|
||||
|
||||
select {
|
||||
case <-stalling.stalled:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the operation never reached a blob read")
|
||||
}
|
||||
|
||||
require.NotEmpty(t, scratchEntries(t, scratch),
|
||||
"expected decrypted scratch files to exist mid-read")
|
||||
|
||||
stopCtx, stopCancel := context.WithTimeout(
|
||||
context.Background(), 10*time.Second)
|
||||
defer stopCancel()
|
||||
|
||||
require.True(t, stop(stopCtx),
|
||||
"stop timed out; the stalled blob read was not interrupted")
|
||||
require.ErrorIs(t, opErr, context.Canceled)
|
||||
require.Empty(t, scratchEntries(t, scratch),
|
||||
"decrypted scratch files remained after the interrupt")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/309: a path
|
||||
// given to restore that matches nothing in the snapshot is named in a
|
||||
// warning and makes the restore fail, after the paths that do match are
|
||||
// restored.
|
||||
|
||||
func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := faultTestConfig()
|
||||
files := writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
store, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
id := fullFaultBackup(ctx, t, fs, store, cfg,
|
||||
database.NewRepositories(db), dataDir, dbPath, "unmatched")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
matched := filepath.Join(dataDir, "small.txt")
|
||||
unmatched := filepath.Join(dataDir, "typo.txt")
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||
v.UI = ui.NewWithColor(&out, false)
|
||||
|
||||
err = v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: id,
|
||||
TargetDir: restoreDir,
|
||||
Paths: []string{matched, unmatched},
|
||||
})
|
||||
|
||||
require.Error(t, err, "a path that matches nothing must fail the restore")
|
||||
assert.Contains(t, out.String(), unmatched,
|
||||
"the path that matches nothing must be named")
|
||||
assert.NotContains(t, out.String(), matched,
|
||||
"the path that matches must not be reported")
|
||||
|
||||
got, err := afero.ReadFile(fs, filepath.Join(restoreDir, matched))
|
||||
require.NoError(t, err, "the path that matches must still be restored")
|
||||
assert.Equal(t, files[matched], got)
|
||||
}
|
||||
|
||||
// When no requested path matches, nothing is restored and each path is
|
||||
// named. One path is relative. The other goes through the symlink that
|
||||
// is the snapshot's configured path, while the entries are stored under
|
||||
// the directory the symlink points to.
|
||||
func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
const snapshotName = "linked"
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
linkDir := filepath.Join(tempDir, "srclink")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
require.NoError(t, os.Symlink(dataDir, linkDir))
|
||||
|
||||
cfg := faultTestConfig()
|
||||
cfg.IndexPath = dbPath
|
||||
cfg.ChunkSize = config.Size(faultChunkSize)
|
||||
cfg.Snapshots = map[string]config.SnapshotConfig{
|
||||
snapshotName: {Paths: []string{linkDir}},
|
||||
}
|
||||
|
||||
store, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||
|
||||
require.NoError(t, backUp(v, snapshotName))
|
||||
|
||||
id := localSnapshotID(ctx, t, repos, snapshotName)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
throughLink := filepath.Join(linkDir, "sub")
|
||||
relative := strings.TrimPrefix(filepath.Join(dataDir, "small.txt"), "/")
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||
reader.UI = ui.NewWithColor(&out, false)
|
||||
|
||||
err = reader.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: id,
|
||||
TargetDir: restoreDir,
|
||||
Paths: []string{throughLink, relative},
|
||||
})
|
||||
|
||||
require.Error(t, err, "a restore that matches nothing must fail")
|
||||
assert.Contains(t, out.String(), throughLink,
|
||||
"the path through the symlink must be named")
|
||||
assert.Contains(t, out.String(), relative,
|
||||
"the relative path must be named")
|
||||
|
||||
_, err = fs.Stat(restoreDir)
|
||||
assert.True(t, os.IsNotExist(err), "nothing must be restored")
|
||||
}
|
||||
Reference in New Issue
Block a user