Compare commits
5
Commits
017877da44
...
60c83844ce
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
60c83844ce | ||
|
|
8c54e0ff75 | ||
|
|
d1213ecb34 | ||
|
|
5f6455ba17 | ||
|
|
0288735aaf |
@@ -375,7 +375,9 @@ on the destination in one go, use `vaultik remote nuke --force`.
|
|||||||
* Optional path arguments to restore specific files/directories (default: all)
|
* Optional path arguments to restore specific files/directories (default: all)
|
||||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
||||||
symlinks, and empty directories
|
symlinks, and empty directories
|
||||||
* `--verify`: After restoring, verify every file's chunk hashes match
|
* `--verify`: After restoring, verify every restored file's chunk hashes
|
||||||
|
match. A file skipped under `--skip-errors` is reported as failed to
|
||||||
|
restore and is not verified.
|
||||||
|
|
||||||
**`prune`**: Tidy up everything that isn't needed. Runs three passes:
|
**`prune`**: Tidy up everything that isn't needed. Runs three passes:
|
||||||
(1) reconcile the local index against the destination store — any
|
(1) reconcile the local index against the destination store — any
|
||||||
@@ -433,7 +435,10 @@ object store. Credentials are read from `s3.access_key_id` and
|
|||||||
`s3.secret_access_key` in the config file.
|
`s3.secret_access_key` in the config file.
|
||||||
|
|
||||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
||||||
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
a local or mounted filesystem. Useful for testing or backing up to a NAS. The
|
||||||
|
path must be absolute; a relative one is refused when the config loads.
|
||||||
|
Percent-escapes in the path are decoded, so `my%20stick` and `my stick` name
|
||||||
|
the same directory.
|
||||||
|
|
||||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||||
providers. Requires rclone to be configured separately (`rclone config`).
|
providers. Requires rclone to be configured separately (`rclone config`).
|
||||||
|
|||||||
@@ -245,6 +245,17 @@ func (p *Packer) Flush(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
||||||
|
// packed, if any, and drops that blob. A caller that stops before Flush
|
||||||
|
// must call it, or the temp file is left behind. Thread-safe.
|
||||||
|
func (p *Packer) DiscardCurrentBlob() {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
|
p.cleanupTempFile()
|
||||||
|
p.currentBlob = nil
|
||||||
|
}
|
||||||
|
|
||||||
// FinalizeBlob finalizes the current blob being assembled.
|
// FinalizeBlob finalizes the current blob being assembled.
|
||||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||||
// the content-addressed hash. The finalized blob is either passed to the
|
// the content-addressed hash. The finalized blob is either passed to the
|
||||||
|
|||||||
@@ -251,6 +251,12 @@ func NewConfigCommand() *cobra.Command {
|
|||||||
Use: "config",
|
Use: "config",
|
||||||
Short: "Manage the configuration file",
|
Short: "Manage the configuration file",
|
||||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(newConfigInitCommand())
|
cmd.AddCommand(newConfigInitCommand())
|
||||||
|
|||||||
@@ -15,6 +15,12 @@ func NewDatabaseCommand() *cobra.Command {
|
|||||||
Use: "database",
|
Use: "database",
|
||||||
Short: "Manage the local state database",
|
Short: "Manage the local state database",
|
||||||
Long: `Commands for managing the local SQLite state database.`,
|
Long: `Commands for managing the local SQLite state database.`,
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package cli //nolint:testpackage // shares runEntry and the command-word constants
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
|
||||||
|
// mistyped subcommand, as a crontab line might carry one. It must fail
|
||||||
|
// with exit 1 and name the word, as `vaultik bogus` does at the top
|
||||||
|
// level, instead of printing the group's help and exiting 0 having done
|
||||||
|
// nothing. The group name alone still prints its help.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||||
|
func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
group string
|
||||||
|
mistyped string
|
||||||
|
}{
|
||||||
|
{group: cmdSnapshot, mistyped: "craete"},
|
||||||
|
{group: "config", mistyped: "inti"},
|
||||||
|
{group: cmdRemote, mistyped: "ifno"},
|
||||||
|
{group: "database", mistyped: "delte"},
|
||||||
|
{group: "completion", mistyped: "nosuchshell"},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.group, func(t *testing.T) {
|
||||||
|
code, stdout, stderr := runEntry(t,
|
||||||
|
testCase.group, testCase.mistyped)
|
||||||
|
|
||||||
|
assert.Equal(t, 1, code)
|
||||||
|
assert.Empty(t, stdout)
|
||||||
|
assert.Contains(t, stderr,
|
||||||
|
`unknown command "`+testCase.mistyped+`"`)
|
||||||
|
|
||||||
|
code, stdout, _ = runEntry(t, testCase.group)
|
||||||
|
|
||||||
|
assert.Equal(t, 0, code)
|
||||||
|
assert.Contains(t, stdout,
|
||||||
|
programName+" "+testCase.group+" [command]")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,12 @@ func NewRemoteCommand() *cobra.Command {
|
|||||||
Use: "remote",
|
Use: "remote",
|
||||||
Short: "Remote storage management commands",
|
Short: "Remote storage management commands",
|
||||||
Long: "Commands for inspecting and managing remote storage",
|
Long: "Commands for inspecting and managing remote storage",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -74,6 +74,21 @@ on the source system.`,
|
|||||||
NewDatabaseCommand(),
|
NewDatabaseCommand(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// cobra adds the completion command only when Execute runs, and with
|
||||||
|
// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
|
||||||
|
// printed help and exited 0. Adding it here lets it have a Run. Its
|
||||||
|
// scripts go to os.Stdout as it is now: a later SetOut does not
|
||||||
|
// redirect them.
|
||||||
|
cmd.InitDefaultCompletionCmd()
|
||||||
|
|
||||||
|
for _, sub := range cmd.Commands() {
|
||||||
|
if sub.Name() == "completion" {
|
||||||
|
sub.Run = func(completion *cobra.Command, _ []string) {
|
||||||
|
_ = completion.Help()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,6 +41,12 @@ func NewSnapshotCommand() *cobra.Command {
|
|||||||
Use: "snapshot",
|
Use: "snapshot",
|
||||||
Short: "Snapshot management commands",
|
Short: "Snapshot management commands",
|
||||||
Long: "Commands for creating, listing, and managing snapshots",
|
Long: "Commands for creating, listing, and managing snapshots",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -61,6 +61,9 @@ var (
|
|||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
errBadStorageScheme = errors.New(
|
errBadStorageScheme = errors.New(
|
||||||
"storage_url must start with s3://, file://, or rclone://")
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
|
errFileURLNotAbsolute = errors.New(
|
||||||
|
"a file:// storage_url needs an absolute path, " +
|
||||||
|
"as in file:///path/to/backup")
|
||||||
errStorageNotConfigured = errors.New(
|
errStorageNotConfigured = errors.New(
|
||||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||||
"s3.bucket + credentials")
|
"s3.bucket + credentials")
|
||||||
@@ -461,11 +464,18 @@ func (c *Config) validateStorage() error {
|
|||||||
|
|
||||||
// validateStorageURL validates URL-based storage configuration. File and
|
// validateStorageURL validates URL-based storage configuration. File and
|
||||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||||
// credential fields.
|
// credential fields. A file URL must name an absolute path: a relative one
|
||||||
|
// would resolve against the working directory, so the destination would
|
||||||
|
// move with it.
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
switch {
|
switch {
|
||||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||||
// File storage doesn't need S3 credentials
|
// storage.ParseStorageURL applies the same rule, but config
|
||||||
|
// cannot import storage, which imports config.
|
||||||
|
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||||
|
return errFileURLNotAbsolute
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||||
// Rclone storage uses rclone's own config
|
// Rclone storage uses rclone's own config
|
||||||
|
|||||||
@@ -337,6 +337,35 @@ func TestLoadS3PartSize(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||||
|
// relative path fails at load. Accepted, it would resolve against the
|
||||||
|
// working directory, so a backup run from another directory would write
|
||||||
|
// to a different destination and still succeed.
|
||||||
|
func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const relativeConfig = "snapshots:\n" +
|
||||||
|
" test:\n" +
|
||||||
|
" paths: [/tmp/vaultik-test-source]\n" +
|
||||||
|
"storage_url: file://dest\n"
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
|
||||||
|
err := os.WriteFile(path, []byte(relativeConfig), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = Load(path)
|
||||||
|
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||||
|
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), "absolute path") {
|
||||||
|
t.Errorf("error %q does not say to use an absolute path", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||||
|
|||||||
@@ -531,3 +531,50 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
|||||||
require.Equal(t, 1, result.FilesScanned,
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
"Unanchored pattern for file should match anywhere")
|
"Unanchored pattern for file should match anywhere")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_DoubleStarDirectoryAtEveryDepth(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, a leading **/ also matches directly under the source dir
|
||||||
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/projectname"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// **/projectname should exclude BOTH:
|
||||||
|
// - /backup/projectname/file.txt
|
||||||
|
// - /backup/otherproject/projectname/file.txt
|
||||||
|
// Total files: 4, excluded: 2
|
||||||
|
require.Equal(t, 2, result.FilesScanned,
|
||||||
|
"**/ directory pattern should match at the top level and nested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, a leading **/ also matches directly under the source dir
|
||||||
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.txt"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// **/*.txt should exclude ALL .txt files:
|
||||||
|
// - /backup/file.txt
|
||||||
|
// - /backup/projectname/file.txt
|
||||||
|
// - /backup/otherproject/projectname/file.txt
|
||||||
|
// Total files: 4, excluded: 3
|
||||||
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
|
"**/ file pattern should match at the top level and nested")
|
||||||
|
}
|
||||||
|
|||||||
@@ -1295,6 +1295,11 @@ func (s *Scanner) batchAddFilesToSnapshot(
|
|||||||
func (s *Scanner) processPhase(
|
func (s *Scanner) processPhase(
|
||||||
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
||||||
) error {
|
) error {
|
||||||
|
// On an error or cancellation the blob being packed is never
|
||||||
|
// finalized, so its temp file must be removed here. After a
|
||||||
|
// successful flush there is no such blob and this does nothing.
|
||||||
|
defer s.packer.DiscardCurrentBlob()
|
||||||
|
|
||||||
// Calculate total bytes to process
|
// Calculate total bytes to process
|
||||||
var totalBytes int64
|
var totalBytes int64
|
||||||
for _, f := range filesToProcess {
|
for _, f := range filesToProcess {
|
||||||
@@ -2013,6 +2018,14 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
// Remove trailing slash if present (directory indicator)
|
// Remove trailing slash if present (directory indicator)
|
||||||
pattern = strings.TrimSuffix(pattern, "/")
|
pattern = strings.TrimSuffix(pattern, "/")
|
||||||
|
|
||||||
|
// A leading "**/" demands a "/" before the name and so misses an
|
||||||
|
// entry directly under the source path. An unanchored pattern
|
||||||
|
// already matches at every depth, so drop the prefix: as in
|
||||||
|
// gitignore, "**/x" then also matches a top-level "x".
|
||||||
|
if !anchored {
|
||||||
|
pattern = strings.TrimPrefix(pattern, "**/")
|
||||||
|
}
|
||||||
|
|
||||||
// Compile the glob pattern
|
// Compile the glob pattern
|
||||||
// For patterns without path separators, we need to match them as components
|
// For patterns without path separators, we need to match them as components
|
||||||
// e.g., ".git" should match ".git" anywhere in the path
|
// e.g., ".git" should match ".git" anywhere in the path
|
||||||
|
|||||||
@@ -381,3 +381,60 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
|
||||||
|
// part-way, by a file it cannot read or by a cancellation, removes the
|
||||||
|
// temp file of the blob it was packing.
|
||||||
|
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// a.txt is backed up first and starts a blob; the run stops at b.txt.
|
||||||
|
const target = "/source/b.txt"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
newFs func(cancel context.CancelFunc) afero.Fs
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file cannot be read",
|
||||||
|
newFs: func(context.CancelFunc) afero.Fs {
|
||||||
|
return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cancelled",
|
||||||
|
newFs: func(cancel context.CancelFunc) afero.Fs {
|
||||||
|
return &cancelOnOpenFs{
|
||||||
|
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
fs := tt.newFs(cancel)
|
||||||
|
writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
|
||||||
|
writeSkipErrorTestFile(t, fs, target, "second file content")
|
||||||
|
|
||||||
|
_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected the scan to stop with an error, got nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listing blob temp files: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(left) != 0 {
|
||||||
|
t.Fatalf("expected no blob temp file left, got %v", left)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+26
-12
@@ -17,8 +17,10 @@ const (
|
|||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
var (
|
var (
|
||||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||||
|
ErrRelativeFilePath = errors.New(
|
||||||
|
"file URL needs an absolute path, as in file:///path/to/backup")
|
||||||
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
@@ -51,17 +53,8 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle file:// URLs
|
|
||||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||||
path := after
|
return parseFileURL(after)
|
||||||
if path == "" {
|
|
||||||
return nil, ErrEmptyFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
return &URL{
|
|
||||||
Scheme: schemeFile,
|
|
||||||
Prefix: path,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(rawURL, "s3://") {
|
if strings.HasPrefix(rawURL, "s3://") {
|
||||||
@@ -75,6 +68,27 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrUnsupportedScheme
|
return nil, ErrUnsupportedScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseFileURL parses the path that follows file://. The path must be
|
||||||
|
// absolute: a relative one would resolve against the working directory,
|
||||||
|
// so the destination would move with it. Percent-escapes are decoded, so
|
||||||
|
// my%20stick names the directory "my stick"; a literal space is kept.
|
||||||
|
func parseFileURL(rawPath string) (*URL, error) {
|
||||||
|
if rawPath == "" {
|
||||||
|
return nil, ErrEmptyFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(rawPath, "/") {
|
||||||
|
return nil, ErrRelativeFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err := url.PathUnescape(rawPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
// the userinfo and any query parameter other than endpoint, region and
|
// the userinfo and any query parameter other than endpoint, region and
|
||||||
// ssl, so a credential-bearing URL is never stored or echoed.
|
// ssl, so a credential-bearing URL is never stored or echoed.
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ import (
|
|||||||
func TestParseStorageURLValid(t *testing.T) {
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const bucket = "mybucket"
|
const (
|
||||||
|
bucket = "mybucket"
|
||||||
|
fileScheme = "file"
|
||||||
|
)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -24,7 +27,17 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file percent-escaped path is decoded",
|
||||||
|
raw: "file:///Volumes/my%20stick/backup",
|
||||||
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file path with a literal space",
|
||||||
|
raw: "file:///Volumes/my stick/backup",
|
||||||
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
@@ -90,6 +103,7 @@ func TestParseStorageURLErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"empty url", "", storage.ErrEmptyStorageURL},
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
|
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
||||||
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
|
|||||||
+27
-12
@@ -103,11 +103,12 @@ type RestoreResult struct {
|
|||||||
BlobsDownloaded int
|
BlobsDownloaded int
|
||||||
BytesDownloaded int64
|
BytesDownloaded int64
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
|
FilesFailed int
|
||||||
|
FailedFiles []string // Paths of files that failed to restore
|
||||||
// Verification results (only populated if Verify option is set)
|
// Verification results (only populated if Verify option is set)
|
||||||
FilesVerified int
|
FilesVerified int
|
||||||
BytesVerified int64
|
BytesVerified int64
|
||||||
FilesFailed int
|
VerifyFailedFiles []string // Paths of restored files that failed verification
|
||||||
FailedFiles []string // Paths of files that failed verification
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Restore restores files from a snapshot to the target directory
|
// Restore restores files from a snapshot to the target directory
|
||||||
@@ -647,20 +648,34 @@ func (v *Vaultik) handleRestoreVerification(
|
|||||||
opts *RestoreOptions,
|
opts *RestoreOptions,
|
||||||
result *RestoreResult,
|
result *RestoreResult,
|
||||||
) error {
|
) error {
|
||||||
err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result)
|
// A file that failed to restore was not written and is already
|
||||||
|
// reported as such, so it is not verified.
|
||||||
|
failedToRestore := make(map[string]bool, len(result.FailedFiles))
|
||||||
|
for _, path := range result.FailedFiles {
|
||||||
|
failedToRestore[path] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
restored := make([]*database.File, 0, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
if !failedToRestore[file.Path.String()] {
|
||||||
|
restored = append(restored, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err := v.verifyRestoredFiles(v.ctx, repos, restored, opts.TargetDir, result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("verification failed: %w", err)
|
return fmt.Errorf("verification failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.FilesFailed > 0 {
|
if len(result.VerifyFailedFiles) > 0 {
|
||||||
v.UI.Errorf("Verification failed: %s files did not match expected checksums.",
|
v.UI.Errorf("Verification failed: %s files did not match expected checksums.",
|
||||||
v.UI.Count(result.FilesFailed))
|
v.UI.Count(len(result.VerifyFailedFiles)))
|
||||||
|
|
||||||
for _, path := range result.FailedFiles {
|
for _, path := range result.VerifyFailedFiles {
|
||||||
v.UI.Detailf("%s", v.UI.Path(path))
|
v.UI.Detailf("%s", v.UI.Path(path))
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedVerify)
|
return fmt.Errorf("%d %w", len(result.VerifyFailedFiles), errFilesFailedVerify)
|
||||||
}
|
}
|
||||||
|
|
||||||
v.UI.Completef("Verified %s files (%s).",
|
v.UI.Completef("Verified %s files (%s).",
|
||||||
@@ -1420,8 +1435,8 @@ func (v *Vaultik) verifyRestoredFiles(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("File verification failed", "path", file.Path, "error", err)
|
log.Error("File verification failed", "path", file.Path, "error", err)
|
||||||
|
|
||||||
result.FilesFailed++
|
result.VerifyFailedFiles = append(
|
||||||
result.FailedFiles = append(result.FailedFiles, file.Path.String())
|
result.VerifyFailedFiles, file.Path.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
bytesProcessed += file.Size
|
bytesProcessed += file.Size
|
||||||
@@ -1437,7 +1452,7 @@ func (v *Vaultik) verifyRestoredFiles(
|
|||||||
log.Info("Verification complete",
|
log.Info("Verification complete",
|
||||||
"files_verified", result.FilesVerified,
|
"files_verified", result.FilesVerified,
|
||||||
"bytes_verified", ubytes(result.BytesVerified),
|
"bytes_verified", ubytes(result.BytesVerified),
|
||||||
"files_failed", result.FilesFailed,
|
"files_failed", len(result.VerifyFailedFiles),
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
@@ -84,6 +86,130 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with
|
||||||
|
// SkipErrors and Verify after one blob of a two-blob snapshot was
|
||||||
|
// deleted. Verification must check only the files that were written, so
|
||||||
|
// a file of the deleted blob is reported as failed to restore and not
|
||||||
|
// also as failing verification, and Restore must still return an error.
|
||||||
|
func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
backup := backupThenDeleteOneBlob(ctx, t)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
||||||
|
v.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: backup.snapshotID,
|
||||||
|
TargetDir: backup.restoreDir,
|
||||||
|
Verify: true,
|
||||||
|
SkipErrors: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.Error(t, err, "restore must fail when files were skipped")
|
||||||
|
assert.Contains(t, err.Error(),
|
||||||
|
fmt.Sprintf("%d file(s) failed to restore", len(backup.lost)))
|
||||||
|
|
||||||
|
written := len(backup.files) - len(backup.lost)
|
||||||
|
|
||||||
|
assert.Contains(t, out.String(), fmt.Sprintf("Verifying %d files", written))
|
||||||
|
assert.Contains(t, out.String(), fmt.Sprintf("Verified %d files", written))
|
||||||
|
assert.NotContains(t, out.String(), "Verification failed")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreSkipErrorsVerifyReportsFailuresApart restores with
|
||||||
|
// SkipErrors and Verify after one blob of a two-blob snapshot was
|
||||||
|
// deleted, while one file that does restore is written one byte too long.
|
||||||
|
// The restore warning must list only the files of the deleted blob, the
|
||||||
|
// verification summary only the long file, each path once, and Restore
|
||||||
|
// must return an error.
|
||||||
|
func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
backup := backupThenDeleteOneBlob(ctx, t)
|
||||||
|
|
||||||
|
var long string
|
||||||
|
|
||||||
|
for path := range backup.files {
|
||||||
|
if !backup.lost[path] {
|
||||||
|
long = path
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
fs := trailingByteFs{Fs: backup.fs, suffix: long}
|
||||||
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, fs)
|
||||||
|
v.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: backup.snapshotID,
|
||||||
|
TargetDir: backup.restoreDir,
|
||||||
|
Verify: true,
|
||||||
|
SkipErrors: true,
|
||||||
|
})
|
||||||
|
require.Error(t, err, "restore must fail when files failed")
|
||||||
|
|
||||||
|
_, afterWarning, found := strings.Cut(out.String(),
|
||||||
|
fmt.Sprintf("%d file(s) failed to restore:", len(backup.lost)))
|
||||||
|
require.Truef(t, found, "no restore warning in:\n%s", out.String())
|
||||||
|
|
||||||
|
warning, summary, found := strings.Cut(afterWarning,
|
||||||
|
"Verification failed: 1 files did not match")
|
||||||
|
require.Truef(t, found, "no verification summary in:\n%s", out.String())
|
||||||
|
|
||||||
|
for path := range backup.lost {
|
||||||
|
assert.Equalf(t, 1, strings.Count(warning, path),
|
||||||
|
"%s must be listed once as failed to restore", path)
|
||||||
|
assert.NotContainsf(t, summary, path,
|
||||||
|
"%s was not restored and must not fail verification", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NotContainsf(t, warning, long,
|
||||||
|
"%s was restored and must not be listed as failed to restore", long)
|
||||||
|
assert.Equalf(t, 1, strings.Count(summary, long),
|
||||||
|
"%s must be listed once as failing verification", long)
|
||||||
|
}
|
||||||
|
|
||||||
|
// trailingByteFs appends one byte, on close, to the file it opens whose
|
||||||
|
// path ends in suffix, so that file is restored one byte too long and
|
||||||
|
// fails verification.
|
||||||
|
type trailingByteFs struct {
|
||||||
|
afero.Fs
|
||||||
|
|
||||||
|
suffix string
|
||||||
|
}
|
||||||
|
|
||||||
|
//nolint:ireturn // afero.Fs.OpenFile is defined to return the interface
|
||||||
|
func (fs trailingByteFs) OpenFile(
|
||||||
|
name string, flag int, perm os.FileMode,
|
||||||
|
) (afero.File, error) {
|
||||||
|
f, err := fs.Fs.OpenFile(name, flag, perm)
|
||||||
|
if err != nil || !strings.HasSuffix(name, fs.suffix) {
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return trailingByteFile{File: f}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type trailingByteFile struct {
|
||||||
|
afero.File
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f trailingByteFile) Close() error {
|
||||||
|
_, _ = f.File.Write([]byte{'x'})
|
||||||
|
|
||||||
|
return f.File.Close()
|
||||||
|
}
|
||||||
|
|
||||||
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
|
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
|
||||||
// still ends the restore with an error when SkipErrors is not set.
|
// still ends the restore with an error when SkipErrors is not set.
|
||||||
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
|
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user