Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a94d2f8795 | ||
|
|
8c54e0ff75 | ||
|
|
d1213ecb34 | ||
|
|
5f6455ba17 | ||
|
|
0288735aaf |
@@ -433,7 +433,10 @@ object store. Credentials are read from `s3.access_key_id` and
|
|||||||
`s3.secret_access_key` in the config file.
|
`s3.secret_access_key` in the config file.
|
||||||
|
|
||||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
||||||
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
a local or mounted filesystem. Useful for testing or backing up to a NAS. The
|
||||||
|
path must be absolute; a relative one is refused when the config loads.
|
||||||
|
Percent-escapes in the path are decoded, so `my%20stick` and `my stick` name
|
||||||
|
the same directory.
|
||||||
|
|
||||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||||
providers. Requires rclone to be configured separately (`rclone config`).
|
providers. Requires rclone to be configured separately (`rclone config`).
|
||||||
|
|||||||
@@ -245,6 +245,17 @@ func (p *Packer) Flush(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
||||||
|
// packed, if any, and drops that blob. A caller that stops before Flush
|
||||||
|
// must call it, or the temp file is left behind. Thread-safe.
|
||||||
|
func (p *Packer) DiscardCurrentBlob() {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
|
p.cleanupTempFile()
|
||||||
|
p.currentBlob = nil
|
||||||
|
}
|
||||||
|
|
||||||
// FinalizeBlob finalizes the current blob being assembled.
|
// FinalizeBlob finalizes the current blob being assembled.
|
||||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||||
// the content-addressed hash. The finalized blob is either passed to the
|
// the content-addressed hash. The finalized blob is either passed to the
|
||||||
|
|||||||
@@ -251,6 +251,12 @@ func NewConfigCommand() *cobra.Command {
|
|||||||
Use: "config",
|
Use: "config",
|
||||||
Short: "Manage the configuration file",
|
Short: "Manage the configuration file",
|
||||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(newConfigInitCommand())
|
cmd.AddCommand(newConfigInitCommand())
|
||||||
|
|||||||
@@ -15,6 +15,12 @@ func NewDatabaseCommand() *cobra.Command {
|
|||||||
Use: "database",
|
Use: "database",
|
||||||
Short: "Manage the local state database",
|
Short: "Manage the local state database",
|
||||||
Long: `Commands for managing the local SQLite state database.`,
|
Long: `Commands for managing the local SQLite state database.`,
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package cli //nolint:testpackage // shares runEntry and the command-word constants
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
|
||||||
|
// mistyped subcommand, as a crontab line might carry one. It must fail
|
||||||
|
// with exit 1 and name the word, as `vaultik bogus` does at the top
|
||||||
|
// level, instead of printing the group's help and exiting 0 having done
|
||||||
|
// nothing. The group name alone still prints its help.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||||
|
func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
group string
|
||||||
|
mistyped string
|
||||||
|
}{
|
||||||
|
{group: cmdSnapshot, mistyped: "craete"},
|
||||||
|
{group: "config", mistyped: "inti"},
|
||||||
|
{group: cmdRemote, mistyped: "ifno"},
|
||||||
|
{group: "database", mistyped: "delte"},
|
||||||
|
{group: "completion", mistyped: "nosuchshell"},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.group, func(t *testing.T) {
|
||||||
|
code, stdout, stderr := runEntry(t,
|
||||||
|
testCase.group, testCase.mistyped)
|
||||||
|
|
||||||
|
assert.Equal(t, 1, code)
|
||||||
|
assert.Empty(t, stdout)
|
||||||
|
assert.Contains(t, stderr,
|
||||||
|
`unknown command "`+testCase.mistyped+`"`)
|
||||||
|
|
||||||
|
code, stdout, _ = runEntry(t, testCase.group)
|
||||||
|
|
||||||
|
assert.Equal(t, 0, code)
|
||||||
|
assert.Contains(t, stdout,
|
||||||
|
programName+" "+testCase.group+" [command]")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,12 @@ func NewRemoteCommand() *cobra.Command {
|
|||||||
Use: "remote",
|
Use: "remote",
|
||||||
Short: "Remote storage management commands",
|
Short: "Remote storage management commands",
|
||||||
Long: "Commands for inspecting and managing remote storage",
|
Long: "Commands for inspecting and managing remote storage",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -74,6 +74,21 @@ on the source system.`,
|
|||||||
NewDatabaseCommand(),
|
NewDatabaseCommand(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// cobra adds the completion command only when Execute runs, and with
|
||||||
|
// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
|
||||||
|
// printed help and exited 0. Adding it here lets it have a Run. Its
|
||||||
|
// scripts go to os.Stdout as it is now: a later SetOut does not
|
||||||
|
// redirect them.
|
||||||
|
cmd.InitDefaultCompletionCmd()
|
||||||
|
|
||||||
|
for _, sub := range cmd.Commands() {
|
||||||
|
if sub.Name() == "completion" {
|
||||||
|
sub.Run = func(completion *cobra.Command, _ []string) {
|
||||||
|
_ = completion.Help()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,6 +41,12 @@ func NewSnapshotCommand() *cobra.Command {
|
|||||||
Use: "snapshot",
|
Use: "snapshot",
|
||||||
Short: "Snapshot management commands",
|
Short: "Snapshot management commands",
|
||||||
Long: "Commands for creating, listing, and managing snapshots",
|
Long: "Commands for creating, listing, and managing snapshots",
|
||||||
|
// cobra checks Args only on a command that has a Run: without
|
||||||
|
// one, a mistyped subcommand printed help and exited 0.
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Help()
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -61,6 +61,9 @@ var (
|
|||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
errBadStorageScheme = errors.New(
|
errBadStorageScheme = errors.New(
|
||||||
"storage_url must start with s3://, file://, or rclone://")
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
|
errFileURLNotAbsolute = errors.New(
|
||||||
|
"a file:// storage_url needs an absolute path, " +
|
||||||
|
"as in file:///path/to/backup")
|
||||||
errStorageNotConfigured = errors.New(
|
errStorageNotConfigured = errors.New(
|
||||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||||
"s3.bucket + credentials")
|
"s3.bucket + credentials")
|
||||||
@@ -461,11 +464,18 @@ func (c *Config) validateStorage() error {
|
|||||||
|
|
||||||
// validateStorageURL validates URL-based storage configuration. File and
|
// validateStorageURL validates URL-based storage configuration. File and
|
||||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||||
// credential fields.
|
// credential fields. A file URL must name an absolute path: a relative one
|
||||||
|
// would resolve against the working directory, so the destination would
|
||||||
|
// move with it.
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
switch {
|
switch {
|
||||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||||
// File storage doesn't need S3 credentials
|
// storage.ParseStorageURL applies the same rule, but config
|
||||||
|
// cannot import storage, which imports config.
|
||||||
|
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||||
|
return errFileURLNotAbsolute
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||||
// Rclone storage uses rclone's own config
|
// Rclone storage uses rclone's own config
|
||||||
|
|||||||
@@ -337,6 +337,35 @@ func TestLoadS3PartSize(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||||
|
// relative path fails at load. Accepted, it would resolve against the
|
||||||
|
// working directory, so a backup run from another directory would write
|
||||||
|
// to a different destination and still succeed.
|
||||||
|
func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const relativeConfig = "snapshots:\n" +
|
||||||
|
" test:\n" +
|
||||||
|
" paths: [/tmp/vaultik-test-source]\n" +
|
||||||
|
"storage_url: file://dest\n"
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
|
||||||
|
err := os.WriteFile(path, []byte(relativeConfig), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = Load(path)
|
||||||
|
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||||
|
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(err.Error(), "absolute path") {
|
||||||
|
t.Errorf("error %q does not say to use an absolute path", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||||
|
|||||||
@@ -531,3 +531,50 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
|||||||
require.Equal(t, 1, result.FilesScanned,
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
"Unanchored pattern for file should match anywhere")
|
"Unanchored pattern for file should match anywhere")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_DoubleStarDirectoryAtEveryDepth(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, a leading **/ also matches directly under the source dir
|
||||||
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/projectname"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// **/projectname should exclude BOTH:
|
||||||
|
// - /backup/projectname/file.txt
|
||||||
|
// - /backup/otherproject/projectname/file.txt
|
||||||
|
// Total files: 4, excluded: 2
|
||||||
|
require.Equal(t, 2, result.FilesScanned,
|
||||||
|
"**/ directory pattern should match at the top level and nested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, a leading **/ also matches directly under the source dir
|
||||||
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.txt"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// **/*.txt should exclude ALL .txt files:
|
||||||
|
// - /backup/file.txt
|
||||||
|
// - /backup/projectname/file.txt
|
||||||
|
// - /backup/otherproject/projectname/file.txt
|
||||||
|
// Total files: 4, excluded: 3
|
||||||
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
|
"**/ file pattern should match at the top level and nested")
|
||||||
|
}
|
||||||
|
|||||||
@@ -1295,6 +1295,11 @@ func (s *Scanner) batchAddFilesToSnapshot(
|
|||||||
func (s *Scanner) processPhase(
|
func (s *Scanner) processPhase(
|
||||||
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
||||||
) error {
|
) error {
|
||||||
|
// On an error or cancellation the blob being packed is never
|
||||||
|
// finalized, so its temp file must be removed here. After a
|
||||||
|
// successful flush there is no such blob and this does nothing.
|
||||||
|
defer s.packer.DiscardCurrentBlob()
|
||||||
|
|
||||||
// Calculate total bytes to process
|
// Calculate total bytes to process
|
||||||
var totalBytes int64
|
var totalBytes int64
|
||||||
for _, f := range filesToProcess {
|
for _, f := range filesToProcess {
|
||||||
@@ -2013,6 +2018,14 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
// Remove trailing slash if present (directory indicator)
|
// Remove trailing slash if present (directory indicator)
|
||||||
pattern = strings.TrimSuffix(pattern, "/")
|
pattern = strings.TrimSuffix(pattern, "/")
|
||||||
|
|
||||||
|
// A leading "**/" demands a "/" before the name and so misses an
|
||||||
|
// entry directly under the source path. An unanchored pattern
|
||||||
|
// already matches at every depth, so drop the prefix: as in
|
||||||
|
// gitignore, "**/x" then also matches a top-level "x".
|
||||||
|
if !anchored {
|
||||||
|
pattern = strings.TrimPrefix(pattern, "**/")
|
||||||
|
}
|
||||||
|
|
||||||
// Compile the glob pattern
|
// Compile the glob pattern
|
||||||
// For patterns without path separators, we need to match them as components
|
// For patterns without path separators, we need to match them as components
|
||||||
// e.g., ".git" should match ".git" anywhere in the path
|
// e.g., ".git" should match ".git" anywhere in the path
|
||||||
|
|||||||
@@ -381,3 +381,60 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
|
||||||
|
// part-way, by a file it cannot read or by a cancellation, removes the
|
||||||
|
// temp file of the blob it was packing.
|
||||||
|
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// a.txt is backed up first and starts a blob; the run stops at b.txt.
|
||||||
|
const target = "/source/b.txt"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
newFs func(cancel context.CancelFunc) afero.Fs
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file cannot be read",
|
||||||
|
newFs: func(context.CancelFunc) afero.Fs {
|
||||||
|
return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cancelled",
|
||||||
|
newFs: func(cancel context.CancelFunc) afero.Fs {
|
||||||
|
return &cancelOnOpenFs{
|
||||||
|
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
fs := tt.newFs(cancel)
|
||||||
|
writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
|
||||||
|
writeSkipErrorTestFile(t, fs, target, "second file content")
|
||||||
|
|
||||||
|
_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected the scan to stop with an error, got nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listing blob temp files: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(left) != 0 {
|
||||||
|
t.Fatalf("expected no blob temp file left, got %v", left)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+26
-12
@@ -17,8 +17,10 @@ const (
|
|||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
var (
|
var (
|
||||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||||
|
ErrRelativeFilePath = errors.New(
|
||||||
|
"file URL needs an absolute path, as in file:///path/to/backup")
|
||||||
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
@@ -51,17 +53,8 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle file:// URLs
|
|
||||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||||
path := after
|
return parseFileURL(after)
|
||||||
if path == "" {
|
|
||||||
return nil, ErrEmptyFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
return &URL{
|
|
||||||
Scheme: schemeFile,
|
|
||||||
Prefix: path,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(rawURL, "s3://") {
|
if strings.HasPrefix(rawURL, "s3://") {
|
||||||
@@ -75,6 +68,27 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrUnsupportedScheme
|
return nil, ErrUnsupportedScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseFileURL parses the path that follows file://. The path must be
|
||||||
|
// absolute: a relative one would resolve against the working directory,
|
||||||
|
// so the destination would move with it. Percent-escapes are decoded, so
|
||||||
|
// my%20stick names the directory "my stick"; a literal space is kept.
|
||||||
|
func parseFileURL(rawPath string) (*URL, error) {
|
||||||
|
if rawPath == "" {
|
||||||
|
return nil, ErrEmptyFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(rawPath, "/") {
|
||||||
|
return nil, ErrRelativeFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err := url.PathUnescape(rawPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
// the userinfo and any query parameter other than endpoint, region and
|
// the userinfo and any query parameter other than endpoint, region and
|
||||||
// ssl, so a credential-bearing URL is never stored or echoed.
|
// ssl, so a credential-bearing URL is never stored or echoed.
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ import (
|
|||||||
func TestParseStorageURLValid(t *testing.T) {
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const bucket = "mybucket"
|
const (
|
||||||
|
bucket = "mybucket"
|
||||||
|
fileScheme = "file"
|
||||||
|
)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -24,7 +27,17 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file percent-escaped path is decoded",
|
||||||
|
raw: "file:///Volumes/my%20stick/backup",
|
||||||
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file path with a literal space",
|
||||||
|
raw: "file:///Volumes/my stick/backup",
|
||||||
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
@@ -90,6 +103,7 @@ func TestParseStorageURLErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"empty url", "", storage.ErrEmptyStorageURL},
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
|
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
||||||
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
|
|||||||
@@ -309,11 +309,22 @@ func (v *Vaultik) restoreAllFiles(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
|
plan, unplanned, err := newRestorePlan(
|
||||||
|
v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("building restore plan: %w", err)
|
return nil, fmt.Errorf("building restore plan: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ranging over files rather than the map reports them in a stable order.
|
||||||
|
for _, file := range files {
|
||||||
|
if planErr, ok := unplanned[file.ID]; ok {
|
||||||
|
err = v.handleRestoreFileError(plan, opts, result, file, file.ID, planErr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
filesByID, totalBytesExpected := indexRestoreFiles(files)
|
filesByID, totalBytesExpected := indexRestoreFiles(files)
|
||||||
|
|
||||||
v.UI.Beginf("Restoring %s files (%s) to %s.",
|
v.UI.Beginf("Restoring %s files (%s) to %s.",
|
||||||
|
|||||||
@@ -61,8 +61,8 @@ func craftedFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
|
// TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
|
||||||
// has no blob_chunks row fails restore planning and the chunk write with
|
// has no blob_chunks row is left out of the restore plan with an error,
|
||||||
// an error.
|
// and fails the chunk write with an error.
|
||||||
func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
|
func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -77,9 +77,11 @@ func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
|
|||||||
chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
|
chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
_, err = newRestorePlan(ctx, repos, []*database.File{file},
|
plan, unplanned, err := newRestorePlan(ctx, repos, []*database.File{file},
|
||||||
chunkToBlobMap, map[string]string{})
|
chunkToBlobMap, map[string]string{})
|
||||||
require.ErrorIs(t, err, errPlanChunkMissing)
|
require.NoError(t, err)
|
||||||
|
require.ErrorIs(t, unplanned[file.ID], errPlanChunkMissing)
|
||||||
|
require.False(t, plan.hasPending())
|
||||||
|
|
||||||
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
|
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -41,20 +41,25 @@ type restorePlan struct {
|
|||||||
|
|
||||||
// newRestorePlan builds the file→blob index for the given files. Files
|
// newRestorePlan builds the file→blob index for the given files. Files
|
||||||
// whose chunks reference no blobs (symlinks, directories) start in the
|
// whose chunks reference no blobs (symlinks, directories) start in the
|
||||||
// ready queue immediately.
|
// ready queue immediately. A file with a chunk that no blob holds cannot
|
||||||
|
// be restored: it is left out of the plan and returned, by ID, with the
|
||||||
|
// reason.
|
||||||
func newRestorePlan(
|
func newRestorePlan(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
repos *database.Repositories,
|
repos *database.Repositories,
|
||||||
files []*database.File,
|
files []*database.File,
|
||||||
chunkToBlobMap map[string]*database.BlobChunk,
|
chunkToBlobMap map[string]*database.BlobChunk,
|
||||||
blobIDToHash map[string]string,
|
blobIDToHash map[string]string,
|
||||||
) (*restorePlan, error) {
|
) (*restorePlan, map[types.FileID]error, error) {
|
||||||
p := &restorePlan{
|
p := &restorePlan{
|
||||||
fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
|
fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
|
||||||
blobFiles: make(map[string]map[types.FileID]struct{}),
|
blobFiles: make(map[string]map[types.FileID]struct{}),
|
||||||
ready: make([]types.FileID, 0, len(files)),
|
ready: make([]types.FileID, 0, len(files)),
|
||||||
cached: make(map[string]struct{}),
|
cached: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
|
unplanned := make(map[types.FileID]error)
|
||||||
|
|
||||||
|
nextFile:
|
||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
|
if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
|
||||||
// No chunks to fetch — restore can run immediately.
|
// No chunks to fetch — restore can run immediately.
|
||||||
@@ -66,7 +71,7 @@ func newRestorePlan(
|
|||||||
|
|
||||||
fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
|
fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("planning %s: %w", f.Path, err)
|
return nil, nil, fmt.Errorf("planning %s: %w", f.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blobs := make(map[string]struct{})
|
blobs := make(map[string]struct{})
|
||||||
@@ -74,14 +79,18 @@ func newRestorePlan(
|
|||||||
for _, fc := range fileChunks {
|
for _, fc := range fileChunks {
|
||||||
bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
|
bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("planning %s: %w: %s",
|
unplanned[f.ID] = fmt.Errorf("%w: %s",
|
||||||
f.Path, errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
|
errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
|
||||||
|
|
||||||
|
continue nextFile
|
||||||
}
|
}
|
||||||
|
|
||||||
hash, ok := blobIDToHash[bc.BlobID.String()]
|
hash, ok := blobIDToHash[bc.BlobID.String()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("planning %s: %w: %s",
|
unplanned[f.ID] = fmt.Errorf("%w: %s",
|
||||||
f.Path, errPlanBlobIDMissing, bc.BlobID)
|
errPlanBlobIDMissing, bc.BlobID)
|
||||||
|
|
||||||
|
continue nextFile
|
||||||
}
|
}
|
||||||
|
|
||||||
blobs[hash] = struct{}{}
|
blobs[hash] = struct{}{}
|
||||||
@@ -103,7 +112,7 @@ func newRestorePlan(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return p, nil
|
return p, unplanned, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// markBlobCached records that the named blob is now resident in the
|
// markBlobCached records that the named blob is now resident in the
|
||||||
|
|||||||
@@ -28,7 +28,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// missingBlobBackup is a snapshot of single-chunk files spread over two
|
// missingBlobBackup is a snapshot of single-chunk files spread over two
|
||||||
// blobs, with one of those blobs deleted from the store.
|
// blobs, with one of those blobs lost: deleted from the store, or missing
|
||||||
|
// from the snapshot metadata.
|
||||||
type missingBlobBackup struct {
|
type missingBlobBackup struct {
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
cfg *config.Config
|
cfg *config.Config
|
||||||
@@ -37,20 +38,39 @@ type missingBlobBackup struct {
|
|||||||
restoreDir string
|
restoreDir string
|
||||||
// files holds the original content by source path.
|
// files holds the original content by source path.
|
||||||
files map[string][]byte
|
files map[string][]byte
|
||||||
// lost holds the source paths whose chunk is in the deleted blob.
|
// lost holds the source paths whose chunk is in the lost blob.
|
||||||
lost map[string]bool
|
lost map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
|
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
|
||||||
// after one blob of a two-blob snapshot was deleted. Every file stored in
|
// after one blob of a two-blob snapshot was deleted from the store.
|
||||||
// that blob must be reported as failed and left absent, every other file
|
|
||||||
// must be restored intact, and Restore must still return an error.
|
|
||||||
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
backup := backupThenDeleteOneBlob(ctx, t)
|
assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
|
||||||
|
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
|
||||||
|
// earlier backup defects left behind.
|
||||||
|
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
|
||||||
|
// stored in the lost blob must be reported as failed and left absent,
|
||||||
|
// every other file must be restored intact, and Restore must still return
|
||||||
|
// an error.
|
||||||
|
func assertRestoreSkipsLostFiles(
|
||||||
|
ctx context.Context, t *testing.T, backup *missingBlobBackup,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
|
|
||||||
@@ -72,14 +92,14 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
|||||||
|
|
||||||
if backup.lost[path] {
|
if backup.lost[path] {
|
||||||
assert.Containsf(t, out.String(), path,
|
assert.Containsf(t, out.String(), path,
|
||||||
"%s needs the deleted blob and must be reported", path)
|
"%s needs the lost blob and must be reported", path)
|
||||||
assert.NoFileExists(t, restored)
|
assert.NoFileExists(t, restored)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := afero.ReadFile(backup.fs, restored)
|
got, err := afero.ReadFile(backup.fs, restored)
|
||||||
require.NoErrorf(t, err, "%s does not need the deleted blob", path)
|
require.NoErrorf(t, err, "%s does not need the lost blob", path)
|
||||||
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,6 +123,30 @@ func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
|
|||||||
require.ErrorIs(t, err, storage.ErrNotFound)
|
require.ErrorIs(t, err, storage.ErrNotFound)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held
|
||||||
|
// by no blob still ends the restore, before any file is written, when
|
||||||
|
// SkipErrors is not set.
|
||||||
|
func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
backup := backupThenDropOneBlobFromMetadata(ctx, t)
|
||||||
|
|
||||||
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: backup.snapshotID,
|
||||||
|
TargetDir: backup.restoreDir,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.ErrorContains(t, err, "chunk missing from blob map")
|
||||||
|
|
||||||
|
for path := range backup.files {
|
||||||
|
assert.NoFileExists(t, filepath.Join(backup.restoreDir, path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
|
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
|
||||||
// files, then deletes from the store the blob holding the first of them.
|
// files, then deletes from the store the blob holding the first of them.
|
||||||
func backupThenDeleteOneBlob(
|
func backupThenDeleteOneBlob(
|
||||||
@@ -110,6 +154,54 @@ func backupThenDeleteOneBlob(
|
|||||||
) *missingBlobBackup {
|
) *missingBlobBackup {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf(
|
||||||
|
"blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)))
|
||||||
|
|
||||||
|
return backup
|
||||||
|
}
|
||||||
|
|
||||||
|
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
|
||||||
|
// single-chunk files, then removes the blob holding the first of them from
|
||||||
|
// the snapshot and exports the snapshot metadata again. The exported
|
||||||
|
// metadata then lists chunks that no blob holds.
|
||||||
|
func backupThenDropOneBlobFromMetadata(
|
||||||
|
ctx context.Context, t *testing.T,
|
||||||
|
) *missingBlobBackup {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
||||||
|
|
||||||
|
// The uploads row refers to the blob row, so it would keep the export
|
||||||
|
// from deleting it.
|
||||||
|
for _, query := range []string{
|
||||||
|
"DELETE FROM snapshot_blobs WHERE blob_hash = ?",
|
||||||
|
"DELETE FROM uploads WHERE blob_hash = ?",
|
||||||
|
} {
|
||||||
|
_, err := db.Conn().ExecContext(ctx, query, lostBlob)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sm := newFaultSnapshotManager(
|
||||||
|
backup.fs, backup.storer, backup.cfg, database.NewRepositories(db))
|
||||||
|
require.NoError(t,
|
||||||
|
sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID))
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
return backup
|
||||||
|
}
|
||||||
|
|
||||||
|
// backupSingleChunkFiles backs up missingBlobFileCount single-chunk files.
|
||||||
|
// It returns the backup, with lost naming the files stored in the same
|
||||||
|
// blob as the first of them, the still open local index, and the hash of
|
||||||
|
// that blob.
|
||||||
|
func backupSingleChunkFiles(
|
||||||
|
ctx context.Context, t *testing.T,
|
||||||
|
) (*missingBlobBackup, *database.DB, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
fs := afero.NewOsFs()
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
@@ -143,16 +235,11 @@ func backupThenDeleteOneBlob(
|
|||||||
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
|
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, db.Close())
|
lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
|
||||||
|
|
||||||
deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
|
|
||||||
require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
|
|
||||||
"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
|
|
||||||
|
|
||||||
lost := make(map[string]bool)
|
lost := make(map[string]bool)
|
||||||
|
|
||||||
for path, hash := range blobOfFile {
|
for path, hash := range blobOfFile {
|
||||||
if hash == deleted {
|
if hash == lostBlob {
|
||||||
lost[path] = true
|
lost[path] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -168,7 +255,7 @@ func backupThenDeleteOneBlob(
|
|||||||
restoreDir: filepath.Join(tempDir, "restored"),
|
restoreDir: filepath.Join(tempDir, "restored"),
|
||||||
files: files,
|
files: files,
|
||||||
lost: lost,
|
lost: lost,
|
||||||
}
|
}, db, lostBlob
|
||||||
}
|
}
|
||||||
|
|
||||||
// blobHashOfSingleChunkFile returns the hash of the blob holding the one
|
// blobHashOfSingleChunkFile returns the hash of the blob holding the one
|
||||||
|
|||||||
Reference in New Issue
Block a user