Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
95add7fc38 |
No files matched your search
@@ -169,7 +169,7 @@ vaultik [--config <path>] config init
|
|||||||
vaultik [--config <path>] config edit
|
vaultik [--config <path>] config edit
|
||||||
vaultik [--config <path>] config get <key>
|
vaultik [--config <path>] config get <key>
|
||||||
vaultik [--config <path>] config set <key> <value>
|
vaultik [--config <path>] config set <key> <value>
|
||||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
|
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>] [--dry-run]
|
||||||
vaultik [--config <path>] snapshot list [--json] # alias: ls
|
vaultik [--config <path>] snapshot list [--json] # alias: ls
|
||||||
vaultik [--config <path>] snapshot files <snapshot-id> [path] [--json]
|
vaultik [--config <path>] snapshot files <snapshot-id> [path] [--json]
|
||||||
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
||||||
@@ -369,6 +369,17 @@ In such a config, an unknown key or a value of the wrong type is refused.
|
|||||||
duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`). Without
|
duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`). Without
|
||||||
`--prune`, or with a value that is not a valid duration, the command fails
|
`--prune`, or with a value that is not a valid duration, the command fails
|
||||||
before the backup starts.
|
before the backup starts.
|
||||||
|
- `--dry-run`: Scan the source paths and print what a backup would store,
|
||||||
|
without storing it. The summary gives the files examined, to back up and
|
||||||
|
unchanged, their sizes, and the count of excluded entries. An excluded
|
||||||
|
directory counts once, since the scan does not enter it. With `--verbose`, it
|
||||||
|
also prints each excluded entry and the exclude pattern that matched it. It
|
||||||
|
writes nothing to the destination and records nothing in the local index, so
|
||||||
|
it does not bind the index to the destination. Like the other commands that
|
||||||
|
read the local index, it creates an empty one if there is none. When the local
|
||||||
|
index holds an incomplete snapshot from an interrupted backup, it says so: the
|
||||||
|
next backup drops that snapshot first and may back up more files than the dry
|
||||||
|
run counts. It cannot be combined with `--prune`.
|
||||||
- Fails before it writes anything when a `file://` destination directory does
|
- Fails before it writes anything when a `file://` destination directory does
|
||||||
not exist, or when the local index holds snapshots and the destination answers
|
not exist, or when the local index holds snapshots and the destination answers
|
||||||
that it has no `vaultik-destination.txt`. When that look-up fails for any
|
that it has no `vaultik-destination.txt`. When that look-up fails for any
|
||||||
|
|||||||
+4
-4
@@ -39,10 +39,10 @@ const (
|
|||||||
// hold the PID lock so that at most one runs at a time.
|
// hold the PID lock so that at most one runs at a time.
|
||||||
mutating lockMode = iota
|
mutating lockMode = iota
|
||||||
// readOnly commands (info, snapshot list, snapshot files, snapshot
|
// readOnly commands (info, snapshot list, snapshot files, snapshot
|
||||||
// verify, remote info, snapshot restore) do not write the local index
|
// verify, remote info, snapshot restore, snapshot create --dry-run)
|
||||||
// or the remote store, so they run without the lock and are never
|
// do not write the local index or the remote store, so they run
|
||||||
// blocked by a running mutator. restore writes only to the target
|
// without the lock and are never blocked by a running mutator.
|
||||||
// directory it is given.
|
// restore writes only to the target directory it is given.
|
||||||
readOnly
|
readOnly
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -282,13 +282,6 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
|||||||
storageURLKey, testS3URL,
|
storageURLKey, testS3URL,
|
||||||
"s3.secret_access_key is required for s3:// URLs; " +
|
"s3.secret_access_key is required for s3:// URLs; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key first"},
|
"set s3.access_key_id and s3.secret_access_key first"},
|
||||||
// rclone:// URLs that no command could open.
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone:///backups",
|
|
||||||
"rclone URL missing remote name"},
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
|
|
||||||
"rclone URL must not carry parameters in the remote name"},
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
|
|
||||||
`unknown query parameter in storage URL: "x"`},
|
|
||||||
// With its reference filled in, the config loads.
|
// With its reference filled in, the config loads.
|
||||||
{levelReference, "blob_size_limit", "1MB",
|
{levelReference, "blob_size_limit", "1MB",
|
||||||
"blob_size_limit must be at least"},
|
"blob_size_limit must be at least"},
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEntrySnapshotCreateDryRun runs `snapshot create --dry-run
|
||||||
|
// --verbose` on a source directory that holds one kept file, one file an
|
||||||
|
// unanchored pattern excludes and one directory an anchored pattern
|
||||||
|
// excludes. The summary counts the kept file and two excluded entries,
|
||||||
|
// the excluded ones are listed with their patterns, and nothing is
|
||||||
|
// written to the destination or recorded in the local index. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/436.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntrySnapshotCreateDryRun(t *testing.T) {
|
||||||
|
configPath, storeDir := writeBackupConfig(t)
|
||||||
|
sourceDir := filepath.Join(filepath.Dir(configPath), "source")
|
||||||
|
cacheDir := filepath.Join(sourceDir, "cache")
|
||||||
|
|
||||||
|
require.NoError(t, os.WriteFile(
|
||||||
|
filepath.Join(sourceDir, "debug.log"), []byte("log"), 0o600))
|
||||||
|
require.NoError(t, os.Mkdir(cacheDir, 0o750))
|
||||||
|
|
||||||
|
for _, name := range []string{"a", "b"} {
|
||||||
|
require.NoError(t, os.WriteFile(
|
||||||
|
filepath.Join(cacheDir, name), []byte(name), 0o600))
|
||||||
|
}
|
||||||
|
|
||||||
|
out := ui.NewWithColor(io.Discard, false)
|
||||||
|
require.NoError(t, writeConfigSet(out, configPath, "exclude.0", "*.log"))
|
||||||
|
require.NoError(t, writeConfigSet(out, configPath, "exclude.1", "/cache"))
|
||||||
|
|
||||||
|
code, stdout, stderr := runEntry(t, flagConfig, configPath, "--verbose",
|
||||||
|
cmdSnapshot, cmdCreate, "--dry-run")
|
||||||
|
|
||||||
|
require.Equal(t, 0, code, stderr)
|
||||||
|
assert.Contains(t, stdout, "Excluded "+filepath.Join(sourceDir, "debug.log")+
|
||||||
|
`: matches exclude pattern "*.log".`)
|
||||||
|
assert.Contains(t, stdout, "Excluded "+cacheDir+
|
||||||
|
`: matches exclude pattern "/cache".`)
|
||||||
|
assert.NotContains(t, stdout, filepath.Join(cacheDir, "a"))
|
||||||
|
assert.Contains(t, stdout, "Files: 1 examined, 1 to back up, 0 unchanged.")
|
||||||
|
assert.Contains(t, stdout, "Excluded: 2 entries.")
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, entries, "a dry run writes nothing to the destination")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
db, err := database.New(ctx,
|
||||||
|
filepath.Join(filepath.Dir(configPath), "index.sqlite"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, db.Close()) }()
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
|
snapshots, err := repos.Snapshots.ListRecent(ctx, 1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, snapshots, "a dry run records no snapshot")
|
||||||
|
|
||||||
|
binding, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, binding, "a dry run does not bind the local index")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEntrySnapshotCreateDryRunRefusesPrune checks that --dry-run with
|
||||||
|
// --prune fails before anything runs.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntrySnapshotCreateDryRunRefusesPrune(t *testing.T) {
|
||||||
|
configPath, _ := writeBackupConfig(t)
|
||||||
|
|
||||||
|
code, _, stderr := runEntry(t, flagConfig, configPath,
|
||||||
|
cmdSnapshot, cmdCreate, "--dry-run", "--prune")
|
||||||
|
|
||||||
|
assert.NotEqual(t, 0, code)
|
||||||
|
assert.Contains(t, stderr, "--dry-run cannot be used with --prune")
|
||||||
|
}
|
||||||
+35
-19
@@ -99,6 +99,13 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
|
|
||||||
// Use the backup functionality from cli package
|
// Use the backup functionality from cli package
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
opts.ListExcluded = rootFlags.Verbose
|
||||||
|
|
||||||
|
// A dry run writes nothing, so it does not take the lock.
|
||||||
|
mode := mutating
|
||||||
|
if opts.DryRun {
|
||||||
|
mode = readOnly
|
||||||
|
}
|
||||||
|
|
||||||
// --cron suppression is wired through v.UI by setupGlobals.
|
// --cron suppression is wired through v.UI by setupGlobals.
|
||||||
return RunOperation(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
@@ -109,28 +116,11 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
Cron: opts.Cron,
|
Cron: opts.Cron,
|
||||||
Quiet: rootFlags.Quiet,
|
Quiet: rootFlags.Quiet,
|
||||||
},
|
},
|
||||||
Mode: mutating,
|
Mode: mode,
|
||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.CreateSnapshot(opts)
|
return v.CreateSnapshot(opts)
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
reportSnapshotCreateFailure(opts.DryRun, err)
|
||||||
log.Info("Post-backup prune failed", "error", err)
|
|
||||||
ReportErrorf("Backup completed, but %v", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
|
||||||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
|
||||||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
|
|
||||||
log.Info("Backup completed with an error", "error", err)
|
|
||||||
ReportErrorf("Backup completed, but %v", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("Snapshot creation failed", "error", err)
|
|
||||||
ReportErrorf("Snapshot creation failed: %v", err)
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -149,10 +139,36 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
"With --prune: keep snapshots newer than this duration "+
|
"With --prune: keep snapshots newer than this duration "+
|
||||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||||
"instead of only the latest")
|
"instead of only the latest")
|
||||||
|
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
|
||||||
|
"Scan the snapshot source paths and print what a backup would store, "+
|
||||||
|
"writing nothing to the local index or the backup destination; "+
|
||||||
|
"with --verbose, also print each excluded entry and the exclude "+
|
||||||
|
"pattern that matched it")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportSnapshotCreateFailure shows the user the error `snapshot create`
|
||||||
|
// failed with. dryRun says whether it was given --dry-run.
|
||||||
|
func reportSnapshotCreateFailure(dryRun bool, err error) {
|
||||||
|
switch {
|
||||||
|
case dryRun:
|
||||||
|
log.Info("Dry run failed", "error", err)
|
||||||
|
ReportErrorf("Dry run failed: %v", err)
|
||||||
|
case errors.Is(err, vaultik.ErrPostBackupPruneFailed):
|
||||||
|
log.Info("Post-backup prune failed", "error", err)
|
||||||
|
ReportErrorf("Backup completed, but %v", err)
|
||||||
|
case errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
||||||
|
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
||||||
|
errors.Is(err, vaultik.ErrVolumeNotMounted):
|
||||||
|
log.Info("Backup completed with an error", "error", err)
|
||||||
|
ReportErrorf("Backup completed, but %v", err)
|
||||||
|
default:
|
||||||
|
log.Info("Snapshot creation failed", "error", err)
|
||||||
|
ReportErrorf("Snapshot creation failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// newSnapshotListCommand creates the 'snapshot list' subcommand
|
// newSnapshotListCommand creates the 'snapshot list' subcommand
|
||||||
func newSnapshotListCommand() *cobra.Command {
|
func newSnapshotListCommand() *cobra.Command {
|
||||||
var jsonOutput bool
|
var jsonOutput bool
|
||||||
|
|||||||
+33
-20
@@ -21,7 +21,6 @@ import (
|
|||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const appName = "vaultik"
|
const appName = "vaultik"
|
||||||
@@ -64,8 +63,13 @@ var (
|
|||||||
errBlobSizeTooSmall = errors.New(
|
errBlobSizeTooSmall = errors.New(
|
||||||
"blob_size_limit must be at least the largest chunk the chunker can " +
|
"blob_size_limit must be at least the largest chunk the chunker can " +
|
||||||
"emit (chunk_size times the FastCDC size spread)")
|
"emit (chunk_size times the FastCDC size spread)")
|
||||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
|
errBadStorageScheme = errors.New(
|
||||||
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
|
errFileURLNotAbsolute = errors.New(
|
||||||
|
"a file:// storage_url needs an absolute path, " +
|
||||||
|
"as in file:///path/to/backup")
|
||||||
errIndexPathNotAbsolute = errors.New(
|
errIndexPathNotAbsolute = errors.New(
|
||||||
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
||||||
"as in ~/.local/share/vaultik/index.sqlite")
|
"as in ~/.local/share/vaultik/index.sqlite")
|
||||||
@@ -553,29 +557,38 @@ func (c *Config) validateStorage() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateStorageURL checks storage_url with the same parser that opens
|
// validateStorageURL validates URL-based storage configuration. File and
|
||||||
// the destination, so a value every command would refuse fails at load.
|
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||||
// File and rclone URLs need no credentials; S3 URLs require the legacy
|
// credential fields. A file URL must name an absolute path: a relative one
|
||||||
// s3.* credential fields.
|
// would resolve against the working directory, so the destination would
|
||||||
|
// move with it.
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
parsed, err := storageurl.Parse(c.StorageURL)
|
switch {
|
||||||
if err != nil {
|
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||||
return fmt.Errorf("invalid storage_url: %w", err)
|
// storage.ParseStorageURL applies the same rule, but config
|
||||||
}
|
// cannot import storage, which imports config.
|
||||||
|
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||||
|
return errFileURLNotAbsolute
|
||||||
|
}
|
||||||
|
|
||||||
if parsed.Scheme != storageurl.SchemeS3 {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||||
|
// Rclone storage uses rclone's own config
|
||||||
|
return nil
|
||||||
|
case strings.HasPrefix(c.StorageURL, "s3://"):
|
||||||
|
// S3 storage needs credentials
|
||||||
|
if c.S3.AccessKeyID == "" {
|
||||||
|
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
||||||
|
}
|
||||||
|
|
||||||
if c.S3.AccessKeyID == "" {
|
if c.S3.SecretAccessKey == "" {
|
||||||
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.SecretAccessKey == "" {
|
return nil
|
||||||
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
|
default:
|
||||||
|
return errBadStorageScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module exports the config module for fx dependency injection.
|
// Module exports the config module for fx dependency injection.
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -503,8 +502,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err = Load(path)
|
_, err = Load(path)
|
||||||
if !errors.Is(err, storageurl.ErrRelativeFilePath) {
|
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||||
t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
|
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "absolute path") {
|
if !strings.Contains(err.Error(), "absolute path") {
|
||||||
|
|||||||
+103
-13
@@ -71,7 +71,10 @@ type Scanner struct {
|
|||||||
// indexAndLockFiles holds the paths of vaultik's own working files,
|
// indexAndLockFiles holds the paths of vaultik's own working files,
|
||||||
// which the scan treats as excluded (ScannerConfig.IndexAndLockFiles).
|
// which the scan treats as excluded (ScannerConfig.IndexAndLockFiles).
|
||||||
indexAndLockFiles map[string]struct{}
|
indexAndLockFiles map[string]struct{}
|
||||||
progress *ProgressReporter
|
// listExcluded prints each excluded entry and why it is excluded
|
||||||
|
// (DryRun).
|
||||||
|
listExcluded bool
|
||||||
|
progress *ProgressReporter
|
||||||
// skipErrors skips files that cannot be opened or read (logged loudly);
|
// skipErrors skips files that cannot be opened or read (logged loudly);
|
||||||
// packer, database, encryption, and upload errors still abort the run.
|
// packer, database, encryption, and upload errors still abort the run.
|
||||||
skipErrors bool
|
skipErrors bool
|
||||||
@@ -190,6 +193,11 @@ type ScanResult struct {
|
|||||||
// itself when it is a directory. It is zero for an empty directory,
|
// itself when it is a directory. It is zero for an empty directory,
|
||||||
// such as the mount point of a volume that is not mounted.
|
// such as the mount point of a volume that is not mounted.
|
||||||
EntriesFound int
|
EntriesFound int
|
||||||
|
|
||||||
|
// EntriesExcluded counts the entries the walk found that an exclude
|
||||||
|
// pattern matches or that are vaultik's own files. An excluded
|
||||||
|
// directory counts once: the walk does not enter it.
|
||||||
|
EntriesExcluded int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new scanner instance
|
// NewScanner creates a new scanner instance
|
||||||
@@ -360,6 +368,42 @@ func (s *Scanner) Scan(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DryRun walks path as Scan does and counts what a backup would store,
|
||||||
|
// without storing it. It reads the local index to tell unchanged files
|
||||||
|
// from new and changed ones, and writes nothing to the local index or
|
||||||
|
// the backup destination store. With listExcluded it prints each
|
||||||
|
// excluded entry and the exclude pattern that matched it.
|
||||||
|
func (s *Scanner) DryRun(
|
||||||
|
ctx context.Context, path string, listExcluded bool,
|
||||||
|
) (*ScanResult, error) {
|
||||||
|
s.currentSourcePath = path
|
||||||
|
s.listExcluded = listExcluded
|
||||||
|
result := &ScanResult{
|
||||||
|
StartTime: time.Now().UTC(),
|
||||||
|
}
|
||||||
|
|
||||||
|
knownFiles, err := s.loadDatabaseState(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
existingFiles := make(map[string]struct{})
|
||||||
|
|
||||||
|
_, err = s.scanPhase(ctx, path, result, existingFiles, knownFiles)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scan phase failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.detectDeletedFiles(ctx, path, existingFiles, result)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("detecting deleted files: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.EndTime = time.Now().UTC()
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetProgress returns the progress reporter for this scanner, or nil when
|
// GetProgress returns the progress reporter for this scanner, or nil when
|
||||||
// progress is off. Scan neither starts nor stops it: the caller does,
|
// progress is off. Scan neither starts nor stops it: the caller does,
|
||||||
// once for all the paths it scans, because a second Stop panics.
|
// once for all the paths it scans, because a second Stop panics.
|
||||||
@@ -944,12 +988,8 @@ func (s *Scanner) scanPhase(
|
|||||||
// or cannot be looked up is neither counted nor reported. For
|
// or cannot be looked up is neither counted nor reported. For
|
||||||
// directories, skip the entire subtree. info is nil when the
|
// directories, skip the entire subtree. info is nil when the
|
||||||
// walk could not look the entry up.
|
// walk could not look the entry up.
|
||||||
if s.shouldExclude(filePath, path) {
|
if excluded, walkErr := s.skipExcluded(filePath, path, info, result); excluded {
|
||||||
if info != nil && info.IsDir() {
|
return walkErr
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The source path is in no directory listing, so its absence is
|
// The source path is in no directory listing, so its absence is
|
||||||
@@ -1016,6 +1056,32 @@ func (s *Scanner) scanPhase(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// skipExcluded reports whether the walk skips filePath, under rootPath,
|
||||||
|
// because it is excluded, and counts it if so. The error is what the walk
|
||||||
|
// function returns for it: filepath.SkipDir for a directory, so the walk
|
||||||
|
// does not enter it, and nil otherwise. info is nil when the walk could
|
||||||
|
// not look the entry up.
|
||||||
|
func (s *Scanner) skipExcluded(
|
||||||
|
filePath, rootPath string, info os.FileInfo, result *ScanResult,
|
||||||
|
) (bool, error) {
|
||||||
|
pattern, excluded := s.excludingPattern(filePath, rootPath)
|
||||||
|
if !excluded {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result.EntriesExcluded++
|
||||||
|
|
||||||
|
if s.listExcluded {
|
||||||
|
s.printExcluded(filePath, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
if info != nil && info.IsDir() {
|
||||||
|
return true, filepath.SkipDir
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
// countNonDirectories returns how many of files are not directories.
|
// countNonDirectories returns how many of files are not directories.
|
||||||
func countNonDirectories(files map[string]*database.File) int64 {
|
func countNonDirectories(files map[string]*database.File) int64 {
|
||||||
var count int64
|
var count int64
|
||||||
@@ -2351,23 +2417,32 @@ func compileExcludePatterns(patterns []string) ([]compiledPattern, error) {
|
|||||||
// filePath is the full path to the file
|
// filePath is the full path to the file
|
||||||
// rootPath is the root of the backup source directory
|
// rootPath is the root of the backup source directory
|
||||||
func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
|
func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
|
||||||
|
_, excluded := s.excludingPattern(filePath, rootPath)
|
||||||
|
|
||||||
|
return excluded
|
||||||
|
}
|
||||||
|
|
||||||
|
// excludingPattern reports whether shouldExclude excludes filePath and
|
||||||
|
// returns the exclude pattern, as configured, that matches it. The
|
||||||
|
// pattern is "" for one of vaultik's own working files.
|
||||||
|
func (s *Scanner) excludingPattern(filePath, rootPath string) (string, bool) {
|
||||||
if _, ok := s.indexAndLockFiles[filePath]; ok {
|
if _, ok := s.indexAndLockFiles[filePath]; ok {
|
||||||
return true
|
return "", true
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(s.compiledExclude) == 0 {
|
if len(s.compiledExclude) == 0 {
|
||||||
return false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the relative path from root
|
// Get the relative path from root
|
||||||
relPath, err := filepath.Rel(rootPath, filePath)
|
relPath, err := filepath.Rel(rootPath, filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Never exclude the root directory itself
|
// Never exclude the root directory itself
|
||||||
if relPath == "." {
|
if relPath == "." {
|
||||||
return false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalize path separators
|
// Normalize path separators
|
||||||
@@ -2376,11 +2451,26 @@ func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
|
|||||||
// Check each pattern
|
// Check each pattern
|
||||||
for _, cp := range s.compiledExclude {
|
for _, cp := range s.compiledExclude {
|
||||||
if cp.matches(relPath) {
|
if cp.matches(relPath) {
|
||||||
return true
|
return cp.original, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return false
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// printExcluded prints that the walk excluded filePath and why: pattern
|
||||||
|
// is the exclude pattern that matched it, "" for one of vaultik's own
|
||||||
|
// working files.
|
||||||
|
func (s *Scanner) printExcluded(filePath, pattern string) {
|
||||||
|
if pattern == "" {
|
||||||
|
s.ui.Infof("Excluded %s: vaultik's own index or lock file.",
|
||||||
|
s.ui.Path(filePath))
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.ui.Infof("Excluded %s: matches exclude pattern %q.",
|
||||||
|
s.ui.Path(filePath), pattern)
|
||||||
}
|
}
|
||||||
|
|
||||||
// matches reports whether the compiled pattern matches the given
|
// matches reports whether the compiled pattern matches the given
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileStorer implements Storer using the local filesystem.
|
// FileStorer implements Storer using the local filesystem.
|
||||||
@@ -289,7 +288,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
|
|||||||
// Info returns human-readable storage location information.
|
// Info returns human-readable storage location information.
|
||||||
func (f *FileStorer) Info() Info {
|
func (f *FileStorer) Info() Info {
|
||||||
return Info{
|
return Info{
|
||||||
Type: storageurl.SchemeFile,
|
Type: schemeFile,
|
||||||
Location: f.basePath,
|
Location: f.basePath,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,20 +2,14 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
"sneak.berlin/go/vaultik/internal/s3"
|
"sneak.berlin/go/vaultik/internal/s3"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
|
|
||||||
// has no storage backend.
|
|
||||||
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
|
||||||
|
|
||||||
// defaultS3Region is used when neither the URL nor the config specify one.
|
// defaultS3Region is used when neither the URL nor the config specify one.
|
||||||
const defaultS3Region = "us-east-1"
|
const defaultS3Region = "us-east-1"
|
||||||
|
|
||||||
@@ -46,19 +40,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
|
|||||||
|
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
||||||
parsed, err := storageurl.Parse(rawURL)
|
parsed, err := ParseStorageURL(rawURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch parsed.Scheme {
|
switch parsed.Scheme {
|
||||||
case storageurl.SchemeFile:
|
case schemeFile:
|
||||||
return NewFileStorer(parsed.Prefix)
|
return NewFileStorer(parsed.Prefix)
|
||||||
|
|
||||||
case storageurl.SchemeS3:
|
case schemeS3:
|
||||||
return storerFromParsedS3URL(parsed, cfg)
|
return storerFromParsedS3URL(parsed, cfg)
|
||||||
|
|
||||||
case storageurl.SchemeRclone:
|
case schemeRclone:
|
||||||
return NewRcloneStorer(
|
return NewRcloneStorer(
|
||||||
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
||||||
|
|
||||||
@@ -71,7 +65,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
|||||||
// filling endpoint protocol and region defaults from the config.
|
// filling endpoint protocol and region defaults from the config.
|
||||||
//
|
//
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
|
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
||||||
// Build endpoint URL
|
// Build endpoint URL
|
||||||
endpoint := parsed.Endpoint
|
endpoint := parsed.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configfile"
|
"github.com/rclone/rclone/fs/config/configfile"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
|
|
||||||
// Import all rclone backends
|
// Import all rclone backends
|
||||||
_ "github.com/rclone/rclone/backend/all"
|
_ "github.com/rclone/rclone/backend/all"
|
||||||
@@ -267,7 +266,7 @@ func (r *RcloneStorer) Info() Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Info{
|
return Info{
|
||||||
Type: storageurl.SchemeRclone,
|
Type: schemeRclone,
|
||||||
Location: location,
|
Location: location,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
// Package storageurl parses the storage_url config value. The config
|
package storage
|
||||||
// package uses it to refuse a bad value when the config loads; the storage
|
|
||||||
// package uses it to open the destination.
|
|
||||||
package storageurl
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -13,9 +10,9 @@ import (
|
|||||||
|
|
||||||
// Storage URL scheme names.
|
// Storage URL scheme names.
|
||||||
const (
|
const (
|
||||||
SchemeFile = "file"
|
schemeFile = "file"
|
||||||
SchemeS3 = "s3"
|
schemeS3 = "s3"
|
||||||
SchemeRclone = "rclone"
|
schemeRclone = "rclone"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
@@ -28,7 +25,8 @@ var (
|
|||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||||
ErrURLCredentials = errors.New(
|
ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
||||||
|
ErrURLCredentials = errors.New(
|
||||||
"storage URL must not carry credentials; " +
|
"storage URL must not carry credentials; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
||||||
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
||||||
@@ -51,12 +49,12 @@ type URL struct {
|
|||||||
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse parses a storage URL string.
|
// ParseStorageURL parses a storage URL string.
|
||||||
// Supported formats:
|
// Supported formats:
|
||||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
||||||
// - file:///absolute/path/to/backup
|
// - file:///absolute/path/to/backup
|
||||||
// - rclone://remote/path/to/backups
|
// - rclone://remote/path/to/backups
|
||||||
func Parse(rawURL string) (*URL, error) {
|
func ParseStorageURL(rawURL string) (*URL, error) {
|
||||||
if rawURL == "" {
|
if rawURL == "" {
|
||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
@@ -94,7 +92,7 @@ func parseFileURL(rawPath string) (*URL, error) {
|
|||||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &URL{Scheme: SchemeFile, Prefix: path}, nil
|
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
@@ -137,7 +135,7 @@ func parseS3URL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: SchemeS3,
|
Scheme: schemeS3,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
Endpoint: query.Get("endpoint"),
|
Endpoint: query.Get("endpoint"),
|
||||||
@@ -191,7 +189,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: SchemeRclone,
|
Scheme: schemeRclone,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
RcloneRemote: remote,
|
RcloneRemote: remote,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -228,9 +226,9 @@ func wrapParseError(err error) error {
|
|||||||
// String returns a human-readable representation of the storage URL.
|
// String returns a human-readable representation of the storage URL.
|
||||||
func (u *URL) String() string {
|
func (u *URL) String() string {
|
||||||
switch u.Scheme {
|
switch u.Scheme {
|
||||||
case SchemeFile:
|
case schemeFile:
|
||||||
return "file://" + u.Prefix
|
return "file://" + u.Prefix
|
||||||
case SchemeS3:
|
case schemeS3:
|
||||||
endpoint := u.Endpoint
|
endpoint := u.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
endpoint = "s3.amazonaws.com"
|
endpoint = "s3.amazonaws.com"
|
||||||
@@ -241,7 +239,7 @@ func (u *URL) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
||||||
case SchemeRclone:
|
case schemeRclone:
|
||||||
if u.Prefix != "" {
|
if u.Prefix != "" {
|
||||||
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package storageurl_test
|
package storage_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -6,12 +6,12 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestParseValid checks that each supported scheme parses into the
|
// TestParseStorageURLValid checks that each supported scheme parses into
|
||||||
// expected fields, since those fields decide which backend is built.
|
// the expected fields, since those fields decide which backend is built.
|
||||||
func TestParseValid(t *testing.T) {
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -22,27 +22,27 @@ func TestParseValid(t *testing.T) {
|
|||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
raw string
|
raw string
|
||||||
want *storageurl.URL
|
want *storage.URL
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file percent-escaped path is decoded",
|
name: "file percent-escaped path is decoded",
|
||||||
raw: "file:///Volumes/my%20stick/backup",
|
raw: "file:///Volumes/my%20stick/backup",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file path with a literal space",
|
name: "file path with a literal space",
|
||||||
raw: "file:///Volumes/my stick/backup",
|
raw: "file:///Volumes/my stick/backup",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
raw: "s3://mybucket/backups/host",
|
raw: "s3://mybucket/backups/host",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Prefix: "backups/host", UseSSL: true,
|
Prefix: "backups/host", UseSSL: true,
|
||||||
},
|
},
|
||||||
@@ -50,12 +50,12 @@ func TestParseValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 bucket only",
|
name: "s3 bucket only",
|
||||||
raw: "s3://mybucket",
|
raw: "s3://mybucket",
|
||||||
want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 with endpoint, region, ssl off",
|
name: "s3 with endpoint, region, ssl off",
|
||||||
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
||||||
},
|
},
|
||||||
@@ -63,14 +63,14 @@ func TestParseValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "rclone remote and path",
|
name: "rclone remote and path",
|
||||||
raw: "rclone://gdrive/backups/host",
|
raw: "rclone://gdrive/backups/host",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone remote only",
|
name: "rclone remote only",
|
||||||
raw: "rclone://gdrive",
|
raw: "rclone://gdrive",
|
||||||
want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,22 +78,22 @@ func TestParseValid(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
got, err := storageurl.Parse(tc.raw)
|
got, err := storage.ParseStorageURL(tc.raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
|
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
if !reflect.DeepEqual(got, tc.want) {
|
||||||
t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseErrors checks that empty, missing, and unknown-scheme inputs
|
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
|
||||||
// fail with the documented sentinel errors instead of parsing to a wrong
|
// inputs fail with the documented sentinel errors instead of parsing to a
|
||||||
// destination.
|
// wrong destination.
|
||||||
func TestParseErrors(t *testing.T) {
|
func TestParseStorageURLErrors(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -101,33 +101,34 @@ func TestParseErrors(t *testing.T) {
|
|||||||
raw string
|
raw string
|
||||||
wantErr error
|
wantErr error
|
||||||
}{
|
}{
|
||||||
{"empty url", "", storageurl.ErrEmptyStorageURL},
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storageurl.ErrEmptyFilePath},
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
{"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
|
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
||||||
{"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
{"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
|
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
|
||||||
{"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
|
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Errorf("Parse(%q) error = %v, want %v",
|
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsCredentials checks that a URL carrying credentials in
|
// TestParseStorageURLRejectsCredentials checks that a URL carrying
|
||||||
// its userinfo or in an unknown query parameter is rejected, and that the
|
// credentials in its userinfo or in an unknown query parameter is
|
||||||
// error never echoes the secret-bearing URL back into logs or output.
|
// rejected, and that the error never echoes the secret-bearing URL back
|
||||||
func TestParseRejectsCredentials(t *testing.T) {
|
// into logs or output.
|
||||||
|
func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Split so the literals never form a "user:pass@" URL pattern that
|
// Split so the literals never form a "user:pass@" URL pattern that
|
||||||
@@ -146,44 +147,44 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 userinfo",
|
name: "s3 userinfo",
|
||||||
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret, "mybucket"},
|
secrets: []string{key, secret, "mybucket"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint",
|
name: "s3 userinfo in endpoint",
|
||||||
raw: "s3://mybucket?endpoint=https://" +
|
raw: "s3://mybucket?endpoint=https://" +
|
||||||
key + ":" + secret + "@minio.example.com",
|
key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint without scheme",
|
name: "s3 userinfo in endpoint without scheme",
|
||||||
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 unknown query param",
|
name: "s3 unknown query param",
|
||||||
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 misspelt endpoint",
|
name: "s3 misspelt endpoint",
|
||||||
raw: "s3://mybucket?endpiont=minio.example.com",
|
raw: "s3://mybucket?endpiont=minio.example.com",
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: nil,
|
secrets: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone userinfo",
|
name: "rclone userinfo",
|
||||||
raw: "rclone://user:" + secret + "@gdrive/backups",
|
raw: "rclone://user:" + secret + "@gdrive/backups",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone query param",
|
name: "rclone query param",
|
||||||
raw: "rclone://gdrive/backups?token=" + secret,
|
raw: "rclone://gdrive/backups?token=" + secret,
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -192,9 +193,9 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Fatalf("Parse(%q) error = %v, want %v",
|
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,11 +216,11 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsRcloneRemoteParams checks that an rclone remote name
|
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
|
||||||
// carrying parameters (rclone's "remote,param=value" form, which can hold
|
// remote name carrying parameters (rclone's "remote,param=value" form,
|
||||||
// a credential) is rejected without echoing the value, including the URLs
|
// which can hold a credential) is rejected without echoing the value,
|
||||||
// that url.Parse itself refuses.
|
// including the URLs that url.Parse itself refuses.
|
||||||
func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -248,10 +249,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
|
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
|
||||||
t.Fatalf("Parse(%q) error = %v, want %v",
|
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, storageurl.ErrRcloneRemoteParams)
|
tc.raw, err, storage.ErrRcloneRemoteParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, secret := range tc.secrets {
|
for _, secret := range tc.secrets {
|
||||||
@@ -263,10 +264,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
|
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
|
||||||
// query cannot be decoded is rejected, not read with that part of the
|
// URL whose query cannot be decoded is rejected, not read with that part
|
||||||
// query left out, and that the error does not echo it.
|
// of the query left out, and that the error does not echo it.
|
||||||
func TestParseRejectsUndecodableQuery(t *testing.T) {
|
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -302,14 +303,14 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("Parse(%q) returned no error", tc.raw)
|
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !errors.Is(err, storageurl.ErrURLBadQuery) {
|
if !errors.Is(err, storage.ErrURLBadQuery) {
|
||||||
t.Errorf("Parse(%q) error = %v, want %v",
|
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, storageurl.ErrURLBadQuery)
|
tc.raw, err, storage.ErrURLBadQuery)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, s := range []string{key, secret, "minio.example.com"} {
|
for _, s := range []string{key, secret, "minio.example.com"} {
|
||||||
@@ -321,17 +322,17 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseFailureHidesURL checks that when url.Parse itself fails, the
|
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
|
||||||
// wrapped error carries only the inner cause, not the *url.Error whose
|
// itself fails, the wrapped error carries only the inner cause, not the
|
||||||
// text embeds the raw (possibly credential-bearing) URL.
|
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
|
||||||
func TestParseFailureHidesURL(t *testing.T) {
|
func TestParseStorageURLParseFailureHidesURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const raw = "s3://mybucket/%zz"
|
const raw = "s3://mybucket/%zz"
|
||||||
|
|
||||||
_, err := storageurl.Parse(raw)
|
_, err := storage.ParseStorageURL(raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("Parse(%q) returned no error", raw)
|
t.Fatalf("ParseStorageURL(%q) returned no error", raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.Contains(err.Error(), "mybucket") {
|
if strings.Contains(err.Error(), "mybucket") {
|
||||||
@@ -33,6 +33,7 @@ var (
|
|||||||
"creating a snapshot needs at least one public key in " +
|
"creating a snapshot needs at least one public key in " +
|
||||||
"age_recipients (generate a keypair with: age-keygen)")
|
"age_recipients (generate a keypair with: age-keygen)")
|
||||||
errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune")
|
errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune")
|
||||||
|
errDryRunWithPrune = errors.New("--dry-run cannot be used with --prune")
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrPostBackupPruneFailed wraps a failure of the post-backup prune,
|
// ErrPostBackupPruneFailed wraps a failure of the post-backup prune,
|
||||||
@@ -77,12 +78,24 @@ type SnapshotCreateOptions struct {
|
|||||||
KeepNewerThan string
|
KeepNewerThan string
|
||||||
SkipErrors bool // Skip file read errors (log them loudly but continue)
|
SkipErrors bool // Skip file read errors (log them loudly but continue)
|
||||||
Snapshots []string // Optional list of snapshot names to process (empty = all)
|
Snapshots []string // Optional list of snapshot names to process (empty = all)
|
||||||
|
// DryRun scans the snapshot source paths and prints what a backup
|
||||||
|
// would store, writing nothing to the local index or the backup
|
||||||
|
// destination store.
|
||||||
|
DryRun bool
|
||||||
|
// ListExcluded makes a dry run print each excluded entry and the
|
||||||
|
// exclude pattern that matched it. A backup ignores it.
|
||||||
|
ListExcluded bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate fails when KeepNewerThan is set without Prune or is not a
|
// Validate fails when DryRun is set with Prune, or when KeepNewerThan is
|
||||||
// valid duration. Only the post-backup prune reads KeepNewerThan, after
|
// set without Prune or is not a valid duration. Only the post-backup
|
||||||
// the backup, so the caller checks it before starting one.
|
// prune reads KeepNewerThan, after the backup, so the caller checks it
|
||||||
|
// before starting one.
|
||||||
func (opts *SnapshotCreateOptions) Validate() error {
|
func (opts *SnapshotCreateOptions) Validate() error {
|
||||||
|
if opts.DryRun && opts.Prune {
|
||||||
|
return errDryRunWithPrune
|
||||||
|
}
|
||||||
|
|
||||||
if opts.KeepNewerThan == "" {
|
if opts.KeepNewerThan == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -107,6 +120,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
|||||||
return errNoAgeRecipients
|
return errNoAgeRecipients
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.DryRun {
|
||||||
|
return v.dryRunSnapshots(opts)
|
||||||
|
}
|
||||||
|
|
||||||
overallStartTime := time.Now()
|
overallStartTime := time.Now()
|
||||||
|
|
||||||
log.Info("Starting snapshot creation",
|
log.Info("Starting snapshot creation",
|
||||||
@@ -358,6 +375,9 @@ type snapshotStats struct {
|
|||||||
// could not be read or listed.
|
// could not be read or listed.
|
||||||
totalFilesUnreadable int
|
totalFilesUnreadable int
|
||||||
|
|
||||||
|
// The entries the scan excluded; an excluded directory counts once.
|
||||||
|
totalEntriesExcluded int
|
||||||
|
|
||||||
// The configured paths in which the scan found nothing.
|
// The configured paths in which the scan found nothing.
|
||||||
emptyPaths []string
|
emptyPaths []string
|
||||||
|
|
||||||
@@ -371,6 +391,29 @@ type snapshotStats struct {
|
|||||||
blobUncompressedSize int64
|
blobUncompressedSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// add adds the result of scanning the configured path dir to the totals.
|
||||||
|
func (stats *snapshotStats) add(dir string, result *snapshot.ScanResult) {
|
||||||
|
stats.totalFiles += result.FilesScanned
|
||||||
|
stats.totalBytes += result.BytesScanned
|
||||||
|
stats.totalChunks += result.ChunksCreated
|
||||||
|
stats.newChunkSize += result.NewChunkSize
|
||||||
|
stats.totalBlobs += result.BlobsCreated
|
||||||
|
stats.totalFilesSkipped += result.FilesSkipped
|
||||||
|
stats.totalFilesFailed += result.FilesFailed
|
||||||
|
stats.totalFilesUnreadable += result.FilesUnreadable
|
||||||
|
stats.totalBytesSkipped += result.BytesSkipped
|
||||||
|
stats.totalFilesDeleted += result.FilesDeleted
|
||||||
|
stats.totalBytesDeleted += result.BytesDeleted
|
||||||
|
stats.totalBlobsUploaded += result.BlobsUploaded
|
||||||
|
stats.totalBytesUploaded += result.BytesUploaded
|
||||||
|
stats.uploadDuration += result.UploadDuration
|
||||||
|
stats.totalEntriesExcluded += result.EntriesExcluded
|
||||||
|
|
||||||
|
if result.EntriesFound == 0 {
|
||||||
|
stats.emptyPaths = append(stats.emptyPaths, dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// createNamedSnapshot creates a single named snapshot and returns the
|
// createNamedSnapshot creates a single named snapshot and returns the
|
||||||
// statistics of its scan.
|
// statistics of its scan.
|
||||||
func (v *Vaultik) createNamedSnapshot(
|
func (v *Vaultik) createNamedSnapshot(
|
||||||
@@ -564,24 +607,7 @@ func (v *Vaultik) scanAllDirectories(
|
|||||||
return nil, fmt.Errorf("failed to scan %s: %w", dir, err)
|
return nil, fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stats.totalFiles += result.FilesScanned
|
stats.add(dir, result)
|
||||||
stats.totalBytes += result.BytesScanned
|
|
||||||
stats.totalChunks += result.ChunksCreated
|
|
||||||
stats.newChunkSize += result.NewChunkSize
|
|
||||||
stats.totalBlobs += result.BlobsCreated
|
|
||||||
stats.totalFilesSkipped += result.FilesSkipped
|
|
||||||
stats.totalFilesFailed += result.FilesFailed
|
|
||||||
stats.totalFilesUnreadable += result.FilesUnreadable
|
|
||||||
stats.totalBytesSkipped += result.BytesSkipped
|
|
||||||
stats.totalFilesDeleted += result.FilesDeleted
|
|
||||||
stats.totalBytesDeleted += result.BytesDeleted
|
|
||||||
stats.totalBlobsUploaded += result.BlobsUploaded
|
|
||||||
stats.totalBytesUploaded += result.BytesUploaded
|
|
||||||
stats.uploadDuration += result.UploadDuration
|
|
||||||
|
|
||||||
if result.EntriesFound == 0 {
|
|
||||||
stats.emptyPaths = append(stats.emptyPaths, dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("Directory scan complete",
|
log.Info("Directory scan complete",
|
||||||
"path", dir,
|
"path", dir,
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package vaultik
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
)
|
||||||
|
|
||||||
|
// dryRunSnapshots scans the source paths of each snapshot named in
|
||||||
|
// opts, or of every configured one, and prints what a backup would
|
||||||
|
// store. It writes nothing to the local index or the backup destination
|
||||||
|
// store. It first makes the read-only checks a backup makes, so it fails
|
||||||
|
// where a backup would: a local index bound to a different destination,
|
||||||
|
// or a destination that answers it has no marker while the local index
|
||||||
|
// holds snapshots stored there.
|
||||||
|
func (v *Vaultik) dryRunSnapshots(opts *SnapshotCreateOptions) error {
|
||||||
|
err := v.checkStorageBinding()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = v.checkDestinationMarker()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup first drops incomplete snapshots and the file records only
|
||||||
|
// they list (PruneDatabase), so it backs up those files again; the
|
||||||
|
// dry run counts them as unchanged.
|
||||||
|
incomplete, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("getting incomplete snapshots: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(incomplete) > 0 {
|
||||||
|
v.UI.Noticef("The local index holds %s incomplete snapshot(s) from "+
|
||||||
|
"an interrupted backup. The next backup drops them first and may "+
|
||||||
|
"back up more files than this dry run counts.",
|
||||||
|
v.UI.Count(len(incomplete)))
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshotNames, err := v.snapshotNamesToCreate(opts.Snapshots)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, snapName := range snapshotNames {
|
||||||
|
err = v.dryRunSnapshot(opts, snapName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dryRunSnapshot scans the source paths of the snapshot named snapName
|
||||||
|
// and prints what a backup of it would store.
|
||||||
|
func (v *Vaultik) dryRunSnapshot(
|
||||||
|
opts *SnapshotCreateOptions, snapName string,
|
||||||
|
) error {
|
||||||
|
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
previousSnapshotID, err := v.previousSnapshotID(snapName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||||
|
UI: v.UI,
|
||||||
|
Fs: v.Fs,
|
||||||
|
Exclude: v.Config.GetExcludes(snapName),
|
||||||
|
SkipErrors: opts.SkipErrors,
|
||||||
|
PreviousSnapshotID: previousSnapshotID,
|
||||||
|
})
|
||||||
|
|
||||||
|
v.UI.Beginf("Dry run of snapshot %s.", snapName)
|
||||||
|
|
||||||
|
stats := &snapshotStats{}
|
||||||
|
|
||||||
|
for i, dir := range resolvedDirs {
|
||||||
|
v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).",
|
||||||
|
v.UI.Path(dir), i+1, len(resolvedDirs))
|
||||||
|
|
||||||
|
result, err := scanner.DryRun(v.ctx, dir, opts.ListExcluded)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats.add(dir, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
v.printDryRunSummary(snapName, stats)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printDryRunSummary prints what a backup of the snapshot named snapName
|
||||||
|
// would store, in the terms of printSnapshotSummary.
|
||||||
|
func (v *Vaultik) printDryRunSummary(snapName string, stats *snapshotStats) {
|
||||||
|
filesToBackUp := stats.totalFiles - stats.totalFilesSkipped -
|
||||||
|
stats.totalFilesFailed
|
||||||
|
|
||||||
|
v.UI.Completef("Dry run of snapshot %s complete; nothing was stored.",
|
||||||
|
snapName)
|
||||||
|
|
||||||
|
filesMsg := fmt.Sprintf("Files: %s examined, %s to back up, %s unchanged",
|
||||||
|
v.UI.Count(stats.totalFiles),
|
||||||
|
v.UI.Count(filesToBackUp),
|
||||||
|
v.UI.Count(stats.totalFilesSkipped))
|
||||||
|
if stats.totalFilesFailed > 0 {
|
||||||
|
filesMsg += fmt.Sprintf(", %s failed", v.UI.Count(stats.totalFilesFailed))
|
||||||
|
}
|
||||||
|
|
||||||
|
if stats.totalFilesDeleted > 0 {
|
||||||
|
filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted))
|
||||||
|
}
|
||||||
|
|
||||||
|
v.UI.Detailf("%s.", filesMsg)
|
||||||
|
|
||||||
|
dataMsg := fmt.Sprintf("Data: %s total (%s to back up)",
|
||||||
|
v.UI.Size(stats.totalBytes+stats.totalBytesSkipped),
|
||||||
|
v.UI.Size(stats.totalBytes))
|
||||||
|
if stats.totalBytesDeleted > 0 {
|
||||||
|
dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted))
|
||||||
|
}
|
||||||
|
|
||||||
|
v.UI.Detailf("%s.", dataMsg)
|
||||||
|
v.UI.Detailf("Excluded: %s entries.", v.UI.Count(stats.totalEntriesExcluded))
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// dryRunSnapshotName is the newSummaryEnv snapshot the dry run test
|
||||||
|
// backs up and then dry-runs.
|
||||||
|
const dryRunSnapshotName = "first"
|
||||||
|
|
||||||
|
// TestDryRunCountsWhatABackupWouldStore runs a dry run after a backup:
|
||||||
|
// the unchanged files count as unchanged, a rewritten one as to back up,
|
||||||
|
// and the dry run records no snapshot. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/436.
|
||||||
|
func TestDryRunCountsWhatABackupWouldStore(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newSummaryEnv(t)
|
||||||
|
env.backUp(t, dryRunSnapshotName, false)
|
||||||
|
|
||||||
|
dryRun := func() string {
|
||||||
|
env.out.Reset()
|
||||||
|
require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
DryRun: true,
|
||||||
|
Snapshots: []string{dryRunSnapshotName},
|
||||||
|
}))
|
||||||
|
|
||||||
|
return env.out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Contains(t, dryRun(),
|
||||||
|
"Files: 3 examined, 0 to back up, 3 unchanged.")
|
||||||
|
|
||||||
|
require.NoError(t, os.WriteFile(env.aPath, []byte("rewritten"), 0o600))
|
||||||
|
|
||||||
|
assert.Contains(t, dryRun(),
|
||||||
|
"Files: 3 examined, 1 to back up, 2 unchanged.")
|
||||||
|
|
||||||
|
// localSnapshotID fails unless exactly one snapshot has the name.
|
||||||
|
localSnapshotID(context.Background(), t, env.repos, dryRunSnapshotName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDryRunNamesIncompleteSnapshots checks that a dry run says so when
|
||||||
|
// the local index holds an incomplete snapshot, which the next backup
|
||||||
|
// drops before it scans.
|
||||||
|
func TestDryRunNamesIncompleteSnapshots(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newSummaryEnv(t)
|
||||||
|
env.backUp(t, dryRunSnapshotName, false)
|
||||||
|
|
||||||
|
_, err := env.v.SnapshotManager.CreateSnapshotWithName(
|
||||||
|
context.Background(), testHostname, "split", "v", "g")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
env.out.Reset()
|
||||||
|
require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
DryRun: true,
|
||||||
|
Snapshots: []string{dryRunSnapshotName},
|
||||||
|
}))
|
||||||
|
assert.Contains(t, env.out.String(),
|
||||||
|
"The local index holds 1 incomplete snapshot(s)")
|
||||||
|
}
|
||||||
Reference in new issue
Block a user