Compare commits

..
1 Commits
Author SHA1 Message Date
sneak 95add7fc38 Add snapshot create --dry-run to show what excludes keep and drop (closes #436)
check / check (push) Successful in 15m19s
The dry run scans the source paths, reads the local index to tell
unchanged files from new and changed ones, and prints the files
examined, to back up and unchanged, their sizes, and the count of
excluded entries. With --verbose it prints each excluded entry and the
pattern that matched it. It writes nothing to the local index or the
destination, takes no lock, and refuses --prune.

Model: opus-5-5
2026-11-19 07:02:34 +00:00
16 changed files with 505 additions and 310 deletions

No files matched your search

+2 -9
View File
@@ -35,16 +35,9 @@ RUN for attempt in 1 2 3; do \
echo "module download failed: giving up after 3 attempts" >&2; \
exit 1
COPY . .
# -race also turns on checkptr, which checks every pointer conversion. In
# the SQLite library's translated C code that is about a third of the test
# CPU time, much of it under one lock all SQLite work shares, and the
# tests that use SQLite time out on a busy host. The gcflags turn it off
# for that library only; our own packages keep it.
RUN go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-cover ./... || \
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-v ./...; exit 1; }
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
+13 -7
View File
@@ -169,7 +169,7 @@ vaultik [--config <path>] config init
vaultik [--config <path>] config edit
vaultik [--config <path>] config get <key>
vaultik [--config <path>] config set <key> <value>
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>] [--dry-run]
vaultik [--config <path>] snapshot list [--json] # alias: ls
vaultik [--config <path>] snapshot files <snapshot-id> [path] [--json]
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
@@ -183,7 +183,6 @@ vaultik [--config <path>] remote nuke --force
vaultik [--config <path>] database delete [--force]
vaultik completion <bash|zsh|fish|powershell>
vaultik version
vaultik --version
```
### global flags
@@ -370,6 +369,17 @@ In such a config, an unknown key or a value of the wrong type is refused.
duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`). Without
`--prune`, or with a value that is not a valid duration, the command fails
before the backup starts.
- `--dry-run`: Scan the source paths and print what a backup would store,
without storing it. The summary gives the files examined, to back up and
unchanged, their sizes, and the count of excluded entries. An excluded
directory counts once, since the scan does not enter it. With `--verbose`, it
also prints each excluded entry and the exclude pattern that matched it. It
writes nothing to the destination and records nothing in the local index, so
it does not bind the index to the destination. Like the other commands that
read the local index, it creates an empty one if there is none. When the local
index holds an incomplete snapshot from an interrupted backup, it says so: the
next backup drops that snapshot first and may back up more files than the dry
run counts. It cannot be combined with `--prune`.
- Fails before it writes anything when a `file://` destination directory does
not exist, or when the local index holds snapshots and the destination answers
that it has no `vaultik-destination.txt`. When that look-up fails for any
@@ -603,11 +613,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
providers. Requires rclone to be configured separately (`rclone config`).
An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem
+4 -4
View File
@@ -39,10 +39,10 @@ const (
// hold the PID lock so that at most one runs at a time.
mutating lockMode = iota
// readOnly commands (info, snapshot list, snapshot files, snapshot
// verify, remote info, snapshot restore) do not write the local index
// or the remote store, so they run without the lock and are never
// blocked by a running mutator. restore writes only to the target
// directory it is given.
// verify, remote info, snapshot restore, snapshot create --dry-run)
// do not write the local index or the remote store, so they run
// without the lock and are never blocked by a running mutator.
// restore writes only to the target directory it is given.
readOnly
)
@@ -0,0 +1,89 @@
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
import (
"context"
"io"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/ui"
)
// TestEntrySnapshotCreateDryRun runs `snapshot create --dry-run
// --verbose` on a source directory that holds one kept file, one file an
// unanchored pattern excludes and one directory an anchored pattern
// excludes. The summary counts the kept file and two excluded entries,
// the excluded ones are listed with their patterns, and nothing is
// written to the destination or recorded in the local index. See
// https://git.eeqj.de/sneak/vaultik/issues/436.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntrySnapshotCreateDryRun(t *testing.T) {
configPath, storeDir := writeBackupConfig(t)
sourceDir := filepath.Join(filepath.Dir(configPath), "source")
cacheDir := filepath.Join(sourceDir, "cache")
require.NoError(t, os.WriteFile(
filepath.Join(sourceDir, "debug.log"), []byte("log"), 0o600))
require.NoError(t, os.Mkdir(cacheDir, 0o750))
for _, name := range []string{"a", "b"} {
require.NoError(t, os.WriteFile(
filepath.Join(cacheDir, name), []byte(name), 0o600))
}
out := ui.NewWithColor(io.Discard, false)
require.NoError(t, writeConfigSet(out, configPath, "exclude.0", "*.log"))
require.NoError(t, writeConfigSet(out, configPath, "exclude.1", "/cache"))
code, stdout, stderr := runEntry(t, flagConfig, configPath, "--verbose",
cmdSnapshot, cmdCreate, "--dry-run")
require.Equal(t, 0, code, stderr)
assert.Contains(t, stdout, "Excluded "+filepath.Join(sourceDir, "debug.log")+
`: matches exclude pattern "*.log".`)
assert.Contains(t, stdout, "Excluded "+cacheDir+
`: matches exclude pattern "/cache".`)
assert.NotContains(t, stdout, filepath.Join(cacheDir, "a"))
assert.Contains(t, stdout, "Files: 1 examined, 1 to back up, 0 unchanged.")
assert.Contains(t, stdout, "Excluded: 2 entries.")
entries, err := os.ReadDir(storeDir)
require.NoError(t, err)
assert.Empty(t, entries, "a dry run writes nothing to the destination")
ctx := context.Background()
db, err := database.New(ctx,
filepath.Join(filepath.Dir(configPath), "index.sqlite"))
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
repos := database.NewRepositories(db)
snapshots, err := repos.Snapshots.ListRecent(ctx, 1)
require.NoError(t, err)
assert.Empty(t, snapshots, "a dry run records no snapshot")
binding, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
require.NoError(t, err)
assert.Empty(t, binding, "a dry run does not bind the local index")
}
// TestEntrySnapshotCreateDryRunRefusesPrune checks that --dry-run with
// --prune fails before anything runs.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntrySnapshotCreateDryRunRefusesPrune(t *testing.T) {
configPath, _ := writeBackupConfig(t)
code, _, stderr := runEntry(t, flagConfig, configPath,
cmdSnapshot, cmdCreate, "--dry-run", "--prune")
assert.NotEqual(t, 0, code)
assert.Contains(t, stderr, "--dry-run cannot be used with --prune")
}
+4 -18
View File
@@ -32,8 +32,6 @@ var rootFlags RootFlags
// It sets up the command structure, global flags, and adds all subcommands.
// This is the main entry point for the CLI command hierarchy.
func NewRootCommand() *cobra.Command {
var showVersion bool
cmd := &cobra.Command{
Use: "vaultik",
Short: "Secure incremental backup tool with asymmetric encryption",
@@ -41,27 +39,15 @@ func NewRootCommand() *cobra.Command {
public keys and uploads to S3-compatible storage. No private keys are needed
on the source system.`,
SilenceUsage: true,
// Bare 'vaultik' (no subcommand): print help, or with --version
// the report of the version command. The banner is printed once
// at process startup by Entry, before cobra parses arguments, so
// it appears even when cobra rejects args (e.g. "requires at
// least 2 arg(s)") and on --help.
// Bare 'vaultik' (no subcommand): print help. The banner is
// printed once at process startup by Entry, before cobra
// parses arguments, so it appears even when cobra rejects
// args (e.g. "requires at least 2 arg(s)") and on --help.
Run: func(cmd *cobra.Command, _ []string) {
if showVersion {
writeVersion(commandUI(cmd))
return
}
_ = cmd.Help()
},
}
// cobra's own --version prints a template, not the version report,
// so vaultik defines the flag itself.
cmd.Flags().BoolVar(&showVersion, "version", false,
"Print version information (same as 'vaultik version')")
// Add global flags
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
+35 -19
View File
@@ -99,6 +99,13 @@ on Linux), then /etc/vaultik/config.yml.`,
// Use the backup functionality from cli package
rootFlags := GetRootFlags()
opts.ListExcluded = rootFlags.Verbose
// A dry run writes nothing, so it does not take the lock.
mode := mutating
if opts.DryRun {
mode = readOnly
}
// --cron suppression is wired through v.UI by setupGlobals.
return RunOperation(cmd.Context(), AppOptions{
@@ -109,28 +116,11 @@ on Linux), then /etc/vaultik/config.yml.`,
Cron: opts.Cron,
Quiet: rootFlags.Quiet,
},
Mode: mutating,
Mode: mode,
}, func(v *vaultik.Vaultik) error {
return v.CreateSnapshot(opts)
}, func(err error) {
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
log.Info("Post-backup prune failed", "error", err)
ReportErrorf("Backup completed, but %v", err)
return
}
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
log.Info("Backup completed with an error", "error", err)
ReportErrorf("Backup completed, but %v", err)
return
}
log.Info("Snapshot creation failed", "error", err)
ReportErrorf("Snapshot creation failed: %v", err)
reportSnapshotCreateFailure(opts.DryRun, err)
})
},
}
@@ -149,10 +139,36 @@ on Linux), then /etc/vaultik/config.yml.`,
"With --prune: keep snapshots newer than this duration "+
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
"instead of only the latest")
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
"Scan the snapshot source paths and print what a backup would store, "+
"writing nothing to the local index or the backup destination; "+
"with --verbose, also print each excluded entry and the exclude "+
"pattern that matched it")
return cmd
}
// reportSnapshotCreateFailure shows the user the error `snapshot create`
// failed with. dryRun says whether it was given --dry-run.
func reportSnapshotCreateFailure(dryRun bool, err error) {
switch {
case dryRun:
log.Info("Dry run failed", "error", err)
ReportErrorf("Dry run failed: %v", err)
case errors.Is(err, vaultik.ErrPostBackupPruneFailed):
log.Info("Post-backup prune failed", "error", err)
ReportErrorf("Backup completed, but %v", err)
case errors.Is(err, vaultik.ErrFilesUnreadable) ||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
errors.Is(err, vaultik.ErrVolumeNotMounted):
log.Info("Backup completed with an error", "error", err)
ReportErrorf("Backup completed, but %v", err)
default:
log.Info("Snapshot creation failed", "error", err)
ReportErrorf("Snapshot creation failed: %v", err)
}
}
// newSnapshotListCommand creates the 'snapshot list' subcommand
func newSnapshotListCommand() *cobra.Command {
var jsonOutput bool
-24
View File
@@ -86,30 +86,6 @@ func TestVersionCommandReportLines(t *testing.T) {
}
}
// TestVersionFlagPrintsVersionReport checks that `vaultik --version`
// prints exactly the report of `vaultik version`.
//
//nolint:paralleltest // executes commands that read the global rootFlags
func TestVersionFlagPrintsVersionReport(t *testing.T) {
cmd := cli.NewRootCommand()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs([]string{"--version"})
err := cmd.Execute()
if err != nil {
t.Fatalf("vaultik --version failed: %v", err)
}
want := runVersionCommand(t)
if out.String() != want {
t.Errorf("vaultik --version printed:\n%s\nwant:\n%s", out.String(), want)
}
}
// TestVersionCommandFlagsDevelopmentBuild is the regression test for
// the thing this command exists to prevent: a build that is not a
// release must say so. The notice used to be gated on the version
-35
View File
@@ -4,10 +4,8 @@ package snapshot
import (
"os"
"path/filepath"
"syscall"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -40,36 +38,3 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
}
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+105 -16
View File
@@ -71,6 +71,9 @@ type Scanner struct {
// indexAndLockFiles holds the paths of vaultik's own working files,
// which the scan treats as excluded (ScannerConfig.IndexAndLockFiles).
indexAndLockFiles map[string]struct{}
// listExcluded prints each excluded entry and why it is excluded
// (DryRun).
listExcluded bool
progress *ProgressReporter
// skipErrors skips files that cannot be opened or read (logged loudly);
// packer, database, encryption, and upload errors still abort the run.
@@ -190,6 +193,11 @@ type ScanResult struct {
// itself when it is a directory. It is zero for an empty directory,
// such as the mount point of a volume that is not mounted.
EntriesFound int
// EntriesExcluded counts the entries the walk found that an exclude
// pattern matches or that are vaultik's own files. An excluded
// directory counts once: the walk does not enter it.
EntriesExcluded int
}
// NewScanner creates a new scanner instance
@@ -360,6 +368,42 @@ func (s *Scanner) Scan(
return result, nil
}
// DryRun walks path as Scan does and counts what a backup would store,
// without storing it. It reads the local index to tell unchanged files
// from new and changed ones, and writes nothing to the local index or
// the backup destination store. With listExcluded it prints each
// excluded entry and the exclude pattern that matched it.
func (s *Scanner) DryRun(
ctx context.Context, path string, listExcluded bool,
) (*ScanResult, error) {
s.currentSourcePath = path
s.listExcluded = listExcluded
result := &ScanResult{
StartTime: time.Now().UTC(),
}
knownFiles, err := s.loadDatabaseState(ctx, path)
if err != nil {
return nil, err
}
existingFiles := make(map[string]struct{})
_, err = s.scanPhase(ctx, path, result, existingFiles, knownFiles)
if err != nil {
return nil, fmt.Errorf("scan phase failed: %w", err)
}
err = s.detectDeletedFiles(ctx, path, existingFiles, result)
if err != nil {
return nil, fmt.Errorf("detecting deleted files: %w", err)
}
result.EndTime = time.Now().UTC()
return result, nil
}
// GetProgress returns the progress reporter for this scanner, or nil when
// progress is off. Scan neither starts nor stops it: the caller does,
// once for all the paths it scans, because a second Stop panics.
@@ -944,12 +988,8 @@ func (s *Scanner) scanPhase(
// or cannot be looked up is neither counted nor reported. For
// directories, skip the entire subtree. info is nil when the
// walk could not look the entry up.
if s.shouldExclude(filePath, path) {
if info != nil && info.IsDir() {
return filepath.SkipDir
}
return nil
if excluded, walkErr := s.skipExcluded(filePath, path, info, result); excluded {
return walkErr
}
// The source path is in no directory listing, so its absence is
@@ -1016,6 +1056,32 @@ func (s *Scanner) scanPhase(
}, nil
}
// skipExcluded reports whether the walk skips filePath, under rootPath,
// because it is excluded, and counts it if so. The error is what the walk
// function returns for it: filepath.SkipDir for a directory, so the walk
// does not enter it, and nil otherwise. info is nil when the walk could
// not look the entry up.
func (s *Scanner) skipExcluded(
filePath, rootPath string, info os.FileInfo, result *ScanResult,
) (bool, error) {
pattern, excluded := s.excludingPattern(filePath, rootPath)
if !excluded {
return false, nil
}
result.EntriesExcluded++
if s.listExcluded {
s.printExcluded(filePath, pattern)
}
if info != nil && info.IsDir() {
return true, filepath.SkipDir
}
return true, nil
}
// countNonDirectories returns how many of files are not directories.
func countNonDirectories(files map[string]*database.File) int64 {
var count int64
@@ -1438,12 +1504,11 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
}
// deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat from a real
// filesystem. The in-memory test filesystem gives either no stat or one
// whose device and inode numbers are both zero, which no real file has.
// file info describes, and false when info carries no stat, as from the
// in-memory test filesystem.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || (stat.Dev == 0 && stat.Ino == 0) {
if !ok {
return 0, false
}
@@ -2352,23 +2417,32 @@ func compileExcludePatterns(patterns []string) ([]compiledPattern, error) {
// filePath is the full path to the file
// rootPath is the root of the backup source directory
func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
_, excluded := s.excludingPattern(filePath, rootPath)
return excluded
}
// excludingPattern reports whether shouldExclude excludes filePath and
// returns the exclude pattern, as configured, that matches it. The
// pattern is "" for one of vaultik's own working files.
func (s *Scanner) excludingPattern(filePath, rootPath string) (string, bool) {
if _, ok := s.indexAndLockFiles[filePath]; ok {
return true
return "", true
}
if len(s.compiledExclude) == 0 {
return false
return "", false
}
// Get the relative path from root
relPath, err := filepath.Rel(rootPath, filePath)
if err != nil {
return false
return "", false
}
// Never exclude the root directory itself
if relPath == "." {
return false
return "", false
}
// Normalize path separators
@@ -2377,11 +2451,26 @@ func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
// Check each pattern
for _, cp := range s.compiledExclude {
if cp.matches(relPath) {
return true
return cp.original, true
}
}
return false
return "", false
}
// printExcluded prints that the walk excluded filePath and why: pattern
// is the exclude pattern that matched it, "" for one of vaultik's own
// working files.
func (s *Scanner) printExcluded(filePath, pattern string) {
if pattern == "" {
s.ui.Infof("Excluded %s: vaultik's own index or lock file.",
s.ui.Path(filePath))
return
}
s.ui.Infof("Excluded %s: matches exclude pattern %q.",
s.ui.Path(filePath), pattern)
}
// matches reports whether the compiled pattern matches the given
+4 -9
View File
@@ -118,22 +118,17 @@ func ShortHostname(hostname string) string {
// hostname_timestamp if name is empty. The timestamp is in whole seconds.
// If the local index already has a snapshot with that ID, from a run of the
// same name that started in the same second, it waits a second and takes a
// new timestamp. The snapshot's start time is the moment in its ID, so a
// later run of one name always has a later start time.
// new timestamp.
func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) {
shortHostname := ShortHostname(hostname)
var (
snapshotID string
startedAt time.Time
)
var snapshotID string
for {
// Build snapshot ID with optional name
startedAt = time.Now().UTC()
timestamp := startedAt.Format("2006-01-02T15:04:05Z")
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -158,7 +153,7 @@ func (sm *SnapshotManager) CreateSnapshotWithName(
Hostname: types.Hostname(hostname),
VaultikVersion: types.Version(version),
VaultikGitRevision: types.GitRevision(gitRevision),
StartedAt: startedAt,
StartedAt: time.Now().UTC(),
CompletedAt: nil, // Not completed yet
FileCount: 0,
ChunkCount: 0,
-52
View File
@@ -492,55 +492,3 @@ func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
t.Fatalf("both creates returned snapshot ID %s", first)
}
}
// Two creates of one snapshot name back to back list newest first. The
// start time is the moment in the ID, so the two cannot share a start
// time even when the clock crosses a second while the first is created.
// See https://git.eeqj.de/sneak/vaultik/issues/482.
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const created = 2
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
sm := &SnapshotManager{repos: repos}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
if err != nil {
t.Fatalf("listing snapshots failed: %v", err)
}
if len(snapshots) != created {
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
}
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
t.Fatalf("start times %v and %v are not in order",
snapshots[1].StartedAt, snapshots[0].StartedAt)
}
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
t.Fatalf("listed %s, %s; want %s, %s",
snapshots[0].ID, snapshots[1].ID, second, first)
}
}
-23
View File
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
@@ -24,15 +23,6 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct {
@@ -63,19 +53,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path
}
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the
-73
View File
@@ -436,79 +436,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
}
}
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one
+47 -21
View File
@@ -33,6 +33,7 @@ var (
"creating a snapshot needs at least one public key in " +
"age_recipients (generate a keypair with: age-keygen)")
errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune")
errDryRunWithPrune = errors.New("--dry-run cannot be used with --prune")
)
// ErrPostBackupPruneFailed wraps a failure of the post-backup prune,
@@ -77,12 +78,24 @@ type SnapshotCreateOptions struct {
KeepNewerThan string
SkipErrors bool // Skip file read errors (log them loudly but continue)
Snapshots []string // Optional list of snapshot names to process (empty = all)
// DryRun scans the snapshot source paths and prints what a backup
// would store, writing nothing to the local index or the backup
// destination store.
DryRun bool
// ListExcluded makes a dry run print each excluded entry and the
// exclude pattern that matched it. A backup ignores it.
ListExcluded bool
}
// Validate fails when KeepNewerThan is set without Prune or is not a
// valid duration. Only the post-backup prune reads KeepNewerThan, after
// the backup, so the caller checks it before starting one.
// Validate fails when DryRun is set with Prune, or when KeepNewerThan is
// set without Prune or is not a valid duration. Only the post-backup
// prune reads KeepNewerThan, after the backup, so the caller checks it
// before starting one.
func (opts *SnapshotCreateOptions) Validate() error {
if opts.DryRun && opts.Prune {
return errDryRunWithPrune
}
if opts.KeepNewerThan == "" {
return nil
}
@@ -107,6 +120,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
return errNoAgeRecipients
}
if opts.DryRun {
return v.dryRunSnapshots(opts)
}
overallStartTime := time.Now()
log.Info("Starting snapshot creation",
@@ -358,6 +375,9 @@ type snapshotStats struct {
// could not be read or listed.
totalFilesUnreadable int
// The entries the scan excluded; an excluded directory counts once.
totalEntriesExcluded int
// The configured paths in which the scan found nothing.
emptyPaths []string
@@ -371,6 +391,29 @@ type snapshotStats struct {
blobUncompressedSize int64
}
// add adds the result of scanning the configured path dir to the totals.
func (stats *snapshotStats) add(dir string, result *snapshot.ScanResult) {
stats.totalFiles += result.FilesScanned
stats.totalBytes += result.BytesScanned
stats.totalChunks += result.ChunksCreated
stats.newChunkSize += result.NewChunkSize
stats.totalBlobs += result.BlobsCreated
stats.totalFilesSkipped += result.FilesSkipped
stats.totalFilesFailed += result.FilesFailed
stats.totalFilesUnreadable += result.FilesUnreadable
stats.totalBytesSkipped += result.BytesSkipped
stats.totalFilesDeleted += result.FilesDeleted
stats.totalBytesDeleted += result.BytesDeleted
stats.totalBlobsUploaded += result.BlobsUploaded
stats.totalBytesUploaded += result.BytesUploaded
stats.uploadDuration += result.UploadDuration
stats.totalEntriesExcluded += result.EntriesExcluded
if result.EntriesFound == 0 {
stats.emptyPaths = append(stats.emptyPaths, dir)
}
}
// createNamedSnapshot creates a single named snapshot and returns the
// statistics of its scan.
func (v *Vaultik) createNamedSnapshot(
@@ -564,24 +607,7 @@ func (v *Vaultik) scanAllDirectories(
return nil, fmt.Errorf("failed to scan %s: %w", dir, err)
}
stats.totalFiles += result.FilesScanned
stats.totalBytes += result.BytesScanned
stats.totalChunks += result.ChunksCreated
stats.newChunkSize += result.NewChunkSize
stats.totalBlobs += result.BlobsCreated
stats.totalFilesSkipped += result.FilesSkipped
stats.totalFilesFailed += result.FilesFailed
stats.totalFilesUnreadable += result.FilesUnreadable
stats.totalBytesSkipped += result.BytesSkipped
stats.totalFilesDeleted += result.FilesDeleted
stats.totalBytesDeleted += result.BytesDeleted
stats.totalBlobsUploaded += result.BlobsUploaded
stats.totalBytesUploaded += result.BytesUploaded
stats.uploadDuration += result.UploadDuration
if result.EntriesFound == 0 {
stats.emptyPaths = append(stats.emptyPaths, dir)
}
stats.add(dir, result)
log.Info("Directory scan complete",
"path", dir,
+133
View File
@@ -0,0 +1,133 @@
package vaultik
import (
"fmt"
"sneak.berlin/go/vaultik/internal/snapshot"
)
// dryRunSnapshots scans the source paths of each snapshot named in
// opts, or of every configured one, and prints what a backup would
// store. It writes nothing to the local index or the backup destination
// store. It first makes the read-only checks a backup makes, so it fails
// where a backup would: a local index bound to a different destination,
// or a destination that answers it has no marker while the local index
// holds snapshots stored there.
func (v *Vaultik) dryRunSnapshots(opts *SnapshotCreateOptions) error {
err := v.checkStorageBinding()
if err != nil {
return err
}
err = v.checkDestinationMarker()
if err != nil {
return err
}
// A backup first drops incomplete snapshots and the file records only
// they list (PruneDatabase), so it backs up those files again; the
// dry run counts them as unchanged.
incomplete, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incomplete) > 0 {
v.UI.Noticef("The local index holds %s incomplete snapshot(s) from "+
"an interrupted backup. The next backup drops them first and may "+
"back up more files than this dry run counts.",
v.UI.Count(len(incomplete)))
}
snapshotNames, err := v.snapshotNamesToCreate(opts.Snapshots)
if err != nil {
return err
}
for _, snapName := range snapshotNames {
err = v.dryRunSnapshot(opts, snapName)
if err != nil {
return err
}
}
return nil
}
// dryRunSnapshot scans the source paths of the snapshot named snapName
// and prints what a backup of it would store.
func (v *Vaultik) dryRunSnapshot(
opts *SnapshotCreateOptions, snapName string,
) error {
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
if err != nil {
return err
}
previousSnapshotID, err := v.previousSnapshotID(snapName)
if err != nil {
return err
}
scanner := v.ScannerFactory(snapshot.ScannerParams{
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
PreviousSnapshotID: previousSnapshotID,
})
v.UI.Beginf("Dry run of snapshot %s.", snapName)
stats := &snapshotStats{}
for i, dir := range resolvedDirs {
v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).",
v.UI.Path(dir), i+1, len(resolvedDirs))
result, err := scanner.DryRun(v.ctx, dir, opts.ListExcluded)
if err != nil {
return fmt.Errorf("failed to scan %s: %w", dir, err)
}
stats.add(dir, result)
}
v.printDryRunSummary(snapName, stats)
return nil
}
// printDryRunSummary prints what a backup of the snapshot named snapName
// would store, in the terms of printSnapshotSummary.
func (v *Vaultik) printDryRunSummary(snapName string, stats *snapshotStats) {
filesToBackUp := stats.totalFiles - stats.totalFilesSkipped -
stats.totalFilesFailed
v.UI.Completef("Dry run of snapshot %s complete; nothing was stored.",
snapName)
filesMsg := fmt.Sprintf("Files: %s examined, %s to back up, %s unchanged",
v.UI.Count(stats.totalFiles),
v.UI.Count(filesToBackUp),
v.UI.Count(stats.totalFilesSkipped))
if stats.totalFilesFailed > 0 {
filesMsg += fmt.Sprintf(", %s failed", v.UI.Count(stats.totalFilesFailed))
}
if stats.totalFilesDeleted > 0 {
filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted))
}
v.UI.Detailf("%s.", filesMsg)
dataMsg := fmt.Sprintf("Data: %s total (%s to back up)",
v.UI.Size(stats.totalBytes+stats.totalBytesSkipped),
v.UI.Size(stats.totalBytes))
if stats.totalBytesDeleted > 0 {
dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted))
}
v.UI.Detailf("%s.", dataMsg)
v.UI.Detailf("Excluded: %s entries.", v.UI.Count(stats.totalEntriesExcluded))
}
+69
View File
@@ -0,0 +1,69 @@
package vaultik_test
import (
"context"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// dryRunSnapshotName is the newSummaryEnv snapshot the dry run test
// backs up and then dry-runs.
const dryRunSnapshotName = "first"
// TestDryRunCountsWhatABackupWouldStore runs a dry run after a backup:
// the unchanged files count as unchanged, a rewritten one as to back up,
// and the dry run records no snapshot. See
// https://git.eeqj.de/sneak/vaultik/issues/436.
func TestDryRunCountsWhatABackupWouldStore(t *testing.T) {
t.Parallel()
env := newSummaryEnv(t)
env.backUp(t, dryRunSnapshotName, false)
dryRun := func() string {
env.out.Reset()
require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
DryRun: true,
Snapshots: []string{dryRunSnapshotName},
}))
return env.out.String()
}
assert.Contains(t, dryRun(),
"Files: 3 examined, 0 to back up, 3 unchanged.")
require.NoError(t, os.WriteFile(env.aPath, []byte("rewritten"), 0o600))
assert.Contains(t, dryRun(),
"Files: 3 examined, 1 to back up, 2 unchanged.")
// localSnapshotID fails unless exactly one snapshot has the name.
localSnapshotID(context.Background(), t, env.repos, dryRunSnapshotName)
}
// TestDryRunNamesIncompleteSnapshots checks that a dry run says so when
// the local index holds an incomplete snapshot, which the next backup
// drops before it scans.
func TestDryRunNamesIncompleteSnapshots(t *testing.T) {
t.Parallel()
env := newSummaryEnv(t)
env.backUp(t, dryRunSnapshotName, false)
_, err := env.v.SnapshotManager.CreateSnapshotWithName(
context.Background(), testHostname, "split", "v", "g")
require.NoError(t, err)
env.out.Reset()
require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
DryRun: true,
Snapshots: []string{dryRunSnapshotName},
}))
assert.Contains(t, env.out.String(),
"The local index holds 1 incomplete snapshot(s)")
}