Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6ed84bf6d8 |
+4
-4
@@ -279,8 +279,8 @@ Key methods:
|
|||||||
- `Scan(ctx, path, snapshotID)` → Main scan operation
|
- `Scan(ctx, path, snapshotID)` → Main scan operation
|
||||||
|
|
||||||
Scan phases:
|
Scan phases:
|
||||||
1. **Phase 1**: Walk directory, identify files needing processing
|
1. **Phase 0**: Detect deleted files from previous snapshots
|
||||||
2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists and the walk did not find; a file that is excluded, or inside an excluded directory, is not counted
|
2. **Phase 1**: Walk directory, identify files needing processing
|
||||||
3. **Phase 2**: Process files (chunk → pack → upload)
|
3. **Phase 2**: Process files (chunk → pack → upload)
|
||||||
|
|
||||||
#### SnapshotManager
|
#### SnapshotManager
|
||||||
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
|
|||||||
│ │
|
│ │
|
||||||
│ ├─► scanner.Scan(ctx, path, snapshotID)
|
│ ├─► scanner.Scan(ctx, path, snapshotID)
|
||||||
│ │ │
|
│ │ │
|
||||||
|
│ │ ├─► Phase 0: detectDeletedFiles()
|
||||||
|
│ │ │
|
||||||
│ │ ├─► Phase 1: scanPhase()
|
│ │ ├─► Phase 1: scanPhase()
|
||||||
│ │ │ Walk directory
|
│ │ │ Walk directory
|
||||||
│ │ │ Check file metadata changes
|
│ │ │ Check file metadata changes
|
||||||
│ │ │ Build list of files to process
|
│ │ │ Build list of files to process
|
||||||
│ │ │
|
│ │ │
|
||||||
│ │ ├─► Phase 1b: detectDeletedFiles()
|
|
||||||
│ │ │
|
|
||||||
│ │ └─► Phase 2: processPhase()
|
│ │ └─► Phase 2: processPhase()
|
||||||
│ │ For each file:
|
│ │ For each file:
|
||||||
│ │ chunker.ChunkReaderStreaming()
|
│ │ chunker.ChunkReaderStreaming()
|
||||||
|
|||||||
@@ -372,7 +372,9 @@ on the destination in one go, use `vaultik remote nuke --force`.
|
|||||||
store — its remote key (or an unambiguous leading part of it) as shown
|
store — its remote key (or an unambiguous leading part of it) as shown
|
||||||
by `snapshot list`. See
|
by `snapshot list`. See
|
||||||
[restoring on another machine](#restoring-on-another-machine).
|
[restoring on another machine](#restoring-on-another-machine).
|
||||||
* Optional path arguments to restore specific files/directories (default: all)
|
* Optional path arguments to restore specific files/directories (default: all).
|
||||||
|
A path that matches nothing in the snapshot is named in a warning, and the
|
||||||
|
command exits non-zero after restoring the paths that match
|
||||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
||||||
symlinks, and empty directories
|
symlinks, and empty directories
|
||||||
* `--verify`: After restoring, verify every file's chunk hashes match
|
* `--verify`: After restoring, verify every file's chunk hashes match
|
||||||
@@ -433,10 +435,7 @@ object store. Credentials are read from `s3.access_key_id` and
|
|||||||
`s3.secret_access_key` in the config file.
|
`s3.secret_access_key` in the config file.
|
||||||
|
|
||||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
||||||
a local or mounted filesystem. Useful for testing or backing up to a NAS. The
|
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
||||||
path must be absolute; a relative one is refused when the config loads.
|
|
||||||
Percent-escapes in the path are decoded, so `my%20stick` and `my stick` name
|
|
||||||
the same directory.
|
|
||||||
|
|
||||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||||
providers. Requires rclone to be configured separately (`rclone config`).
|
providers. Requires rclone to be configured separately (`rclone config`).
|
||||||
|
|||||||
@@ -245,17 +245,6 @@ func (p *Packer) Flush(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
|
||||||
// packed, if any, and drops that blob. A caller that stops before Flush
|
|
||||||
// must call it, or the temp file is left behind. Thread-safe.
|
|
||||||
func (p *Packer) DiscardCurrentBlob() {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.cleanupTempFile()
|
|
||||||
p.currentBlob = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FinalizeBlob finalizes the current blob being assembled.
|
// FinalizeBlob finalizes the current blob being assembled.
|
||||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||||
// the content-addressed hash. The finalized blob is either passed to the
|
// the content-addressed hash. The finalized blob is either passed to the
|
||||||
|
|||||||
@@ -251,12 +251,6 @@ func NewConfigCommand() *cobra.Command {
|
|||||||
Use: "config",
|
Use: "config",
|
||||||
Short: "Manage the configuration file",
|
Short: "Manage the configuration file",
|
||||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||||
// cobra checks Args only on a command that has a Run: without
|
|
||||||
// one, a mistyped subcommand printed help and exited 0.
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
_ = cmd.Help()
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(newConfigInitCommand())
|
cmd.AddCommand(newConfigInitCommand())
|
||||||
|
|||||||
@@ -15,12 +15,6 @@ func NewDatabaseCommand() *cobra.Command {
|
|||||||
Use: "database",
|
Use: "database",
|
||||||
Short: "Manage the local state database",
|
Short: "Manage the local state database",
|
||||||
Long: `Commands for managing the local SQLite state database.`,
|
Long: `Commands for managing the local SQLite state database.`,
|
||||||
// cobra checks Args only on a command that has a Run: without
|
|
||||||
// one, a mistyped subcommand printed help and exited 0.
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
_ = cmd.Help()
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
package cli //nolint:testpackage // shares runEntry and the command-word constants
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
|
|
||||||
// mistyped subcommand, as a crontab line might carry one. It must fail
|
|
||||||
// with exit 1 and name the word, as `vaultik bogus` does at the top
|
|
||||||
// level, instead of printing the group's help and exiting 0 having done
|
|
||||||
// nothing. The group name alone still prints its help.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
|
||||||
func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
|
|
||||||
for _, testCase := range []struct {
|
|
||||||
group string
|
|
||||||
mistyped string
|
|
||||||
}{
|
|
||||||
{group: cmdSnapshot, mistyped: "craete"},
|
|
||||||
{group: "config", mistyped: "inti"},
|
|
||||||
{group: cmdRemote, mistyped: "ifno"},
|
|
||||||
{group: "database", mistyped: "delte"},
|
|
||||||
{group: "completion", mistyped: "nosuchshell"},
|
|
||||||
} {
|
|
||||||
t.Run(testCase.group, func(t *testing.T) {
|
|
||||||
code, stdout, stderr := runEntry(t,
|
|
||||||
testCase.group, testCase.mistyped)
|
|
||||||
|
|
||||||
assert.Equal(t, 1, code)
|
|
||||||
assert.Empty(t, stdout)
|
|
||||||
assert.Contains(t, stderr,
|
|
||||||
`unknown command "`+testCase.mistyped+`"`)
|
|
||||||
|
|
||||||
code, stdout, _ = runEntry(t, testCase.group)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, code)
|
|
||||||
assert.Contains(t, stdout,
|
|
||||||
programName+" "+testCase.group+" [command]")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -18,12 +18,6 @@ func NewRemoteCommand() *cobra.Command {
|
|||||||
Use: "remote",
|
Use: "remote",
|
||||||
Short: "Remote storage management commands",
|
Short: "Remote storage management commands",
|
||||||
Long: "Commands for inspecting and managing remote storage",
|
Long: "Commands for inspecting and managing remote storage",
|
||||||
// cobra checks Args only on a command that has a Run: without
|
|
||||||
// one, a mistyped subcommand printed help and exited 0.
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
_ = cmd.Help()
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -74,21 +74,6 @@ on the source system.`,
|
|||||||
NewDatabaseCommand(),
|
NewDatabaseCommand(),
|
||||||
)
|
)
|
||||||
|
|
||||||
// cobra adds the completion command only when Execute runs, and with
|
|
||||||
// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
|
|
||||||
// printed help and exited 0. Adding it here lets it have a Run. Its
|
|
||||||
// scripts go to os.Stdout as it is now: a later SetOut does not
|
|
||||||
// redirect them.
|
|
||||||
cmd.InitDefaultCompletionCmd()
|
|
||||||
|
|
||||||
for _, sub := range cmd.Commands() {
|
|
||||||
if sub.Name() == "completion" {
|
|
||||||
sub.Run = func(completion *cobra.Command, _ []string) {
|
|
||||||
_ = completion.Help()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,12 +41,6 @@ func NewSnapshotCommand() *cobra.Command {
|
|||||||
Use: "snapshot",
|
Use: "snapshot",
|
||||||
Short: "Snapshot management commands",
|
Short: "Snapshot management commands",
|
||||||
Long: "Commands for creating, listing, and managing snapshots",
|
Long: "Commands for creating, listing, and managing snapshots",
|
||||||
// cobra checks Args only on a command that has a Run: without
|
|
||||||
// one, a mistyped subcommand printed help and exited 0.
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
_ = cmd.Help()
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ This command will restore files from the specified snapshot to the
|
|||||||
target directory.
|
target directory.
|
||||||
If no paths are specified, all files are restored.
|
If no paths are specified, all files are restored.
|
||||||
If paths are specified, only matching files/directories are restored.
|
If paths are specified, only matching files/directories are restored.
|
||||||
|
A path that matches nothing in the snapshot is named in a warning, and
|
||||||
|
the command exits non-zero after restoring the paths that match.
|
||||||
|
|
||||||
The snapshot may be named by its ID or, when restoring on a host with no
|
The snapshot may be named by its ID or, when restoring on a host with no
|
||||||
local index, by the remote key that 'snapshot list' prints for a
|
local index, by the remote key that 'snapshot list' prints for a
|
||||||
|
|||||||
@@ -61,9 +61,6 @@ var (
|
|||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
errBadStorageScheme = errors.New(
|
errBadStorageScheme = errors.New(
|
||||||
"storage_url must start with s3://, file://, or rclone://")
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
errFileURLNotAbsolute = errors.New(
|
|
||||||
"a file:// storage_url needs an absolute path, " +
|
|
||||||
"as in file:///path/to/backup")
|
|
||||||
errStorageNotConfigured = errors.New(
|
errStorageNotConfigured = errors.New(
|
||||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||||
"s3.bucket + credentials")
|
"s3.bucket + credentials")
|
||||||
@@ -464,18 +461,11 @@ func (c *Config) validateStorage() error {
|
|||||||
|
|
||||||
// validateStorageURL validates URL-based storage configuration. File and
|
// validateStorageURL validates URL-based storage configuration. File and
|
||||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||||
// credential fields. A file URL must name an absolute path: a relative one
|
// credential fields.
|
||||||
// would resolve against the working directory, so the destination would
|
|
||||||
// move with it.
|
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
switch {
|
switch {
|
||||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||||
// storage.ParseStorageURL applies the same rule, but config
|
// File storage doesn't need S3 credentials
|
||||||
// cannot import storage, which imports config.
|
|
||||||
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
|
||||||
return errFileURLNotAbsolute
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||||
// Rclone storage uses rclone's own config
|
// Rclone storage uses rclone's own config
|
||||||
|
|||||||
@@ -337,35 +337,6 @@ func TestLoadS3PartSize(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
|
||||||
// relative path fails at load. Accepted, it would resolve against the
|
|
||||||
// working directory, so a backup run from another directory would write
|
|
||||||
// to a different destination and still succeed.
|
|
||||||
func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const relativeConfig = "snapshots:\n" +
|
|
||||||
" test:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" +
|
|
||||||
"storage_url: file://dest\n"
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte(relativeConfig), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = Load(path)
|
|
||||||
if !errors.Is(err, errFileURLNotAbsolute) {
|
|
||||||
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "absolute path") {
|
|
||||||
t.Errorf("error %q does not say to use an absolute path", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||||
|
|||||||
@@ -277,50 +277,6 @@ func (r *FileRepository) ListUnderPath(
|
|||||||
return files, rows.Err()
|
return files, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListUnderPathInSnapshot returns the files that ListUnderPath returns
|
|
||||||
// for path and that the snapshot lists, in no particular order.
|
|
||||||
func (r *FileRepository) ListUnderPathInSnapshot(
|
|
||||||
ctx context.Context, path, snapshotID string,
|
|
||||||
) ([]*File, error) {
|
|
||||||
path = strings.TrimRight(path, "/")
|
|
||||||
dirPrefix := path + "/"
|
|
||||||
|
|
||||||
query := `
|
|
||||||
SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
|
|
||||||
f.mode, f.uid, f.gid, f.link_target
|
|
||||||
FROM files f
|
|
||||||
JOIN snapshot_files sf ON sf.file_id = f.id
|
|
||||||
WHERE sf.snapshot_id = ?
|
|
||||||
AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
|
|
||||||
`
|
|
||||||
|
|
||||||
rows, err := r.db.conn.QueryContext(ctx, query,
|
|
||||||
snapshotID, path, dirPrefix, dirPrefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("querying files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
err := rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
Fatalf("failed to close rows: %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
var files []*File
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
file, err := r.scanFileRows(rows)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("scanning file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files = append(files, file)
|
|
||||||
}
|
|
||||||
|
|
||||||
return files, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
|
// ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
|
||||||
// path starts with prefix and that list at least one chunk held by no
|
// path starts with prefix and that list at least one chunk held by no
|
||||||
// blob whose upload has completed (uploaded_ts set). A new snapshot
|
// blob whose upload has completed (uploaded_ts set). A new snapshot
|
||||||
|
|||||||
@@ -212,65 +212,6 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListUnderPathInSnapshot leaves out files outside path and files the
|
|
||||||
// snapshot does not list, though another snapshot lists them.
|
|
||||||
func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, cleanup := setupTestDB(t)
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
files := database.NewFileRepository(db)
|
|
||||||
snapshots := database.NewSnapshotRepository(db)
|
|
||||||
|
|
||||||
listed := map[string][]string{
|
|
||||||
"one": {"/src/a.txt", "/src2/c.txt"},
|
|
||||||
"two": {"/src/b.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for snapshotID, paths := range listed {
|
|
||||||
err := snapshots.Create(ctx, nil, &database.Snapshot{
|
|
||||||
ID: types.SnapshotID(snapshotID),
|
|
||||||
Hostname: "host",
|
|
||||||
StartedAt: time.Now(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range paths {
|
|
||||||
err := files.Create(ctx, nil, &database.File{
|
|
||||||
Path: types.FilePath(path),
|
|
||||||
MTime: time.Now().Truncate(time.Second),
|
|
||||||
Mode: 0644,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = snapshots.AddFile(ctx, nil, snapshotID, path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to list files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := make([]string, 0, len(found))
|
|
||||||
for _, f := range found {
|
|
||||||
got = append(got, f.Path.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, []string{"/src/a.txt"}) {
|
|
||||||
t.Errorf("got %q, want only /src/a.txt", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileRepositorySymlink(t *testing.T) {
|
func TestFileRepositorySymlink(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -531,50 +531,3 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
|||||||
require.Equal(t, 1, result.FilesScanned,
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
"Unanchored pattern for file should match anywhere")
|
"Unanchored pattern for file should match anywhere")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExcludePatterns_DoubleStarDirectoryAtEveryDepth(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// As in gitignore, a leading **/ also matches directly under the source dir
|
|
||||||
fs := setupAnchoredTestFS(t)
|
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/projectname"})
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
|
||||||
|
|
||||||
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// **/projectname should exclude BOTH:
|
|
||||||
// - /backup/projectname/file.txt
|
|
||||||
// - /backup/otherproject/projectname/file.txt
|
|
||||||
// Total files: 4, excluded: 2
|
|
||||||
require.Equal(t, 2, result.FilesScanned,
|
|
||||||
"**/ directory pattern should match at the top level and nested")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// As in gitignore, a leading **/ also matches directly under the source dir
|
|
||||||
fs := setupAnchoredTestFS(t)
|
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.txt"})
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
|
||||||
|
|
||||||
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// **/*.txt should exclude ALL .txt files:
|
|
||||||
// - /backup/file.txt
|
|
||||||
// - /backup/projectname/file.txt
|
|
||||||
// - /backup/otherproject/projectname/file.txt
|
|
||||||
// Total files: 4, excluded: 3
|
|
||||||
require.Equal(t, 1, result.FilesScanned,
|
|
||||||
"**/ file pattern should match at the top level and nested")
|
|
||||||
}
|
|
||||||
|
|||||||
+11
-15
@@ -16,9 +16,6 @@ type ScannerParams struct {
|
|||||||
Fs afero.Fs
|
Fs afero.Fs
|
||||||
Exclude []string // Exclude patterns (combined global + snapshot-specific)
|
Exclude []string // Exclude patterns (combined global + snapshot-specific)
|
||||||
SkipErrors bool // Skip file read errors (log loudly but continue)
|
SkipErrors bool // Skip file read errors (log loudly but continue)
|
||||||
// PreviousSnapshotID is the most recent completed snapshot of the
|
|
||||||
// same name, "" for none.
|
|
||||||
PreviousSnapshotID string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module exports backup functionality as an fx module.
|
// Module exports backup functionality as an fx module.
|
||||||
@@ -47,18 +44,17 @@ func provideScannerFactory(
|
|||||||
}
|
}
|
||||||
|
|
||||||
return NewScanner(ScannerConfig{
|
return NewScanner(ScannerConfig{
|
||||||
FS: params.Fs,
|
FS: params.Fs,
|
||||||
ChunkSize: cfg.ChunkSize.Int64(),
|
ChunkSize: cfg.ChunkSize.Int64(),
|
||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
Storage: storer,
|
Storage: storer,
|
||||||
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
|
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
|
||||||
CompressionLevel: cfg.CompressionLevel,
|
CompressionLevel: cfg.CompressionLevel,
|
||||||
AgeRecipients: cfg.AgeRecipients,
|
AgeRecipients: cfg.AgeRecipients,
|
||||||
EnableProgress: params.EnableProgress,
|
EnableProgress: params.EnableProgress,
|
||||||
UI: params.UI,
|
UI: params.UI,
|
||||||
Exclude: excludes,
|
Exclude: excludes,
|
||||||
SkipErrors: params.SkipErrors,
|
SkipErrors: params.SkipErrors,
|
||||||
PreviousSnapshotID: params.PreviousSnapshotID,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,10 +58,6 @@ type Scanner struct {
|
|||||||
compressionLevel int
|
compressionLevel int
|
||||||
ageRecipient string
|
ageRecipient string
|
||||||
snapshotID string // Current snapshot being processed
|
snapshotID string // Current snapshot being processed
|
||||||
// previousSnapshotID is the most recent completed snapshot of the
|
|
||||||
// same name, "" for none. Only a file it lists can be counted as
|
|
||||||
// deleted.
|
|
||||||
previousSnapshotID string
|
|
||||||
// currentSourcePath is the source directory being scanned, stored with
|
// currentSourcePath is the source directory being scanned, stored with
|
||||||
// each file record.
|
// each file record.
|
||||||
currentSourcePath string
|
currentSourcePath string
|
||||||
@@ -133,9 +129,6 @@ type ScannerConfig struct {
|
|||||||
// SkipErrors skips files that cannot be opened or read (log loudly but
|
// SkipErrors skips files that cannot be opened or read (log loudly but
|
||||||
// continue); packer, database, encryption, and upload errors still abort.
|
// continue); packer, database, encryption, and upload errors still abort.
|
||||||
SkipErrors bool
|
SkipErrors bool
|
||||||
// PreviousSnapshotID is the most recent completed snapshot of the
|
|
||||||
// same name, "" for none.
|
|
||||||
PreviousSnapshotID string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ScanResult contains the results of a scan operation. Files and bytes
|
// ScanResult contains the results of a scan operation. Files and bytes
|
||||||
@@ -211,7 +204,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
|||||||
compiledExclude: compiledExclude,
|
compiledExclude: compiledExclude,
|
||||||
progress: progress,
|
progress: progress,
|
||||||
skipErrors: cfg.SkipErrors,
|
skipErrors: cfg.SkipErrors,
|
||||||
previousSnapshotID: cfg.PreviousSnapshotID,
|
|
||||||
ui: uiw,
|
ui: uiw,
|
||||||
pendingChunkHashes: make(map[string]struct{}),
|
pendingChunkHashes: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
@@ -263,9 +255,8 @@ func (s *Scanner) Scan(
|
|||||||
|
|
||||||
filesToProcess := scanResult.FilesToProcess
|
filesToProcess := scanResult.FilesToProcess
|
||||||
|
|
||||||
// Phase 1b: Detect deleted files by comparing the previous snapshot
|
// Phase 1b: Detect deleted files by comparing DB against scanned files
|
||||||
// of this name against scanned files
|
err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
|
||||||
err = s.detectDeletedFiles(ctx, path, existingFiles, result)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("detecting deleted files: %w", err)
|
return nil, fmt.Errorf("detecting deleted files: %w", err)
|
||||||
}
|
}
|
||||||
@@ -445,8 +436,9 @@ func (s *Scanner) summarizeScanPhase(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// loadKnownFiles loads the known files at and beneath path from the
|
// loadKnownFiles loads the known files at and beneath path from the
|
||||||
// database into a map for fast lookup. This avoids per-file database
|
// database into a map for fast lookup. Every loaded file the scan does
|
||||||
// queries during the scan phase.
|
// not find is counted as deleted. This avoids per-file database queries
|
||||||
|
// during the scan phase.
|
||||||
func (s *Scanner) loadKnownFiles(
|
func (s *Scanner) loadKnownFiles(
|
||||||
ctx context.Context, path string,
|
ctx context.Context, path string,
|
||||||
) (map[string]*database.File, error) {
|
) (map[string]*database.File, error) {
|
||||||
@@ -1303,11 +1295,6 @@ func (s *Scanner) batchAddFilesToSnapshot(
|
|||||||
func (s *Scanner) processPhase(
|
func (s *Scanner) processPhase(
|
||||||
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
||||||
) error {
|
) error {
|
||||||
// On an error or cancellation the blob being packed is never
|
|
||||||
// finalized, so its temp file must be removed here. After a
|
|
||||||
// successful flush there is no such blob and this does nothing.
|
|
||||||
defer s.packer.DiscardCurrentBlob()
|
|
||||||
|
|
||||||
// Calculate total bytes to process
|
// Calculate total bytes to process
|
||||||
var totalBytes int64
|
var totalBytes int64
|
||||||
for _, f := range filesToProcess {
|
for _, f := range filesToProcess {
|
||||||
@@ -1946,28 +1933,21 @@ func (s *Scanner) queueFileForBatchInsert(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// detectDeletedFiles counts as deleted each file at or beneath path that
|
// detectDeletedFilesFromMap finds files that existed in previous snapshots
|
||||||
// the previous snapshot of this name lists and the walk did not find.
|
// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
|
||||||
// A file that is excluded, or inside an excluded directory, is not
|
// database access.
|
||||||
// counted: the walk skips it whether or not it is still on disk.
|
func (s *Scanner) detectDeletedFilesFromMap(
|
||||||
func (s *Scanner) detectDeletedFiles(
|
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
path string,
|
knownFiles map[string]*database.File,
|
||||||
existingFiles map[string]struct{},
|
existingFiles map[string]struct{},
|
||||||
result *ScanResult,
|
result *ScanResult,
|
||||||
) error {
|
) error {
|
||||||
if s.previousSnapshotID == "" {
|
if len(knownFiles) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
|
// Check each known file against the enumerated set (no filesystem access needed)
|
||||||
ctx, path, s.previousSnapshotID)
|
for path, file := range knownFiles {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("listing files of snapshot %s: %w",
|
|
||||||
s.previousSnapshotID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, file := range previousFiles {
|
|
||||||
// Check context cancellation periodically
|
// Check context cancellation periodically
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -1975,16 +1955,13 @@ func (s *Scanner) detectDeletedFiles(
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
filePath := file.Path.String()
|
// Check if the file exists in our enumerated set
|
||||||
|
if _, exists := existingFiles[path]; !exists {
|
||||||
_, exists := existingFiles[filePath]
|
// File has been deleted
|
||||||
if exists || s.excludedOrInExcludedDir(filePath, path) {
|
result.FilesDeleted++
|
||||||
continue
|
result.BytesDeleted += file.Size
|
||||||
|
log.Debug("Detected deleted file", "path", path, "size", file.Size)
|
||||||
}
|
}
|
||||||
|
|
||||||
result.FilesDeleted++
|
|
||||||
result.BytesDeleted += file.Size
|
|
||||||
log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.FilesDeleted > 0 {
|
if result.FilesDeleted > 0 {
|
||||||
@@ -1995,20 +1972,6 @@ func (s *Scanner) detectDeletedFiles(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// excludedOrInExcludedDir reports whether filePath, which is at or
|
|
||||||
// beneath rootPath, or any directory between rootPath and it, matches an
|
|
||||||
// exclude pattern. A pattern such as */.local/share/Trash matches the
|
|
||||||
// directory but not the files inside it, which the walk skips too.
|
|
||||||
func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
|
|
||||||
for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
|
|
||||||
if s.shouldExclude(p, rootPath) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// wrapPermissionError augments permission errors with platform-specific
|
// wrapPermissionError augments permission errors with platform-specific
|
||||||
// remediation instructions. On macOS, TCC-protected directories (Calendars,
|
// remediation instructions. On macOS, TCC-protected directories (Calendars,
|
||||||
// Reminders, Photos, etc.) return EPERM unless the running application has
|
// Reminders, Photos, etc.) return EPERM unless the running application has
|
||||||
@@ -2050,14 +2013,6 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
// Remove trailing slash if present (directory indicator)
|
// Remove trailing slash if present (directory indicator)
|
||||||
pattern = strings.TrimSuffix(pattern, "/")
|
pattern = strings.TrimSuffix(pattern, "/")
|
||||||
|
|
||||||
// A leading "**/" demands a "/" before the name and so misses an
|
|
||||||
// entry directly under the source path. An unanchored pattern
|
|
||||||
// already matches at every depth, so drop the prefix: as in
|
|
||||||
// gitignore, "**/x" then also matches a top-level "x".
|
|
||||||
if !anchored {
|
|
||||||
pattern = strings.TrimPrefix(pattern, "**/")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compile the glob pattern
|
// Compile the glob pattern
|
||||||
// For patterns without path separators, we need to match them as components
|
// For patterns without path separators, we need to match them as components
|
||||||
// e.g., ".git" should match ".git" anywhere in the path
|
// e.g., ".git" should match ".git" anywhere in the path
|
||||||
|
|||||||
@@ -381,60 +381,3 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
|
|
||||||
// part-way, by a file it cannot read or by a cancellation, removes the
|
|
||||||
// temp file of the blob it was packing.
|
|
||||||
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// a.txt is backed up first and starts a blob; the run stops at b.txt.
|
|
||||||
const target = "/source/b.txt"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
newFs func(cancel context.CancelFunc) afero.Fs
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "file cannot be read",
|
|
||||||
newFs: func(context.CancelFunc) afero.Fs {
|
|
||||||
return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cancelled",
|
|
||||||
newFs: func(cancel context.CancelFunc) afero.Fs {
|
|
||||||
return &cancelOnOpenFs{
|
|
||||||
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
fs := tt.newFs(cancel)
|
|
||||||
writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
|
|
||||||
writeSkipErrorTestFile(t, fs, target, "second file content")
|
|
||||||
|
|
||||||
_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected the scan to stop with an error, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listing blob temp files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(left) != 0 {
|
|
||||||
t.Fatalf("expected no blob temp file left, got %v", left)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+12
-26
@@ -17,10 +17,8 @@ const (
|
|||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
var (
|
var (
|
||||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||||
ErrRelativeFilePath = errors.New(
|
|
||||||
"file URL needs an absolute path, as in file:///path/to/backup")
|
|
||||||
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
@@ -53,8 +51,17 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle file:// URLs
|
||||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||||
return parseFileURL(after)
|
path := after
|
||||||
|
if path == "" {
|
||||||
|
return nil, ErrEmptyFilePath
|
||||||
|
}
|
||||||
|
|
||||||
|
return &URL{
|
||||||
|
Scheme: schemeFile,
|
||||||
|
Prefix: path,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(rawURL, "s3://") {
|
if strings.HasPrefix(rawURL, "s3://") {
|
||||||
@@ -68,27 +75,6 @@ func ParseStorageURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrUnsupportedScheme
|
return nil, ErrUnsupportedScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseFileURL parses the path that follows file://. The path must be
|
|
||||||
// absolute: a relative one would resolve against the working directory,
|
|
||||||
// so the destination would move with it. Percent-escapes are decoded, so
|
|
||||||
// my%20stick names the directory "my stick"; a literal space is kept.
|
|
||||||
func parseFileURL(rawPath string) (*URL, error) {
|
|
||||||
if rawPath == "" {
|
|
||||||
return nil, ErrEmptyFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasPrefix(rawPath, "/") {
|
|
||||||
return nil, ErrRelativeFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
path, err := url.PathUnescape(rawPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
// the userinfo and any query parameter other than endpoint, region and
|
// the userinfo and any query parameter other than endpoint, region and
|
||||||
// ssl, so a credential-bearing URL is never stored or echoed.
|
// ssl, so a credential-bearing URL is never stored or echoed.
|
||||||
|
|||||||
@@ -14,10 +14,7 @@ import (
|
|||||||
func TestParseStorageURLValid(t *testing.T) {
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const bucket = "mybucket"
|
||||||
bucket = "mybucket"
|
|
||||||
fileScheme = "file"
|
|
||||||
)
|
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -27,17 +24,7 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "file percent-escaped path is decoded",
|
|
||||||
raw: "file:///Volumes/my%20stick/backup",
|
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "file path with a literal space",
|
|
||||||
raw: "file:///Volumes/my stick/backup",
|
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
@@ -103,7 +90,6 @@ func TestParseStorageURLErrors(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"empty url", "", storage.ErrEmptyStorageURL},
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
|
||||||
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
|
|||||||
@@ -661,18 +661,17 @@ func faultScannerFactory(
|
|||||||
) snapshot.ScannerFactory {
|
) snapshot.ScannerFactory {
|
||||||
return func(params snapshot.ScannerParams) *snapshot.Scanner {
|
return func(params snapshot.ScannerParams) *snapshot.Scanner {
|
||||||
return snapshot.NewScanner(snapshot.ScannerConfig{
|
return snapshot.NewScanner(snapshot.ScannerConfig{
|
||||||
FS: params.Fs,
|
FS: params.Fs,
|
||||||
Storage: storer,
|
Storage: storer,
|
||||||
ChunkSize: faultChunkSize,
|
ChunkSize: faultChunkSize,
|
||||||
MaxBlobSize: faultMaxBlobSize,
|
MaxBlobSize: faultMaxBlobSize,
|
||||||
CompressionLevel: cfg.CompressionLevel,
|
CompressionLevel: cfg.CompressionLevel,
|
||||||
AgeRecipients: cfg.AgeRecipients,
|
AgeRecipients: cfg.AgeRecipients,
|
||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
EnableProgress: params.EnableProgress,
|
EnableProgress: params.EnableProgress,
|
||||||
UI: params.UI,
|
UI: params.UI,
|
||||||
Exclude: params.Exclude,
|
Exclude: params.Exclude,
|
||||||
SkipErrors: params.SkipErrors,
|
SkipErrors: params.SkipErrors,
|
||||||
PreviousSnapshotID: params.PreviousSnapshotID,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+43
-15
@@ -27,6 +27,7 @@ import (
|
|||||||
// Sentinel errors for restore failures.
|
// Sentinel errors for restore failures.
|
||||||
var (
|
var (
|
||||||
errFilesFailedRestore = errors.New("file(s) failed to restore")
|
errFilesFailedRestore = errors.New("file(s) failed to restore")
|
||||||
|
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
||||||
errFilesFailedVerify = errors.New("files failed verification")
|
errFilesFailedVerify = errors.New("files failed verification")
|
||||||
errDecryptionKeyRequired = errors.New(
|
errDecryptionKeyRequired = errors.New(
|
||||||
"decryption key required for restore\n\n" +
|
"decryption key required for restore\n\n" +
|
||||||
@@ -149,7 +150,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
repos := database.NewRepositories(tempDB)
|
repos := database.NewRepositories(tempDB)
|
||||||
|
|
||||||
// Step 2: Get list of files to restore
|
// Step 2: Get list of files to restore
|
||||||
files, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
|
files, unmatched, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("getting files to restore: %w", err)
|
return fmt.Errorf("getting files to restore: %w", err)
|
||||||
}
|
}
|
||||||
@@ -158,7 +159,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
log.Warn("No files found to restore")
|
log.Warn("No files found to restore")
|
||||||
v.UI.Warningf("No files found to restore.")
|
v.UI.Warningf("No files found to restore.")
|
||||||
|
|
||||||
return nil
|
return v.reportUnmatchedPaths(unmatched)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Found files to restore", "count", len(files))
|
log.Info("Found files to restore", "count", len(files))
|
||||||
@@ -198,14 +199,16 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
v.UI.Duration(result.Duration),
|
v.UI.Duration(result.Duration),
|
||||||
)
|
)
|
||||||
|
|
||||||
return v.finishRestore(repos, files, opts, result)
|
return v.finishRestore(repos, files, unmatched, opts, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
// finishRestore emits the post-restore warnings, runs optional
|
// finishRestore emits the post-restore warnings, runs optional
|
||||||
// verification, and converts any failed-file count into an error.
|
// verification, and converts any failed-file count or requested path
|
||||||
|
// that matched nothing into an error.
|
||||||
func (v *Vaultik) finishRestore(
|
func (v *Vaultik) finishRestore(
|
||||||
repos *database.Repositories,
|
repos *database.Repositories,
|
||||||
files []*database.File,
|
files []*database.File,
|
||||||
|
unmatched []string,
|
||||||
opts *RestoreOptions,
|
opts *RestoreOptions,
|
||||||
result *RestoreResult,
|
result *RestoreResult,
|
||||||
) error {
|
) error {
|
||||||
@@ -223,6 +226,8 @@ func (v *Vaultik) finishRestore(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unmatchedErr := v.reportUnmatchedPaths(unmatched)
|
||||||
|
|
||||||
// Run verification if requested
|
// Run verification if requested
|
||||||
if opts.Verify {
|
if opts.Verify {
|
||||||
err := v.handleRestoreVerification(repos, files, opts, result)
|
err := v.handleRestoreVerification(repos, files, opts, result)
|
||||||
@@ -235,7 +240,23 @@ func (v *Vaultik) finishRestore(
|
|||||||
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
|
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return unmatchedErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportUnmatchedPaths warns, by name, about each requested path that
|
||||||
|
// matched nothing in the snapshot, and returns an error counting them,
|
||||||
|
// or nil when there are none.
|
||||||
|
func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
|
||||||
|
if len(unmatched) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range unmatched {
|
||||||
|
log.Warn("Requested path matches nothing in the snapshot", "path", path)
|
||||||
|
v.UI.Warningf("%s matches nothing in the snapshot.", v.UI.Path(path))
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("%d %w", len(unmatched), errPathsNotInSnapshot)
|
||||||
}
|
}
|
||||||
|
|
||||||
// restoreIdentities parses the configured age secret key once into every
|
// restoreIdentities parses the configured age secret key once into every
|
||||||
@@ -812,27 +833,34 @@ func (v *Vaultik) materializeSnapshotDB(
|
|||||||
return db, tempDir, nil
|
return db, tempDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getFilesToRestore returns the list of files to restore based on path filters
|
// getFilesToRestore returns the list of files to restore based on path
|
||||||
|
// filters, and the filters, as given, that matched no file.
|
||||||
func (v *Vaultik) getFilesToRestore(
|
func (v *Vaultik) getFilesToRestore(
|
||||||
ctx context.Context, repos *database.Repositories, pathFilters []string,
|
ctx context.Context, repos *database.Repositories, pathFilters []string,
|
||||||
) ([]*database.File, error) {
|
) ([]*database.File, []string, error) {
|
||||||
// If no filters, get all files
|
// If no filters, get all files
|
||||||
if len(pathFilters) == 0 {
|
if len(pathFilters) == 0 {
|
||||||
return repos.Files.ListAll(ctx)
|
files, err := repos.Files.ListAll(ctx)
|
||||||
|
|
||||||
|
return files, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get files matching the path filters
|
// Get files matching the path filters
|
||||||
var result []*database.File
|
var (
|
||||||
|
result []*database.File
|
||||||
|
unmatched []string
|
||||||
|
)
|
||||||
|
|
||||||
seen := make(map[string]bool)
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
for _, filter := range pathFilters {
|
for _, filter := range pathFilters {
|
||||||
// Normalize the filter path
|
files, err := repos.Files.ListUnderPath(ctx, filepath.Clean(filter))
|
||||||
filter = filepath.Clean(filter)
|
|
||||||
|
|
||||||
files, err := repos.Files.ListUnderPath(ctx, filter)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("listing files under %s: %w", filter, err)
|
return nil, nil, fmt.Errorf("listing files under %s: %w", filter, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(files) == 0 {
|
||||||
|
unmatched = append(unmatched, filter)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
@@ -843,7 +871,7 @@ func (v *Vaultik) getFilesToRestore(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, unmatched, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildChunkToBlobMap creates a mapping from chunk hash to blob information
|
// buildChunkToBlobMap creates a mapping from chunk hash to blob information
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/309: a path
|
||||||
|
// given to restore that matches nothing in the snapshot is named in a
|
||||||
|
// warning and makes the restore fail, after the paths that do match are
|
||||||
|
// restored.
|
||||||
|
|
||||||
|
func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
restoreDir := filepath.Join(tempDir, "restored")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg := faultTestConfig()
|
||||||
|
files := writeFaultSourceTree(t, fs, dataDir)
|
||||||
|
|
||||||
|
store, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
id := fullFaultBackup(ctx, t, fs, store, cfg,
|
||||||
|
database.NewRepositories(db), dataDir, dbPath, "unmatched")
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
matched := filepath.Join(dataDir, "small.txt")
|
||||||
|
unmatched := filepath.Join(dataDir, "typo.txt")
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
v := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||||
|
v.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
err = v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: id,
|
||||||
|
TargetDir: restoreDir,
|
||||||
|
Paths: []string{matched, unmatched},
|
||||||
|
})
|
||||||
|
|
||||||
|
require.Error(t, err, "a path that matches nothing must fail the restore")
|
||||||
|
assert.Contains(t, out.String(), unmatched,
|
||||||
|
"the path that matches nothing must be named")
|
||||||
|
assert.NotContains(t, out.String(), matched,
|
||||||
|
"the path that matches must not be reported")
|
||||||
|
|
||||||
|
got, err := afero.ReadFile(fs, filepath.Join(restoreDir, matched))
|
||||||
|
require.NoError(t, err, "the path that matches must still be restored")
|
||||||
|
assert.Equal(t, files[matched], got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// When no requested path matches, nothing is restored and each path is
|
||||||
|
// named. One path is relative. The other goes through the symlink that
|
||||||
|
// is the snapshot's configured path, while the entries are stored under
|
||||||
|
// the directory the symlink points to.
|
||||||
|
func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const snapshotName = "linked"
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
|
linkDir := filepath.Join(tempDir, "srclink")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
restoreDir := filepath.Join(tempDir, "restored")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
writeFaultSourceTree(t, fs, dataDir)
|
||||||
|
require.NoError(t, os.Symlink(dataDir, linkDir))
|
||||||
|
|
||||||
|
cfg := faultTestConfig()
|
||||||
|
cfg.IndexPath = dbPath
|
||||||
|
cfg.ChunkSize = config.Size(faultChunkSize)
|
||||||
|
cfg.Snapshots = map[string]config.SnapshotConfig{
|
||||||
|
snapshotName: {Paths: []string{linkDir}},
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||||
|
|
||||||
|
require.NoError(t, backUp(v, snapshotName))
|
||||||
|
|
||||||
|
id := localSnapshotID(ctx, t, repos, snapshotName)
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
throughLink := filepath.Join(linkDir, "sub")
|
||||||
|
relative := strings.TrimPrefix(filepath.Join(dataDir, "small.txt"), "/")
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||||
|
reader.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
err = reader.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: id,
|
||||||
|
TargetDir: restoreDir,
|
||||||
|
Paths: []string{throughLink, relative},
|
||||||
|
})
|
||||||
|
|
||||||
|
require.Error(t, err, "a restore that matches nothing must fail")
|
||||||
|
assert.Contains(t, out.String(), throughLink,
|
||||||
|
"the path through the symlink must be named")
|
||||||
|
assert.Contains(t, out.String(), relative,
|
||||||
|
"the relative path must be named")
|
||||||
|
|
||||||
|
_, err = fs.Stat(restoreDir)
|
||||||
|
assert.True(t, os.IsNotExist(err), "nothing must be restored")
|
||||||
|
}
|
||||||
@@ -240,18 +240,12 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
previousSnapshotID, err := v.previousSnapshotID(snapName)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||||
EnableProgress: !opts.Cron,
|
EnableProgress: !opts.Cron,
|
||||||
UI: v.UI,
|
UI: v.UI,
|
||||||
Fs: v.Fs,
|
Fs: v.Fs,
|
||||||
Exclude: v.Config.GetExcludes(snapName),
|
Exclude: v.Config.GetExcludes(snapName),
|
||||||
SkipErrors: opts.SkipErrors,
|
SkipErrors: opts.SkipErrors,
|
||||||
PreviousSnapshotID: previousSnapshotID,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
|
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
|
||||||
@@ -286,24 +280,6 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// previousSnapshotID returns the ID of the most recent completed snapshot
|
|
||||||
// named snapName in the local index, or "" when there is none.
|
|
||||||
func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
|
|
||||||
snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("listing snapshots: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, s := range snapshots {
|
|
||||||
if s.CompletedAt != nil &&
|
|
||||||
parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
|
|
||||||
return s.ID.String(), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveSnapshotPaths resolves source directories to absolute paths
|
// resolveSnapshotPaths resolves source directories to absolute paths
|
||||||
// with symlink resolution.
|
// with symlink resolution.
|
||||||
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
||||||
|
|||||||
@@ -1,196 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
|
|
||||||
// backup counts a file as deleted only when the previous snapshot of
|
|
||||||
// the same name had it and it is gone, and never counts an excluded
|
|
||||||
// file.
|
|
||||||
|
|
||||||
// deletedCountEnv is a backup setup over the source directory src whose
|
|
||||||
// user-facing output is kept in out. Each test configures its snapshots
|
|
||||||
// with setSnapshot.
|
|
||||||
type deletedCountEnv struct {
|
|
||||||
v *vaultik.Vaultik
|
|
||||||
out *bytes.Buffer
|
|
||||||
src string
|
|
||||||
}
|
|
||||||
|
|
||||||
// newDeletedCountEnv writes files, keyed by path relative to src.
|
|
||||||
func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
srcDir := filepath.Join(tempDir, "src")
|
|
||||||
|
|
||||||
for rel, content := range files {
|
|
||||||
path := filepath.Join(srcDir, rel)
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
|
|
||||||
cfg.Snapshots = map[string]config.SnapshotConfig{}
|
|
||||||
|
|
||||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, cfg.IndexPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
out := &bytes.Buffer{}
|
|
||||||
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
|
||||||
v.UI = ui.NewWithColor(out, false)
|
|
||||||
|
|
||||||
return &deletedCountEnv{v: v, out: out, src: srcDir}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setSnapshot configures the snapshot name to back up the path rel
|
|
||||||
// beneath src, with the given excludes.
|
|
||||||
func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
|
|
||||||
e.v.Config.Snapshots[name] = config.SnapshotConfig{
|
|
||||||
Paths: []string{filepath.Join(e.src, rel)},
|
|
||||||
Exclude: exclude,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// backUp runs a backup of the named snapshot and returns its output.
|
|
||||||
func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
e.out.Reset()
|
|
||||||
require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
|
||||||
Snapshots: []string{name},
|
|
||||||
}))
|
|
||||||
|
|
||||||
return e.out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// A file this snapshot excludes is not counted as deleted: neither one
|
|
||||||
// that another snapshot backs up, nor one that the previous snapshot of
|
|
||||||
// this name had before the exclude was added.
|
|
||||||
func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newDeletedCountEnv(t, map[string][]byte{
|
|
||||||
"top.txt": []byte("top"),
|
|
||||||
"cache/tmp.bin": []byte("cache"),
|
|
||||||
"photos/a.jpg": []byte("photo a"),
|
|
||||||
"photos/b.jpg": []byte("photo b"),
|
|
||||||
"photos/c.jpg": []byte("photo c"),
|
|
||||||
})
|
|
||||||
env.setSnapshot("photos", "photos")
|
|
||||||
env.setSnapshot("home", "", "/photos")
|
|
||||||
|
|
||||||
env.backUp(t, "photos")
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
|
|
||||||
|
|
||||||
env.setSnapshot("home", "", "/photos", "/cache")
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A file inside a directory that an exclude added after a backup
|
|
||||||
// matches is not counted as deleted, though the pattern matches only the
|
|
||||||
// directory.
|
|
||||||
func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newDeletedCountEnv(t, map[string][]byte{
|
|
||||||
"top.txt": []byte("top"),
|
|
||||||
"user/.local/share/Trash/old.txt": []byte("old"),
|
|
||||||
})
|
|
||||||
env.setSnapshot("home", "")
|
|
||||||
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
|
|
||||||
|
|
||||||
env.setSnapshot("home", "", "*/.local/share/Trash")
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A backup counts only files that the previous snapshot of its own name
|
|
||||||
// had, not one that a later snapshot of another name over the same path
|
|
||||||
// had and that is since removed.
|
|
||||||
func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newDeletedCountEnv(t, map[string][]byte{
|
|
||||||
"kept.txt": []byte("kept"),
|
|
||||||
})
|
|
||||||
env.setSnapshot("home", "")
|
|
||||||
env.setSnapshot("other", "")
|
|
||||||
|
|
||||||
env.backUp(t, "home")
|
|
||||||
|
|
||||||
// Snapshot start times are kept in whole seconds: wait so that other's
|
|
||||||
// snapshot is strictly the most recent.
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
|
|
||||||
extra := filepath.Join(env.src, "extra.txt")
|
|
||||||
require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
|
|
||||||
env.backUp(t, "other")
|
|
||||||
require.NoError(t, os.Remove(extra))
|
|
||||||
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A file removed from disk is counted as deleted by the next backup of
|
|
||||||
// the snapshot that had it, and not by any later backup of that
|
|
||||||
// snapshot, nor by the first backup of another name over the same path.
|
|
||||||
func TestBackupCountsDeletedFileOnce(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
aContent := bytes.Repeat([]byte("a"), 1000)
|
|
||||||
bContent := bytes.Repeat([]byte("b"), 2000)
|
|
||||||
keptContent := bytes.Repeat([]byte("k"), 500)
|
|
||||||
|
|
||||||
env := newDeletedCountEnv(t, map[string][]byte{
|
|
||||||
"a.txt": aContent,
|
|
||||||
"b.txt": bContent,
|
|
||||||
"kept.txt": keptContent,
|
|
||||||
})
|
|
||||||
env.setSnapshot("home", "")
|
|
||||||
env.setSnapshot("other", "")
|
|
||||||
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
|
|
||||||
|
|
||||||
require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
|
|
||||||
require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
|
|
||||||
|
|
||||||
summary := env.backUp(t, "home")
|
|
||||||
assert.Contains(t, summary,
|
|
||||||
"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
|
|
||||||
assert.Contains(t, summary,
|
|
||||||
fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
|
|
||||||
env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
|
|
||||||
env.v.UI.Size(int64(len(aContent)+len(bContent)))))
|
|
||||||
|
|
||||||
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
|
|
||||||
assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user