Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
141ca1c68a |
No files matched your search
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
`vaultik` is an incremental backup tool written in Go. It encrypts data using an
|
`vaultik` is an incremental backup tool written in Go. It encrypts data using an
|
||||||
`age` public key and uploads each encrypted blob directly to a remote
|
`age` public key and uploads each encrypted blob directly to a remote
|
||||||
S3-compatible object store. The backed-up system never needs the private key. A
|
S3-compatible object store. It requires no private keys, secrets, or credentials
|
||||||
backup needs only the public key and the credentials to write to the object
|
(other than those required to PUT to encrypted object storage, such as S3 API
|
||||||
store, such as S3 API keys.
|
keys) stored on the backed-up system.
|
||||||
|
|
||||||
## quickstart
|
## quickstart
|
||||||
|
|
||||||
@@ -171,7 +171,6 @@ vaultik [--config <path>] config get <key>
|
|||||||
vaultik [--config <path>] config set <key> <value>
|
vaultik [--config <path>] config set <key> <value>
|
||||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
|
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
|
||||||
vaultik [--config <path>] snapshot list [--json] # alias: ls
|
vaultik [--config <path>] snapshot list [--json] # alias: ls
|
||||||
vaultik [--config <path>] snapshot files <snapshot-id> [path] [--json]
|
|
||||||
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
||||||
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--snapshot <name>...] [--force]
|
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--snapshot <name>...] [--force]
|
||||||
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force] [--local-only] [--json] # alias: rm
|
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force] [--local-only] [--json] # alias: rm
|
||||||
@@ -224,24 +223,12 @@ the backup can delete it; the new snapshot then cannot be restored.
|
|||||||
`remote nuke` deletes everything regardless. To not depend on the grace period,
|
`remote nuke` deletes everything regardless. To not depend on the grace period,
|
||||||
give each host or user its own prefix (or directory) on the destination.
|
give each host or user its own prefix (or directory) on the destination.
|
||||||
|
|
||||||
These read-only commands do not take the lock, so they are never blocked, even
|
Read-only commands — `info`, `snapshot list`, `snapshot verify`, and
|
||||||
while a backup is in progress:
|
`remote info` — do not take the lock and are never blocked, so they run even
|
||||||
|
while a backup is in progress. `snapshot restore` does not take the lock either:
|
||||||
- `info`
|
it writes only to the target directory you name, not the local index or the
|
||||||
- `snapshot list`
|
destination store. `config`, `database delete`, `completion`, and `version` do
|
||||||
- `snapshot files`
|
not take the lock.
|
||||||
- `snapshot verify`
|
|
||||||
- `remote info`
|
|
||||||
|
|
||||||
`snapshot restore` does not take the lock either: it writes only to the target
|
|
||||||
directory you name, not the local index or the destination store.
|
|
||||||
|
|
||||||
These commands do not take the lock either:
|
|
||||||
|
|
||||||
- `config`
|
|
||||||
- `database delete`
|
|
||||||
- `completion`
|
|
||||||
- `version`
|
|
||||||
|
|
||||||
### stdout and stderr
|
### stdout and stderr
|
||||||
|
|
||||||
@@ -274,9 +261,9 @@ work as written.
|
|||||||
### environment variables
|
### environment variables
|
||||||
|
|
||||||
- `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for
|
- `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for
|
||||||
`snapshot restore`, `snapshot files` and `snapshot verify --deep`). May hold
|
`snapshot restore` and `snapshot verify --deep`). May hold the whole
|
||||||
the whole `age-keygen` file — comments and every identity in it are accepted.
|
`age-keygen` file — comments and every identity in it are accepted. Set it
|
||||||
Set it from the file, e.g.
|
from the file, e.g.
|
||||||
`export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"`, so
|
`export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"`, so
|
||||||
the key is not typed into your shell history.
|
the key is not typed into your shell history.
|
||||||
- `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
- `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
||||||
@@ -421,26 +408,6 @@ point behind.
|
|||||||
truncated listing all go to stderr through the logger, so stdout stays a
|
truncated listing all go to stderr through the logger, so stdout stays a
|
||||||
single parseable document.
|
single parseable document.
|
||||||
|
|
||||||
**`snapshot files`**: List the files in a snapshot without restoring it. Each
|
|
||||||
row of the table shows mode, size in bytes, modification time and path, sorted
|
|
||||||
by path.
|
|
||||||
|
|
||||||
- Requires `VAULTIK_AGE_SECRET_KEY`: the file list is only in the snapshot's
|
|
||||||
encrypted database, which the command downloads and decrypts.
|
|
||||||
- Accepts the same identifiers as `snapshot restore`.
|
|
||||||
- An optional path limits the list to that path and what is under it. A path
|
|
||||||
that matches nothing in the snapshot is an error.
|
|
||||||
- `--json`: Output an array of objects with `path`, `size`, `mode` and `mtime`.
|
|
||||||
- In the table, a path that holds a newline or another non-printable character
|
|
||||||
is shown in double quotes, with that character escaped, as in `"a\nb"`.
|
|
||||||
- The mode is in Go's file-mode notation, in the table and in `--json`. Letters
|
|
||||||
before the nine permission characters show the file type and any special bit,
|
|
||||||
as in `dtrwxrwxrwx` for a sticky directory. These letters differ from `ls -l`:
|
|
||||||
- `L`: a symbolic link, which `ls -l` shows as `l`.
|
|
||||||
- `u`: setuid, which `ls -l` shows as `s` in place of the owner's `x`.
|
|
||||||
- `g`: setgid, which `ls -l` shows as `s` in place of the group's `x`.
|
|
||||||
- `t`: sticky, which `ls -l` shows as `t` in place of the others' `x`.
|
|
||||||
|
|
||||||
**`snapshot verify`**: Verify snapshot integrity.
|
**`snapshot verify`**: Verify snapshot integrity.
|
||||||
|
|
||||||
- Default (shallow): checks that every blob the manifest lists is present in
|
- Default (shallow): checks that every blob the manifest lists is present in
|
||||||
@@ -468,12 +435,10 @@ destination store. Blobs are NOT touched — deleting blobs requires reading eve
|
|||||||
remaining remote manifest (the destination store may hold snapshots this host
|
remaining remote manifest (the destination store may hold snapshots this host
|
||||||
doesn't know about), which is what `vaultik prune` does. The command prints the
|
doesn't know about), which is what `vaultik prune` does. The command prints the
|
||||||
`vaultik prune` invocation to run as a follow-up only when it removed the
|
`vaultik prune` invocation to run as a follow-up only when it removed the
|
||||||
snapshot's metadata from the destination store, and never under `--json`. It
|
snapshot's metadata from the destination store, and never under `--json`. An ID
|
||||||
accepts the same identifiers as `snapshot restore`: a snapshot ID, or a
|
that names no snapshot in the local index or on the destination store is an
|
||||||
remote-only snapshot's remote key (or an unambiguous leading part of it). An
|
error; with `--local-only`, the snapshot must be in the local index. Local row
|
||||||
identifier that names no snapshot in the local index or on the destination store
|
cleanup (files, chunks, blobs the snapshot was the last referrer for) runs
|
||||||
is an error; with `--local-only`, the snapshot must be in the local index. Local
|
|
||||||
row cleanup (files, chunks, blobs the snapshot was the last referrer for) runs
|
|
||||||
automatically. If removing the snapshot's metadata from the destination store
|
automatically. If removing the snapshot's metadata from the destination store
|
||||||
fails (for example because the store is unreachable, or the local index holds
|
fails (for example because the store is unreachable, or the local index holds
|
||||||
snapshots and the destination answers that it has no `vaultik-destination.txt`),
|
snapshots and the destination answers that it has no `vaultik-destination.txt`),
|
||||||
@@ -483,6 +448,8 @@ remove the snapshot's metadata from it (`vaultik prune` does not). For a
|
|||||||
snapshot that is only on the destination store, that failure is an error. To
|
snapshot that is only on the destination store, that failure is an error. To
|
||||||
wipe everything on the destination in one go, use `vaultik remote nuke --force`.
|
wipe everything on the destination in one go, use `vaultik remote nuke --force`.
|
||||||
|
|
||||||
|
- Accepts a snapshot ID, or — for a snapshot only on the destination store — its
|
||||||
|
remote key (or an unambiguous leading part of it) as shown by `snapshot list`.
|
||||||
- `--local-only`: Skip remote cleanup; only touch the local index
|
- `--local-only`: Skip remote cleanup; only touch the local index
|
||||||
- `--dry-run`: Show what would be deleted without deleting
|
- `--dry-run`: Show what would be deleted without deleting
|
||||||
- `--force`: Skip confirmation prompt
|
- `--force`: Skip confirmation prompt
|
||||||
@@ -757,7 +724,7 @@ annotated example also lives in
|
|||||||
| Field | Default | Description |
|
| Field | Default | Description |
|
||||||
| -------------------- | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| -------------------- | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty |
|
| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty |
|
||||||
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot files`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
|
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
|
||||||
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
||||||
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
||||||
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
|
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
|
||||||
@@ -874,27 +841,15 @@ environment variable is unset (https://no-color.org/).
|
|||||||
`database delete`.
|
`database delete`.
|
||||||
- **The data a command exists to produce** is written plain, with no marker and
|
- **The data a command exists to produce** is written plain, with no marker and
|
||||||
no color, because a marker would corrupt a table or a parsed document. This
|
no color, because a marker would corrupt a table or a parsed document. This
|
||||||
covers the report or table of each of these commands:
|
covers the `version`, `info`, `remote info` and `snapshot verify` reports, the
|
||||||
- `version`
|
`snapshot list` table, `config get` values, and every `--json` document.
|
||||||
- `info`
|
`--quiet` silences the human reports and tables (`version`, `info`,
|
||||||
- `remote info`
|
`remote info`, `snapshot verify`, `snapshot list`) but never the
|
||||||
- `snapshot verify`
|
machine-consumed `config get` value or the `--json` documents, which a script
|
||||||
- `snapshot list`
|
depends on. The confirmation prompts of `database delete`, `prune`,
|
||||||
- `snapshot files`
|
`snapshot purge` and `snapshot remove`, each with what it asks to delete, are
|
||||||
|
also written this way and always shown: they are an interactive exchange the
|
||||||
`config get` values and every `--json` document are written plain too.
|
operator must see.
|
||||||
`--quiet` silences the reports and tables above, but never a `config get`
|
|
||||||
value or a `--json` document, which a script depends on.
|
|
||||||
|
|
||||||
These commands write their confirmation prompt plain too, with what it asks
|
|
||||||
to delete:
|
|
||||||
- `database delete`
|
|
||||||
- `prune`
|
|
||||||
- `snapshot purge`
|
|
||||||
- `snapshot remove`
|
|
||||||
|
|
||||||
`--quiet` never silences these prompts: each is an interactive exchange the
|
|
||||||
operator must see.
|
|
||||||
|
|
||||||
`internal/ui` writes to stdout; it is the output the user asked for. The
|
`internal/ui` writes to stdout; it is the output the user asked for. The
|
||||||
exceptions are the startup banner and the error a failed command ends with,
|
exceptions are the startup banner and the error a failed command ends with,
|
||||||
|
|||||||
+9
-17
@@ -265,23 +265,15 @@ so an operator can weigh them rather than discover them.
|
|||||||
blob holds hundreds of chunks, so individual chunk lengths are not visible in
|
blob holds hundreds of chunks, so individual chunk lengths are not visible in
|
||||||
the blob's size; lowering the limit toward the chunk size begins to expose
|
the blob's size; lowering the limit toward the chunk size begins to expose
|
||||||
them.
|
them.
|
||||||
4. **Decrypted data on local disk.** These commands stage plaintext under
|
4. **Decrypted data on local disk.** Several commands stage plaintext under
|
||||||
`$TMPDIR`:
|
`$TMPDIR`: `snapshot restore` writes decrypted blobs under
|
||||||
- `snapshot restore`: decrypted blobs under `vaultik-blobcache-*/`, with no
|
`vaultik-blobcache-*/` (no size cap) and the decrypted metadata database at
|
||||||
size cap, and the decrypted metadata database at
|
`vaultik-restore-*/snapshot.db`; `verify --deep` writes that database at
|
||||||
`vaultik-restore-*/snapshot.db`.
|
`vaultik-verify-*/snapshot.db`; `snapshot create` keeps a plaintext copy of
|
||||||
- `snapshot files`: the decrypted metadata database at
|
the index at `vaultik-snapshot-*/snapshot.db`. These files are created `0600`
|
||||||
`vaultik-restore-*/snapshot.db`.
|
and removed on success, but a `kill -9` or a power loss leaves them behind —
|
||||||
- `verify --deep`: the decrypted metadata database at
|
delete any leftover `vaultik-*` directory under `$TMPDIR` by hand. `$TMPDIR`
|
||||||
`vaultik-verify-*/snapshot.db`.
|
should be trusted to the same degree as the restore target.
|
||||||
- `snapshot create`: a plaintext copy of the index at
|
|
||||||
`vaultik-snapshot-*/snapshot.db`.
|
|
||||||
|
|
||||||
These files are created `0600` and removed on success, but a `kill -9` or a
|
|
||||||
power loss leaves them behind — delete any leftover `vaultik-*` directory
|
|
||||||
under `$TMPDIR` by hand. `$TMPDIR` should be trusted to the same degree as
|
|
||||||
the restore target.
|
|
||||||
|
|
||||||
5. **Store permissions differ per command.** The backed-up host needs only PUT
|
5. **Store permissions differ per command.** The backed-up host needs only PUT
|
||||||
to run `snapshot create`: it writes blobs and metadata and neither reads nor
|
to run `snapshot create`: it writes blobs and metadata and neither reads nor
|
||||||
deletes them. Other commands need more — `snapshot verify`,
|
deletes them. Other commands need more — `snapshot verify`,
|
||||||
|
|||||||
+6
-7
@@ -38,11 +38,10 @@ const (
|
|||||||
// prune, remote nuke) write the local index or the remote store. They
|
// prune, remote nuke) write the local index or the remote store. They
|
||||||
// hold the PID lock so that at most one runs at a time.
|
// hold the PID lock so that at most one runs at a time.
|
||||||
mutating lockMode = iota
|
mutating lockMode = iota
|
||||||
// readOnly commands (info, snapshot list, snapshot files, snapshot
|
// readOnly commands (info, snapshot list, snapshot verify, remote info,
|
||||||
// verify, remote info, snapshot restore) do not write the local index
|
// snapshot restore) do not write the local index or the remote store,
|
||||||
// or the remote store, so they run without the lock and are never
|
// so they run without the lock and are never blocked by a running
|
||||||
// blocked by a running mutator. restore writes only to the target
|
// mutator. restore writes only to the target directory it is given.
|
||||||
// directory it is given.
|
|
||||||
readOnly
|
readOnly
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -311,7 +310,7 @@ func RunOperation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runVaultikApp runs the standard single-operation command lifecycle
|
// runVaultikApp runs the standard single-operation command lifecycle
|
||||||
// shared by the snapshot list/files/purge/remove and remote nuke subcommands:
|
// shared by the snapshot list/purge/remove and remote nuke subcommands:
|
||||||
// resolve the config, then run op against the Vaultik instance through
|
// resolve the config, then run op against the Vaultik instance through
|
||||||
// RunOperation, reporting a failure prefixed with failMsg on stderr. mode
|
// RunOperation, reporting a failure prefixed with failMsg on stderr. mode
|
||||||
// says whether the command takes the PID lock. jsonOutput marks a command
|
// says whether the command takes the PID lock. jsonOutput marks a command
|
||||||
@@ -338,7 +337,7 @@ func runVaultikApp(
|
|||||||
},
|
},
|
||||||
Mode: mode,
|
Mode: mode,
|
||||||
}, op, func(err error) {
|
}, op, func(err error) {
|
||||||
log.Info(failMsg, "error", err)
|
log.Error(failMsg, "error", err)
|
||||||
ReportErrorf("%s: %v", failMsg, err)
|
ReportErrorf("%s: %v", failMsg, err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -282,13 +282,6 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
|||||||
storageURLKey, testS3URL,
|
storageURLKey, testS3URL,
|
||||||
"s3.secret_access_key is required for s3:// URLs; " +
|
"s3.secret_access_key is required for s3:// URLs; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key first"},
|
"set s3.access_key_id and s3.secret_access_key first"},
|
||||||
// rclone:// URLs that no command could open.
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone:///backups",
|
|
||||||
"rclone URL missing remote name"},
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
|
|
||||||
"rclone URL must not carry parameters in the remote name"},
|
|
||||||
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
|
|
||||||
`unknown query parameter in storage URL: "x"`},
|
|
||||||
// With its reference filled in, the config loads.
|
// With its reference filled in, the config loads.
|
||||||
{levelReference, "blob_size_limit", "1MB",
|
{levelReference, "blob_size_limit", "1MB",
|
||||||
"blob_size_limit must be at least"},
|
"blob_size_limit must be at least"},
|
||||||
|
|||||||
@@ -85,10 +85,6 @@ func emitStartupBanner(args []string, w io.Writer) {
|
|||||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
||||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
// an error to cobra isn't an option) and anywhere else a CLI command
|
||||||
// must surface a failure outside the normal RunE return path.
|
// must surface a failure outside the normal RunE return path.
|
||||||
//
|
|
||||||
// This line is the one report of the failure. A caller that also logs
|
|
||||||
// the failure logs it at info level, so the record appears only under
|
|
||||||
// --verbose or --debug and the error is not printed twice.
|
|
||||||
func ReportErrorf(format string, args ...any) {
|
func ReportErrorf(format string, args ...any) {
|
||||||
ui.New(os.Stderr).Errorf(format, args...)
|
ui.New(os.Stderr).Errorf(format, args...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,205 +0,0 @@
|
|||||||
package cli //nolint:testpackage // shares the fixtures and capture helpers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"filippo.io/age"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// errorLinePrefix starts the final line that reports a failed command.
|
|
||||||
errorLinePrefix = "ERROR: "
|
|
||||||
|
|
||||||
flagVerbose = "--verbose"
|
|
||||||
cmdRestore = "restore"
|
|
||||||
|
|
||||||
// blobsDir is the directory of a destination store that holds the
|
|
||||||
// blobs.
|
|
||||||
blobsDir = "blobs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestEntryFailureIsPrintedOnce runs commands that fail after startup
|
|
||||||
// against a destination they cannot use. `info` is not among them: it
|
|
||||||
// cannot fail after startup.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
|
||||||
func TestEntryFailureIsPrintedOnce(t *testing.T) {
|
|
||||||
for _, command := range [][]string{
|
|
||||||
{cmdSnapshot, cmdCreate},
|
|
||||||
{cmdSnapshot, cmdRemove, someSnapshotID},
|
|
||||||
{cmdSnapshot, cmdVerify, someSnapshotID},
|
|
||||||
{cmdRemote, cmdInfo},
|
|
||||||
{cmdPrune, flagJSON},
|
|
||||||
} {
|
|
||||||
for _, verbose := range []bool{false, true} {
|
|
||||||
name := fmt.Sprintf("%s verbose=%t", strings.Join(command, " "), verbose)
|
|
||||||
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
requireErrorPrintedOnce(t, verbose, append([]string{flagConfig,
|
|
||||||
writeUnusableDestinationConfig(t)}, command...)...)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntryBlobUploadFailureIsPrintedOnce backs up to a destination
|
|
||||||
// whose blobs directory is a file, so the blob upload fails.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
|
||||||
func TestEntryBlobUploadFailureIsPrintedOnce(t *testing.T) {
|
|
||||||
for _, verbose := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
|
|
||||||
configPath, storeDir := writeBackupConfig(t)
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(
|
|
||||||
filepath.Join(storeDir, blobsDir), nil, 0o600))
|
|
||||||
|
|
||||||
requireErrorPrintedOnce(t, verbose,
|
|
||||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntryBlobDownloadFailureIsPrintedOnce backs up, deletes the blobs
|
|
||||||
// from the destination, and restores, so the blob download fails.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
|
||||||
func TestEntryBlobDownloadFailureIsPrintedOnce(t *testing.T) {
|
|
||||||
for _, verbose := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
|
|
||||||
configPath, storeDir := writeBackupConfigWithPrivateKey(t)
|
|
||||||
|
|
||||||
code, _, _ := runEntry(t,
|
|
||||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
|
||||||
require.Equal(t, 0, code)
|
|
||||||
|
|
||||||
require.NoError(t, os.RemoveAll(filepath.Join(storeDir, blobsDir)))
|
|
||||||
|
|
||||||
requireErrorPrintedOnce(t, verbose, flagConfig, configPath,
|
|
||||||
cmdSnapshot, cmdRestore, onlySnapshotID(t, configPath), t.TempDir())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireErrorPrintedOnce runs Entry with args, adding --verbose when
|
|
||||||
// verbose is set. The command must fail, and stderr must carry its error
|
|
||||||
// once, as the final ERROR: line. A log record carrying the same error
|
|
||||||
// appears only under --verbose.
|
|
||||||
func requireErrorPrintedOnce(t *testing.T, verbose bool, args ...string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if verbose {
|
|
||||||
args = append(args, flagVerbose)
|
|
||||||
}
|
|
||||||
|
|
||||||
code, _, stderr := runEntry(t, args...)
|
|
||||||
require.Equal(t, 1, code)
|
|
||||||
|
|
||||||
errorLine, records := failureOutput(t, stderr)
|
|
||||||
|
|
||||||
if !verbose {
|
|
||||||
assert.Zero(t, records,
|
|
||||||
"without --verbose no log record repeats the error:\n%s", stderr)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotZero(t, records,
|
|
||||||
"--verbose must keep the log record of %q", errorLine)
|
|
||||||
}
|
|
||||||
|
|
||||||
// failureOutput requires stderr to hold exactly one ERROR: line and
|
|
||||||
// returns it, with the number of log records whose error attribute is
|
|
||||||
// the error that line reports. stderr is a pipe here, so every log
|
|
||||||
// record is a JSON object on its own line.
|
|
||||||
func failureOutput(t *testing.T, stderr string) (string, int) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
errorLines []string
|
|
||||||
records []map[string]any
|
|
||||||
)
|
|
||||||
|
|
||||||
for line := range strings.Lines(stderr) {
|
|
||||||
var record map[string]any
|
|
||||||
if json.Unmarshal([]byte(line), &record) == nil {
|
|
||||||
records = append(records, record)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(line, errorLinePrefix) {
|
|
||||||
errorLines = append(errorLines, strings.TrimSpace(line))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Len(t, errorLines, 1,
|
|
||||||
"the error must be printed once as an ERROR: line:\n%s", stderr)
|
|
||||||
|
|
||||||
// An error can run over several lines; the ERROR: line holds the
|
|
||||||
// first of them.
|
|
||||||
repeats := 0
|
|
||||||
|
|
||||||
for _, record := range records {
|
|
||||||
failure, ok := record["error"].(string)
|
|
||||||
firstLine, _, _ := strings.Cut(failure, "\n")
|
|
||||||
|
|
||||||
if ok && strings.Contains(errorLines[0], firstLine) {
|
|
||||||
repeats++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errorLines[0], repeats
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeBackupConfigWithPrivateKey builds what writeBackupConfig builds,
|
|
||||||
// with a generated key pair in place of its recipient. The config also
|
|
||||||
// holds the private key, so that restore can decrypt the backup. Returns
|
|
||||||
// the config path and the store directory.
|
|
||||||
func writeBackupConfigWithPrivateKey(t *testing.T) (string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
configPath, storeDir := writeBackupConfig(t)
|
|
||||||
|
|
||||||
identity, err := age.GenerateX25519Identity()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
out := ui.NewWithColor(io.Discard, false)
|
|
||||||
|
|
||||||
require.NoError(t, writeConfigSet(out, configPath,
|
|
||||||
"age_recipients.0", identity.Recipient().String()))
|
|
||||||
require.NoError(t, writeConfigSet(out, configPath,
|
|
||||||
"age_secret_key", identity.String()))
|
|
||||||
|
|
||||||
return configPath, storeDir
|
|
||||||
}
|
|
||||||
|
|
||||||
// onlySnapshotID returns the ID of the one snapshot in the local index
|
|
||||||
// that writeBackupConfig puts beside the config file.
|
|
||||||
func onlySnapshotID(t *testing.T, configPath string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
db, err := database.New(ctx,
|
|
||||||
filepath.Join(filepath.Dir(configPath), "index.sqlite"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, db.Close()) }()
|
|
||||||
|
|
||||||
backedUp, err := database.NewRepositories(db).Snapshots.GetOnlySnapshot(ctx)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return backedUp.ID.String()
|
|
||||||
}
|
|
||||||
@@ -61,8 +61,8 @@ func TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup(t *testing.T) {
|
|||||||
|
|
||||||
// TestEntrySnapshotCreateReportsPostBackupPruneFailure makes the
|
// TestEntrySnapshotCreateReportsPostBackupPruneFailure makes the
|
||||||
// post-backup prune fail after a good backup, by planting a manifest
|
// post-backup prune fail after a good backup, by planting a manifest
|
||||||
// that cannot be decoded, and checks that the failure is reported once,
|
// that cannot be decoded, and checks that the failure is reported as the
|
||||||
// as the prune's and not as a failed snapshot creation.
|
// prune's and not as a failed snapshot creation.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||||
@@ -83,9 +83,6 @@ func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
|||||||
assert.Contains(t, stderr, "Backup completed, but post-backup prune failed")
|
assert.Contains(t, stderr, "Backup completed, but post-backup prune failed")
|
||||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||||
|
|
||||||
_, repeats := failureOutput(t, stderr)
|
|
||||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
|
||||||
|
|
||||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, snapshots, 2,
|
assert.Len(t, snapshots, 2,
|
||||||
@@ -96,8 +93,7 @@ func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
|||||||
// empties the source directory, as a volume that is not mounted leaves
|
// empties the source directory, as a volume that is not mounted leaves
|
||||||
// its mount point, and runs `snapshot create --prune`. The command exits
|
// its mount point, and runs `snapshot create --prune`. The command exits
|
||||||
// non-zero and reports the backup as completed, not as a failed snapshot
|
// non-zero and reports the backup as completed, not as a failed snapshot
|
||||||
// creation. The error is printed once. See
|
// creation. See https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
// https://git.eeqj.de/sneak/vaultik/issues/388.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||||
@@ -118,9 +114,6 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
|||||||
"Backup completed, but 1 source path(s) were empty")
|
"Backup completed, but 1 source path(s) were empty")
|
||||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||||
|
|
||||||
_, repeats := failureOutput(t, stderr)
|
|
||||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
|
||||||
|
|
||||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||||
@@ -131,8 +124,7 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
|||||||
// directory as a mount point, and runs `snapshot create --prune`, which
|
// directory as a mount point, and runs `snapshot create --prune`, which
|
||||||
// does not find it as one. The command exits non-zero, reports the
|
// does not find it as one. The command exits non-zero, reports the
|
||||||
// backup as completed, not as a failed snapshot creation, and keeps the
|
// backup as completed, not as a failed snapshot creation, and keeps the
|
||||||
// older snapshot. The error is printed once. See
|
// older snapshot. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||||
// https://git.eeqj.de/sneak/vaultik/issues/395.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
||||||
@@ -167,9 +159,6 @@ func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
|||||||
"an earlier snapshot recorded were not found")
|
"an earlier snapshot recorded were not found")
|
||||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||||
|
|
||||||
_, repeats := failureOutput(t, stderr)
|
|
||||||
assert.Zero(t, repeats, "no log record may repeat the error:\n%s", stderr)
|
|
||||||
|
|
||||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
package cli //nolint:testpackage // shares runEntry and the backup config helpers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// cmdFiles is the word of `snapshot files`, used to build argument
|
|
||||||
// vectors below.
|
|
||||||
const cmdFiles = "files"
|
|
||||||
|
|
||||||
// TestEntrySnapshotFilesListsOnlyThePath backs up two files and runs
|
|
||||||
// `snapshot files` with the snapshot ID and the path of one of them,
|
|
||||||
// which must be the only file listed.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
|
||||||
func TestEntrySnapshotFilesListsOnlyThePath(t *testing.T) {
|
|
||||||
configPath, _ := writeBackupConfigWithPrivateKey(t)
|
|
||||||
|
|
||||||
// writeBackupConfig puts file.txt in the source directory beside the
|
|
||||||
// config file.
|
|
||||||
sourceDir := filepath.Join(filepath.Dir(configPath), "source")
|
|
||||||
listed := filepath.Join(sourceDir, "file.txt")
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(
|
|
||||||
filepath.Join(sourceDir, "other.txt"), []byte("other"), 0o600))
|
|
||||||
|
|
||||||
code, _, stderr := runEntry(t,
|
|
||||||
flagConfig, configPath, cmdSnapshot, cmdCreate)
|
|
||||||
require.Equal(t, 0, code, stderr)
|
|
||||||
|
|
||||||
code, stdout, stderr := runEntry(t, flagConfig, configPath, cmdSnapshot,
|
|
||||||
cmdFiles, onlySnapshotID(t, configPath), listed, flagJSON)
|
|
||||||
require.Equal(t, 0, code, stderr)
|
|
||||||
|
|
||||||
var entries []vaultik.ListedFile
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal([]byte(stdout), &entries))
|
|
||||||
require.Len(t, entries, 1, "only the path given may be listed")
|
|
||||||
assert.Equal(t, listed, entries[0].Path)
|
|
||||||
}
|
|
||||||
@@ -48,7 +48,7 @@ func TestCLIEntry(t *testing.T) {
|
|||||||
} else {
|
} else {
|
||||||
// Check snapshot subcommands
|
// Check snapshot subcommands
|
||||||
expectedSubCommands := []string{
|
expectedSubCommands := []string{
|
||||||
"create", "list", "files", "purge", "verify", "remove", "restore",
|
"create", "list", "purge", "verify", "remove", "restore",
|
||||||
}
|
}
|
||||||
for _, expected := range expectedSubCommands {
|
for _, expected := range expectedSubCommands {
|
||||||
found := false
|
found := false
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ func NewInfoCommand() *cobra.Command {
|
|||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.ShowInfo()
|
return v.ShowInfo()
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
log.Info("Failed to show info", "error", err)
|
log.Error("Failed to show info", "error", err)
|
||||||
ReportErrorf("Failed to show info: %v", err)
|
ReportErrorf("Failed to show info: %v", err)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ referenced.`,
|
|||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.Prune(opts)
|
return v.Prune(opts)
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
log.Info("Prune operation failed", "error", err)
|
log.Error("Prune operation failed", "error", err)
|
||||||
ReportErrorf("Prune failed: %v", err)
|
ReportErrorf("Prune failed: %v", err)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
}, func(v *vaultik.Vaultik) error {
|
}, func(v *vaultik.Vaultik) error {
|
||||||
return v.RemoteInfo(jsonOutput)
|
return v.RemoteInfo(jsonOutput)
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
log.Info("Failed to get remote info", "error", err)
|
log.Error("Failed to get remote info", "error", err)
|
||||||
ReportErrorf("Failed to get remote info: %v", err)
|
ReportErrorf("Failed to get remote info: %v", err)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
|||||||
+10
-51
@@ -9,10 +9,6 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
|
|
||||||
// filesMaxArgs is the maximum positional argument count of
|
|
||||||
// `snapshot files <snapshot-id> [path]`.
|
|
||||||
const filesMaxArgs = 2
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errSnapshotIDRequired = errors.New("snapshot ID required")
|
errSnapshotIDRequired = errors.New("snapshot ID required")
|
||||||
errWrongArgCount = errors.New("wrong argument count")
|
errWrongArgCount = errors.New("wrong argument count")
|
||||||
@@ -56,7 +52,6 @@ func NewSnapshotCommand() *cobra.Command {
|
|||||||
// Add subcommands
|
// Add subcommands
|
||||||
cmd.AddCommand(newSnapshotCreateCommand())
|
cmd.AddCommand(newSnapshotCreateCommand())
|
||||||
cmd.AddCommand(newSnapshotListCommand())
|
cmd.AddCommand(newSnapshotListCommand())
|
||||||
cmd.AddCommand(newSnapshotFilesCommand())
|
|
||||||
cmd.AddCommand(newSnapshotPurgeCommand())
|
cmd.AddCommand(newSnapshotPurgeCommand())
|
||||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||||
cmd.AddCommand(newSnapshotRemoveCommand())
|
cmd.AddCommand(newSnapshotRemoveCommand())
|
||||||
@@ -114,7 +109,7 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
return v.CreateSnapshot(opts)
|
return v.CreateSnapshot(opts)
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
||||||
log.Info("Post-backup prune failed", "error", err)
|
log.Error("Post-backup prune failed", "error", err)
|
||||||
ReportErrorf("Backup completed, but %v", err)
|
ReportErrorf("Backup completed, but %v", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -123,13 +118,13 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
||||||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
||||||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
|
errors.Is(err, vaultik.ErrVolumeNotMounted) {
|
||||||
log.Info("Backup completed with an error", "error", err)
|
log.Error("Backup completed with an error", "error", err)
|
||||||
ReportErrorf("Backup completed, but %v", err)
|
ReportErrorf("Backup completed, but %v", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Snapshot creation failed", "error", err)
|
log.Error("Snapshot creation failed", "error", err)
|
||||||
ReportErrorf("Snapshot creation failed: %v", err)
|
ReportErrorf("Snapshot creation failed: %v", err)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
@@ -177,44 +172,6 @@ func newSnapshotListCommand() *cobra.Command {
|
|||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// newSnapshotFilesCommand creates the 'snapshot files' subcommand
|
|
||||||
func newSnapshotFilesCommand() *cobra.Command {
|
|
||||||
var jsonOutput bool
|
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
|
||||||
Use: "files <snapshot-id> [path]",
|
|
||||||
Short: "List the files in a snapshot",
|
|
||||||
Long: `Lists the path, size, mode and modification time of every file in a
|
|
||||||
snapshot, without restoring it. With a path, lists only that path and
|
|
||||||
what is under it; a path that matches nothing is an error.
|
|
||||||
|
|
||||||
The snapshot may be named by its ID or, on a host with no local index,
|
|
||||||
by the remote key that 'snapshot list' prints for a remote-only snapshot
|
|
||||||
(an unambiguous leading part is enough).
|
|
||||||
|
|
||||||
The file list is only in the snapshot's encrypted database, so this
|
|
||||||
downloads and decrypts it, and requires the age private key in the
|
|
||||||
VAULTIK_AGE_SECRET_KEY environment variable, as 'snapshot restore' does.`,
|
|
||||||
Args: cobra.RangeArgs(1, filesMaxArgs),
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
path := ""
|
|
||||||
if len(args) == filesMaxArgs {
|
|
||||||
path = args[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
return runVaultikApp(cmd, readOnly, jsonOutput,
|
|
||||||
"Failed to list snapshot files",
|
|
||||||
func(v *vaultik.Vaultik) error {
|
|
||||||
return v.ListSnapshotFiles(args[0], path, jsonOutput)
|
|
||||||
})
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
|
|
||||||
|
|
||||||
return cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
// newSnapshotPurgeCommand creates the 'snapshot purge' subcommand
|
// newSnapshotPurgeCommand creates the 'snapshot purge' subcommand
|
||||||
func newSnapshotPurgeCommand() *cobra.Command {
|
func newSnapshotPurgeCommand() *cobra.Command {
|
||||||
opts := &vaultik.SnapshotPurgeOptions{}
|
opts := &vaultik.SnapshotPurgeOptions{}
|
||||||
@@ -301,7 +258,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Verification failed", "error", err)
|
log.Error("Verification failed", "error", err)
|
||||||
ReportErrorf("Verification failed: %v", err)
|
ReportErrorf("Verification failed: %v", err)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
@@ -331,10 +288,12 @@ know about), which is what 'vaultik prune' does. The command prints the
|
|||||||
'vaultik prune' invocation to run as a follow-up only when it removed the
|
'vaultik prune' invocation to run as a follow-up only when it removed the
|
||||||
snapshot's metadata from the destination store, and never under --json.
|
snapshot's metadata from the destination store, and never under --json.
|
||||||
|
|
||||||
The snapshot may be named by its ID or by its remote key, which
|
The snapshot may be named by its ID or, on a host with no local index, by
|
||||||
'snapshot list' prints for a remote-only snapshot. An unambiguous
|
the remote key that 'snapshot list' prints for a remote-only snapshot (an
|
||||||
leading part of the key is enough. An identifier that names no snapshot
|
unambiguous leading part is enough).
|
||||||
in the local index or on the destination store is an error.
|
|
||||||
|
An ID that names no snapshot in the local index or on the destination
|
||||||
|
store is an error.
|
||||||
|
|
||||||
Use --local-only to skip the remote half (e.g. when you want to forget a
|
Use --local-only to skip the remote half (e.g. when you want to forget a
|
||||||
snapshot locally without touching the destination store); the snapshot
|
snapshot locally without touching the destination store); the snapshot
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
|||||||
SkipErrors: rootFlags.SkipErrors,
|
SkipErrors: rootFlags.SkipErrors,
|
||||||
})
|
})
|
||||||
}, func(err error) {
|
}, func(err error) {
|
||||||
log.Info("Restore operation failed", "error", err)
|
log.Error("Restore operation failed", "error", err)
|
||||||
ReportErrorf("Restore failed: %v", err)
|
ReportErrorf("Restore failed: %v", err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
+40
-45
@@ -21,7 +21,6 @@ import (
|
|||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const appName = "vaultik"
|
const appName = "vaultik"
|
||||||
@@ -64,8 +63,13 @@ var (
|
|||||||
errBlobSizeTooSmall = errors.New(
|
errBlobSizeTooSmall = errors.New(
|
||||||
"blob_size_limit must be at least the largest chunk the chunker can " +
|
"blob_size_limit must be at least the largest chunk the chunker can " +
|
||||||
"emit (chunk_size times the FastCDC size spread)")
|
"emit (chunk_size times the FastCDC size spread)")
|
||||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
|
errBadStorageScheme = errors.New(
|
||||||
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
|
errFileURLNotAbsolute = errors.New(
|
||||||
|
"a file:// storage_url needs an absolute path, " +
|
||||||
|
"as in file:///path/to/backup")
|
||||||
errIndexPathNotAbsolute = errors.New(
|
errIndexPathNotAbsolute = errors.New(
|
||||||
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
||||||
"as in ~/.local/share/vaultik/index.sqlite")
|
"as in ~/.local/share/vaultik/index.sqlite")
|
||||||
@@ -392,8 +396,6 @@ func Parse(data []byte) (*Config, error) {
|
|||||||
// never the value. An empty list is accepted, because only snapshot
|
// never the value. An empty list is accepted, because only snapshot
|
||||||
// create needs a recipient and it checks for one itself
|
// create needs a recipient and it checks for one itself
|
||||||
// - At least one snapshot must be configured with at least one path
|
// - At least one snapshot must be configured with at least one path
|
||||||
// - Every exclude pattern, global or in a snapshot, must compile; errors
|
|
||||||
// name the setting and quote the pattern
|
|
||||||
// - Storage must be configured (either storage_url or s3.* fields)
|
// - Storage must be configured (either storage_url or s3.* fields)
|
||||||
// - The index path must be absolute: a relative one would resolve against
|
// - The index path must be absolute: a relative one would resolve against
|
||||||
// the working directory, so each directory would get its own index
|
// the working directory, so each directory would get its own index
|
||||||
@@ -414,18 +416,18 @@ func (c *Config) Validate() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := c.validateSnapshots()
|
if len(c.Snapshots) == 0 {
|
||||||
if err != nil {
|
return errNoSnapshots
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = c.validateExcludes()
|
for name, snap := range c.Snapshots {
|
||||||
if err != nil {
|
if len(snap.Paths) == 0 {
|
||||||
return err
|
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate storage configuration
|
// Validate storage configuration
|
||||||
err = c.validateStorage()
|
err := c.validateStorage()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -464,22 +466,6 @@ func (c *Config) Validate() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateSnapshots refuses a config with no snapshot or with a snapshot
|
|
||||||
// that has no path.
|
|
||||||
func (c *Config) validateSnapshots() error {
|
|
||||||
if len(c.Snapshots) == 0 {
|
|
||||||
return errNoSnapshots
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, snap := range c.Snapshots {
|
|
||||||
if len(snap.Paths) == 0 {
|
|
||||||
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// validateAgeRecipient parses one age_recipients entry with the age library
|
// validateAgeRecipient parses one age_recipients entry with the age library
|
||||||
// and returns a value-free error on failure. A recipient string can be
|
// and returns a value-free error on failure. A recipient string can be
|
||||||
// sensitive (an operator may paste a secret key by mistake), so neither the
|
// sensitive (an operator may paste a secret key by mistake), so neither the
|
||||||
@@ -553,29 +539,38 @@ func (c *Config) validateStorage() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateStorageURL checks storage_url with the same parser that opens
|
// validateStorageURL validates URL-based storage configuration. File and
|
||||||
// the destination, so a value every command would refuse fails at load.
|
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||||
// File and rclone URLs need no credentials; S3 URLs require the legacy
|
// credential fields. A file URL must name an absolute path: a relative one
|
||||||
// s3.* credential fields.
|
// would resolve against the working directory, so the destination would
|
||||||
|
// move with it.
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
parsed, err := storageurl.Parse(c.StorageURL)
|
switch {
|
||||||
if err != nil {
|
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||||
return fmt.Errorf("invalid storage_url: %w", err)
|
// storage.ParseStorageURL applies the same rule, but config
|
||||||
}
|
// cannot import storage, which imports config.
|
||||||
|
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||||
|
return errFileURLNotAbsolute
|
||||||
|
}
|
||||||
|
|
||||||
if parsed.Scheme != storageurl.SchemeS3 {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||||
|
// Rclone storage uses rclone's own config
|
||||||
|
return nil
|
||||||
|
case strings.HasPrefix(c.StorageURL, "s3://"):
|
||||||
|
// S3 storage needs credentials
|
||||||
|
if c.S3.AccessKeyID == "" {
|
||||||
|
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
||||||
|
}
|
||||||
|
|
||||||
if c.S3.AccessKeyID == "" {
|
if c.S3.SecretAccessKey == "" {
|
||||||
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.SecretAccessKey == "" {
|
return nil
|
||||||
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
|
default:
|
||||||
|
return errBadStorageScheme
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module exports the config module for fx dependency injection.
|
// Module exports the config module for fx dependency injection.
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -503,8 +502,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err = Load(path)
|
_, err = Load(path)
|
||||||
if !errors.Is(err, storageurl.ErrRelativeFilePath) {
|
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||||
t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
|
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "absolute path") {
|
if !strings.Contains(err.Error(), "absolute path") {
|
||||||
|
|||||||
@@ -1,39 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/exclude"
|
|
||||||
)
|
|
||||||
|
|
||||||
// validateExcludes refuses an exclude pattern, global or in a snapshot,
|
|
||||||
// that does not compile, naming the setting and quoting the pattern.
|
|
||||||
func (c *Config) validateExcludes() error {
|
|
||||||
err := validateExcludeList("exclude", c.Exclude)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range c.SnapshotNames() {
|
|
||||||
err = validateExcludeList("snapshots."+name+".exclude",
|
|
||||||
c.Snapshots[name].Exclude)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// validateExcludeList refuses the first pattern in patterns that does not
|
|
||||||
// compile. setting is the name of the list, as in "exclude".
|
|
||||||
func validateExcludeList(setting string, patterns []string) error {
|
|
||||||
for i, pattern := range patterns {
|
|
||||||
_, _, err := exclude.Compile(pattern)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s[%d]: %w", setting, i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
package config_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestLoadRefusesBadExcludePattern checks that an exclude pattern that does
|
|
||||||
// not compile, global or in a snapshot, fails at load with the setting and
|
|
||||||
// the pattern named, rather than the backup storing what it meant to leave
|
|
||||||
// out.
|
|
||||||
func TestLoadRefusesBadExcludePattern(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const storage = "storage_url: file:///tmp/vaultik-test-storage\n"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
config string
|
|
||||||
wantErr string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "global",
|
|
||||||
config: "exclude: ['*.tmp', '.cache', 'build[']\n" +
|
|
||||||
"snapshots:\n" +
|
|
||||||
" home:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" + storage,
|
|
||||||
wantErr: `exclude[2]: "build[" is not a valid exclude pattern`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "snapshot",
|
|
||||||
config: "snapshots:\n" +
|
|
||||||
" home:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" +
|
|
||||||
" exclude: ['/a/**/x[']\n" + storage,
|
|
||||||
wantErr: `snapshots.home.exclude[0]: "/a/**/x[" is not a valid exclude pattern`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte(tt.config), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = config.Load(path)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("Load() returned nil, want an error containing %q", tt.wantErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), tt.wantErr) {
|
|
||||||
t.Fatalf("Load() error = %q, want it to contain %q", err, tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -219,8 +219,7 @@ func retryOpen(ctx context.Context, path string) (*DB, error) {
|
|||||||
|
|
||||||
// errUntrustedSnapshotSchema is returned when a downloaded snapshot
|
// errUntrustedSnapshotSchema is returned when a downloaded snapshot
|
||||||
// database carries schema objects the real schema never defines, or is
|
// database carries schema objects the real schema never defines, or is
|
||||||
// missing a table the restore, deep-verify and `snapshot files` queries
|
// missing a table the restore and deep-verify queries read.
|
||||||
// read.
|
|
||||||
var errUntrustedSnapshotSchema = errors.New(
|
var errUntrustedSnapshotSchema = errors.New(
|
||||||
"downloaded snapshot database has an untrusted schema")
|
"downloaded snapshot database has an untrusted schema")
|
||||||
|
|
||||||
@@ -242,12 +241,12 @@ func snapshotReadOnlyDSN(path string) string {
|
|||||||
|
|
||||||
// OpenReadOnly opens an already-materialized SQLite file for read-only
|
// OpenReadOnly opens an already-materialized SQLite file for read-only
|
||||||
// querying of a snapshot database downloaded from the store, used by
|
// querying of a snapshot database downloaded from the store, used by
|
||||||
// restore, deep verify and `snapshot files`. Unlike New it never applies
|
// restore and deep verify. Unlike New it never applies schema migrations
|
||||||
// schema migrations and never writes: the connection is opened read-only
|
// and never writes: the connection is opened read-only with query_only
|
||||||
// with query_only and trusted_schema=OFF. It refuses any file whose schema
|
// and trusted_schema=OFF. It refuses any file whose schema carries a
|
||||||
// carries a trigger, view or virtual table, or lacks an expected table, so
|
// trigger, view or virtual table, or lacks an expected table, so a forged
|
||||||
// a forged file cannot redefine what the restore queries return. The
|
// file cannot redefine what the restore queries return. The caller owns
|
||||||
// caller owns the file and must remove it.
|
// the file and must remove it.
|
||||||
func OpenReadOnly(ctx context.Context, path string) (*DB, error) {
|
func OpenReadOnly(ctx context.Context, path string) (*DB, error) {
|
||||||
conn, err := sql.Open("sqlite", snapshotReadOnlyDSN(path))
|
conn, err := sql.Open("sqlite", snapshotReadOnlyDSN(path))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -277,9 +276,9 @@ func OpenReadOnly(ctx context.Context, path string) (*DB, error) {
|
|||||||
// the plain table set the real schema defines. Any trigger, view or
|
// the plain table set the real schema defines. Any trigger, view or
|
||||||
// virtual table, or a missing expected table, fails the open.
|
// virtual table, or a missing expected table, fails the open.
|
||||||
func verifySnapshotSchema(ctx context.Context, conn *sql.DB) error {
|
func verifySnapshotSchema(ctx context.Context, conn *sql.DB) error {
|
||||||
// expectedSnapshotTables are the tables the restore, deep-verify and
|
// expectedSnapshotTables are the tables the restore and deep-verify
|
||||||
// `snapshot files` queries read. A downloaded database missing any of
|
// queries read. A downloaded database missing any of them is not a
|
||||||
// them is not a genuine snapshot database and is refused.
|
// genuine snapshot database and is refused.
|
||||||
expectedSnapshotTables := []string{
|
expectedSnapshotTables := []string{
|
||||||
"blob_chunks",
|
"blob_chunks",
|
||||||
"blobs",
|
"blobs",
|
||||||
|
|||||||
@@ -238,11 +238,10 @@ func (r *SnapshotRepository) GetByID(
|
|||||||
|
|
||||||
// GetOnlySnapshot returns the sole snapshot in an exported per-snapshot
|
// GetOnlySnapshot returns the sole snapshot in an exported per-snapshot
|
||||||
// metadata database. The backup path writes each snapshot's database with
|
// metadata database. The backup path writes each snapshot's database with
|
||||||
// exactly one snapshot row (see cleanSnapshotDB), so restore, deep verify
|
// exactly one snapshot row (see cleanSnapshotDB), so restore and deep
|
||||||
// and `snapshot files` expect exactly one. Zero rows return
|
// verify expect exactly one. Zero rows return ErrNoSnapshotInDatabase and
|
||||||
// ErrNoSnapshotInDatabase and more than one returns
|
// more than one returns ErrMultipleSnapshotsInDatabase; callers treat
|
||||||
// ErrMultipleSnapshotsInDatabase; callers treat either as a failed
|
// either as a failed identity check on the downloaded database.
|
||||||
// identity check on the downloaded database.
|
|
||||||
func (r *SnapshotRepository) GetOnlySnapshot(ctx context.Context) (*Snapshot, error) {
|
func (r *SnapshotRepository) GetOnlySnapshot(ctx context.Context) (*Snapshot, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
// Package exclude compiles exclude patterns into globs.
|
|
||||||
package exclude
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/gobwas/glob"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errBadExcludePattern = errors.New("is not a valid exclude pattern")
|
|
||||||
|
|
||||||
// Compile compiles one exclude pattern into the globs that match it.
|
|
||||||
// anchored reports whether the pattern matches only from the root of the
|
|
||||||
// source directory. An empty pattern yields no globs. Config.Validate and
|
|
||||||
// the scanner both compile patterns with it, so a pattern that loads is one
|
|
||||||
// the scanner can use.
|
|
||||||
func Compile(pattern string) ([]glob.Glob, bool, error) {
|
|
||||||
if pattern == "" {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// A leading "/" anchors the pattern to the source directory.
|
|
||||||
trimmed, anchored := strings.CutPrefix(pattern, "/")
|
|
||||||
|
|
||||||
// A trailing "/" marks a directory and is not matched.
|
|
||||||
trimmed = strings.TrimSuffix(trimmed, "/")
|
|
||||||
|
|
||||||
// A leading "**/" demands a "/" before the name and so misses an
|
|
||||||
// entry directly under the source path. It means "at every depth",
|
|
||||||
// which is what an unanchored pattern already does, so drop the
|
|
||||||
// prefix and match unanchored: as in gitignore, "**/x" and
|
|
||||||
// "/**/x" then also match a top-level "x".
|
|
||||||
if rest, found := strings.CutPrefix(trimmed, "**/"); found {
|
|
||||||
trimmed = rest
|
|
||||||
anchored = false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Likewise "/**/" inside a pattern demands two slashes, so
|
|
||||||
// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
|
|
||||||
// zero or more directories, so also compile the pattern with it
|
|
||||||
// made a single "/". The glob library's {a,b} is no help: with a
|
|
||||||
// "**" inside it in mid-pattern, it misses paths it should match.
|
|
||||||
var globs []glob.Glob
|
|
||||||
|
|
||||||
for _, variant := range doubleStarVariants(trimmed) {
|
|
||||||
g, err := glob.Compile(variant, '/')
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, fmt.Errorf("%q %w: %w",
|
|
||||||
pattern, errBadExcludePattern, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
globs = append(globs, g)
|
|
||||||
}
|
|
||||||
|
|
||||||
return globs, anchored, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// doubleStarVariants returns pattern once for every way of keeping each
|
|
||||||
// "/**/" in it or making it a single "/".
|
|
||||||
func doubleStarVariants(pattern string) []string {
|
|
||||||
const doubleStar = "/**/"
|
|
||||||
|
|
||||||
before, after, found := strings.Cut(pattern, doubleStar)
|
|
||||||
if !found {
|
|
||||||
return []string{pattern}
|
|
||||||
}
|
|
||||||
|
|
||||||
var variants []string
|
|
||||||
|
|
||||||
for _, rest := range doubleStarVariants(after) {
|
|
||||||
variants = append(variants, before+doubleStar+rest, before+"/"+rest)
|
|
||||||
}
|
|
||||||
|
|
||||||
return variants
|
|
||||||
}
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
package exclude_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/gobwas/glob"
|
|
||||||
"sneak.berlin/go/vaultik/internal/exclude"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestCompile checks the error, the anchoring and the globs that Compile
|
|
||||||
// returns for one exclude pattern.
|
|
||||||
func TestCompile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
pattern string
|
|
||||||
wantErr string
|
|
||||||
wantAnchored bool
|
|
||||||
// wantMatches lists paths that one of the globs must match. When
|
|
||||||
// it is empty, Compile must return no globs.
|
|
||||||
wantMatches []string
|
|
||||||
}{
|
|
||||||
{name: "does not compile", pattern: "build[",
|
|
||||||
wantErr: `"build[" is not a valid exclude pattern`},
|
|
||||||
{name: "empty", pattern: ""},
|
|
||||||
{name: "leading slash anchors", pattern: "/build",
|
|
||||||
wantAnchored: true, wantMatches: []string{"build"}},
|
|
||||||
{name: "double star after the slash clears anchoring",
|
|
||||||
pattern: "/**/build", wantMatches: []string{"build"}},
|
|
||||||
{name: "double star in the middle", pattern: "a/**/*.log",
|
|
||||||
wantMatches: []string{"a/x.log", "a/b/x.log"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
globs, anchored, err := exclude.Compile(tt.pattern)
|
|
||||||
if tt.wantErr == "" && err != nil {
|
|
||||||
t.Fatalf("Compile(%q) error = %v, want none", tt.pattern, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.wantErr != "" &&
|
|
||||||
(err == nil || !strings.HasPrefix(err.Error(), tt.wantErr)) {
|
|
||||||
t.Fatalf("Compile(%q) error = %v, want one starting with %q",
|
|
||||||
tt.pattern, err, tt.wantErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if anchored != tt.wantAnchored {
|
|
||||||
t.Errorf("Compile(%q) anchored = %t, want %t",
|
|
||||||
tt.pattern, anchored, tt.wantAnchored)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(tt.wantMatches) == 0 && len(globs) != 0 {
|
|
||||||
t.Errorf("Compile(%q) returned %d globs, want none",
|
|
||||||
tt.pattern, len(globs))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range tt.wantMatches {
|
|
||||||
matched := slices.ContainsFunc(globs, func(g glob.Glob) bool {
|
|
||||||
return g.Match(path)
|
|
||||||
})
|
|
||||||
if !matched {
|
|
||||||
t.Errorf("Compile(%q) globs do not match %q",
|
|
||||||
tt.pattern, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -20,7 +20,6 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/blob"
|
"sneak.berlin/go/vaultik/internal/blob"
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/exclude"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
@@ -222,12 +221,8 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
|||||||
progress = NewProgressReporter()
|
progress = NewProgressReporter()
|
||||||
}
|
}
|
||||||
|
|
||||||
compiledExclude, err := compileExcludePatterns(cfg.Exclude)
|
// Compile exclude patterns
|
||||||
if err != nil {
|
compiledExclude := compileExcludePatterns(cfg.Exclude)
|
||||||
log.Error("Failed to compile exclude patterns", "error", err)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
uiw := cfg.UI
|
uiw := cfg.UI
|
||||||
if uiw == nil {
|
if uiw == nil {
|
||||||
@@ -1880,12 +1875,10 @@ func (s *Scanner) uploadBlobIfNeeded(
|
|||||||
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
|
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
|
||||||
finishedBlob.Compressed, progressCallback)
|
finishedBlob.Compressed, progressCallback)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The record is at info level: the returned error ends the backup
|
// Only an interrupt cancels ctx, and the user is told about it,
|
||||||
// and is printed as its ERROR: line. Only an interrupt cancels
|
// so an upload it stopped is not logged as a storage failure.
|
||||||
// ctx, and the user is told about it, so an upload it stopped is
|
|
||||||
// not logged as a storage failure.
|
|
||||||
if ctx.Err() == nil {
|
if ctx.Err() == nil {
|
||||||
log.Info("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
|
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||||
@@ -2323,18 +2316,51 @@ func wrapPermissionError(path string, err error) error {
|
|||||||
"with --skip-errors to continue past unreadable files)", path, err)
|
"with --skip-errors to continue past unreadable files)", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// compileExcludePatterns compiles the exclude patterns into glob matchers.
|
// compileExcludePatterns compiles the exclude patterns into glob matchers
|
||||||
// Config.Validate has already refused a pattern that does not compile.
|
func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||||
func compileExcludePatterns(patterns []string) ([]compiledPattern, error) {
|
|
||||||
var compiled []compiledPattern
|
var compiled []compiledPattern
|
||||||
|
|
||||||
for _, p := range patterns {
|
for _, p := range patterns {
|
||||||
globs, anchored, err := exclude.Compile(p)
|
if p == "" {
|
||||||
if err != nil {
|
continue
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, g := range globs {
|
// Check if pattern is anchored (starts with /)
|
||||||
|
anchored := strings.HasPrefix(p, "/")
|
||||||
|
|
||||||
|
pattern := p
|
||||||
|
if anchored {
|
||||||
|
pattern = p[1:] // Remove leading /
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove trailing slash if present (directory indicator)
|
||||||
|
pattern = strings.TrimSuffix(pattern, "/")
|
||||||
|
|
||||||
|
// A leading "**/" demands a "/" before the name and so misses an
|
||||||
|
// entry directly under the source path. It means "at every depth",
|
||||||
|
// which is what an unanchored pattern already does, so drop the
|
||||||
|
// prefix and match unanchored: as in gitignore, "**/x" and
|
||||||
|
// "/**/x" then also match a top-level "x".
|
||||||
|
if rest, found := strings.CutPrefix(pattern, "**/"); found {
|
||||||
|
pattern = rest
|
||||||
|
anchored = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Likewise "/**/" inside a pattern demands two slashes, so
|
||||||
|
// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
|
||||||
|
// zero or more directories, so also compile the pattern with it
|
||||||
|
// made a single "/". The glob library's {a,b} is no help: with a
|
||||||
|
// "**" inside it in mid-pattern, it misses paths it should match.
|
||||||
|
for _, variant := range doubleStarVariants(pattern) {
|
||||||
|
// For patterns without path separators, we need to match them as components
|
||||||
|
// e.g., ".git" should match ".git" anywhere in the path
|
||||||
|
g, err := glob.Compile(variant, '/')
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
compiled = append(compiled, compiledPattern{
|
compiled = append(compiled, compiledPattern{
|
||||||
pattern: g,
|
pattern: g,
|
||||||
anchored: anchored,
|
anchored: anchored,
|
||||||
@@ -2343,7 +2369,26 @@ func compileExcludePatterns(patterns []string) ([]compiledPattern, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return compiled, nil
|
return compiled
|
||||||
|
}
|
||||||
|
|
||||||
|
// doubleStarVariants returns pattern once for every way of keeping each
|
||||||
|
// "/**/" in it or making it a single "/".
|
||||||
|
func doubleStarVariants(pattern string) []string {
|
||||||
|
const doubleStar = "/**/"
|
||||||
|
|
||||||
|
before, after, found := strings.Cut(pattern, doubleStar)
|
||||||
|
if !found {
|
||||||
|
return []string{pattern}
|
||||||
|
}
|
||||||
|
|
||||||
|
var variants []string
|
||||||
|
|
||||||
|
for _, rest := range doubleStarVariants(after) {
|
||||||
|
variants = append(variants, before+doubleStar+rest, before+"/"+rest)
|
||||||
|
}
|
||||||
|
|
||||||
|
return variants
|
||||||
}
|
}
|
||||||
|
|
||||||
// shouldExclude checks if a path should be excluded based on exclude patterns
|
// shouldExclude checks if a path should be excluded based on exclude patterns
|
||||||
|
|||||||
@@ -37,9 +37,9 @@ func uploadTestBlob(ctx context.Context, store storage.Storer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// captureLog runs fn and returns what the logger wrote to stderr
|
// captureLog runs fn and returns what the logger wrote to stderr
|
||||||
// meanwhile, at the --verbose level. The logger writes to the os.Stderr
|
// meanwhile. The logger writes to the os.Stderr it finds when it is
|
||||||
// it finds when it is initialized, so os.Stderr is pointed at a file
|
// initialized, so os.Stderr is pointed at a file first. Not
|
||||||
// first. Not parallel-safe: os.Stderr and the logger are process-global.
|
// parallel-safe: os.Stderr and the logger are process-global.
|
||||||
func captureLog(t *testing.T, fn func()) string {
|
func captureLog(t *testing.T, fn func()) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -53,7 +53,7 @@ func captureLog(t *testing.T, fn func()) string {
|
|||||||
previous := os.Stderr
|
previous := os.Stderr
|
||||||
os.Stderr = stderrFile
|
os.Stderr = stderrFile
|
||||||
|
|
||||||
log.Initialize(log.Config{Verbose: true})
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
fn()
|
fn()
|
||||||
|
|
||||||
@@ -107,10 +107,10 @@ func TestInterruptedUploadIsNotLoggedAsError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An upload that fails for any other reason is still logged.
|
// An upload that fails for any other reason is still logged as an error.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureLog replaces os.Stderr
|
//nolint:paralleltest // captureLog replaces os.Stderr
|
||||||
func TestFailedUploadIsLogged(t *testing.T) {
|
func TestFailedUploadIsLoggedAsError(t *testing.T) {
|
||||||
inner, err := storage.NewFileStorer(t.TempDir())
|
inner, err := storage.NewFileStorer(t.TempDir())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileStorer implements Storer using the local filesystem.
|
// FileStorer implements Storer using the local filesystem.
|
||||||
@@ -289,7 +288,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
|
|||||||
// Info returns human-readable storage location information.
|
// Info returns human-readable storage location information.
|
||||||
func (f *FileStorer) Info() Info {
|
func (f *FileStorer) Info() Info {
|
||||||
return Info{
|
return Info{
|
||||||
Type: storageurl.SchemeFile,
|
Type: schemeFile,
|
||||||
Location: f.basePath,
|
Location: f.basePath,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,20 +2,14 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
"sneak.berlin/go/vaultik/internal/s3"
|
"sneak.berlin/go/vaultik/internal/s3"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
|
|
||||||
// has no storage backend.
|
|
||||||
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
|
||||||
|
|
||||||
// defaultS3Region is used when neither the URL nor the config specify one.
|
// defaultS3Region is used when neither the URL nor the config specify one.
|
||||||
const defaultS3Region = "us-east-1"
|
const defaultS3Region = "us-east-1"
|
||||||
|
|
||||||
@@ -46,19 +40,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
|
|||||||
|
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
||||||
parsed, err := storageurl.Parse(rawURL)
|
parsed, err := ParseStorageURL(rawURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch parsed.Scheme {
|
switch parsed.Scheme {
|
||||||
case storageurl.SchemeFile:
|
case schemeFile:
|
||||||
return NewFileStorer(parsed.Prefix)
|
return NewFileStorer(parsed.Prefix)
|
||||||
|
|
||||||
case storageurl.SchemeS3:
|
case schemeS3:
|
||||||
return storerFromParsedS3URL(parsed, cfg)
|
return storerFromParsedS3URL(parsed, cfg)
|
||||||
|
|
||||||
case storageurl.SchemeRclone:
|
case schemeRclone:
|
||||||
return NewRcloneStorer(
|
return NewRcloneStorer(
|
||||||
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
||||||
|
|
||||||
@@ -71,7 +65,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
|||||||
// filling endpoint protocol and region defaults from the config.
|
// filling endpoint protocol and region defaults from the config.
|
||||||
//
|
//
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
|
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
||||||
// Build endpoint URL
|
// Build endpoint URL
|
||||||
endpoint := parsed.Endpoint
|
endpoint := parsed.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configfile"
|
"github.com/rclone/rclone/fs/config/configfile"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
|
||||||
|
|
||||||
// Import all rclone backends
|
// Import all rclone backends
|
||||||
_ "github.com/rclone/rclone/backend/all"
|
_ "github.com/rclone/rclone/backend/all"
|
||||||
@@ -267,7 +266,7 @@ func (r *RcloneStorer) Info() Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Info{
|
return Info{
|
||||||
Type: storageurl.SchemeRclone,
|
Type: schemeRclone,
|
||||||
Location: location,
|
Location: location,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
// Package storageurl parses the storage_url config value. The config
|
package storage
|
||||||
// package uses it to refuse a bad value when the config loads; the storage
|
|
||||||
// package uses it to open the destination.
|
|
||||||
package storageurl
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -13,9 +10,9 @@ import (
|
|||||||
|
|
||||||
// Storage URL scheme names.
|
// Storage URL scheme names.
|
||||||
const (
|
const (
|
||||||
SchemeFile = "file"
|
schemeFile = "file"
|
||||||
SchemeS3 = "s3"
|
schemeS3 = "s3"
|
||||||
SchemeRclone = "rclone"
|
schemeRclone = "rclone"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
@@ -28,7 +25,8 @@ var (
|
|||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||||
ErrURLCredentials = errors.New(
|
ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
||||||
|
ErrURLCredentials = errors.New(
|
||||||
"storage URL must not carry credentials; " +
|
"storage URL must not carry credentials; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
||||||
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
||||||
@@ -51,12 +49,12 @@ type URL struct {
|
|||||||
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse parses a storage URL string.
|
// ParseStorageURL parses a storage URL string.
|
||||||
// Supported formats:
|
// Supported formats:
|
||||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
||||||
// - file:///absolute/path/to/backup
|
// - file:///absolute/path/to/backup
|
||||||
// - rclone://remote/path/to/backups
|
// - rclone://remote/path/to/backups
|
||||||
func Parse(rawURL string) (*URL, error) {
|
func ParseStorageURL(rawURL string) (*URL, error) {
|
||||||
if rawURL == "" {
|
if rawURL == "" {
|
||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
@@ -94,7 +92,7 @@ func parseFileURL(rawPath string) (*URL, error) {
|
|||||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &URL{Scheme: SchemeFile, Prefix: path}, nil
|
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
@@ -137,7 +135,7 @@ func parseS3URL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: SchemeS3,
|
Scheme: schemeS3,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
Endpoint: query.Get("endpoint"),
|
Endpoint: query.Get("endpoint"),
|
||||||
@@ -191,7 +189,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: SchemeRclone,
|
Scheme: schemeRclone,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
RcloneRemote: remote,
|
RcloneRemote: remote,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -228,9 +226,9 @@ func wrapParseError(err error) error {
|
|||||||
// String returns a human-readable representation of the storage URL.
|
// String returns a human-readable representation of the storage URL.
|
||||||
func (u *URL) String() string {
|
func (u *URL) String() string {
|
||||||
switch u.Scheme {
|
switch u.Scheme {
|
||||||
case SchemeFile:
|
case schemeFile:
|
||||||
return "file://" + u.Prefix
|
return "file://" + u.Prefix
|
||||||
case SchemeS3:
|
case schemeS3:
|
||||||
endpoint := u.Endpoint
|
endpoint := u.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
endpoint = "s3.amazonaws.com"
|
endpoint = "s3.amazonaws.com"
|
||||||
@@ -241,7 +239,7 @@ func (u *URL) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
||||||
case SchemeRclone:
|
case schemeRclone:
|
||||||
if u.Prefix != "" {
|
if u.Prefix != "" {
|
||||||
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package storageurl_test
|
package storage_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -6,12 +6,12 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/storageurl"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestParseValid checks that each supported scheme parses into the
|
// TestParseStorageURLValid checks that each supported scheme parses into
|
||||||
// expected fields, since those fields decide which backend is built.
|
// the expected fields, since those fields decide which backend is built.
|
||||||
func TestParseValid(t *testing.T) {
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -22,27 +22,27 @@ func TestParseValid(t *testing.T) {
|
|||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
raw string
|
raw string
|
||||||
want *storageurl.URL
|
want *storage.URL
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file percent-escaped path is decoded",
|
name: "file percent-escaped path is decoded",
|
||||||
raw: "file:///Volumes/my%20stick/backup",
|
raw: "file:///Volumes/my%20stick/backup",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file path with a literal space",
|
name: "file path with a literal space",
|
||||||
raw: "file:///Volumes/my stick/backup",
|
raw: "file:///Volumes/my stick/backup",
|
||||||
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
raw: "s3://mybucket/backups/host",
|
raw: "s3://mybucket/backups/host",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Prefix: "backups/host", UseSSL: true,
|
Prefix: "backups/host", UseSSL: true,
|
||||||
},
|
},
|
||||||
@@ -50,12 +50,12 @@ func TestParseValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 bucket only",
|
name: "s3 bucket only",
|
||||||
raw: "s3://mybucket",
|
raw: "s3://mybucket",
|
||||||
want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 with endpoint, region, ssl off",
|
name: "s3 with endpoint, region, ssl off",
|
||||||
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
||||||
},
|
},
|
||||||
@@ -63,14 +63,14 @@ func TestParseValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "rclone remote and path",
|
name: "rclone remote and path",
|
||||||
raw: "rclone://gdrive/backups/host",
|
raw: "rclone://gdrive/backups/host",
|
||||||
want: &storageurl.URL{
|
want: &storage.URL{
|
||||||
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone remote only",
|
name: "rclone remote only",
|
||||||
raw: "rclone://gdrive",
|
raw: "rclone://gdrive",
|
||||||
want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,22 +78,22 @@ func TestParseValid(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
got, err := storageurl.Parse(tc.raw)
|
got, err := storage.ParseStorageURL(tc.raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
|
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
if !reflect.DeepEqual(got, tc.want) {
|
||||||
t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseErrors checks that empty, missing, and unknown-scheme inputs
|
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
|
||||||
// fail with the documented sentinel errors instead of parsing to a wrong
|
// inputs fail with the documented sentinel errors instead of parsing to a
|
||||||
// destination.
|
// wrong destination.
|
||||||
func TestParseErrors(t *testing.T) {
|
func TestParseStorageURLErrors(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -101,33 +101,34 @@ func TestParseErrors(t *testing.T) {
|
|||||||
raw string
|
raw string
|
||||||
wantErr error
|
wantErr error
|
||||||
}{
|
}{
|
||||||
{"empty url", "", storageurl.ErrEmptyStorageURL},
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storageurl.ErrEmptyFilePath},
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
{"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
|
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
||||||
{"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
{"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
|
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
|
||||||
{"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
|
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Errorf("Parse(%q) error = %v, want %v",
|
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsCredentials checks that a URL carrying credentials in
|
// TestParseStorageURLRejectsCredentials checks that a URL carrying
|
||||||
// its userinfo or in an unknown query parameter is rejected, and that the
|
// credentials in its userinfo or in an unknown query parameter is
|
||||||
// error never echoes the secret-bearing URL back into logs or output.
|
// rejected, and that the error never echoes the secret-bearing URL back
|
||||||
func TestParseRejectsCredentials(t *testing.T) {
|
// into logs or output.
|
||||||
|
func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Split so the literals never form a "user:pass@" URL pattern that
|
// Split so the literals never form a "user:pass@" URL pattern that
|
||||||
@@ -146,44 +147,44 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 userinfo",
|
name: "s3 userinfo",
|
||||||
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret, "mybucket"},
|
secrets: []string{key, secret, "mybucket"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint",
|
name: "s3 userinfo in endpoint",
|
||||||
raw: "s3://mybucket?endpoint=https://" +
|
raw: "s3://mybucket?endpoint=https://" +
|
||||||
key + ":" + secret + "@minio.example.com",
|
key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint without scheme",
|
name: "s3 userinfo in endpoint without scheme",
|
||||||
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 unknown query param",
|
name: "s3 unknown query param",
|
||||||
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 misspelt endpoint",
|
name: "s3 misspelt endpoint",
|
||||||
raw: "s3://mybucket?endpiont=minio.example.com",
|
raw: "s3://mybucket?endpiont=minio.example.com",
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: nil,
|
secrets: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone userinfo",
|
name: "rclone userinfo",
|
||||||
raw: "rclone://user:" + secret + "@gdrive/backups",
|
raw: "rclone://user:" + secret + "@gdrive/backups",
|
||||||
wantErr: storageurl.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone query param",
|
name: "rclone query param",
|
||||||
raw: "rclone://gdrive/backups?token=" + secret,
|
raw: "rclone://gdrive/backups?token=" + secret,
|
||||||
wantErr: storageurl.ErrURLUnknownParam,
|
wantErr: storage.ErrURLUnknownParam,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -192,9 +193,9 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Fatalf("Parse(%q) error = %v, want %v",
|
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,11 +216,11 @@ func TestParseRejectsCredentials(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsRcloneRemoteParams checks that an rclone remote name
|
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
|
||||||
// carrying parameters (rclone's "remote,param=value" form, which can hold
|
// remote name carrying parameters (rclone's "remote,param=value" form,
|
||||||
// a credential) is rejected without echoing the value, including the URLs
|
// which can hold a credential) is rejected without echoing the value,
|
||||||
// that url.Parse itself refuses.
|
// including the URLs that url.Parse itself refuses.
|
||||||
func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -248,10 +249,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
|
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
|
||||||
t.Fatalf("Parse(%q) error = %v, want %v",
|
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, storageurl.ErrRcloneRemoteParams)
|
tc.raw, err, storage.ErrRcloneRemoteParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, secret := range tc.secrets {
|
for _, secret := range tc.secrets {
|
||||||
@@ -263,10 +264,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
|
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
|
||||||
// query cannot be decoded is rejected, not read with that part of the
|
// URL whose query cannot be decoded is rejected, not read with that part
|
||||||
// query left out, and that the error does not echo it.
|
// of the query left out, and that the error does not echo it.
|
||||||
func TestParseRejectsUndecodableQuery(t *testing.T) {
|
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -302,14 +303,14 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storageurl.Parse(tc.raw)
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("Parse(%q) returned no error", tc.raw)
|
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !errors.Is(err, storageurl.ErrURLBadQuery) {
|
if !errors.Is(err, storage.ErrURLBadQuery) {
|
||||||
t.Errorf("Parse(%q) error = %v, want %v",
|
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
tc.raw, err, storageurl.ErrURLBadQuery)
|
tc.raw, err, storage.ErrURLBadQuery)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, s := range []string{key, secret, "minio.example.com"} {
|
for _, s := range []string{key, secret, "minio.example.com"} {
|
||||||
@@ -321,17 +322,17 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseFailureHidesURL checks that when url.Parse itself fails, the
|
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
|
||||||
// wrapped error carries only the inner cause, not the *url.Error whose
|
// itself fails, the wrapped error carries only the inner cause, not the
|
||||||
// text embeds the raw (possibly credential-bearing) URL.
|
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
|
||||||
func TestParseFailureHidesURL(t *testing.T) {
|
func TestParseStorageURLParseFailureHidesURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const raw = "s3://mybucket/%zz"
|
const raw = "s3://mybucket/%zz"
|
||||||
|
|
||||||
_, err := storageurl.Parse(raw)
|
_, err := storage.ParseStorageURL(raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("Parse(%q) returned no error", raw)
|
t.Fatalf("ParseStorageURL(%q) returned no error", raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.Contains(err.Error(), "mybucket") {
|
if strings.Contains(err.Error(), "mybucket") {
|
||||||
+6
-10
@@ -17,9 +17,6 @@
|
|||||||
// Count, Percent) return ANSI-colored strings the caller composes into
|
// Count, Percent) return ANSI-colored strings the caller composes into
|
||||||
// the message body. When color is disabled (non-TTY output or NO_COLOR
|
// the message body. When color is disabled (non-TTY output or NO_COLOR
|
||||||
// set) all formatters return plain text.
|
// set) all formatters return plain text.
|
||||||
//
|
|
||||||
// QuoteUnprintable is the same escaping without colour, for plain tables
|
|
||||||
// that print paths from the snapshot database.
|
|
||||||
package ui
|
package ui
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -230,17 +227,17 @@ func (w *Writer) Hex(s string) string {
|
|||||||
short = s[:hexAbbrevLen] + "..."
|
short = s[:hexAbbrevLen] + "..."
|
||||||
}
|
}
|
||||||
|
|
||||||
return w.paint(ansiCyan, QuoteUnprintable(short))
|
return w.paint(ansiCyan, sanitize(short))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Snapshot colorizes a snapshot ID (full, no abbreviation).
|
// Snapshot colorizes a snapshot ID (full, no abbreviation).
|
||||||
func (w *Writer) Snapshot(id string) string {
|
func (w *Writer) Snapshot(id string) string {
|
||||||
return w.paint(ansiCyan+ansiBold, QuoteUnprintable(id))
|
return w.paint(ansiCyan+ansiBold, sanitize(id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Path colorizes a filesystem path.
|
// Path colorizes a filesystem path.
|
||||||
func (w *Writer) Path(p string) string {
|
func (w *Writer) Path(p string) string {
|
||||||
return w.paint(ansiBlue, QuoteUnprintable(p))
|
return w.paint(ansiBlue, sanitize(p))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Size colorizes a byte count using humanize.Bytes.
|
// Size colorizes a byte count using humanize.Bytes.
|
||||||
@@ -315,15 +312,14 @@ func (w *Writer) paint(color, s string) string {
|
|||||||
return color + s + ansiReset
|
return color + s + ansiReset
|
||||||
}
|
}
|
||||||
|
|
||||||
// QuoteUnprintable returns s unchanged when every rune in it is printable,
|
// sanitize returns s unchanged when every rune in it is printable, and a
|
||||||
// and a double-quoted, backslash-escaped form (\n, \x1b, …) otherwise. The
|
// double-quoted, backslash-escaped form (\n, \x1b, …) otherwise. The
|
||||||
// string value formatters escape their argument through this before
|
// string value formatters escape their argument through this before
|
||||||
// painting: identifiers, paths and symlink targets they render come from
|
// painting: identifiers, paths and symlink targets they render come from
|
||||||
// the snapshot database, which is not trusted, and escaping must happen
|
// the snapshot database, which is not trusted, and escaping must happen
|
||||||
// before colour is applied — the painted result already contains the
|
// before colour is applied — the painted result already contains the
|
||||||
// escape codes the raw text would otherwise be indistinguishable from.
|
// escape codes the raw text would otherwise be indistinguishable from.
|
||||||
// Plain tables call it directly.
|
func sanitize(s string) string {
|
||||||
func QuoteUnprintable(s string) string {
|
|
||||||
for _, r := range s {
|
for _, r := range s {
|
||||||
if !unicode.IsPrint(r) {
|
if !unicode.IsPrint(r) {
|
||||||
return strconv.Quote(s)
|
return strconv.Quote(s)
|
||||||
|
|||||||
@@ -160,8 +160,7 @@ func (v *Vaultik) FetchBlob(
|
|||||||
// Not every backend's reader stops on cancel by itself (the file
|
// Not every backend's reader stops on cancel by itself (the file
|
||||||
// backend's does not), and a read stalled on a slow or hung source would
|
// backend's does not), and a read stalled on a slow or hung source would
|
||||||
// otherwise outlast the shutdown timeout, so the process would exit
|
// otherwise outlast the shutdown timeout, so the process would exit
|
||||||
// before restore, deep verify or `snapshot files` removed its decrypted
|
// before restore or deep verify removed its decrypted temporary files.
|
||||||
// temporary files.
|
|
||||||
type closeOnCancelReader struct {
|
type closeOnCancelReader struct {
|
||||||
io.ReadCloser
|
io.ReadCloser
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -472,25 +473,30 @@ func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
|
|||||||
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
|
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRemoveSnapshot_ByRemoteKey checks that a snapshot only the
|
// TestRemoveSnapshot_RemoteOnlyByRemoteKey checks that a snapshot only
|
||||||
// destination store holds can be removed by the remote key that
|
// on the destination store, on a host with no local index, is removed by
|
||||||
// `snapshot list` shows for it, or by a leading part of that key.
|
// the remote key that 'snapshot list' prints for such a snapshot, and by
|
||||||
func TestRemoveSnapshot_ByRemoteKey(t *testing.T) {
|
// an unambiguous leading part of it — the same identifiers verify and
|
||||||
|
// restore accept.
|
||||||
|
func TestRemoveSnapshot_RemoteOnlyByRemoteKey(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
|
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
|
||||||
|
|
||||||
for name, identifier := range map[string]string{
|
identifiers := map[string]string{
|
||||||
"full key": remoteKey,
|
"full key": remoteKey,
|
||||||
"leading part": remoteKey[:8],
|
"leading part": remoteKey[:8],
|
||||||
} {
|
}
|
||||||
|
|
||||||
|
for name, identifier := range identifiers {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
store := newTestStorer()
|
store := newTestStorer()
|
||||||
addManifest(t, store, "snapshot-001", nil)
|
addManifest(t, store, "snapshot-001", nil)
|
||||||
|
|
||||||
|
// No local index: a host that sees the snapshot only remotely.
|
||||||
tv := vaultik.NewForTesting(store)
|
tv := vaultik.NewForTesting(store)
|
||||||
tv.Repositories = newLocalIndex(t)
|
tv.Repositories = newLocalIndex(t)
|
||||||
|
|
||||||
@@ -500,17 +506,116 @@ func TestRemoveSnapshot_ByRemoteKey(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.True(t, result.RemoteRemoved)
|
assert.True(t, result.RemoteRemoved)
|
||||||
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||||
|
|
||||||
|
out := tv.Stdout.String()
|
||||||
|
assert.NotContains(t, out, "from local database")
|
||||||
|
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRemoveSnapshot_ByRemoteKeyInLocalIndex checks that a snapshot the
|
// TestRemoveSnapshot_LocalIndexByRemoteKey checks that a remote key (or
|
||||||
// local index holds, named by its remote key, is removed from the local
|
// its unambiguous leading part) that names a snapshot the local index
|
||||||
// index as well as from the destination store.
|
// holds removes that snapshot's local row and its remote metadata
|
||||||
func TestRemoveSnapshot_ByRemoteKeyInLocalIndex(t *testing.T) {
|
// together. `snapshot list --json` prints a remote key for every
|
||||||
|
// snapshot, including local ones.
|
||||||
|
func TestRemoveSnapshot_LocalIndexByRemoteKey(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
|
||||||
|
|
||||||
|
identifiers := map[string]string{
|
||||||
|
"full key": remoteKey,
|
||||||
|
"leading part": remoteKey[:8],
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, identifier := range identifiers {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newTestStorer()
|
||||||
|
addManifest(t, store, "snapshot-001", nil)
|
||||||
|
|
||||||
|
// A backup wrote to this destination, so it carries the
|
||||||
|
// marker, and the local index holds the snapshot.
|
||||||
|
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||||
|
bytes.NewReader(nil)))
|
||||||
|
|
||||||
|
tv := vaultik.NewForTesting(store)
|
||||||
|
repos := newLocalIndex(t, "snapshot-001")
|
||||||
|
tv.Repositories = repos
|
||||||
|
|
||||||
|
result, err := tv.RemoveSnapshot(identifier,
|
||||||
|
&vaultik.RemoveOptions{Force: true})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, result.RemoteRemoved)
|
||||||
|
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||||
|
|
||||||
|
row, err := repos.Snapshots.GetByID(
|
||||||
|
context.Background(), "snapshot-001")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, row)
|
||||||
|
|
||||||
|
out := tv.Stdout.String()
|
||||||
|
assert.Contains(t, out,
|
||||||
|
"Removed snapshot 'snapshot-001' from local database")
|
||||||
|
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRemoveSnapshot_DryRunByLeadingPart checks that a dry run by a
|
||||||
|
// leading part of a remote key names the snapshot that part resolved to:
|
||||||
|
// by its ID when the local index holds the snapshot, otherwise by its
|
||||||
|
// full remote key.
|
||||||
|
func TestRemoveSnapshot_DryRunByLeadingPart(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
|
||||||
|
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
localIDs []string
|
||||||
|
shown string
|
||||||
|
}{
|
||||||
|
"in local index": {localIDs: []string{"snapshot-001"}, shown: "snapshot-001"},
|
||||||
|
"remote only": {localIDs: nil, shown: remoteKey},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newTestStorer()
|
||||||
|
addManifest(t, store, "snapshot-001", nil)
|
||||||
|
|
||||||
|
// A backup wrote to this destination, so it carries the marker.
|
||||||
|
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||||
|
bytes.NewReader(nil)))
|
||||||
|
|
||||||
|
tv := vaultik.NewForTesting(store)
|
||||||
|
tv.Repositories = newLocalIndex(t, tc.localIDs...)
|
||||||
|
|
||||||
|
_, err := tv.RemoveSnapshot(remoteKey[:8],
|
||||||
|
&vaultik.RemoveOptions{DryRun: true})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, tv.Stdout.String(),
|
||||||
|
"Would remove snapshot: "+tc.shown+".")
|
||||||
|
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRemoveSnapshot_JSONDryRunByLeadingPart checks that a JSON dry run
|
||||||
|
// by a leading part of a remote key prints the ID and the full remote key
|
||||||
|
// of the snapshot that part resolved to.
|
||||||
|
func TestRemoveSnapshot_JSONDryRunByLeadingPart(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
|
||||||
|
|
||||||
store := newTestStorer()
|
store := newTestStorer()
|
||||||
addManifest(t, store, "snapshot-001", nil)
|
addManifest(t, store, "snapshot-001", nil)
|
||||||
|
|
||||||
@@ -521,17 +626,16 @@ func TestRemoveSnapshot_ByRemoteKeyInLocalIndex(t *testing.T) {
|
|||||||
tv := vaultik.NewForTesting(store)
|
tv := vaultik.NewForTesting(store)
|
||||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||||
|
|
||||||
result, err := tv.RemoveSnapshot(snapshot.RemoteSnapshotKey("snapshot-001"),
|
_, err := tv.RemoveSnapshot(remoteKey[:8],
|
||||||
&vaultik.RemoveOptions{Force: true})
|
&vaultik.RemoveOptions{DryRun: true, JSON: true})
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.True(t, result.RemoteRemoved)
|
|
||||||
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
|
||||||
|
|
||||||
snap, err := tv.Repositories.Snapshots.GetByID(context.Background(),
|
var printed map[string]any
|
||||||
"snapshot-001")
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, json.Unmarshal(tv.Stdout.Bytes(), &printed))
|
||||||
assert.Nil(t, snap)
|
assert.Equal(t, "snapshot-001", printed["snapshot_id"])
|
||||||
|
assert.Equal(t, remoteKey, printed["remote_key"])
|
||||||
|
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
|
// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
|
||||||
@@ -568,6 +672,38 @@ func TestRemoveSnapshot_NotInLocalIndexDeleteFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRemoveSnapshot_LocalIndexByLeadingPartDeleteFails checks the
|
||||||
|
// warning when a snapshot the local index holds is removed by a leading
|
||||||
|
// part of its remote key and the delete on the destination store fails.
|
||||||
|
// The warning names the snapshot by its ID, not by the part given.
|
||||||
|
func TestRemoveSnapshot_LocalIndexByLeadingPartDeleteFails(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newTestStorer()
|
||||||
|
addManifest(t, store, "snapshot-001", nil)
|
||||||
|
|
||||||
|
// A backup wrote to this destination, so it carries the marker.
|
||||||
|
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||||
|
bytes.NewReader(nil)))
|
||||||
|
|
||||||
|
faulty := faultstore.New(store)
|
||||||
|
faulty.OnDelete = func(string) bool { return true }
|
||||||
|
|
||||||
|
tv := vaultik.NewForTesting(faulty)
|
||||||
|
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||||
|
|
||||||
|
result, err := tv.RemoveSnapshot(
|
||||||
|
snapshot.RemoteSnapshotKey("snapshot-001")[:8],
|
||||||
|
&vaultik.RemoveOptions{Force: true})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, result.RemoteRemoved)
|
||||||
|
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||||
|
assert.Contains(t, tv.Stdout.String(),
|
||||||
|
"Run 'vaultik snapshot remove snapshot-001' again")
|
||||||
|
}
|
||||||
|
|
||||||
func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
|
func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
+26
-32
@@ -31,7 +31,7 @@ var (
|
|||||||
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
|
||||||
errFilesFailedVerify = errors.New("files failed verification")
|
errFilesFailedVerify = errors.New("files failed verification")
|
||||||
errDecryptionKeyRequired = errors.New(
|
errDecryptionKeyRequired = errors.New(
|
||||||
"decryption key required\n\n" +
|
"decryption key required for restore\n\n" +
|
||||||
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
|
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
|
||||||
"age private key file:\n" +
|
"age private key file:\n" +
|
||||||
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
|
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
|
||||||
@@ -153,7 +153,18 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer v.closeSnapshotDB(tempDB, tempDir)
|
defer func() {
|
||||||
|
err := tempDB.Close()
|
||||||
|
if err != nil {
|
||||||
|
log.Debug("Failed to close temp database", "error", err)
|
||||||
|
}
|
||||||
|
// Remove the whole private directory, so the decrypted database
|
||||||
|
// and any SQLite side files it produced are gone on every path.
|
||||||
|
err = v.Fs.RemoveAll(tempDir)
|
||||||
|
if err != nil {
|
||||||
|
log.Debug("Failed to remove temp database directory", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
repos := database.NewRepositories(tempDB)
|
repos := database.NewRepositories(tempDB)
|
||||||
|
|
||||||
@@ -314,10 +325,9 @@ func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
|
|||||||
// identity it contains. The value may be a single key line or a whole
|
// identity it contains. The value may be a single key line or a whole
|
||||||
// age-keygen file with several identities; all of them are returned so
|
// age-keygen file with several identities; all of them are returned so
|
||||||
// blobgen (via age.Decrypt) can read a blob encrypted to any of their
|
// blobgen (via age.Decrypt) can read a blob encrypted to any of their
|
||||||
// recipients. This is the first step of restore, deep verify and
|
// recipients. This is the first step of both restore and deep verify, so
|
||||||
// `snapshot files`, so a missing or unparseable key fails before anything
|
// a missing or unparseable key fails before anything is downloaded. The
|
||||||
// is downloaded. The error names the configuration source but never the
|
// error names the configuration source but never the key value.
|
||||||
// key value.
|
|
||||||
func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
||||||
if v.Config.AgeSecretKey == "" {
|
if v.Config.AgeSecretKey == "" {
|
||||||
return nil, errDecryptionKeyRequired
|
return nil, errDecryptionKeyRequired
|
||||||
@@ -657,19 +667,19 @@ func (v *Vaultik) restoreProgressTick(
|
|||||||
|
|
||||||
// handleRestoreFileError records a per-file restore failure: fatal unless
|
// handleRestoreFileError records a per-file restore failure: fatal unless
|
||||||
// --skip-errors is set, in which case the file is counted as failed and
|
// --skip-errors is set, in which case the file is counted as failed and
|
||||||
// dropped from the plan. A fatal failure is not logged here: the returned
|
// dropped from the plan.
|
||||||
// error is printed as the command's ERROR: line.
|
|
||||||
func (v *Vaultik) handleRestoreFileError(
|
func (v *Vaultik) handleRestoreFileError(
|
||||||
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
|
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
|
||||||
file *database.File, fileID types.FileID, err error,
|
file *database.File, fileID types.FileID, err error,
|
||||||
) error {
|
) error {
|
||||||
|
log.Error("Failed to restore file", "path", file.Path, "error", err)
|
||||||
|
|
||||||
if !opts.SkipErrors {
|
if !opts.SkipErrors {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"restoring %s: %w (pass --skip-errors to continue past "+
|
"restoring %s: %w (pass --skip-errors to continue past "+
|
||||||
"restore failures)", file.Path, err)
|
"restore failures)", file.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Error("Failed to restore file", "path", file.Path, "error", err)
|
|
||||||
v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).",
|
v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).",
|
||||||
v.UI.Path(file.Path.String()), err)
|
v.UI.Path(file.Path.String()), err)
|
||||||
|
|
||||||
@@ -870,34 +880,18 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
return db, tempDir, nil
|
return db, tempDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// closeSnapshotDB closes a database returned by downloadSnapshotDB and
|
|
||||||
// removes its private directory, so the decrypted database and any SQLite
|
|
||||||
// side files it produced are gone.
|
|
||||||
func (v *Vaultik) closeSnapshotDB(db *database.DB, tempDir string) {
|
|
||||||
err := db.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Debug("Failed to close temp database", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Fs.RemoveAll(tempDir)
|
|
||||||
if err != nil {
|
|
||||||
log.Debug("Failed to remove temp database directory", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifySnapshotDBIdentity confirms the decrypted metadata database really
|
// verifySnapshotDBIdentity confirms the decrypted metadata database really
|
||||||
// is the snapshot named by remoteKey. age decryption proves the database
|
// is the snapshot named by remoteKey. age decryption proves the database
|
||||||
// is readable, not that the object served at
|
// is readable, not that the object served at
|
||||||
// metadata/<remoteKey>/db.zst.age is the snapshot that was requested: an
|
// metadata/<remoteKey>/db.zst.age is the snapshot that was requested: an
|
||||||
// attacker who swaps in another valid db.zst.age (which needs no key
|
// attacker who swaps in another valid db.zst.age (which needs no key
|
||||||
// material) would otherwise redirect restore, deep verify and
|
// material) would otherwise redirect restore and deep verify to a
|
||||||
// `snapshot files` to a different snapshot's contents. The exported
|
// different snapshot's contents. The exported per-snapshot database holds
|
||||||
// per-snapshot database holds exactly one snapshot row, and a snapshot's
|
// exactly one snapshot row, and a snapshot's remote key is derived from
|
||||||
// remote key is derived from that row's ID, so the database is the
|
// that row's ID, so the database is the requested one exactly when its
|
||||||
// requested one exactly when its sole snapshot hashes back to remoteKey.
|
// sole snapshot hashes back to remoteKey. Comparing the requested
|
||||||
// Comparing the requested identifier directly would not do: it may be a
|
// identifier directly would not do: it may be a remote-key prefix a
|
||||||
// remote-key prefix a recovery host uses in place of a human snapshot ID
|
// recovery host uses in place of a human snapshot ID it cannot know.
|
||||||
// it cannot know.
|
|
||||||
func (v *Vaultik) verifySnapshotDBIdentity(
|
func (v *Vaultik) verifySnapshotDBIdentity(
|
||||||
db *database.DB, requested, remoteKey string,
|
db *database.DB, requested, remoteKey string,
|
||||||
) error {
|
) error {
|
||||||
|
|||||||
@@ -1424,11 +1424,14 @@ type RemoveOptions struct {
|
|||||||
LocalOnly bool // Skip remote cleanup; only touch the local index
|
LocalOnly bool // Skip remote cleanup; only touch the local index
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemoveResult contains the result of a snapshot removal
|
// RemoveResult contains the result of a snapshot removal. For a single
|
||||||
|
// snapshot, SnapshotID is its ID, empty when the ID is not known, and
|
||||||
|
// RemoteKey is the full remote key of its metadata.
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // snake_case is the established JSON output format
|
//nolint:tagliatelle // snake_case is the established JSON output format
|
||||||
type RemoveResult struct {
|
type RemoveResult struct {
|
||||||
SnapshotID string `json:"snapshot_id,omitempty"`
|
SnapshotID string `json:"snapshot_id,omitempty"`
|
||||||
|
RemoteKey string `json:"remote_key,omitempty"`
|
||||||
SnapshotsRemoved []string `json:"snapshots_removed,omitempty"`
|
SnapshotsRemoved []string `json:"snapshots_removed,omitempty"`
|
||||||
RemoteRemoved bool `json:"remote_removed,omitempty"`
|
RemoteRemoved bool `json:"remote_removed,omitempty"`
|
||||||
DryRun bool `json:"dry_run,omitempty"`
|
DryRun bool `json:"dry_run,omitempty"`
|
||||||
@@ -1458,26 +1461,35 @@ const snapshotRemoveCommandHint = "vaultik snapshot remove"
|
|||||||
func (v *Vaultik) RemoveSnapshot(
|
func (v *Vaultik) RemoveSnapshot(
|
||||||
snapshotID string, opts *RemoveOptions,
|
snapshotID string, opts *RemoveOptions,
|
||||||
) (*RemoveResult, error) {
|
) (*RemoveResult, error) {
|
||||||
result := &RemoveResult{
|
result := &RemoveResult{}
|
||||||
SnapshotID: snapshotID,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := v.checkStorageBinding()
|
err := v.checkStorageBinding()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotID, remoteKey, inLocalIndex, err := v.findSnapshotToRemove(
|
inLocalIndex, resolvedID, remoteKey, err := v.findSnapshotToRemove(
|
||||||
snapshotID, opts.LocalOnly)
|
snapshotID, opts.LocalOnly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.DryRun {
|
result.SnapshotID = resolvedID
|
||||||
return v.removeSnapshotDryRun(result, snapshotID, opts)
|
result.RemoteKey = remoteKey
|
||||||
|
|
||||||
|
// The dry run, the prompt and removeSnapshotRemote name the snapshot
|
||||||
|
// that snapshotID resolved to: by its ID, or by its full remote key
|
||||||
|
// when the ID is not known.
|
||||||
|
shown := resolvedID
|
||||||
|
if shown == "" {
|
||||||
|
shown = remoteKey
|
||||||
}
|
}
|
||||||
|
|
||||||
if !v.confirmRemoveSnapshot(snapshotID, opts) {
|
if opts.DryRun {
|
||||||
|
return v.removeSnapshotDryRun(result, shown, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !v.confirmRemoveSnapshot(shown, opts) {
|
||||||
return result, ErrNotConfirmed
|
return result, ErrNotConfirmed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1490,16 +1502,16 @@ func (v *Vaultik) RemoveSnapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if inLocalIndex {
|
if inLocalIndex {
|
||||||
log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)
|
log.Info("Removing snapshot from local database", "snapshot_id", resolvedID)
|
||||||
|
|
||||||
err = v.deleteSnapshotFromLocalDB(snapshotID)
|
err = v.deleteSnapshotFromLocalDB(resolvedID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, fmt.Errorf("removing from local database: %w", err)
|
return result, fmt.Errorf("removing from local database: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result.RemoteRemoved, err = v.removeSnapshotRemote(
|
result.RemoteRemoved, err = v.removeSnapshotRemote(
|
||||||
snapshotID, remoteKey, inLocalIndex, markerErr, opts)
|
shown, remoteKey, inLocalIndex, markerErr, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
@@ -1518,7 +1530,7 @@ func (v *Vaultik) RemoveSnapshot(
|
|||||||
|
|
||||||
if inLocalIndex {
|
if inLocalIndex {
|
||||||
v.UI.Infof("Removed snapshot '%s' from local database.",
|
v.UI.Infof("Removed snapshot '%s' from local database.",
|
||||||
v.UI.Snapshot(snapshotID))
|
v.UI.Snapshot(resolvedID))
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.RemoteRemoved {
|
if result.RemoteRemoved {
|
||||||
@@ -1531,39 +1543,50 @@ func (v *Vaultik) RemoveSnapshot(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// findSnapshotToRemove returns the ID of the snapshot that identifier
|
// findSnapshotToRemove finds the snapshot that identifier names. It
|
||||||
// names, its remote key, and whether the local index holds it. A remote
|
// returns whether the local index holds that snapshot, its snapshot ID,
|
||||||
// key, or the leading part of one, is resolved on the destination store;
|
// and the remote key of its metadata on the destination store. The ID is
|
||||||
// when the local index holds the snapshot with that key, its ID is
|
// known when identifier is a snapshot ID or the local index holds the
|
||||||
// returned, so it is removed from both.
|
// snapshot; otherwise it is empty.
|
||||||
|
//
|
||||||
|
// A snapshot ID is looked up by checkSnapshotExists. Under localOnly the
|
||||||
|
// store is not consulted, so every identifier is looked up that way and
|
||||||
|
// a remote key is not found. Otherwise a remote key, or a leading part
|
||||||
|
// of one, is resolved by resolveSnapshotRemoteKey, as restore and verify
|
||||||
|
// do. That fails with errSnapshotKeyNotFound or errSnapshotKeyAmbiguous
|
||||||
|
// when the identifier matches no snapshot on the store, or more than
|
||||||
|
// one. The store is listed only after checkDestinationMarker passes, as
|
||||||
|
// in checkSnapshotExists. The local row is the one whose snapshot ID
|
||||||
|
// hashes to the resolved key, as in RemoveAllSnapshots, so removing by
|
||||||
|
// remote key removes that row too.
|
||||||
func (v *Vaultik) findSnapshotToRemove(
|
func (v *Vaultik) findSnapshotToRemove(
|
||||||
identifier string, localOnly bool,
|
identifier string, localOnly bool,
|
||||||
) (string, string, bool, error) {
|
) (bool, string, string, error) {
|
||||||
if !isRemoteKeyOrPrefix(identifier) || localOnly {
|
if localOnly || !isRemoteKeyOrPrefix(identifier) {
|
||||||
inLocalIndex, err := v.checkSnapshotExists(identifier, localOnly)
|
inLocalIndex, err := v.checkSnapshotExists(identifier, localOnly)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
return identifier, snapshot.RemoteSnapshotKey(identifier),
|
return inLocalIndex, identifier, snapshot.RemoteSnapshotKey(identifier), nil
|
||||||
inLocalIndex, err
|
}
|
||||||
|
|
||||||
|
err := v.checkDestinationMarker()
|
||||||
|
if err != nil {
|
||||||
|
return false, "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
|
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", false, err
|
return false, "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
localIDs, err := v.localSnapshotIDs()
|
localID, err := v.localSnapshotIDForRemoteKey(remoteKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", false, fmt.Errorf(
|
return false, "", "", err
|
||||||
"looking up snapshot in local index: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, id := range localIDs {
|
return localID != "", localID, remoteKey, nil
|
||||||
if snapshot.RemoteSnapshotKey(id) == remoteKey {
|
|
||||||
return id, remoteKey, true, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return identifier, remoteKey, false, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkSnapshotExists returns errSnapshotNotFound when the local index
|
// checkSnapshotExists returns errSnapshotNotFound when the local index
|
||||||
@@ -1674,20 +1697,23 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeSnapshotRemote strips the snapshot's metadata, under remoteKey,
|
// removeSnapshotRemote strips the snapshot's metadata from the
|
||||||
// from the destination store. It returns true only when the delete finished
|
// destination store. remoteKey is the resolved metadata directory name
|
||||||
// without an error and removed at least one object; under LocalOnly it
|
// to delete, as returned by findSnapshotToRemove. shown names the
|
||||||
// does nothing. markerErr is the result of checkDestinationMarker; when
|
// snapshot in the log records and the warning: its ID, or its full
|
||||||
// it is set, nothing is deleted and it is the failure. When the snapshot
|
// remote key when the ID is not known. It returns true only when the
|
||||||
// is only on the destination store, a failure is returned as an error,
|
// delete finished without an error and removed at least one object;
|
||||||
// because that removal is all the command does. When the snapshot was in
|
// under LocalOnly it does nothing. markerErr is the result of
|
||||||
// the local index, its removal there has already happened, so a failure
|
// checkDestinationMarker; when it is set, nothing is deleted and it is
|
||||||
// is only a warning telling the user to run `vaultik snapshot remove`
|
// the failure. When the snapshot is only on the destination store, a
|
||||||
// for the snapshot again once the destination store is reachable
|
// failure is returned as an error, because that removal is all the
|
||||||
// (`vaultik prune` never removes snapshot metadata that has its
|
// command does. When the snapshot was in the local index, its removal
|
||||||
// manifest).
|
// there has already happened, so a failure is only a warning telling
|
||||||
|
// the user to run `vaultik snapshot remove` for the snapshot again once
|
||||||
|
// the destination store is reachable (`vaultik prune` never removes
|
||||||
|
// snapshot metadata that has its manifest).
|
||||||
func (v *Vaultik) removeSnapshotRemote(
|
func (v *Vaultik) removeSnapshotRemote(
|
||||||
snapshotID, remoteKey string, inLocalIndex bool, markerErr error,
|
shown, remoteKey string, inLocalIndex bool, markerErr error,
|
||||||
opts *RemoveOptions,
|
opts *RemoveOptions,
|
||||||
) (bool, error) {
|
) (bool, error) {
|
||||||
if opts.LocalOnly {
|
if opts.LocalOnly {
|
||||||
@@ -1695,7 +1721,7 @@ func (v *Vaultik) removeSnapshotRemote(
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Removing snapshot metadata from remote storage",
|
log.Info("Removing snapshot metadata from remote storage",
|
||||||
"snapshot_id", snapshotID)
|
"snapshot_id", shown)
|
||||||
|
|
||||||
var deleted int
|
var deleted int
|
||||||
|
|
||||||
@@ -1713,14 +1739,14 @@ func (v *Vaultik) removeSnapshotRemote(
|
|||||||
log.Warn("Could not remove snapshot metadata from remote storage; "+
|
log.Warn("Could not remove snapshot metadata from remote storage; "+
|
||||||
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
|
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
|
||||||
"again once the remote is reachable",
|
"again once the remote is reachable",
|
||||||
"snapshot_id", snapshotID, "error", err)
|
"snapshot_id", shown, "error", err)
|
||||||
|
|
||||||
// The UI writes to stdout, which under --json holds only the
|
// The UI writes to stdout, which under --json holds only the
|
||||||
// document; the log record above is the warning on stderr.
|
// document; the log record above is the warning on stderr.
|
||||||
if v.UI != nil && !opts.JSON {
|
if v.UI != nil && !opts.JSON {
|
||||||
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
|
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
|
||||||
"%v. Run '%s %s' again once the remote is reachable.",
|
"%v. Run '%s %s' again once the remote is reachable.",
|
||||||
err, snapshotRemoveCommandHint, snapshotID)
|
err, snapshotRemoveCommandHint, shown)
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, nil
|
return false, nil
|
||||||
@@ -1815,6 +1841,23 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) {
|
|||||||
return ids, nil
|
return ids, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// localSnapshotIDForRemoteKey returns the human snapshot ID of the local
|
||||||
|
// index row whose RemoteSnapshotKey is key, or "" when no row has it.
|
||||||
|
func (v *Vaultik) localSnapshotIDForRemoteKey(key string) (string, error) {
|
||||||
|
ids, err := v.localSnapshotIDs()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("listing local snapshots: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, id := range ids {
|
||||||
|
if snapshot.RemoteSnapshotKey(id) == key {
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
// listAllRemoteSnapshotKeys collects the hashed remote keys
|
// listAllRemoteSnapshotKeys collects the hashed remote keys
|
||||||
// (subdirectories under metadata/) currently present in the
|
// (subdirectories under metadata/) currently present in the
|
||||||
// destination store. Returns (nil, err) when the store cannot be
|
// destination store. Returns (nil, err) when the store cannot be
|
||||||
|
|||||||
@@ -1,115 +0,0 @@
|
|||||||
package vaultik
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"text/tabwriter"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ListedFile is one entry printed by `snapshot files`. Mode is the Go
|
|
||||||
// file mode string, which shows the entry type and the permission bits
|
|
||||||
// (for example -rw-r--r-- or drwxr-xr-x).
|
|
||||||
type ListedFile struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
MTime time.Time `json:"mtime"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListSnapshotFiles prints the path, size, mode and modification time of
|
|
||||||
// every file the snapshot holds at or under path, or of every file when
|
|
||||||
// path is empty. The file list is only in the snapshot's encrypted
|
|
||||||
// database, so this downloads and decrypts it as restore does, and needs
|
|
||||||
// the private key.
|
|
||||||
func (v *Vaultik) ListSnapshotFiles(snapshotID, path string, jsonOutput bool) error {
|
|
||||||
identities, err := v.restoreIdentities()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
db, tempDir, err := v.downloadSnapshotDB(snapshotID, remoteKey, identities)
|
|
||||||
if errors.Is(err, errWrongPrivateKey) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer v.closeSnapshotDB(db, tempDir)
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
var files []*database.File
|
|
||||||
|
|
||||||
if path == "" {
|
|
||||||
files, err = repos.Files.ListAll(v.ctx)
|
|
||||||
} else {
|
|
||||||
files, err = repos.Files.ListUnderPath(v.ctx, filepath.Clean(path))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("listing files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if path != "" && len(files) == 0 {
|
|
||||||
return fmt.Errorf("%w: %s", errPathsNotInSnapshot, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := make([]ListedFile, 0, len(files))
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
entries = append(entries, ListedFile{
|
|
||||||
Path: file.Path.String(),
|
|
||||||
Size: file.Size,
|
|
||||||
Mode: os.FileMode(file.Mode).String(),
|
|
||||||
MTime: file.MTime,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if jsonOutput {
|
|
||||||
encoder := json.NewEncoder(v.Stdout)
|
|
||||||
encoder.SetIndent("", " ")
|
|
||||||
|
|
||||||
return encoder.Encode(entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
if v.UI.Quiet() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return v.printListedFileTable(entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// printListedFileTable writes the entries as a plain table to stdout.
|
|
||||||
func (v *Vaultik) printListedFileTable(entries []ListedFile) error {
|
|
||||||
w := tabwriter.NewWriter(v.Stdout, 0, 0, tabPadding, ' ', 0)
|
|
||||||
|
|
||||||
_, err := fmt.Fprintln(w, "MODE\tSIZE\tMODIFIED\tPATH")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
_, err = fmt.Fprintf(w, "%s\t%d\t%s\t%s\n",
|
|
||||||
entry.Mode, entry.Size,
|
|
||||||
entry.MTime.Format(time.RFC3339), ui.QuoteUnprintable(entry.Path))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.Flush()
|
|
||||||
}
|
|
||||||
@@ -1,197 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/435:
|
|
||||||
// `snapshot files` lists what a snapshot holds without restoring it.
|
|
||||||
|
|
||||||
// unprintableFileName is a file name holding a newline and an escape
|
|
||||||
// sequence, which the table must not print raw.
|
|
||||||
const unprintableFileName = "odd\nname\x1b[31m"
|
|
||||||
|
|
||||||
// backUpForFileListing backs up the fault source tree, plus a file named
|
|
||||||
// unprintableFileName at its top, and returns a reader whose stdout is
|
|
||||||
// out, the snapshot ID, and the source directory.
|
|
||||||
func backUpForFileListing(
|
|
||||||
t *testing.T, out *bytes.Buffer,
|
|
||||||
) (*vaultik.Vaultik, string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
cfg := faultTestConfig()
|
|
||||||
|
|
||||||
writeFaultSourceTree(t, fs, dataDir)
|
|
||||||
require.NoError(t, afero.WriteFile(fs,
|
|
||||||
filepath.Join(dataDir, unprintableFileName), []byte("odd"), 0o644))
|
|
||||||
|
|
||||||
store, err := storage.NewFileStorer(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
id := fullFaultBackup(ctx, t, fs, store, cfg,
|
|
||||||
database.NewRepositories(db), dataDir, dbPath, "files")
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
v := newReaderVaultik(ctx, cfg, store, nil, fs)
|
|
||||||
v.Stdout = out
|
|
||||||
|
|
||||||
return v, id, dataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSnapshotFilesJSONListsEveryFileWithItsMetadata(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, dataDir := backUpForFileListing(t, &out)
|
|
||||||
|
|
||||||
require.NoError(t, v.ListSnapshotFiles(id, "", true))
|
|
||||||
|
|
||||||
var entries []vaultik.ListedFile
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(out.Bytes(), &entries))
|
|
||||||
|
|
||||||
byPath := make(map[string]vaultik.ListedFile, len(entries))
|
|
||||||
for _, entry := range entries {
|
|
||||||
byPath[entry.Path] = entry
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rel := range []string{"small.txt", "a.bin", "sub", "sub/b.bin", "sub/c.bin"} {
|
|
||||||
path := filepath.Join(dataDir, rel)
|
|
||||||
|
|
||||||
entry, ok := byPath[path]
|
|
||||||
require.True(t, ok, "%s must be listed", path)
|
|
||||||
|
|
||||||
info, err := afero.NewOsFs().Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, info.Mode().String(), entry.Mode, path)
|
|
||||||
assert.True(t, info.ModTime().Equal(entry.MTime),
|
|
||||||
"%s: mtime %v, listed %v", path, info.ModTime(), entry.MTime)
|
|
||||||
|
|
||||||
if !info.IsDir() {
|
|
||||||
assert.Equal(t, info.Size(), entry.Size, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSnapshotFilesListsOnlyWhatIsUnderThePath(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, dataDir := backUpForFileListing(t, &out)
|
|
||||||
|
|
||||||
require.NoError(t, v.ListSnapshotFiles(id, filepath.Join(dataDir, "sub"), false))
|
|
||||||
|
|
||||||
text := out.String()
|
|
||||||
assert.Contains(t, text, filepath.Join(dataDir, "sub", "b.bin"))
|
|
||||||
assert.Contains(t, text, filepath.Join(dataDir, "sub", "c.bin"))
|
|
||||||
assert.NotContains(t, text, filepath.Join(dataDir, "small.txt"))
|
|
||||||
assert.True(t, strings.HasPrefix(text, "MODE"), "the table must have a header")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSnapshotFilesTableEscapesAnUnprintablePath checks that the table
|
|
||||||
// prints a path holding a newline and an escape sequence quoted and
|
|
||||||
// escaped on one row: the newline must not start a forged row, and the
|
|
||||||
// escape sequence must not reach the terminal.
|
|
||||||
func TestSnapshotFilesTableEscapesAnUnprintablePath(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, dataDir := backUpForFileListing(t, &out)
|
|
||||||
|
|
||||||
require.NoError(t, v.ListSnapshotFiles(id, "", false))
|
|
||||||
|
|
||||||
text := out.String()
|
|
||||||
assert.Contains(t, text,
|
|
||||||
strconv.Quote(filepath.Join(dataDir, unprintableFileName)))
|
|
||||||
assert.NotContains(t, text, "\x1b", "no escape byte may reach the output")
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(text, "\n") {
|
|
||||||
assert.False(t, strings.HasPrefix(line, "name"),
|
|
||||||
"the newline in the name must not start a row: %q", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSnapshotFilesQuietSuppressesTableNotJSON is the --quiet contract
|
|
||||||
// for `snapshot files`: the table is silenced, but the --json document a
|
|
||||||
// script depends on is still written.
|
|
||||||
func TestSnapshotFilesQuietSuppressesTableNotJSON(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, _ := backUpForFileListing(t, &out)
|
|
||||||
v.UI.SetQuiet(true)
|
|
||||||
|
|
||||||
require.NoError(t, v.ListSnapshotFiles(id, "", false))
|
|
||||||
require.Empty(t, out.String(), "the table must be silenced under --quiet")
|
|
||||||
|
|
||||||
require.NoError(t, v.ListSnapshotFiles(id, "", true))
|
|
||||||
|
|
||||||
var entries []vaultik.ListedFile
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(out.Bytes(), &entries))
|
|
||||||
assert.NotEmpty(t, entries,
|
|
||||||
"the --json document must still be written under --quiet")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSnapshotFilesFailsOnAPathThatMatchesNothing(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, dataDir := backUpForFileListing(t, &out)
|
|
||||||
missing := filepath.Join(dataDir, "typo.txt")
|
|
||||||
|
|
||||||
err := v.ListSnapshotFiles(id, missing, false)
|
|
||||||
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), missing, "the error must name the path")
|
|
||||||
assert.Empty(t, out.String(), "nothing must be printed")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSnapshotFilesRequiresThePrivateKey(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
v, id, _ := backUpForFileListing(t, &out)
|
|
||||||
v.Config.AgeSecretKey = ""
|
|
||||||
|
|
||||||
err := v.ListSnapshotFiles(id, "", false)
|
|
||||||
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "VAULTIK_AGE_SECRET_KEY")
|
|
||||||
}
|
|
||||||
@@ -39,8 +39,7 @@ const maxRemoteOnlyRows = 1000
|
|||||||
// manifest reads need throttling.
|
// manifest reads need throttling.
|
||||||
const remoteManifestFetchConcurrency = 8
|
const remoteManifestFetchConcurrency = 8
|
||||||
|
|
||||||
// tabPadding is the tabwriter cell padding for the `snapshot list` and
|
// tabPadding is the tabwriter cell padding for the snapshot table.
|
||||||
// `snapshot files` tables.
|
|
||||||
const tabPadding = 3
|
const tabPadding = 3
|
||||||
|
|
||||||
// ListSnapshots prints the table of snapshots known to this host: the
|
// ListSnapshots prints the table of snapshots known to this host: the
|
||||||
|
|||||||
Reference in new issue
Block a user