Compare commits
3
Commits
075c5e9733
...
ae07981902
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae07981902 | ||
|
|
d88ed64489 | ||
|
|
bd9656dbd4 |
@@ -74,11 +74,16 @@ Requirements that no existing tool meets:
|
|||||||
# verify a snapshot (shallow: checks all blobs are present with the listed size)
|
# verify a snapshot (shallow: checks all blobs are present with the listed size)
|
||||||
vaultik snapshot verify <snapshot-id>
|
vaultik snapshot verify <snapshot-id>
|
||||||
|
|
||||||
|
# put the private key file in the environment (reading it from the file
|
||||||
|
# keeps the key out of your shell history); the whole age-keygen file,
|
||||||
|
# with one or more identities, is accepted
|
||||||
|
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||||
|
|
||||||
# deep verify (downloads and cryptographically verifies every blob)
|
# deep verify (downloads and cryptographically verifies every blob)
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot verify --deep <snapshot-id>
|
vaultik snapshot verify --deep <snapshot-id>
|
||||||
|
|
||||||
# restore (requires the private key)
|
# restore (requires the private key)
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot restore <snapshot-id> /tmp/restored
|
vaultik snapshot restore <snapshot-id> /tmp/restored
|
||||||
|
|
||||||
# daily cron job: back up, keep a 4-week rolling window of snapshots
|
# daily cron job: back up, keep a 4-week rolling window of snapshots
|
||||||
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
||||||
@@ -119,15 +124,17 @@ Use that remote key — the hex printed inside `<remote only:...>`, or the
|
|||||||
full `remote_key` from `snapshot list --json` — to restore and verify:
|
full `remote_key` from `snapshot list --json` — to restore and verify:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
# put the private key file in the environment (reading it from the file
|
||||||
|
# keeps the key out of your shell history)
|
||||||
|
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||||
|
|
||||||
# restore everything to /tmp/restored, then check every restored file's
|
# restore everything to /tmp/restored, then check every restored file's
|
||||||
# chunk hashes
|
# chunk hashes
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \
|
vaultik snapshot restore --verify <remote-key> /tmp/restored
|
||||||
vaultik snapshot restore --verify <remote-key> /tmp/restored
|
|
||||||
|
|
||||||
# optionally, deep-verify the snapshot against the store (downloads and
|
# optionally, deep-verify the snapshot against the store (downloads and
|
||||||
# cryptographically checks every blob)
|
# cryptographically checks every blob)
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \
|
vaultik snapshot verify --deep <remote-key>
|
||||||
vaultik snapshot verify --deep <remote-key>
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`age_recipients` (the public key) is not needed to restore — only the
|
`age_recipients` (the public key) is not needed to restore — only the
|
||||||
@@ -217,7 +224,7 @@ and `vaultik prune --json | jq .` both work as written.
|
|||||||
|
|
||||||
### environment variables
|
### environment variables
|
||||||
|
|
||||||
* `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for `snapshot restore` and `snapshot verify --deep`)
|
* `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for `snapshot restore` and `snapshot verify --deep`). May hold the whole `age-keygen` file — comments and every identity in it are accepted. Set it from the file, e.g. `export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"`, so the key is not typed into your shell history.
|
||||||
* `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
* `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
||||||
* `VAULTIK_INDEX_PATH`: Override local SQLite index path
|
* `VAULTIK_INDEX_PATH`: Override local SQLite index path
|
||||||
* `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the run (development/debugging)
|
* `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the run (development/debugging)
|
||||||
|
|||||||
@@ -25,6 +25,29 @@ release" is exactly the contradiction
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-09-22: Validated blob hashes, offsets and lengths read back from
|
||||||
|
the destination before using them
|
||||||
|
([issue #155](https://git.eeqj.de/sneak/vaultik/issues/155)). A blob
|
||||||
|
hash taken from the downloaded database or the store listing was
|
||||||
|
trusted unchecked, so a hostile remote could drive a decrypted blob to
|
||||||
|
be written outside the cache directory (a hash like `aa/../../etc`) or
|
||||||
|
crash a command with a short or negative value. `blobDiskCache.path`
|
||||||
|
now refuses any key containing a path separator, and `ReadAt` rejects a
|
||||||
|
negative offset or length and bounds with `length > size-offset` so a
|
||||||
|
sum cannot overflow past the check. A new `isBlobHash` helper (a plain
|
||||||
|
function, not a method — the packer stores `temp-placeholder-{uuid}` as
|
||||||
|
a hash) gates `FetchBlob`, shallow and deep verify; the `blobs/` and
|
||||||
|
`metadata/` listings skip a non-conforming name with a warning; and
|
||||||
|
short-hash prefixes in log and error text go through a `shortHash`
|
||||||
|
helper that cannot panic. `verify`'s chunk reader also rejects a
|
||||||
|
negative `blob_chunks` length and streams the chunk instead of
|
||||||
|
allocating a database-supplied size. `restore.go` and
|
||||||
|
`internal/database` were left untouched to avoid colliding with the
|
||||||
|
in-flight [issue #156](https://git.eeqj.de/sneak/vaultik/issues/156)
|
||||||
|
work; the cache-path and `FetchBlob` guards already stop the unsafe
|
||||||
|
write and fetch, so restore's own `buildBlobIndexes` early check is
|
||||||
|
deferred as fail-fast defense in depth.
|
||||||
|
|
||||||
- 2026-09-21: Stopped an interrupted blob upload from making a later
|
- 2026-09-21: Stopped an interrupted blob upload from making a later
|
||||||
backup deduplicate against data that was never stored
|
backup deduplicate against data that was never stored
|
||||||
([issue #148](https://git.eeqj.de/sneak/vaultik/issues/148)). The
|
([issue #148](https://git.eeqj.de/sneak/vaultik/issues/148)). The
|
||||||
|
|||||||
@@ -20,10 +20,12 @@ type Reader struct {
|
|||||||
bytesRead int64
|
bytesRead int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies data
|
// NewReader creates a new Reader that decrypts, decompresses, and verifies
|
||||||
func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
// data. Every supplied identity is offered to age.Decrypt, so a blob
|
||||||
|
// encrypted to any one of them can be read.
|
||||||
|
func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||||
// Create decryption reader
|
// Create decryption reader
|
||||||
decReader, err := age.Decrypt(r, identity)
|
decReader, err := age.Decrypt(r, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,8 +35,12 @@ The snapshot may be named by its ID or, when restoring on a host with no
|
|||||||
local index, by the remote key that 'snapshot list' prints for a
|
local index, by the remote key that 'snapshot list' prints for a
|
||||||
remote-only snapshot (an unambiguous leading part is enough).
|
remote-only snapshot (an unambiguous leading part is enough).
|
||||||
|
|
||||||
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with
|
Requires the age private key in the VAULTIK_AGE_SECRET_KEY environment
|
||||||
the age private key.
|
variable. The variable may hold the whole age-keygen file (comments and
|
||||||
|
all of its identities are accepted); read it from the file rather than
|
||||||
|
typing the key, so it does not land in your shell history:
|
||||||
|
|
||||||
|
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Restore entire snapshot
|
# Restore entire snapshot
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package cli //nolint:testpackage // exercises the unexported command constructor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRestoreCommandDoesNotTakeKeyAsArgument guards the fix for the age
|
||||||
|
// key being echoed on the command line: restore must take the key only
|
||||||
|
// from the environment, never as a flag value, and its help must show the
|
||||||
|
// file-based form rather than a literal key that would land in shell
|
||||||
|
// history.
|
||||||
|
func TestRestoreCommandDoesNotTakeKeyAsArgument(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cmd := newSnapshotRestoreCommand()
|
||||||
|
|
||||||
|
cmd.Flags().VisitAll(func(f *pflag.Flag) {
|
||||||
|
lower := strings.ToLower(f.Name)
|
||||||
|
for _, banned := range []string{"key", "secret", "age", "identity"} {
|
||||||
|
if strings.Contains(lower, banned) {
|
||||||
|
t.Errorf("restore must not accept the key as a flag; found --%s", f.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
help := cmd.Long
|
||||||
|
if strings.Contains(help, "AGE-SECRET-KEY-") {
|
||||||
|
t.Error("restore help must not show a literal age private key to type")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(help, "$(cat ") {
|
||||||
|
t.Error("restore help should read the key from a file, e.g. $(cat ...)")
|
||||||
|
}
|
||||||
|
}
|
||||||
+43
-22
@@ -135,6 +135,26 @@ func (c *Config) SnapshotNames() []string {
|
|||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Names of the two places the age secret key can be configured, used by
|
||||||
|
// AgeSecretKeySourceName for error messages that must not echo the value.
|
||||||
|
//
|
||||||
|
//nolint:gosec // G101: these are the names of the config sources, not a key
|
||||||
|
const (
|
||||||
|
ageSecretKeySourceEnv = "VAULTIK_AGE_SECRET_KEY"
|
||||||
|
ageSecretKeySourceConfig = "age_secret_key"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AgeSecretKeySourceName returns the human name of where AgeSecretKey was
|
||||||
|
// configured. A Config built directly (as in tests) has no recorded
|
||||||
|
// source, so it reports the config-file field name.
|
||||||
|
func (c *Config) AgeSecretKeySourceName() string {
|
||||||
|
if c.AgeSecretKeySource != "" {
|
||||||
|
return c.AgeSecretKeySource
|
||||||
|
}
|
||||||
|
|
||||||
|
return ageSecretKeySourceConfig
|
||||||
|
}
|
||||||
|
|
||||||
// Config represents the application configuration for Vaultik.
|
// Config represents the application configuration for Vaultik.
|
||||||
// It defines all settings for backup operations, including source directories,
|
// It defines all settings for backup operations, including source directories,
|
||||||
// encryption recipients, storage configuration, and performance tuning parameters.
|
// encryption recipients, storage configuration, and performance tuning parameters.
|
||||||
@@ -144,8 +164,13 @@ func (c *Config) SnapshotNames() []string {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
AgeRecipients []string `yaml:"age_recipients"`
|
AgeRecipients []string `yaml:"age_recipients"`
|
||||||
AgeSecretKey string `yaml:"age_secret_key"`
|
AgeSecretKey string `yaml:"age_secret_key"`
|
||||||
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
// AgeSecretKeySource names where AgeSecretKey was configured
|
||||||
ChunkSize Size `yaml:"chunk_size"`
|
// ("VAULTIK_AGE_SECRET_KEY" or "age_secret_key") so a later parse
|
||||||
|
// failure can name the source without echoing the secret value. It is
|
||||||
|
// set by Load and never read from or written to the config file.
|
||||||
|
AgeSecretKeySource string `yaml:"-"`
|
||||||
|
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
||||||
|
ChunkSize Size `yaml:"chunk_size"`
|
||||||
// Exclude holds global excludes applied to all snapshots.
|
// Exclude holds global excludes applied to all snapshots.
|
||||||
Exclude []string `yaml:"exclude"`
|
Exclude []string `yaml:"exclude"`
|
||||||
Hostname string `yaml:"hostname"`
|
Hostname string `yaml:"hostname"`
|
||||||
@@ -254,10 +279,7 @@ func Load(path string) (*Config, error) {
|
|||||||
cfg.IndexPath = expandTilde(envIndexPath)
|
cfg.IndexPath = expandTilde(envIndexPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for environment variable override for AgeSecretKey
|
cfg.setAgeSecretKey()
|
||||||
if envAgeSecretKey := os.Getenv("VAULTIK_AGE_SECRET_KEY"); envAgeSecretKey != "" {
|
|
||||||
cfg.AgeSecretKey = extractAgeSecretKey(envAgeSecretKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get hostname if not set
|
// Get hostname if not set
|
||||||
if cfg.Hostname == "" {
|
if cfg.Hostname == "" {
|
||||||
@@ -379,6 +401,21 @@ func validateAgeRecipient(recipient string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setAgeSecretKey records the age secret key and where it came from. The
|
||||||
|
// value is stored raw and parsed only where decryption happens
|
||||||
|
// (internal/vaultik), so backup, list and prune keep working whatever the
|
||||||
|
// field holds. The environment variable overrides the config-file field.
|
||||||
|
func (c *Config) setAgeSecretKey() {
|
||||||
|
if c.AgeSecretKey != "" {
|
||||||
|
c.AgeSecretKeySource = ageSecretKeySourceConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
if env := os.Getenv("VAULTIK_AGE_SECRET_KEY"); env != "" {
|
||||||
|
c.AgeSecretKey = env
|
||||||
|
c.AgeSecretKeySource = ageSecretKeySourceEnv
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// validateStorage validates storage configuration.
|
// validateStorage validates storage configuration.
|
||||||
// If StorageURL is set, it takes precedence. S3 URLs require credentials.
|
// If StorageURL is set, it takes precedence. S3 URLs require credentials.
|
||||||
// File URLs don't require any S3 configuration.
|
// File URLs don't require any S3 configuration.
|
||||||
@@ -435,22 +472,6 @@ func (c *Config) validateStorageURL() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using
|
|
||||||
// the age library's parser, which handles comments and whitespace.
|
|
||||||
func extractAgeSecretKey(input string) string {
|
|
||||||
identities, err := age.ParseIdentities(strings.NewReader(input))
|
|
||||||
if err != nil || len(identities) == 0 {
|
|
||||||
// Fall back to trimmed input if parsing fails
|
|
||||||
return strings.TrimSpace(input)
|
|
||||||
}
|
|
||||||
// Return the string representation of the first identity
|
|
||||||
if id, ok := identities[0].(*age.X25519Identity); ok {
|
|
||||||
return id.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.TrimSpace(input)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Module exports the config module for fx dependency injection.
|
// Module exports the config module for fx dependency injection.
|
||||||
// It provides the Config type to other modules in the application.
|
// It provides the Config type to other modules in the application.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package config //nolint:testpackage // exercises unexported extractAgeSecretKey
|
package config //nolint:testpackage // exercises unexported source constants
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -298,53 +298,31 @@ func TestValidateAgeRecipients(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs
|
// TestAgeSecretKeySourceName checks the name reported for the configured
|
||||||
func TestExtractAgeSecretKey(t *testing.T) {
|
// age secret key: the recorded source when Load set one, and the
|
||||||
|
// config-file field name for a Config built directly (as in tests).
|
||||||
|
func TestAgeSecretKeySourceName(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
source string
|
||||||
expected string
|
want string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "plain key",
|
name: "unset defaults to config field",
|
||||||
input: testIntegrationAgePrivateKey,
|
source: "",
|
||||||
expected: testIntegrationAgePrivateKey,
|
want: ageSecretKeySourceConfig,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "key with trailing newline",
|
name: "environment source",
|
||||||
input: testIntegrationAgePrivateKey + "\n",
|
source: ageSecretKeySourceEnv,
|
||||||
expected: testIntegrationAgePrivateKey,
|
want: ageSecretKeySourceEnv,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "full age-keygen output",
|
name: "config-file source",
|
||||||
input: "# created: 2025-01-14T12:00:00Z\n" +
|
source: ageSecretKeySourceConfig,
|
||||||
"# public key: " + testIntegrationAgePublicKey + "\n" +
|
want: ageSecretKeySourceConfig,
|
||||||
testIntegrationAgePrivateKey + "\n",
|
|
||||||
expected: testIntegrationAgePrivateKey,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "age-keygen output with extra blank lines",
|
|
||||||
input: "# created: 2025-01-14T12:00:00Z\n" +
|
|
||||||
"# public key: " + testIntegrationAgePublicKey + "\n\n" +
|
|
||||||
testIntegrationAgePrivateKey + "\n\n",
|
|
||||||
expected: testIntegrationAgePrivateKey,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "key with leading whitespace",
|
|
||||||
input: " " + testIntegrationAgePrivateKey + " ",
|
|
||||||
expected: testIntegrationAgePrivateKey,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty input",
|
|
||||||
input: "",
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "only comments",
|
|
||||||
input: "# this is a comment\n# another comment",
|
|
||||||
expected: "# this is a comment\n# another comment",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -352,10 +330,9 @@ func TestExtractAgeSecretKey(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
result := extractAgeSecretKey(tt.input)
|
cfg := &Config{AgeSecretKeySource: tt.source}
|
||||||
if result != tt.expected {
|
if got := cfg.AgeSecretKeySourceName(); got != tt.want {
|
||||||
t.Errorf("extractAgeSecretKey(%q) = %q, want %q",
|
t.Errorf("AgeSecretKeySourceName() = %q, want %q", got, tt.want)
|
||||||
tt.input, result, tt.expected)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,18 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/types"
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Sentinel errors for the single-snapshot invariant that an exported
|
||||||
|
// per-snapshot metadata database must satisfy.
|
||||||
|
var (
|
||||||
|
// ErrNoSnapshotInDatabase means the metadata database has no snapshot
|
||||||
|
// row at all.
|
||||||
|
ErrNoSnapshotInDatabase = errors.New("database contains no snapshot")
|
||||||
|
// ErrMultipleSnapshotsInDatabase means the metadata database holds
|
||||||
|
// more than the single snapshot an export is supposed to contain.
|
||||||
|
ErrMultipleSnapshotsInDatabase = errors.New(
|
||||||
|
"database contains more than one snapshot")
|
||||||
|
)
|
||||||
|
|
||||||
// SnapshotRepository provides access to the snapshots table and its
|
// SnapshotRepository provides access to the snapshots table and its
|
||||||
// snapshot_files / snapshot_blobs association tables.
|
// snapshot_files / snapshot_blobs association tables.
|
||||||
type SnapshotRepository struct {
|
type SnapshotRepository struct {
|
||||||
@@ -206,6 +218,48 @@ func (r *SnapshotRepository) GetByID(
|
|||||||
return &snapshot, nil
|
return &snapshot, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetOnlySnapshot returns the sole snapshot in an exported per-snapshot
|
||||||
|
// metadata database. The backup path writes each snapshot's database with
|
||||||
|
// exactly one snapshot row (see cleanSnapshotDB), so restore and deep
|
||||||
|
// verify expect exactly one. Zero rows return ErrNoSnapshotInDatabase and
|
||||||
|
// more than one returns ErrMultipleSnapshotsInDatabase; callers treat
|
||||||
|
// either as a failed identity check on the downloaded database.
|
||||||
|
func (r *SnapshotRepository) GetOnlySnapshot(ctx context.Context) (*Snapshot, error) {
|
||||||
|
query := `
|
||||||
|
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
||||||
|
started_at, completed_at, file_count, chunk_count, blob_count,
|
||||||
|
total_size, blob_size, compression_ratio
|
||||||
|
FROM snapshots
|
||||||
|
LIMIT 2
|
||||||
|
`
|
||||||
|
|
||||||
|
rows, err := r.db.conn.QueryContext(ctx, query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("querying snapshots: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
err := rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
Fatalf("failed to close rows: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
snapshots, err := r.scanSnapshotRows(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch len(snapshots) {
|
||||||
|
case 1:
|
||||||
|
return snapshots[0], nil
|
||||||
|
case 0:
|
||||||
|
return nil, ErrNoSnapshotInDatabase
|
||||||
|
default:
|
||||||
|
return nil, ErrMultipleSnapshotsInDatabase
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ListRecent returns up to limit snapshots, most recently started first.
|
// ListRecent returns up to limit snapshots, most recently started first.
|
||||||
func (r *SnapshotRepository) ListRecent(
|
func (r *SnapshotRepository) ListRecent(
|
||||||
ctx context.Context, limit int,
|
ctx context.Context, limit int,
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ func (h *hashVerifyReader) Close() error {
|
|||||||
actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
|
actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
|
||||||
if actualHashHex != h.blobHash {
|
if actualHashHex != h.blobHash {
|
||||||
return fmt.Errorf("%w: expected %s, got %s",
|
return fmt.Errorf("%w: expected %s, got %s",
|
||||||
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
|
errBlobHashMismatch, shortHash(h.blobHash), shortHash(actualHashHex))
|
||||||
}
|
}
|
||||||
|
|
||||||
if readerErr != nil {
|
if readerErr != nil {
|
||||||
@@ -74,14 +74,15 @@ func (h *hashVerifyReader) Close() error {
|
|||||||
// The hash is verified when the returned reader is closed (after fully reading).
|
// The hash is verified when the returned reader is closed (after fully reading).
|
||||||
// This avoids buffering the entire blob in memory.
|
// This avoids buffering the entire blob in memory.
|
||||||
func (v *Vaultik) FetchAndDecryptBlob(
|
func (v *Vaultik) FetchAndDecryptBlob(
|
||||||
ctx context.Context, blobHash string, expectedSize int64, identity age.Identity,
|
ctx context.Context, blobHash string, expectedSize int64,
|
||||||
|
identities ...age.Identity,
|
||||||
) (io.ReadCloser, error) {
|
) (io.ReadCloser, error) {
|
||||||
rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize)
|
rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, err := blobgen.NewReader(rc, identity)
|
reader, err := blobgen.NewReader(rc, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = rc.Close()
|
_ = rc.Close()
|
||||||
|
|
||||||
@@ -102,6 +103,13 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
|||||||
func (v *Vaultik) FetchBlob(
|
func (v *Vaultik) FetchBlob(
|
||||||
ctx context.Context, blobHash string, expectedSize int64,
|
ctx context.Context, blobHash string, expectedSize int64,
|
||||||
) (io.ReadCloser, int64, error) {
|
) (io.ReadCloser, int64, error) {
|
||||||
|
// blobHash reaches here from the snapshot database, which is not
|
||||||
|
// trusted. Reject a malformed hash before it is spliced into a storage
|
||||||
|
// path (blobHash[:2]/blobHash[2:4]) or a fetch is attempted.
|
||||||
|
if !isBlobHash(blobHash) {
|
||||||
|
return nil, 0, fmt.Errorf("%w: %s", errInvalidBlobHash, shortHash(blobHash))
|
||||||
|
}
|
||||||
|
|
||||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||||
|
|
||||||
t0 := time.Now()
|
t0 := time.Now()
|
||||||
@@ -109,7 +117,7 @@ func (v *Vaultik) FetchBlob(
|
|||||||
getDur := time.Since(t0)
|
getDur := time.Since(t0)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
|
return nil, 0, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t0 = time.Now()
|
t0 = time.Now()
|
||||||
@@ -119,11 +127,11 @@ func (v *Vaultik) FetchBlob(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = rc.Close()
|
_ = rc.Close()
|
||||||
|
|
||||||
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
|
return nil, 0, fmt.Errorf("stat blob %s: %w", shortHash(blobHash), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("FetchBlob round-trips",
|
log.Debug("FetchBlob round-trips",
|
||||||
"hash", blobHash[:16],
|
"hash", shortHash(blobHash),
|
||||||
"ms_storage_get", getDur.Milliseconds(),
|
"ms_storage_get", getDur.Milliseconds(),
|
||||||
"ms_storage_stat", statDur.Milliseconds(),
|
"ms_storage_stat", statDur.Milliseconds(),
|
||||||
"expected_size", expectedSize,
|
"expected_size", expectedSize,
|
||||||
|
|||||||
@@ -55,6 +55,35 @@ func buildHashTestBlob(
|
|||||||
return encBuf.Bytes(), correctHash
|
return encBuf.Bytes(), correctHash
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFetchBlobRejectsMalformedHash verifies FetchBlob refuses a blob hash
|
||||||
|
// that is not 64 lowercase hex characters before it builds a storage path
|
||||||
|
// or issues any request. The hash reaches FetchBlob from the snapshot
|
||||||
|
// database, which is not trusted, so a value such as one containing "/.."
|
||||||
|
// must never reach the store.
|
||||||
|
func TestFetchBlobRejectsMalformedHash(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mockStorage := NewMockStorer()
|
||||||
|
tv := vaultik.NewForTesting(mockStorage)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, bad := range []string{
|
||||||
|
"aa/../../../home/u/.profile",
|
||||||
|
"abc",
|
||||||
|
strings.Repeat("A", 64), // uppercase hex is not accepted
|
||||||
|
strings.Repeat("g", 64), // not hex
|
||||||
|
} {
|
||||||
|
_, _, err := tv.FetchBlob(ctx, bad, 0)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error for malformed hash %q, got nil", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if calls := mockStorage.GetCalls(); len(calls) != 0 {
|
||||||
|
t.Fatalf("storage was accessed for a malformed hash: %v", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks
|
// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks
|
||||||
// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash.
|
// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash.
|
||||||
func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||||
|
|||||||
@@ -6,13 +6,17 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for blob cache lookups.
|
// Sentinel errors for blob cache lookups.
|
||||||
var (
|
var (
|
||||||
errCacheKeyMissing = errors.New("key not in cache")
|
errCacheKeyMissing = errors.New("key not in cache")
|
||||||
errCacheReadBeyondBlob = errors.New("read beyond blob size")
|
errCacheReadBeyondBlob = errors.New("read beyond blob size")
|
||||||
|
errCacheKeyHasSeparator = errors.New(
|
||||||
|
"cache key contains a path separator")
|
||||||
|
errCacheNegativeRead = errors.New("negative offset or length")
|
||||||
)
|
)
|
||||||
|
|
||||||
// blobCacheFileMode is the permission mode for cached blob files.
|
// blobCacheFileMode is the permission mode for cached blob files.
|
||||||
@@ -74,6 +78,11 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) {
|
|||||||
// Put writes blob data to disk cache. Entries larger than maxBytes are
|
// Put writes blob data to disk cache. Entries larger than maxBytes are
|
||||||
// silently skipped.
|
// silently skipped.
|
||||||
func (c *blobDiskCache) Put(key string, data []byte) error {
|
func (c *blobDiskCache) Put(key string, data []byte) error {
|
||||||
|
p, err := c.path(key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
entrySize := int64(len(data))
|
entrySize := int64(len(data))
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
@@ -87,11 +96,12 @@ func (c *blobDiskCache) Put(key string, data []byte) error {
|
|||||||
if e, ok := c.items[key]; ok {
|
if e, ok := c.items[key]; ok {
|
||||||
c.unlink(e)
|
c.unlink(e)
|
||||||
c.curBytes -= e.size
|
c.curBytes -= e.size
|
||||||
_ = os.Remove(c.path(key))
|
_ = os.Remove(p)
|
||||||
|
|
||||||
delete(c.items, key)
|
delete(c.items, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := os.WriteFile(c.path(key), data, blobCacheFileMode)
|
err = os.WriteFile(p, data, blobCacheFileMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("writing blob to cache: %w", err)
|
return fmt.Errorf("writing blob to cache: %w", err)
|
||||||
}
|
}
|
||||||
@@ -119,19 +129,26 @@ func (c *blobDiskCache) Put(key string, data []byte) error {
|
|||||||
// disk without buffering its entire plaintext (which may be tens of GB)
|
// disk without buffering its entire plaintext (which may be tens of GB)
|
||||||
// in RAM.
|
// in RAM.
|
||||||
func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
||||||
|
p, err := c.path(key)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
// Remove any prior entry first; we'll re-link after the file is
|
// Remove any prior entry first; we'll re-link after the file is
|
||||||
// written successfully.
|
// written successfully.
|
||||||
if e, ok := c.items[key]; ok {
|
if e, ok := c.items[key]; ok {
|
||||||
c.unlink(e)
|
c.unlink(e)
|
||||||
c.curBytes -= e.size
|
c.curBytes -= e.size
|
||||||
_ = os.Remove(c.path(key))
|
_ = os.Remove(p)
|
||||||
|
|
||||||
delete(c.items, key)
|
delete(c.items, key)
|
||||||
}
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
//nolint:gosec // G304: path() rejects keys with a separator
|
||||||
f, err := os.OpenFile(
|
f, err := os.OpenFile(
|
||||||
c.path(key), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, blobCacheFileMode)
|
p, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, blobCacheFileMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("creating cache file: %w", err)
|
return 0, fmt.Errorf("creating cache file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -140,13 +157,13 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
|||||||
closeErr := f.Close()
|
closeErr := f.Close()
|
||||||
|
|
||||||
if copyErr != nil {
|
if copyErr != nil {
|
||||||
_ = os.Remove(c.path(key))
|
_ = os.Remove(p)
|
||||||
|
|
||||||
return written, fmt.Errorf("streaming to cache file: %w", copyErr)
|
return written, fmt.Errorf("streaming to cache file: %w", copyErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
if closeErr != nil {
|
if closeErr != nil {
|
||||||
_ = os.Remove(c.path(key))
|
_ = os.Remove(p)
|
||||||
|
|
||||||
return written, fmt.Errorf("closing cache file: %w", closeErr)
|
return written, fmt.Errorf("closing cache file: %w", closeErr)
|
||||||
}
|
}
|
||||||
@@ -158,7 +175,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
|||||||
// floor — but the restore path passes math.MaxInt64 as maxBytes
|
// floor — but the restore path passes math.MaxInt64 as maxBytes
|
||||||
// so this branch is effectively unreachable there.
|
// so this branch is effectively unreachable there.
|
||||||
if written > c.maxBytes {
|
if written > c.maxBytes {
|
||||||
_ = os.Remove(c.path(key))
|
_ = os.Remove(p)
|
||||||
|
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
@@ -181,6 +198,11 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
|||||||
|
|
||||||
// Get reads a cached blob from disk. Returns data and true on hit.
|
// Get reads a cached blob from disk. Returns data and true on hit.
|
||||||
func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
||||||
|
p, err := c.path(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.getCalls++
|
c.getCalls++
|
||||||
|
|
||||||
@@ -195,7 +217,8 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
|||||||
c.pushFront(e)
|
c.pushFront(e)
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
data, err := os.ReadFile(c.path(key))
|
//nolint:gosec // G304: path() rejects keys with a separator
|
||||||
|
data, err := os.ReadFile(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
if e2, ok2 := c.items[key]; ok2 && e2 == e {
|
if e2, ok2 := c.items[key]; ok2 && e2 == e {
|
||||||
@@ -213,6 +236,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
|||||||
|
|
||||||
// ReadAt reads a slice of a cached blob without loading the entire blob into memory.
|
// ReadAt reads a slice of a cached blob without loading the entire blob into memory.
|
||||||
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
|
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
|
||||||
|
p, err := c.path(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// offset and length come from a blob_chunks row read back from the
|
||||||
|
// destination. A negative value must be rejected outright; the upper
|
||||||
|
// bound is checked as length > size-offset (a subtraction) so a huge
|
||||||
|
// offset+length cannot overflow int64 and slip past the check.
|
||||||
|
if offset < 0 || length < 0 {
|
||||||
|
return nil, fmt.Errorf("%w: offset=%d length=%d",
|
||||||
|
errCacheNegativeRead, offset, length)
|
||||||
|
}
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.readAtCalls++
|
c.readAtCalls++
|
||||||
|
|
||||||
@@ -223,7 +260,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
|
|||||||
return nil, fmt.Errorf("%w: %q", errCacheKeyMissing, key)
|
return nil, fmt.Errorf("%w: %q", errCacheKeyMissing, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
if offset+length > e.size {
|
if length > e.size-offset {
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w: offset=%d length=%d size=%d",
|
return nil, fmt.Errorf("%w: offset=%d length=%d size=%d",
|
||||||
@@ -234,7 +271,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
|
|||||||
c.pushFront(e)
|
c.pushFront(e)
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
f, err := os.Open(c.path(key))
|
f, err := os.Open(p) //nolint:gosec // G304: path() rejects keys with a separator
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -276,7 +313,13 @@ func (c *blobDiskCache) Delete(key string) {
|
|||||||
c.unlink(e)
|
c.unlink(e)
|
||||||
delete(c.items, key)
|
delete(c.items, key)
|
||||||
c.curBytes -= e.size
|
c.curBytes -= e.size
|
||||||
_ = os.Remove(c.path(key))
|
|
||||||
|
// The key is already in the map, so it passed path() when it was
|
||||||
|
// inserted; the error cannot occur here.
|
||||||
|
p, err := c.path(key)
|
||||||
|
if err == nil {
|
||||||
|
_ = os.Remove(p)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keys returns a snapshot of all cached keys. Safe for iteration without
|
// Keys returns a snapshot of all cached keys. Safe for iteration without
|
||||||
@@ -347,8 +390,18 @@ func (c *blobDiskCache) Close() error {
|
|||||||
return os.RemoveAll(c.dir)
|
return os.RemoveAll(c.dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *blobDiskCache) path(key string) string {
|
// path returns the on-disk location of the cache file for key. The key is
|
||||||
return filepath.Join(c.dir, key)
|
// a blob hash read back from the destination and is not trusted: a value
|
||||||
|
// such as "aa/../../../home/u/.profile" would otherwise make filepath.Join
|
||||||
|
// escape the cache directory, so a key containing a path separator is
|
||||||
|
// refused rather than joined.
|
||||||
|
func (c *blobDiskCache) path(key string) (string, error) {
|
||||||
|
if strings.ContainsRune(key, '/') ||
|
||||||
|
strings.ContainsRune(key, filepath.Separator) {
|
||||||
|
return "", fmt.Errorf("%w: %q", errCacheKeyHasSeparator, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(c.dir, key), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
|
func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
|
||||||
@@ -391,5 +444,10 @@ func (c *blobDiskCache) evictLRU() {
|
|||||||
c.unlink(victim)
|
c.unlink(victim)
|
||||||
delete(c.items, victim.key)
|
delete(c.items, victim.key)
|
||||||
c.curBytes -= victim.size
|
c.curBytes -= victim.size
|
||||||
_ = os.Remove(c.path(victim.key))
|
|
||||||
|
// victim.key was validated by path() on insertion, so this cannot err.
|
||||||
|
p, err := c.path(victim.key)
|
||||||
|
if err == nil {
|
||||||
|
_ = os.Remove(p)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package vaultik
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
// blobHashHexLen is the length of a blob hash written as lowercase hex: a
|
||||||
|
// SHA-256 digest is 32 bytes, so 64 characters. Remote snapshot keys are
|
||||||
|
// SHA-256 hashes too and share this exact form.
|
||||||
|
const blobHashHexLen = 64
|
||||||
|
|
||||||
|
// shortHashLen is how many leading characters of a hash appear in log and
|
||||||
|
// error text.
|
||||||
|
const shortHashLen = 16
|
||||||
|
|
||||||
|
// errInvalidBlobHash reports a value used as a blob hash that is not
|
||||||
|
// exactly 64 lowercase hex characters. Restore, verify and prune read
|
||||||
|
// these values back from the destination, which is not trusted, so each
|
||||||
|
// one is checked before it is used to build a path or drive a read.
|
||||||
|
var errInvalidBlobHash = errors.New(
|
||||||
|
"blob hash is not 64 lowercase hex characters")
|
||||||
|
|
||||||
|
// isBlobHash reports whether s is exactly 64 lowercase hex characters.
|
||||||
|
// Every real blob hash and remote snapshot key has this form.
|
||||||
|
//
|
||||||
|
// The check is a plain function, not a method on types.BlobHash: the
|
||||||
|
// packer stores "temp-placeholder-{uuid}" as the hash of an unfinished
|
||||||
|
// blob in the local index, so the type itself must keep accepting values
|
||||||
|
// that are not hashes.
|
||||||
|
func isBlobHash(s string) bool {
|
||||||
|
if len(s) != blobHashHexLen {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range s {
|
||||||
|
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// shortHash returns the leading part of a hash for log and error text. It
|
||||||
|
// never panics: a string shorter than the prefix is returned whole. A hash
|
||||||
|
// read from the destination may be malformed, and formatting one for a
|
||||||
|
// message must not crash the command.
|
||||||
|
func shortHash(s string) string {
|
||||||
|
if len(s) <= shortHashLen {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
return s[:shortHashLen]
|
||||||
|
}
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
package vaultik //nolint:testpackage // drives unexported input validation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These tests treat every hash, offset and length read back from the
|
||||||
|
// destination as hostile. A blob hash comes from the downloaded snapshot
|
||||||
|
// database or the store listing, neither of which is authenticated (see
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/155), so each is validated
|
||||||
|
// before it is used to build a path or size an allocation.
|
||||||
|
|
||||||
|
// TestBlobCacheRejectsKeyWithSeparator proves the arbitrary-file-write
|
||||||
|
// hole is closed: a blob hash that climbs out of the cache directory is
|
||||||
|
// refused and nothing is written outside it. This is the exact write the
|
||||||
|
// restore path performs, keyed by the hash from the snapshot database.
|
||||||
|
func TestBlobCacheRejectsKeyWithSeparator(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, err := newBlobDiskCache(1 << 20)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { _ = cache.Close() }()
|
||||||
|
|
||||||
|
target := filepath.Join(t.TempDir(), "pwned")
|
||||||
|
|
||||||
|
// A hash whose relative form escapes the cache directory to target.
|
||||||
|
key, err := filepath.Rel(cache.dir, target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Contains(t, key, "..")
|
||||||
|
|
||||||
|
err = cache.Put(key, []byte("secret"))
|
||||||
|
require.ErrorIs(t, err, errCacheKeyHasSeparator)
|
||||||
|
|
||||||
|
_, err = cache.PutFromReader(key, strings.NewReader("secret"))
|
||||||
|
require.ErrorIs(t, err, errCacheKeyHasSeparator)
|
||||||
|
|
||||||
|
_, statErr := os.Stat(target)
|
||||||
|
require.Truef(t, os.IsNotExist(statErr),
|
||||||
|
"cache wrote outside its directory at %s", target)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBuildBlobIndexesRejectsHostileHash proves restore refuses a snapshot
|
||||||
|
// database whose blob_hash escapes the cache directory. buildBlobIndexes is
|
||||||
|
// the first place restore reads these hashes back, and it fails there, before
|
||||||
|
// any blob is fetched or written, so a hash containing /../ cannot steer a
|
||||||
|
// later write outside the cache directory.
|
||||||
|
func TestBuildBlobIndexesRejectsHostileHash(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
// A blob cache and a target file just outside it. The hostile hash is
|
||||||
|
// the relative path from the cache to that target, so an unguarded
|
||||||
|
// restore keyed by this hash would write there.
|
||||||
|
cache, err := newBlobDiskCache(1 << 20)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { _ = cache.Close() }()
|
||||||
|
|
||||||
|
target := filepath.Join(t.TempDir(), "pwned")
|
||||||
|
hostile, err := filepath.Rel(cache.dir, target)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Contains(t, hostile, "..")
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
require.NoError(t, repos.Blobs.Create(ctx, nil, &database.Blob{
|
||||||
|
ID: types.NewBlobID(),
|
||||||
|
Hash: types.BlobHash(hostile),
|
||||||
|
CreatedTS: time.Now().UTC(),
|
||||||
|
}))
|
||||||
|
|
||||||
|
v := NewForTesting(nil)
|
||||||
|
v.SetContext(ctx)
|
||||||
|
|
||||||
|
_, _, err = v.buildBlobIndexes(repos)
|
||||||
|
require.ErrorIs(t, err, errInvalidBlobHash)
|
||||||
|
|
||||||
|
_, statErr := os.Stat(target)
|
||||||
|
require.Truef(t, os.IsNotExist(statErr),
|
||||||
|
"restore wrote outside the cache directory at %s", target)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBlobCacheReadAtRejectsBadBounds proves a blob_chunks row cannot
|
||||||
|
// drive an out-of-range or negative read. offset/length reach ReadAt
|
||||||
|
// straight from the database.
|
||||||
|
func TestBlobCacheReadAtRejectsBadBounds(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, err := newBlobDiskCache(1 << 20)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { _ = cache.Close() }()
|
||||||
|
|
||||||
|
require.NoError(t, cache.Put("blob", make([]byte, 100)))
|
||||||
|
|
||||||
|
_, err = cache.ReadAt("blob", -1, 10)
|
||||||
|
require.ErrorIs(t, err, errCacheNegativeRead)
|
||||||
|
|
||||||
|
_, err = cache.ReadAt("blob", 0, -1)
|
||||||
|
require.ErrorIs(t, err, errCacheNegativeRead)
|
||||||
|
|
||||||
|
// A length past the end is rejected via the subtraction bound, so a
|
||||||
|
// huge offset+length cannot overflow past the check.
|
||||||
|
_, err = cache.ReadAt("blob", 50, 60)
|
||||||
|
require.ErrorIs(t, err, errCacheReadBeyondBlob)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestListAllRemoteBlobsSkipsNonConformingName proves a bogus object name
|
||||||
|
// under blobs/ (here a three-character name) is skipped rather than
|
||||||
|
// entering the blob map, so prune's later hash[:2]/hash[2:4] path build
|
||||||
|
// cannot panic on it.
|
||||||
|
func TestListAllRemoteBlobsSkipsNonConformingName(t *testing.T) {
|
||||||
|
// Initialize the global logger before t.Parallel() so the write lands
|
||||||
|
// in the serial phase and cannot race other parallel tests reading it.
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
good := strings.Repeat("a", blobHashHexLen)
|
||||||
|
store := &stubLister{objects: []storage.ObjectInfo{
|
||||||
|
{Key: "blobs/" + good[:2] + "/" + good[2:4] + "/" + good, Size: 10},
|
||||||
|
{Key: "blobs/a/b/c", Size: 3},
|
||||||
|
}}
|
||||||
|
|
||||||
|
v := &Vaultik{Storage: store}
|
||||||
|
v.SetContext(context.Background())
|
||||||
|
|
||||||
|
blobs, err := v.listAllRemoteBlobs()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Contains(t, blobs, good)
|
||||||
|
require.NotContains(t, blobs, "c")
|
||||||
|
require.Len(t, blobs, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVerifyManifestBlobsRejectsShortHash proves a manifest (which is not
|
||||||
|
// authenticated) with a short blob hash fails cleanly instead of panicking
|
||||||
|
// on blob.Hash[:2].
|
||||||
|
func TestVerifyManifestBlobsRejectsShortHash(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
v := &Vaultik{Stdout: io.Discard}
|
||||||
|
manifest := &snapshot.Manifest{
|
||||||
|
Blobs: []snapshot.BlobInfo{{Hash: "abc", CompressedSize: 1}},
|
||||||
|
}
|
||||||
|
|
||||||
|
verified, missing, mismatched, missingSize, err :=
|
||||||
|
v.verifyManifestBlobs(manifest, &VerifyOptions{JSON: true})
|
||||||
|
require.ErrorIs(t, err, errInvalidBlobHash)
|
||||||
|
require.Zero(t, verified)
|
||||||
|
require.Zero(t, missing)
|
||||||
|
require.Zero(t, mismatched)
|
||||||
|
require.Zero(t, missingSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVerifyBlobChunksRejectsNegativeLength proves a blob_chunks row with a
|
||||||
|
// negative length returns an error rather than reaching make([]byte,
|
||||||
|
// length) or streaming an untrusted size.
|
||||||
|
func TestVerifyBlobChunksRejectsNegativeLength(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
blobHash := strings.Repeat("b", blobHashHexLen)
|
||||||
|
blob := &database.Blob{
|
||||||
|
ID: types.NewBlobID(),
|
||||||
|
Hash: types.BlobHash(blobHash),
|
||||||
|
CreatedTS: time.Now().UTC(),
|
||||||
|
}
|
||||||
|
require.NoError(t, repos.Blobs.Create(ctx, nil, blob))
|
||||||
|
|
||||||
|
chunkHash := strings.Repeat("c", blobHashHexLen)
|
||||||
|
require.NoError(t, repos.Chunks.Create(ctx, nil,
|
||||||
|
&database.Chunk{ChunkHash: types.ChunkHash(chunkHash), Size: 1024}))
|
||||||
|
require.NoError(t, repos.BlobChunks.Create(ctx, nil, &database.BlobChunk{
|
||||||
|
BlobID: blob.ID,
|
||||||
|
ChunkHash: types.ChunkHash(chunkHash),
|
||||||
|
Offset: 0,
|
||||||
|
Length: -1,
|
||||||
|
}))
|
||||||
|
|
||||||
|
v := NewForTesting(nil)
|
||||||
|
|
||||||
|
_, err = v.verifyBlobChunks(db.Conn(), blobHash, strings.NewReader(""))
|
||||||
|
require.ErrorIs(t, err, errNegativeChunkLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
// errStubUnused marks a stubLister method a test never exercises.
|
||||||
|
var errStubUnused = errors.New("stubLister method not used in test")
|
||||||
|
|
||||||
|
// stubLister is a storage.Storer whose ListStream yields a fixed set of
|
||||||
|
// objects; every other method is unused by the tests here.
|
||||||
|
type stubLister struct {
|
||||||
|
objects []storage.ObjectInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) ListStream(
|
||||||
|
_ context.Context, prefix string,
|
||||||
|
) <-chan storage.ObjectInfo {
|
||||||
|
ch := make(chan storage.ObjectInfo, len(s.objects))
|
||||||
|
for _, o := range s.objects {
|
||||||
|
if strings.HasPrefix(o.Key, prefix) {
|
||||||
|
ch <- o
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
close(ch)
|
||||||
|
|
||||||
|
return ch
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) Put(_ context.Context, _ string, _ io.Reader) error {
|
||||||
|
return errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) PutWithProgress(
|
||||||
|
_ context.Context, _ string, _ io.Reader, _ int64, _ storage.ProgressCallback,
|
||||||
|
) error {
|
||||||
|
return errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) Get(_ context.Context, _ string) (io.ReadCloser, error) {
|
||||||
|
return nil, errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) Stat(_ context.Context, _ string) (*storage.ObjectInfo, error) {
|
||||||
|
return nil, errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) Delete(_ context.Context, _ string) error {
|
||||||
|
return errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) List(_ context.Context, _ string) ([]string, error) {
|
||||||
|
return nil, errStubUnused
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubLister) Info() storage.Info {
|
||||||
|
return storage.Info{}
|
||||||
|
}
|
||||||
@@ -247,9 +247,22 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
parts := strings.Split(object.Key, "/")
|
parts := strings.Split(object.Key, "/")
|
||||||
if len(parts) == blobKeyParts && parts[0] == "blobs" {
|
if len(parts) != blobKeyParts || parts[0] != "blobs" {
|
||||||
allBlobs[parts[3]] = object.Size
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The object name is read from the destination store and is not
|
||||||
|
// trusted. A name that is not a blob hash (e.g. a short or
|
||||||
|
// non-hex string) would panic the later hash[:2]/hash[2:4] path
|
||||||
|
// build, so skip it with a warning rather than delete it.
|
||||||
|
if !isBlobHash(parts[3]) {
|
||||||
|
log.Warn("Skipping non-conforming object under blobs/",
|
||||||
|
"key", object.Key)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
allBlobs[parts[3]] = object.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Found blobs in storage", "count", len(allBlobs))
|
log.Info("Found blobs in storage", "count", len(allBlobs))
|
||||||
|
|||||||
+93
-22
@@ -19,6 +19,7 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,8 +30,13 @@ var (
|
|||||||
errDecryptionKeyRequired = errors.New(
|
errDecryptionKeyRequired = errors.New(
|
||||||
"decryption key required for restore\n\n" +
|
"decryption key required for restore\n\n" +
|
||||||
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
|
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
|
||||||
"age private key:\n" +
|
"age private key file:\n" +
|
||||||
" export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'")
|
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
|
||||||
|
// errInvalidAgeSecretKey is returned when the configured key does not
|
||||||
|
// parse as any age identity. It names the source but never the value,
|
||||||
|
// which is secret, so the message is safe to print and log.
|
||||||
|
errInvalidAgeSecretKey = errors.New(
|
||||||
|
"configured age secret key holds no usable age identity")
|
||||||
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
||||||
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
||||||
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
||||||
@@ -41,6 +47,8 @@ var (
|
|||||||
"restored file has trailing data after its last chunk")
|
"restored file has trailing data after its last chunk")
|
||||||
errRestoreIncomplete = errors.New(
|
errRestoreIncomplete = errors.New(
|
||||||
"restore loop ended with files still pending")
|
"restore loop ended with files still pending")
|
||||||
|
errSnapshotDBMismatch = errors.New(
|
||||||
|
"decrypted database is not the requested snapshot")
|
||||||
)
|
)
|
||||||
|
|
||||||
// snapshotDBFilename is the name the decrypted snapshot database is
|
// snapshotDBFilename is the name the decrypted snapshot database is
|
||||||
@@ -94,7 +102,7 @@ type RestoreResult struct {
|
|||||||
func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
identity, err := v.prepareRestoreIdentity()
|
identities, err := v.restoreIdentities()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -108,7 +116,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
// Step 1: Download and decrypt the snapshot metadata database
|
// Step 1: Download and decrypt the snapshot metadata database
|
||||||
log.Info("Downloading snapshot metadata...")
|
log.Info("Downloading snapshot metadata...")
|
||||||
|
|
||||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identity)
|
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
@@ -157,7 +165,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Step 5: Restore files
|
// Step 5: Restore files
|
||||||
result, err := v.restoreAllFiles(files, repos, opts, identity, chunkToBlobMap)
|
result, err := v.restoreAllFiles(files, repos, opts, identities, chunkToBlobMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -218,21 +226,28 @@ func (v *Vaultik) finishRestore(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepareRestoreIdentity validates that an age secret key is configured
|
// restoreIdentities parses the configured age secret key once into every
|
||||||
// and parses it.
|
// identity it contains. The value may be a single key line or a whole
|
||||||
//
|
// age-keygen file with several identities; all of them are returned so
|
||||||
//nolint:ireturn // age.Identity is the decryption abstraction by design
|
// blobgen (via age.Decrypt) can read a blob encrypted to any of their
|
||||||
func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) {
|
// recipients. This is the first step of both restore and deep verify, so
|
||||||
|
// a missing or unparseable key fails before anything is downloaded. The
|
||||||
|
// error names the configuration source but never the key value.
|
||||||
|
func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
||||||
if v.Config.AgeSecretKey == "" {
|
if v.Config.AgeSecretKey == "" {
|
||||||
return nil, errDecryptionKeyRequired
|
return nil, errDecryptionKeyRequired
|
||||||
}
|
}
|
||||||
|
|
||||||
identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey)
|
// age.ParseIdentities skips comment and blank lines and rejects a
|
||||||
|
// malformed key. Its error can quote the offending line, so it is not
|
||||||
|
// wrapped here — that would leak the secret into the message.
|
||||||
|
identities, err := age.ParseIdentities(strings.NewReader(v.Config.AgeSecretKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing age secret key: %w", err)
|
return nil, fmt.Errorf("%w (source: %s)",
|
||||||
|
errInvalidAgeSecretKey, v.Config.AgeSecretKeySourceName())
|
||||||
}
|
}
|
||||||
|
|
||||||
return identity, nil
|
return identities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// restoreAllFiles processes files in blob-locality order: drain every
|
// restoreAllFiles processes files in blob-locality order: drain every
|
||||||
@@ -245,7 +260,7 @@ func (v *Vaultik) restoreAllFiles(
|
|||||||
files []*database.File,
|
files []*database.File,
|
||||||
repos *database.Repositories,
|
repos *database.Repositories,
|
||||||
opts *RestoreOptions,
|
opts *RestoreOptions,
|
||||||
identity age.Identity,
|
identities []age.Identity,
|
||||||
chunkToBlobMap map[string]*database.BlobChunk,
|
chunkToBlobMap map[string]*database.BlobChunk,
|
||||||
) (*RestoreResult, error) {
|
) (*RestoreResult, error) {
|
||||||
result := &RestoreResult{}
|
result := &RestoreResult{}
|
||||||
@@ -299,7 +314,7 @@ func (v *Vaultik) restoreAllFiles(
|
|||||||
ctx: v.ctx,
|
ctx: v.ctx,
|
||||||
repos: repos,
|
repos: repos,
|
||||||
opts: opts,
|
opts: opts,
|
||||||
identity: identity,
|
identities: identities,
|
||||||
chunkToBlobMap: chunkToBlobMap,
|
chunkToBlobMap: chunkToBlobMap,
|
||||||
blobByHash: blobByHash,
|
blobByHash: blobByHash,
|
||||||
blobIDToHash: blobIDToHash,
|
blobIDToHash: blobIDToHash,
|
||||||
@@ -410,12 +425,12 @@ func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
|
|||||||
|
|
||||||
blob, ok := s.blobByHash[hash]
|
blob, ok := s.blobByHash[hash]
|
||||||
if !ok {
|
if !ok {
|
||||||
return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, hash[:16])
|
return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, shortHash(hash))
|
||||||
}
|
}
|
||||||
|
|
||||||
err := s.downloadBlobToCache(hash, blob.CompressedSize)
|
err := s.downloadBlobToCache(hash, blob.CompressedSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("downloading blob %s: %w", hash[:16], err)
|
return false, fmt.Errorf("downloading blob %s: %w", shortHash(hash), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.result.BlobsDownloaded++
|
s.result.BlobsDownloaded++
|
||||||
@@ -459,6 +474,16 @@ func (v *Vaultik) buildBlobIndexes(
|
|||||||
blobByHash := make(map[string]*database.Blob, len(blobsByID))
|
blobByHash := make(map[string]*database.Blob, len(blobsByID))
|
||||||
for id, blob := range blobsByID {
|
for id, blob := range blobsByID {
|
||||||
hash := blob.Hash.String()
|
hash := blob.Hash.String()
|
||||||
|
|
||||||
|
// The snapshot database is untrusted. A hash that is not 64
|
||||||
|
// lowercase hex characters could steer a later fetch to a path
|
||||||
|
// outside the cache directory, so reject it here, before any
|
||||||
|
// blob is downloaded.
|
||||||
|
if !isBlobHash(hash) {
|
||||||
|
return nil, nil, fmt.Errorf(
|
||||||
|
"%w: %s", errInvalidBlobHash, shortHash(hash))
|
||||||
|
}
|
||||||
|
|
||||||
blobIDToHash[id] = hash
|
blobIDToHash[id] = hash
|
||||||
blobByHash[hash] = blob
|
blobByHash[hash] = blob
|
||||||
}
|
}
|
||||||
@@ -613,7 +638,7 @@ func (v *Vaultik) handleRestoreVerification(
|
|||||||
// for a remote-only snapshot) is used as-is, so a host with no local
|
// for a remote-only snapshot) is used as-is, so a host with no local
|
||||||
// index can restore the snapshots it can only see on the store.
|
// index can restore the snapshots it can only see on the store.
|
||||||
func (v *Vaultik) downloadSnapshotDB(
|
func (v *Vaultik) downloadSnapshotDB(
|
||||||
snapshotID string, identity age.Identity,
|
snapshotID string, identities []age.Identity,
|
||||||
) (*database.DB, string, error) {
|
) (*database.DB, string, error) {
|
||||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -640,7 +665,7 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
"size", ubytes(int64(len(encryptedData))))
|
"size", ubytes(int64(len(encryptedData))))
|
||||||
|
|
||||||
// Decrypt and decompress using blobgen.Reader
|
// Decrypt and decompress using blobgen.Reader
|
||||||
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
|
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
@@ -655,7 +680,53 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
|
|
||||||
log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
|
log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
|
||||||
|
|
||||||
return v.materializeSnapshotDB(dbData)
|
db, tempDir, err := v.materializeSnapshotDB(dbData)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Confirm the decrypted database really is the snapshot named by
|
||||||
|
// remoteKey before any files are read from it. On mismatch, close the
|
||||||
|
// database and remove its private directory so nothing is left behind.
|
||||||
|
err = v.verifySnapshotDBIdentity(db, snapshotID, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
_ = db.Close()
|
||||||
|
_ = v.Fs.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return db, tempDir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifySnapshotDBIdentity confirms the decrypted metadata database really
|
||||||
|
// is the snapshot named by remoteKey. age decryption proves the database
|
||||||
|
// is readable, not that the object served at
|
||||||
|
// metadata/<remoteKey>/db.zst.age is the snapshot that was requested: an
|
||||||
|
// attacker who swaps in another valid db.zst.age (which needs no key
|
||||||
|
// material) would otherwise redirect restore and deep verify to a
|
||||||
|
// different snapshot's contents. The exported per-snapshot database holds
|
||||||
|
// exactly one snapshot row, and a snapshot's remote key is derived from
|
||||||
|
// that row's ID, so the database is the requested one exactly when its
|
||||||
|
// sole snapshot hashes back to remoteKey. Comparing the requested
|
||||||
|
// identifier directly would not do: it may be a remote-key prefix a
|
||||||
|
// recovery host uses in place of a human snapshot ID it cannot know.
|
||||||
|
func (v *Vaultik) verifySnapshotDBIdentity(
|
||||||
|
db *database.DB, requested, remoteKey string,
|
||||||
|
) error {
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
|
snap, err := repos.Snapshots.GetOnlySnapshot(v.ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("checking identity of database for %s: %w", requested, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if snapshot.RemoteSnapshotKey(snap.ID.String()) != remoteKey {
|
||||||
|
return fmt.Errorf("%w: requested %s but the database is snapshot %s",
|
||||||
|
errSnapshotDBMismatch, requested, snap.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// materializeSnapshotDB writes the decrypted snapshot database bytes into
|
// materializeSnapshotDB writes the decrypted snapshot database bytes into
|
||||||
@@ -784,7 +855,7 @@ type restoreSession struct {
|
|||||||
ctx context.Context //nolint:containedctx // per-restore state by design
|
ctx context.Context //nolint:containedctx // per-restore state by design
|
||||||
repos *database.Repositories
|
repos *database.Repositories
|
||||||
opts *RestoreOptions
|
opts *RestoreOptions
|
||||||
identity age.Identity
|
identities []age.Identity
|
||||||
chunkToBlobMap map[string]*database.BlobChunk
|
chunkToBlobMap map[string]*database.BlobChunk
|
||||||
blobByHash map[string]*database.Blob
|
blobByHash map[string]*database.Blob
|
||||||
blobIDToHash map[string]string
|
blobIDToHash map[string]string
|
||||||
@@ -1146,7 +1217,7 @@ func (s *restoreSession) downloadBlobToCache(
|
|||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
t0 := time.Now()
|
t0 := time.Now()
|
||||||
rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identity)
|
rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identities...)
|
||||||
fetchSetupDur := time.Since(t0)
|
fetchSetupDur := time.Since(t0)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package vaultik //nolint:testpackage // exercises unexported restoreIdentities
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"filippo.io/age"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// encryptBlobTo returns a blobgen blob of plaintext encrypted to exactly
|
||||||
|
// one recipient, so a decryptor succeeds only if it holds that recipient's
|
||||||
|
// identity.
|
||||||
|
func encryptBlobTo(t *testing.T, recipient string, plaintext []byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
writer, err := blobgen.NewWriter(&buf, 1, []string{recipient})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = writer.Write(plaintext)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// decryptBlobWith reads a blob back through the identities and returns its
|
||||||
|
// plaintext.
|
||||||
|
func decryptBlobWith(t *testing.T, blob []byte, identities []age.Identity) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
reader, err := blobgen.NewReader(bytes.NewReader(blob), identities...)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
plaintext, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, reader.Close())
|
||||||
|
|
||||||
|
return plaintext
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreIdentitiesAcceptsEveryIdentity proves a key file holding two
|
||||||
|
// identities yields both, so a blob encrypted only to the second
|
||||||
|
// recipient — the one the previous single-identity parse dropped — still
|
||||||
|
// decrypts.
|
||||||
|
func TestRestoreIdentitiesAcceptsEveryIdentity(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
first, err := age.GenerateX25519Identity()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
second, err := age.GenerateX25519Identity()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// A whole age-keygen-style file: comment lines plus two identity lines.
|
||||||
|
keyFile := "# public key: " + first.Recipient().String() + "\n" +
|
||||||
|
first.String() + "\n" +
|
||||||
|
"# public key: " + second.Recipient().String() + "\n" +
|
||||||
|
second.String() + "\n"
|
||||||
|
|
||||||
|
v := &Vaultik{Config: &config.Config{AgeSecretKey: keyFile}}
|
||||||
|
|
||||||
|
identities, err := v.restoreIdentities()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, identities, 2)
|
||||||
|
|
||||||
|
plaintext := []byte("payload encrypted only to the second identity")
|
||||||
|
blob := encryptBlobTo(t, second.Recipient().String(), plaintext)
|
||||||
|
|
||||||
|
require.Equal(t, plaintext, decryptBlobWith(t, blob, identities))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreIdentitiesAcceptsTrailingNewline mirrors a YAML
|
||||||
|
// age_secret_key value that carries a trailing newline: it must still
|
||||||
|
// parse to its one identity and decrypt a blob encrypted to it.
|
||||||
|
func TestRestoreIdentitiesAcceptsTrailingNewline(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
id, err := age.GenerateX25519Identity()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
v := &Vaultik{Config: &config.Config{AgeSecretKey: id.String() + "\n"}}
|
||||||
|
|
||||||
|
identities, err := v.restoreIdentities()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, identities, 1)
|
||||||
|
|
||||||
|
plaintext := []byte("value with a trailing newline")
|
||||||
|
blob := encryptBlobTo(t, id.Recipient().String(), plaintext)
|
||||||
|
|
||||||
|
require.Equal(t, plaintext, decryptBlobWith(t, blob, identities))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreIdentitiesMissingKey reports the dedicated missing-key error
|
||||||
|
// rather than a parse failure, so the user is told to set the key.
|
||||||
|
func TestRestoreIdentitiesMissingKey(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
v := &Vaultik{Config: &config.Config{}}
|
||||||
|
|
||||||
|
_, err := v.restoreIdentities()
|
||||||
|
require.ErrorIs(t, err, errDecryptionKeyRequired)
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRestoreRejectsMalformedKeyBeforeDownload verifies that a malformed
|
||||||
|
// age secret key stops restore at the parse step: nothing is fetched from
|
||||||
|
// the store, and the error does not echo the key value (which is secret).
|
||||||
|
func TestRestoreRejectsMalformedKeyBeforeDownload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const malformed = "this-is-not-a-valid-age-key"
|
||||||
|
|
||||||
|
mock := NewMockStorer()
|
||||||
|
|
||||||
|
v := &vaultik.Vaultik{
|
||||||
|
Config: &config.Config{AgeSecretKey: malformed},
|
||||||
|
Storage: mock,
|
||||||
|
Stdout: io.Discard,
|
||||||
|
Stderr: io.Discard,
|
||||||
|
UI: ui.NewWithColor(io.Discard, false),
|
||||||
|
}
|
||||||
|
v.SetContext(context.Background())
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: "any-snapshot",
|
||||||
|
TargetDir: t.TempDir(),
|
||||||
|
})
|
||||||
|
require.Error(t, err)
|
||||||
|
require.NotContains(t, err.Error(), malformed,
|
||||||
|
"error must not echo the key value")
|
||||||
|
require.Contains(t, err.Error(), "age_secret_key",
|
||||||
|
"error should name the configuration source")
|
||||||
|
require.Empty(t, mock.GetCalls(),
|
||||||
|
"a malformed key must fail before anything is fetched")
|
||||||
|
}
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRestoreAndDeepVerifyRejectSwappedDatabase proves that swapping two
|
||||||
|
// snapshots' encrypted databases on the store is caught. age decryption
|
||||||
|
// alone proves only that a database is readable; without an identity check
|
||||||
|
// restore would happily write the wrong snapshot's files and deep verify
|
||||||
|
// would report success. After the swap, restore and deep verify of A both
|
||||||
|
// fail, and the error names the snapshot the database actually holds (B).
|
||||||
|
func TestRestoreAndDeepVerifyRejectSwappedDatabase(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
|
||||||
|
chunkSize := int64(64 * 1024)
|
||||||
|
|
||||||
|
storer, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Two snapshots with different content, backed up into one shared
|
||||||
|
// store. Different names give them different remote keys, so their
|
||||||
|
// metadata directories are distinct and can be tampered with alone.
|
||||||
|
dataA := filepath.Join(tempDir, "srcA")
|
||||||
|
require.NoError(t, fs.MkdirAll(dataA, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, filepath.Join(dataA, "a.bin"),
|
||||||
|
bytesPattern("alpha-", int(chunkSize*2)), 0o644))
|
||||||
|
|
||||||
|
dataB := filepath.Join(tempDir, "srcB")
|
||||||
|
require.NoError(t, fs.MkdirAll(dataB, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, filepath.Join(dataB, "b.bin"),
|
||||||
|
bytesPattern("beta-", int(chunkSize*2)), 0o644))
|
||||||
|
|
||||||
|
idA := backupNamedSnapshotToStore(ctx, t, fs, dataA, storer,
|
||||||
|
filepath.Join(tempDir, "idxA.sqlite"), "alpha")
|
||||||
|
idB := backupNamedSnapshotToStore(ctx, t, fs, dataB, storer,
|
||||||
|
filepath.Join(tempDir, "idxB.sqlite"), "beta")
|
||||||
|
require.NotEqual(t, idA, idB)
|
||||||
|
|
||||||
|
keyA := snapshot.RemoteSnapshotKey(idA)
|
||||||
|
keyB := snapshot.RemoteSnapshotKey(idB)
|
||||||
|
require.NotEqual(t, keyA, keyB)
|
||||||
|
|
||||||
|
// Baseline: each snapshot verifies against its own intact metadata.
|
||||||
|
require.NoError(t, newStoreClient(ctx, t, fs, storer).RunDeepVerify(
|
||||||
|
idA, &vaultik.VerifyOptions{Deep: true}))
|
||||||
|
require.NoError(t, newStoreClient(ctx, t, fs, storer).RunDeepVerify(
|
||||||
|
idB, &vaultik.VerifyOptions{Deep: true}))
|
||||||
|
|
||||||
|
// Swap the two snapshots' encrypted databases on the store.
|
||||||
|
swapStoreFiles(t, fs,
|
||||||
|
filepath.Join(storeDir, "metadata", keyA, "db.zst.age"),
|
||||||
|
filepath.Join(storeDir, "metadata", keyB, "db.zst.age"))
|
||||||
|
|
||||||
|
// Restore of A now decrypts B's database; the identity check must
|
||||||
|
// reject it and name the snapshot it actually found.
|
||||||
|
restoreErr := newStoreClient(ctx, t, fs, storer).Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: idA,
|
||||||
|
TargetDir: filepath.Join(tempDir, "restoreA"),
|
||||||
|
})
|
||||||
|
require.Error(t, restoreErr)
|
||||||
|
require.ErrorContains(t, restoreErr, idB)
|
||||||
|
|
||||||
|
// Deep verify of A must reject the swapped database for the same reason.
|
||||||
|
verifyErr := newStoreClient(ctx, t, fs, storer).RunDeepVerify(
|
||||||
|
idA, &vaultik.VerifyOptions{Deep: true})
|
||||||
|
require.Error(t, verifyErr)
|
||||||
|
require.ErrorContains(t, verifyErr, idB)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDeepVerifyRejectsSwappedDatabaseWithEmptyManifest covers the case the
|
||||||
|
// issue calls out: swapping in a database whose blob set is empty and
|
||||||
|
// pairing it with an equally empty manifest. The manifest then agrees with
|
||||||
|
// the database, so every blob-level check passes and deep verify used to
|
||||||
|
// report success with zero blobs verified. The identity check rejects it
|
||||||
|
// before any blob check runs.
|
||||||
|
func TestDeepVerifyRejectsSwappedDatabaseWithEmptyManifest(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
|
||||||
|
chunkSize := int64(64 * 1024)
|
||||||
|
|
||||||
|
storer, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Snapshot A: real content, so its manifest lists blobs.
|
||||||
|
dataA := filepath.Join(tempDir, "srcA")
|
||||||
|
require.NoError(t, fs.MkdirAll(dataA, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, filepath.Join(dataA, "a.bin"),
|
||||||
|
bytesPattern("alpha-", int(chunkSize*2)), 0o644))
|
||||||
|
idA := backupNamedSnapshotToStore(ctx, t, fs, dataA, storer,
|
||||||
|
filepath.Join(tempDir, "idxA.sqlite"), "alpha")
|
||||||
|
|
||||||
|
// Snapshot C: a single empty file, so it references no blobs and its
|
||||||
|
// manifest is empty. This is the database/manifest pair an attacker
|
||||||
|
// would swap in to make the blob checks vacuously pass.
|
||||||
|
dataC := filepath.Join(tempDir, "srcC")
|
||||||
|
require.NoError(t, fs.MkdirAll(dataC, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs,
|
||||||
|
filepath.Join(dataC, "empty.bin"), []byte{}, 0o644))
|
||||||
|
idC := backupNamedSnapshotToStore(ctx, t, fs, dataC, storer,
|
||||||
|
filepath.Join(tempDir, "idxC.sqlite"), "charlie")
|
||||||
|
|
||||||
|
keyA := snapshot.RemoteSnapshotKey(idA)
|
||||||
|
keyC := snapshot.RemoteSnapshotKey(idC)
|
||||||
|
|
||||||
|
// Replace A's database and manifest with C's empty pair.
|
||||||
|
copyStoreFile(t, fs,
|
||||||
|
filepath.Join(storeDir, "metadata", keyC, "db.zst.age"),
|
||||||
|
filepath.Join(storeDir, "metadata", keyA, "db.zst.age"))
|
||||||
|
copyStoreFile(t, fs,
|
||||||
|
filepath.Join(storeDir, "metadata", keyC, "manifest.json.zst"),
|
||||||
|
filepath.Join(storeDir, "metadata", keyA, "manifest.json.zst"))
|
||||||
|
|
||||||
|
verifyErr := newStoreClient(ctx, t, fs, storer).RunDeepVerify(
|
||||||
|
idA, &vaultik.VerifyOptions{Deep: true})
|
||||||
|
require.Error(t, verifyErr)
|
||||||
|
require.ErrorContains(t, verifyErr, idC)
|
||||||
|
}
|
||||||
|
|
||||||
|
// backupNamedSnapshotToStore backs up dataDir into the shared storer under
|
||||||
|
// the given snapshot name and returns the human snapshot ID. Two snapshots
|
||||||
|
// backed up under different names get different remote keys, so their
|
||||||
|
// metadata directories on the store are distinct.
|
||||||
|
func backupNamedSnapshotToStore(
|
||||||
|
ctx context.Context, t *testing.T, fs afero.Fs,
|
||||||
|
dataDir string, storer storage.Storer, dbPath, name string,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
const (
|
||||||
|
chunkSize = int64(64 * 1024)
|
||||||
|
maxBlobSize = int64(512 * 1024)
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
AgeRecipients: []string{testAgePublicKey},
|
||||||
|
AgeSecretKey: testAgeSecretKey,
|
||||||
|
CompressionLevel: 3,
|
||||||
|
Hostname: testHostname,
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
|
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
||||||
|
Repos: repos,
|
||||||
|
Storage: storer,
|
||||||
|
Config: cfg,
|
||||||
|
})
|
||||||
|
sm.SetFilesystem(fs)
|
||||||
|
|
||||||
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
||||||
|
FS: fs,
|
||||||
|
Storage: storer,
|
||||||
|
ChunkSize: chunkSize,
|
||||||
|
MaxBlobSize: maxBlobSize,
|
||||||
|
CompressionLevel: cfg.CompressionLevel,
|
||||||
|
AgeRecipients: cfg.AgeRecipients,
|
||||||
|
Repositories: repos,
|
||||||
|
})
|
||||||
|
|
||||||
|
snapshotID, err := sm.CreateSnapshotWithName(
|
||||||
|
ctx, cfg.Hostname, name, "test-version", "test-git")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = scanner.Scan(ctx, dataDir, snapshotID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
||||||
|
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
return snapshotID
|
||||||
|
}
|
||||||
|
|
||||||
|
// newStoreClient builds a Vaultik that reads only from the store: the
|
||||||
|
// secret key, the storer, and a filesystem, with no local index. This is
|
||||||
|
// what restore and deep verify need.
|
||||||
|
func newStoreClient(
|
||||||
|
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
||||||
|
) *vaultik.Vaultik {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
v := &vaultik.Vaultik{
|
||||||
|
Config: &config.Config{
|
||||||
|
AgeSecretKey: testAgeSecretKey,
|
||||||
|
Hostname: testHostname,
|
||||||
|
},
|
||||||
|
Storage: storer,
|
||||||
|
Fs: fs,
|
||||||
|
Stdout: io.Discard,
|
||||||
|
Stderr: io.Discard,
|
||||||
|
UI: ui.NewWithColor(io.Discard, false),
|
||||||
|
}
|
||||||
|
v.SetContext(ctx)
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// swapStoreFiles exchanges the contents of two files on the store.
|
||||||
|
func swapStoreFiles(t *testing.T, fs afero.Fs, a, b string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dataA, err := afero.ReadFile(fs, a)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
dataB, err := afero.ReadFile(fs, b)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, afero.WriteFile(fs, a, dataB, 0o644))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, b, dataA, 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyStoreFile overwrites dst with the contents of src on the store.
|
||||||
|
func copyStoreFile(t *testing.T, fs afero.Fs, src, dst string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
data, err := afero.ReadFile(fs, src)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, afero.WriteFile(fs, dst, data, 0o644))
|
||||||
|
}
|
||||||
@@ -730,8 +730,18 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
|||||||
result.DatabaseMissing = true
|
result.DatabaseMissing = true
|
||||||
}
|
}
|
||||||
|
|
||||||
result.Verified, result.Missing, result.Mismatched, result.MissingSize =
|
result.Verified, result.Missing, result.Mismatched, result.MissingSize, err =
|
||||||
v.verifyManifestBlobs(manifest, opts)
|
v.verifyManifestBlobs(manifest, opts)
|
||||||
|
if err != nil {
|
||||||
|
if opts.JSON {
|
||||||
|
result.Status = verifyStatusFailed
|
||||||
|
result.ErrorMessage = fmt.Sprintf("verifying manifest blobs: %v", err)
|
||||||
|
|
||||||
|
return v.outputVerifyJSON(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("verifying manifest blobs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return v.formatVerifyResult(result, opts)
|
return v.formatVerifyResult(result, opts)
|
||||||
}
|
}
|
||||||
@@ -766,13 +776,20 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
|
|||||||
// comparison matches the deep path (see verifyBlobExistenceFromDB).
|
// comparison matches the deep path (see verifyBlobExistenceFromDB).
|
||||||
func (v *Vaultik) verifyManifestBlobs(
|
func (v *Vaultik) verifyManifestBlobs(
|
||||||
manifest *snapshot.Manifest, opts *VerifyOptions,
|
manifest *snapshot.Manifest, opts *VerifyOptions,
|
||||||
) (int, int, int, int64) {
|
) (int, int, int, int64, error) {
|
||||||
var (
|
var (
|
||||||
verified, missing, mismatched int
|
verified, missing, mismatched int
|
||||||
missingSize int64
|
missingSize int64
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, blob := range manifest.Blobs {
|
for _, blob := range manifest.Blobs {
|
||||||
|
// The manifest is unauthenticated, so its blob hashes are checked
|
||||||
|
// before being spliced into a storage path.
|
||||||
|
if !isBlobHash(blob.Hash) {
|
||||||
|
return 0, 0, 0, 0, fmt.Errorf("%w: %s",
|
||||||
|
errInvalidBlobHash, shortHash(blob.Hash))
|
||||||
|
}
|
||||||
|
|
||||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
|
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
|
||||||
blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
||||||
|
|
||||||
@@ -798,7 +815,7 @@ func (v *Vaultik) verifyManifestBlobs(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return verified, missing, mismatched, missingSize
|
return verified, missing, mismatched, missingSize, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// formatVerifyResult outputs the final verification results as JSON or
|
// formatVerifyResult outputs the final verification results as JSON or
|
||||||
@@ -1313,21 +1330,36 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
parts := strings.Split(object.Key, "/")
|
parts := strings.Split(object.Key, "/")
|
||||||
if len(parts) >= minSnapshotIDParts &&
|
if len(parts) < minSnapshotIDParts ||
|
||||||
parts[0] == metadataDirName && parts[1] != "" {
|
parts[0] != metadataDirName || parts[1] == "" {
|
||||||
// Skip macOS resource fork files (._*) and other hidden files
|
continue
|
||||||
if strings.HasPrefix(parts[1], ".") {
|
}
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasSuffix(object.Key, "/") ||
|
// Skip macOS resource fork files (._*) and other hidden files
|
||||||
strings.Contains(object.Key, "/manifest.json.zst") {
|
if strings.HasPrefix(parts[1], ".") {
|
||||||
key := parts[1]
|
continue
|
||||||
if !seen[key] {
|
}
|
||||||
seen[key] = true
|
|
||||||
keys = append(keys, key)
|
if !strings.HasSuffix(object.Key, "/") &&
|
||||||
}
|
!strings.Contains(object.Key, "/manifest.json.zst") {
|
||||||
}
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := parts[1]
|
||||||
|
|
||||||
|
// A remote snapshot key is a SHA-256 hash: 64 lowercase hex
|
||||||
|
// characters. The listing comes from the untrusted destination,
|
||||||
|
// so accept a key only in that form.
|
||||||
|
if !isBlobHash(key) {
|
||||||
|
log.Warn("Skipping non-conforming key under metadata/",
|
||||||
|
"key", object.Key)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !seen[key] {
|
||||||
|
seen[key] = true
|
||||||
|
keys = append(keys, key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+78
-44
@@ -24,9 +24,10 @@ var (
|
|||||||
errVerificationFailed = errors.New("verification failed")
|
errVerificationFailed = errors.New("verification failed")
|
||||||
errSecretKeyRequired = errors.New(
|
errSecretKeyRequired = errors.New(
|
||||||
"VAULTIK_AGE_SECRET_KEY not set; required for deep verification")
|
"VAULTIK_AGE_SECRET_KEY not set; required for deep verification")
|
||||||
errChunksOutOfOrder = errors.New("chunks out of order")
|
errChunksOutOfOrder = errors.New("chunks out of order")
|
||||||
errChunkHashMismatch = errors.New("chunk hash mismatch")
|
errChunkHashMismatch = errors.New("chunk hash mismatch")
|
||||||
errTrailingBlobData = errors.New(
|
errNegativeChunkLength = errors.New("chunk length is negative")
|
||||||
|
errTrailingBlobData = errors.New(
|
||||||
"blob has unexpected trailing bytes not covered by chunk list")
|
"blob has unexpected trailing bytes not covered by chunk list")
|
||||||
errManifestExtraBlob = errors.New("manifest contains blob not in database")
|
errManifestExtraBlob = errors.New("manifest contains blob not in database")
|
||||||
errManifestMissingBlob = errors.New(
|
errManifestMissingBlob = errors.New(
|
||||||
@@ -94,11 +95,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse the age secret key once, the same way restore does, and reuse
|
// Parse the age secret key once, the same way restore does, and reuse
|
||||||
// the identity for the database and every blob.
|
// the identities for the database and every blob.
|
||||||
identity, err := v.prepareRestoreIdentity()
|
identities, err := v.restoreIdentities()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v.deepVerifyFailure(result, opts,
|
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
fmt.Sprintf("parsing age secret key: %v", err), err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||||
@@ -108,7 +108,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
||||||
snapshotID, opts, result, identity)
|
snapshotID, opts, result, identities)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -129,7 +129,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
result.TotalSize = totalSize
|
result.TotalSize = totalSize
|
||||||
|
|
||||||
err = v.runVerificationSteps(
|
err = v.runVerificationSteps(
|
||||||
manifest, dbBlobs, tempDB, opts, result, totalSize, identity)
|
manifest, dbBlobs, tempDB, opts, result, totalSize, identities)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -154,7 +154,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
// loadVerificationData downloads manifest, database, and blob list for verification
|
// loadVerificationData downloads manifest, database, and blob list for verification
|
||||||
func (v *Vaultik) loadVerificationData(
|
func (v *Vaultik) loadVerificationData(
|
||||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||||
identity age.Identity,
|
identities []age.Identity,
|
||||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||||
// hashed; a remote key (or its abbreviation, as printed for a
|
// hashed; a remote key (or its abbreviation, as printed for a
|
||||||
@@ -191,24 +191,10 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
v.stdoutf("Downloading and decrypting database...\n")
|
v.stdoutf("Downloading and decrypting database...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download and decrypt database
|
tdb, err := v.downloadVerifiedSnapshotDB(
|
||||||
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
snapshotID, remoteKey, opts, result, identities)
|
||||||
log.Info("Downloading encrypted database", "path", dbPath)
|
|
||||||
|
|
||||||
dbReader, err := v.Storage.Get(v.ctx, dbPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
return nil, nil, nil, err
|
||||||
fmt.Sprintf("failed to download database: %v", err),
|
|
||||||
fmt.Errorf("failed to download database: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = dbReader.Close() }()
|
|
||||||
|
|
||||||
tdb, err := v.decryptAndLoadDatabase(dbReader, identity)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
|
||||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
|
||||||
fmt.Errorf("failed to decrypt database: %w", err))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dbBlobs, err := v.getBlobsFromDatabase(tdb.db.Conn())
|
dbBlobs, err := v.getBlobsFromDatabase(tdb.db.Conn())
|
||||||
@@ -237,6 +223,45 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
return manifest, tdb, dbBlobs, nil
|
return manifest, tdb, dbBlobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// downloadVerifiedSnapshotDB downloads and decrypts the snapshot metadata
|
||||||
|
// database and confirms it really is the snapshot named by remoteKey
|
||||||
|
// before any of its rows are trusted (see verifySnapshotDBIdentity). On
|
||||||
|
// any failure it records the failure in result and returns the error the
|
||||||
|
// caller should propagate; the temp database is closed on a rejected
|
||||||
|
// identity so nothing is left on disk.
|
||||||
|
func (v *Vaultik) downloadVerifiedSnapshotDB(
|
||||||
|
snapshotID, remoteKey string, opts *VerifyOptions, result *VerifyResult,
|
||||||
|
identities []age.Identity,
|
||||||
|
) (*tempDB, error) {
|
||||||
|
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||||
|
log.Info("Downloading encrypted database", "path", dbPath)
|
||||||
|
|
||||||
|
dbReader, err := v.Storage.Get(v.ctx, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, v.deepVerifyFailure(result, opts,
|
||||||
|
fmt.Sprintf("failed to download database: %v", err),
|
||||||
|
fmt.Errorf("failed to download database: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = dbReader.Close() }()
|
||||||
|
|
||||||
|
tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
|
||||||
|
if err != nil {
|
||||||
|
return nil, v.deepVerifyFailure(result, opts,
|
||||||
|
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||||
|
fmt.Errorf("failed to decrypt database: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
err = v.verifySnapshotDBIdentity(tdb.db, snapshotID, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
_ = tdb.Close()
|
||||||
|
|
||||||
|
return nil, v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return tdb, nil
|
||||||
|
}
|
||||||
|
|
||||||
// runVerificationSteps executes manifest verification, blob existence
|
// runVerificationSteps executes manifest verification, blob existence
|
||||||
// check, and deep content verification.
|
// check, and deep content verification.
|
||||||
func (v *Vaultik) runVerificationSteps(
|
func (v *Vaultik) runVerificationSteps(
|
||||||
@@ -246,7 +271,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
opts *VerifyOptions,
|
opts *VerifyOptions,
|
||||||
result *VerifyResult,
|
result *VerifyResult,
|
||||||
totalSize int64,
|
totalSize int64,
|
||||||
identity age.Identity,
|
identities []age.Identity,
|
||||||
) error {
|
) error {
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
v.stdoutf("Verifying manifest against database...\n")
|
v.stdoutf("Verifying manifest against database...\n")
|
||||||
@@ -273,7 +298,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
len(dbBlobs), ubytes(totalSize))
|
len(dbBlobs), ubytes(totalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity)
|
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identities)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
}
|
}
|
||||||
@@ -301,10 +326,10 @@ func (t *tempDB) Close() error {
|
|||||||
// from the encrypted stream. It reads through the same blobgen reader restore
|
// from the encrypted stream. It reads through the same blobgen reader restore
|
||||||
// uses, streaming the decrypted, decompressed database to a temp file.
|
// uses, streaming the decrypted, decompressed database to a temp file.
|
||||||
func (v *Vaultik) decryptAndLoadDatabase(
|
func (v *Vaultik) decryptAndLoadDatabase(
|
||||||
reader io.ReadCloser, identity age.Identity,
|
reader io.ReadCloser, identities []age.Identity,
|
||||||
) (*tempDB, error) {
|
) (*tempDB, error) {
|
||||||
// Decrypt and decompress through the shared blobgen reader.
|
// Decrypt and decompress through the shared blobgen reader.
|
||||||
blobReader, err := blobgen.NewReader(reader, identity)
|
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
@@ -365,7 +390,7 @@ func (v *Vaultik) decryptAndLoadDatabase(
|
|||||||
|
|
||||||
// verifyBlob downloads and verifies a single blob
|
// verifyBlob downloads and verifies a single blob
|
||||||
func (v *Vaultik) verifyBlob(
|
func (v *Vaultik) verifyBlob(
|
||||||
blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity,
|
blobInfo snapshot.BlobInfo, db *sql.DB, identities []age.Identity,
|
||||||
) error {
|
) error {
|
||||||
// Download blob using shared fetch method
|
// Download blob using shared fetch method
|
||||||
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
|
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
|
||||||
@@ -379,7 +404,7 @@ func (v *Vaultik) verifyBlob(
|
|||||||
// the plaintext as it is read. A blob's hash — its remote name — is the
|
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||||
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||||
// encrypted bytes.
|
// encrypted bytes.
|
||||||
blobReader, err := blobgen.NewReader(reader, identity)
|
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create blob reader: %w", err)
|
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||||
}
|
}
|
||||||
@@ -397,7 +422,7 @@ func (v *Vaultik) verifyBlob(
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Blob verified",
|
log.Info("Blob verified",
|
||||||
"hash", blobInfo.Hash[:16]+"...",
|
"hash", shortHash(blobInfo.Hash)+"...",
|
||||||
"chunks", chunkCount,
|
"chunks", chunkCount,
|
||||||
"size", ubytes(blobInfo.CompressedSize),
|
"size", ubytes(blobInfo.CompressedSize),
|
||||||
)
|
)
|
||||||
@@ -463,21 +488,24 @@ func (v *Vaultik) verifyBlobChunks(
|
|||||||
totalRead = offset
|
totalRead = offset
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read chunk data
|
// length comes from an untrusted blob_chunks row: reject a
|
||||||
chunkData := make([]byte, length)
|
// negative value, and hash by streaming exactly length bytes
|
||||||
|
// rather than allocating a database-supplied size up front.
|
||||||
|
if length < 0 {
|
||||||
|
return 0, fmt.Errorf("%w: offset %d length %d",
|
||||||
|
errNegativeChunkLength, offset, length)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = io.ReadFull(decompressor, chunkData)
|
hasher := sha256.New()
|
||||||
|
|
||||||
|
n, err := io.CopyN(hasher, decompressor, length)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
|
return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
totalRead += length
|
totalRead += n
|
||||||
|
|
||||||
// Verify chunk hash
|
|
||||||
hasher := sha256.New()
|
|
||||||
hasher.Write(chunkData)
|
|
||||||
calculatedHash := hex.EncodeToString(hasher.Sum(nil))
|
calculatedHash := hex.EncodeToString(hasher.Sum(nil))
|
||||||
|
|
||||||
if calculatedHash != chunkHash {
|
if calculatedHash != chunkHash {
|
||||||
return 0, fmt.Errorf("%w at offset %d: calculated %s, expected %s",
|
return 0, fmt.Errorf("%w at offset %d: calculated %s, expected %s",
|
||||||
errChunkHashMismatch, offset, calculatedHash, chunkHash)
|
errChunkHashMismatch, offset, calculatedHash, chunkHash)
|
||||||
@@ -627,6 +655,12 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
|
|||||||
log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
|
log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
|
||||||
|
|
||||||
for i, blob := range blobs {
|
for i, blob := range blobs {
|
||||||
|
// The hash is read from the snapshot database, which is not
|
||||||
|
// trusted; check it before it is spliced into a storage path.
|
||||||
|
if !isBlobHash(blob.Hash) {
|
||||||
|
return fmt.Errorf("%w: %s", errInvalidBlobHash, shortHash(blob.Hash))
|
||||||
|
}
|
||||||
|
|
||||||
// Construct blob path
|
// Construct blob path
|
||||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
||||||
|
|
||||||
@@ -663,7 +697,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
|
|||||||
// each blob using the database as source.
|
// each blob using the database as source.
|
||||||
func (v *Vaultik) performDeepVerificationFromDB(
|
func (v *Vaultik) performDeepVerificationFromDB(
|
||||||
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
||||||
identity age.Identity,
|
identities []age.Identity,
|
||||||
) error {
|
) error {
|
||||||
// Calculate total bytes for ETA
|
// Calculate total bytes for ETA
|
||||||
var totalBytesExpected int64
|
var totalBytesExpected int64
|
||||||
@@ -681,7 +715,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
|||||||
|
|
||||||
for i, blobInfo := range blobs {
|
for i, blobInfo := range blobs {
|
||||||
// Verify individual blob
|
// Verify individual blob
|
||||||
err := v.verifyBlob(blobInfo, db, identity)
|
err := v.verifyBlob(blobInfo, db, identities)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user