Compare commits

..
Author SHA1 Message Date
sneak 4e084a2746 Check storage_url with the storage parser at config load (closes #468)
check / check (push) Waiting to run
config set accepted rclone:// values that every command then refused:
no remote name, parameters in the remote name, a query parameter. The
config only checked the URL's prefix, because config could not import
the parser from storage, which imports config.

The parser moves from internal/storage to a new package,
internal/storageurl, that both import. ParseStorageURL becomes
storageurl.Parse. Loading the config now runs it on every storage_url,
so config set refuses such a value and leaves the file unchanged.

Model: opus-5-5
2026-11-19 07:24:47 +00:00
clawbot d747ddef3a Let snapshot remove accept a remote-only snapshot's remote key (closes #400)
check / check (push) Successful in 13m35s
snapshot remove hashed its argument into a remote key, so the remote
key that snapshot list shows for a remote-only snapshot never matched.
A remote key or a leading part of it is now resolved with
resolveSnapshotRemoteKey, as snapshot verify and snapshot restore do.

Model: opus-5-5
2026-11-17 01:18:19 +01:00
16 changed files with 265 additions and 168 deletions

No files matched your search

+6 -4
View File
@@ -468,10 +468,12 @@ destination store. Blobs are NOT touched — deleting blobs requires reading eve
remaining remote manifest (the destination store may hold snapshots this host remaining remote manifest (the destination store may hold snapshots this host
doesn't know about), which is what `vaultik prune` does. The command prints the doesn't know about), which is what `vaultik prune` does. The command prints the
`vaultik prune` invocation to run as a follow-up only when it removed the `vaultik prune` invocation to run as a follow-up only when it removed the
snapshot's metadata from the destination store, and never under `--json`. An ID snapshot's metadata from the destination store, and never under `--json`. It
that names no snapshot in the local index or on the destination store is an accepts the same identifiers as `snapshot restore`: a snapshot ID, or a
error; with `--local-only`, the snapshot must be in the local index. Local row remote-only snapshot's remote key (or an unambiguous leading part of it). An
cleanup (files, chunks, blobs the snapshot was the last referrer for) runs identifier that names no snapshot in the local index or on the destination store
is an error; with `--local-only`, the snapshot must be in the local index. Local
row cleanup (files, chunks, blobs the snapshot was the last referrer for) runs
automatically. If removing the snapshot's metadata from the destination store automatically. If removing the snapshot's metadata from the destination store
fails (for example because the store is unreachable, or the local index holds fails (for example because the store is unreachable, or the local index holds
snapshots and the destination answers that it has no `vaultik-destination.txt`), snapshots and the destination answers that it has no `vaultik-destination.txt`),
+7
View File
@@ -282,6 +282,13 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
storageURLKey, testS3URL, storageURLKey, testS3URL,
"s3.secret_access_key is required for s3:// URLs; " + "s3.secret_access_key is required for s3:// URLs; " +
"set s3.access_key_id and s3.secret_access_key first"}, "set s3.access_key_id and s3.secret_access_key first"},
// rclone:// URLs that no command could open.
{configWithoutRecipients, storageURLKey, "rclone:///backups",
"rclone URL missing remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
"rclone URL must not carry parameters in the remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
`unknown query parameter in storage URL: "x"`},
// With its reference filled in, the config loads. // With its reference filled in, the config loads.
{levelReference, "blob_size_limit", "1MB", {levelReference, "blob_size_limit", "1MB",
"blob_size_limit must be at least"}, "blob_size_limit must be at least"},
+4 -2
View File
@@ -331,8 +331,10 @@ know about), which is what 'vaultik prune' does. The command prints the
'vaultik prune' invocation to run as a follow-up only when it removed the 'vaultik prune' invocation to run as a follow-up only when it removed the
snapshot's metadata from the destination store, and never under --json. snapshot's metadata from the destination store, and never under --json.
An ID that names no snapshot in the local index or on the destination The snapshot may be named by its ID or by its remote key, which
store is an error. 'snapshot list' prints for a remote-only snapshot. An unambiguous
leading part of the key is enough. An identifier that names no snapshot
in the local index or on the destination store is an error.
Use --local-only to skip the remote half (e.g. when you want to forget a Use --local-only to skip the remote half (e.g. when you want to forget a
snapshot locally without touching the destination store); the snapshot snapshot locally without touching the destination store); the snapshot
+24 -37
View File
@@ -21,6 +21,7 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
const appName = "vaultik" const appName = "vaultik"
@@ -63,13 +64,8 @@ var (
errBlobSizeTooSmall = errors.New( errBlobSizeTooSmall = errors.New(
"blob_size_limit must be at least the largest chunk the chunker can " + "blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)") "emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19") errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB") errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://")
errFileURLNotAbsolute = errors.New(
"a file:// storage_url needs an absolute path, " +
"as in file:///path/to/backup")
errIndexPathNotAbsolute = errors.New( errIndexPathNotAbsolute = errors.New(
"index_path and VAULTIK_INDEX_PATH need an absolute path, " + "index_path and VAULTIK_INDEX_PATH need an absolute path, " +
"as in ~/.local/share/vaultik/index.sqlite") "as in ~/.local/share/vaultik/index.sqlite")
@@ -557,38 +553,29 @@ func (c *Config) validateStorage() error {
return nil return nil
} }
// validateStorageURL validates URL-based storage configuration. File and // validateStorageURL checks storage_url with the same parser that opens
// rclone URLs need no credentials; S3 URLs require the legacy s3.* // the destination, so a value every command would refuse fails at load.
// credential fields. A file URL must name an absolute path: a relative one // File and rclone URLs need no credentials; S3 URLs require the legacy
// would resolve against the working directory, so the destination would // s3.* credential fields.
// move with it.
func (c *Config) validateStorageURL() error { func (c *Config) validateStorageURL() error {
switch { parsed, err := storageurl.Parse(c.StorageURL)
case strings.HasPrefix(c.StorageURL, "file://"): if err != nil {
// storage.ParseStorageURL applies the same rule, but config return fmt.Errorf("invalid storage_url: %w", err)
// cannot import storage, which imports config.
if !strings.HasPrefix(c.StorageURL, "file:///") {
return errFileURLNotAbsolute
}
return nil
case strings.HasPrefix(c.StorageURL, "rclone://"):
// Rclone storage uses rclone's own config
return nil
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
}
return nil
default:
return errBadStorageScheme
} }
if parsed.Scheme != storageurl.SchemeS3 {
return nil
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
}
return nil
} }
// Module exports the config module for fx dependency injection. // Module exports the config module for fx dependency injection.
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
const ( const (
@@ -502,8 +503,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
} }
_, err = Load(path) _, err = Load(path)
if !errors.Is(err, errFileURLNotAbsolute) { if !errors.Is(err, storageurl.ErrRelativeFilePath) {
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err) t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
} }
if !strings.Contains(err.Error(), "absolute path") { if !strings.Contains(err.Error(), "absolute path") {
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
// FileStorer implements Storer using the local filesystem. // FileStorer implements Storer using the local filesystem.
@@ -288,7 +289,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
// Info returns human-readable storage location information. // Info returns human-readable storage location information.
func (f *FileStorer) Info() Info { func (f *FileStorer) Info() Info {
return Info{ return Info{
Type: schemeFile, Type: storageurl.SchemeFile,
Location: f.basePath, Location: f.basePath,
} }
} }
+11 -5
View File
@@ -2,14 +2,20 @@ package storage
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"strings" "strings"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3" "sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
// has no storage backend.
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
// defaultS3Region is used when neither the URL nor the config specify one. // defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1" const defaultS3Region = "us-east-1"
@@ -40,19 +46,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
//nolint:ireturn // factory intentionally returns the Storer interface //nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := ParseStorageURL(rawURL) parsed, err := storageurl.Parse(rawURL)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing storage URL: %w", err) return nil, fmt.Errorf("parsing storage URL: %w", err)
} }
switch parsed.Scheme { switch parsed.Scheme {
case schemeFile: case storageurl.SchemeFile:
return NewFileStorer(parsed.Prefix) return NewFileStorer(parsed.Prefix)
case schemeS3: case storageurl.SchemeS3:
return storerFromParsedS3URL(parsed, cfg) return storerFromParsedS3URL(parsed, cfg)
case schemeRclone: case storageurl.SchemeRclone:
return NewRcloneStorer( return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix) context.Background(), parsed.RcloneRemote, parsed.Prefix)
@@ -65,7 +71,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
// filling endpoint protocol and region defaults from the config. // filling endpoint protocol and region defaults from the config.
// //
//nolint:ireturn // factory intentionally returns the Storer interface //nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) { func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL // Build endpoint URL
endpoint := parsed.Endpoint endpoint := parsed.Endpoint
if endpoint == "" { if endpoint == "" {
+2 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/operations"
"sneak.berlin/go/vaultik/internal/storageurl"
// Import all rclone backends // Import all rclone backends
_ "github.com/rclone/rclone/backend/all" _ "github.com/rclone/rclone/backend/all"
@@ -266,7 +267,7 @@ func (r *RcloneStorer) Info() Info {
} }
return Info{ return Info{
Type: schemeRclone, Type: storageurl.SchemeRclone,
Location: location, Location: location,
} }
} }
@@ -1,4 +1,7 @@
package storage // Package storageurl parses the storage_url config value. The config
// package uses it to refuse a bad value when the config loads; the storage
// package uses it to open the destination.
package storageurl
import ( import (
"errors" "errors"
@@ -10,9 +13,9 @@ import (
// Storage URL scheme names. // Storage URL scheme names.
const ( const (
schemeFile = "file" SchemeFile = "file"
schemeS3 = "s3" SchemeS3 = "s3"
schemeRclone = "rclone" SchemeRclone = "rclone"
) )
// Sentinel errors for storage URL parsing. // Sentinel errors for storage URL parsing.
@@ -25,8 +28,7 @@ var (
ErrMissingRemote = errors.New("rclone URL missing remote name") ErrMissingRemote = errors.New("rclone URL missing remote name")
ErrUnsupportedScheme = errors.New( ErrUnsupportedScheme = errors.New(
"unsupported URL scheme: must start with s3://, file://, or rclone://") "unsupported URL scheme: must start with s3://, file://, or rclone://")
ErrUnsupportedStorage = errors.New("unsupported storage scheme") ErrURLCredentials = errors.New(
ErrURLCredentials = errors.New(
"storage URL must not carry credentials; " + "storage URL must not carry credentials; " +
"set s3.access_key_id and s3.secret_access_key in the config instead") "set s3.access_key_id and s3.secret_access_key in the config instead")
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL") ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
@@ -49,12 +51,12 @@ type URL struct {
RcloneRemote string // rclone remote name (for rclone:// URLs) RcloneRemote string // rclone remote name (for rclone:// URLs)
} }
// ParseStorageURL parses a storage URL string. // Parse parses a storage URL string.
// Supported formats: // Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true // - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true
// - file:///absolute/path/to/backup // - file:///absolute/path/to/backup
// - rclone://remote/path/to/backups // - rclone://remote/path/to/backups
func ParseStorageURL(rawURL string) (*URL, error) { func Parse(rawURL string) (*URL, error) {
if rawURL == "" { if rawURL == "" {
return nil, ErrEmptyStorageURL return nil, ErrEmptyStorageURL
} }
@@ -92,7 +94,7 @@ func parseFileURL(rawPath string) (*URL, error) {
return nil, fmt.Errorf("invalid file URL path: %w", err) return nil, fmt.Errorf("invalid file URL path: %w", err)
} }
return &URL{Scheme: schemeFile, Prefix: path}, nil return &URL{Scheme: SchemeFile, Prefix: path}, nil
} }
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
@@ -135,7 +137,7 @@ func parseS3URL(rawURL string) (*URL, error) {
} }
return &URL{ return &URL{
Scheme: schemeS3, Scheme: SchemeS3,
Bucket: bucket, Bucket: bucket,
Prefix: strings.TrimPrefix(u.Path, "/"), Prefix: strings.TrimPrefix(u.Path, "/"),
Endpoint: query.Get("endpoint"), Endpoint: query.Get("endpoint"),
@@ -189,7 +191,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
} }
return &URL{ return &URL{
Scheme: schemeRclone, Scheme: SchemeRclone,
Prefix: strings.TrimPrefix(u.Path, "/"), Prefix: strings.TrimPrefix(u.Path, "/"),
RcloneRemote: remote, RcloneRemote: remote,
}, nil }, nil
@@ -226,9 +228,9 @@ func wrapParseError(err error) error {
// String returns a human-readable representation of the storage URL. // String returns a human-readable representation of the storage URL.
func (u *URL) String() string { func (u *URL) String() string {
switch u.Scheme { switch u.Scheme {
case schemeFile: case SchemeFile:
return "file://" + u.Prefix return "file://" + u.Prefix
case schemeS3: case SchemeS3:
endpoint := u.Endpoint endpoint := u.Endpoint
if endpoint == "" { if endpoint == "" {
endpoint = "s3.amazonaws.com" endpoint = "s3.amazonaws.com"
@@ -239,7 +241,7 @@ func (u *URL) String() string {
} }
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case schemeRclone: case SchemeRclone:
if u.Prefix != "" { if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
} }
@@ -1,4 +1,4 @@
package storage_test package storageurl_test
import ( import (
"errors" "errors"
@@ -6,12 +6,12 @@ import (
"strings" "strings"
"testing" "testing"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storageurl"
) )
// TestParseStorageURLValid checks that each supported scheme parses into // TestParseValid checks that each supported scheme parses into the
// the expected fields, since those fields decide which backend is built. // expected fields, since those fields decide which backend is built.
func TestParseStorageURLValid(t *testing.T) { func TestParseValid(t *testing.T) {
t.Parallel() t.Parallel()
const ( const (
@@ -22,27 +22,27 @@ func TestParseStorageURLValid(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
raw string raw string
want *storage.URL want *storageurl.URL
}{ }{
{ {
name: "file absolute path", name: "file absolute path",
raw: "file:///var/backups/vaultik", raw: "file:///var/backups/vaultik",
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
}, },
{ {
name: "file percent-escaped path is decoded", name: "file percent-escaped path is decoded",
raw: "file:///Volumes/my%20stick/backup", raw: "file:///Volumes/my%20stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
}, },
{ {
name: "file path with a literal space", name: "file path with a literal space",
raw: "file:///Volumes/my stick/backup", raw: "file:///Volumes/my stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
}, },
{ {
name: "s3 bucket and prefix, ssl defaults on", name: "s3 bucket and prefix, ssl defaults on",
raw: "s3://mybucket/backups/host", raw: "s3://mybucket/backups/host",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "s3", Bucket: bucket, Scheme: "s3", Bucket: bucket,
Prefix: "backups/host", UseSSL: true, Prefix: "backups/host", UseSSL: true,
}, },
@@ -50,12 +50,12 @@ func TestParseStorageURLValid(t *testing.T) {
{ {
name: "s3 bucket only", name: "s3 bucket only",
raw: "s3://mybucket", raw: "s3://mybucket",
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true}, want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
}, },
{ {
name: "s3 with endpoint, region, ssl off", name: "s3 with endpoint, region, ssl off",
raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false", raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "s3", Bucket: bucket, Scheme: "s3", Bucket: bucket,
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false, Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
}, },
@@ -63,14 +63,14 @@ func TestParseStorageURLValid(t *testing.T) {
{ {
name: "rclone remote and path", name: "rclone remote and path",
raw: "rclone://gdrive/backups/host", raw: "rclone://gdrive/backups/host",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host", Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
}, },
}, },
{ {
name: "rclone remote only", name: "rclone remote only",
raw: "rclone://gdrive", raw: "rclone://gdrive",
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"}, want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
}, },
} }
@@ -78,22 +78,22 @@ func TestParseStorageURLValid(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
got, err := storage.ParseStorageURL(tc.raw) got, err := storageurl.Parse(tc.raw)
if err != nil { if err != nil {
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err) t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
} }
if !reflect.DeepEqual(got, tc.want) { if !reflect.DeepEqual(got, tc.want) {
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want) t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
} }
}) })
} }
} }
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme // TestParseErrors checks that empty, missing, and unknown-scheme inputs
// inputs fail with the documented sentinel errors instead of parsing to a // fail with the documented sentinel errors instead of parsing to a wrong
// wrong destination. // destination.
func TestParseStorageURLErrors(t *testing.T) { func TestParseErrors(t *testing.T) {
t.Parallel() t.Parallel()
cases := []struct { cases := []struct {
@@ -101,34 +101,33 @@ func TestParseStorageURLErrors(t *testing.T) {
raw string raw string
wantErr error wantErr error
}{ }{
{"empty url", "", storage.ErrEmptyStorageURL}, {"empty url", "", storageurl.ErrEmptyStorageURL},
{"file empty path", "file://", storage.ErrEmptyFilePath}, {"file empty path", "file://", storageurl.ErrEmptyFilePath},
{"file relative path", "file://dest", storage.ErrRelativeFilePath}, {"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
{"s3 missing bucket", "s3://", storage.ErrMissingBucket}, {"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket}, {"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
{"rclone missing remote", "rclone://", storage.ErrMissingRemote}, {"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme}, {"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
{"no scheme", "/local/path", storage.ErrUnsupportedScheme}, {"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
} }
for _, tc := range cases { for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, tc.wantErr) { if !errors.Is(err, tc.wantErr) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v", t.Errorf("Parse(%q) error = %v, want %v",
tc.raw, err, tc.wantErr) tc.raw, err, tc.wantErr)
} }
}) })
} }
} }
// TestParseStorageURLRejectsCredentials checks that a URL carrying // TestParseRejectsCredentials checks that a URL carrying credentials in
// credentials in its userinfo or in an unknown query parameter is // its userinfo or in an unknown query parameter is rejected, and that the
// rejected, and that the error never echoes the secret-bearing URL back // error never echoes the secret-bearing URL back into logs or output.
// into logs or output. func TestParseRejectsCredentials(t *testing.T) {
func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Parallel() t.Parallel()
// Split so the literals never form a "user:pass@" URL pattern that // Split so the literals never form a "user:pass@" URL pattern that
@@ -147,44 +146,44 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
{ {
name: "s3 userinfo", name: "s3 userinfo",
raw: "s3://" + key + ":" + secret + "@mybucket/prefix", raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret, "mybucket"}, secrets: []string{key, secret, "mybucket"},
}, },
{ {
name: "s3 userinfo in endpoint", name: "s3 userinfo in endpoint",
raw: "s3://mybucket?endpoint=https://" + raw: "s3://mybucket?endpoint=https://" +
key + ":" + secret + "@minio.example.com", key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 userinfo in endpoint without scheme", name: "s3 userinfo in endpoint without scheme",
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com", raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 unknown query param", name: "s3 unknown query param",
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret, raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 misspelt endpoint", name: "s3 misspelt endpoint",
raw: "s3://mybucket?endpiont=minio.example.com", raw: "s3://mybucket?endpiont=minio.example.com",
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: nil, secrets: nil,
}, },
{ {
name: "rclone userinfo", name: "rclone userinfo",
raw: "rclone://user:" + secret + "@gdrive/backups", raw: "rclone://user:" + secret + "@gdrive/backups",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{secret}, secrets: []string{secret},
}, },
{ {
name: "rclone query param", name: "rclone query param",
raw: "rclone://gdrive/backups?token=" + secret, raw: "rclone://gdrive/backups?token=" + secret,
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: []string{secret}, secrets: []string{secret},
}, },
} }
@@ -193,9 +192,9 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, tc.wantErr) { if !errors.Is(err, tc.wantErr) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v", t.Fatalf("Parse(%q) error = %v, want %v",
tc.raw, err, tc.wantErr) tc.raw, err, tc.wantErr)
} }
@@ -216,11 +215,11 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
} }
} }
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone // TestParseRejectsRcloneRemoteParams checks that an rclone remote name
// remote name carrying parameters (rclone's "remote,param=value" form, // carrying parameters (rclone's "remote,param=value" form, which can hold
// which can hold a credential) is rejected without echoing the value, // a credential) is rejected without echoing the value, including the URLs
// including the URLs that url.Parse itself refuses. // that url.Parse itself refuses.
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) { func TestParseRejectsRcloneRemoteParams(t *testing.T) {
t.Parallel() t.Parallel()
cases := []struct { cases := []struct {
@@ -249,10 +248,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, storage.ErrRcloneRemoteParams) { if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v", t.Fatalf("Parse(%q) error = %v, want %v",
tc.raw, err, storage.ErrRcloneRemoteParams) tc.raw, err, storageurl.ErrRcloneRemoteParams)
} }
for _, secret := range tc.secrets { for _, secret := range tc.secrets {
@@ -264,10 +263,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
} }
} }
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone // TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
// URL whose query cannot be decoded is rejected, not read with that part // query cannot be decoded is rejected, not read with that part of the
// of the query left out, and that the error does not echo it. // query left out, and that the error does not echo it.
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) { func TestParseRejectsUndecodableQuery(t *testing.T) {
t.Parallel() t.Parallel()
const ( const (
@@ -303,14 +302,14 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if err == nil { if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw) t.Fatalf("Parse(%q) returned no error", tc.raw)
} }
if !errors.Is(err, storage.ErrURLBadQuery) { if !errors.Is(err, storageurl.ErrURLBadQuery) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v", t.Errorf("Parse(%q) error = %v, want %v",
tc.raw, err, storage.ErrURLBadQuery) tc.raw, err, storageurl.ErrURLBadQuery)
} }
for _, s := range []string{key, secret, "minio.example.com"} { for _, s := range []string{key, secret, "minio.example.com"} {
@@ -322,17 +321,17 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
} }
} }
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse // TestParseFailureHidesURL checks that when url.Parse itself fails, the
// itself fails, the wrapped error carries only the inner cause, not the // wrapped error carries only the inner cause, not the *url.Error whose
// *url.Error whose text embeds the raw (possibly credential-bearing) URL. // text embeds the raw (possibly credential-bearing) URL.
func TestParseStorageURLParseFailureHidesURL(t *testing.T) { func TestParseFailureHidesURL(t *testing.T) {
t.Parallel() t.Parallel()
const raw = "s3://mybucket/%zz" const raw = "s3://mybucket/%zz"
_, err := storage.ParseStorageURL(raw) _, err := storageurl.Parse(raw)
if err == nil { if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", raw) t.Fatalf("Parse(%q) returned no error", raw)
} }
if strings.Contains(err.Error(), "mybucket") { if strings.Contains(err.Error(), "mybucket") {
+62
View File
@@ -472,6 +472,68 @@ func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
assert.Contains(t, out, "Removed snapshot metadata from remote storage") assert.Contains(t, out, "Removed snapshot metadata from remote storage")
} }
// TestRemoveSnapshot_ByRemoteKey checks that a snapshot only the
// destination store holds can be removed by the remote key that
// `snapshot list` shows for it, or by a leading part of that key.
func TestRemoveSnapshot_ByRemoteKey(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
for name, identifier := range map[string]string{
"full key": remoteKey,
"leading part": remoteKey[:8],
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t)
result, err := tv.RemoveSnapshot(identifier,
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
})
}
}
// TestRemoveSnapshot_ByRemoteKeyInLocalIndex checks that a snapshot the
// local index holds, named by its remote key, is removed from the local
// index as well as from the destination store.
func TestRemoveSnapshot_ByRemoteKeyInLocalIndex(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, "snapshot-001")
result, err := tv.RemoveSnapshot(snapshot.RemoteSnapshotKey("snapshot-001"),
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
snap, err := tv.Repositories.Snapshots.GetByID(context.Background(),
"snapshot-001")
require.NoError(t, err)
assert.Nil(t, snap)
}
// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a // TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
// snapshot only the destination store holds fails when its metadata // snapshot only the destination store holds fails when its metadata
// cannot be deleted there, under --force and under --json: nothing was // cannot be deleted there, under --force and under --json: nothing was
+4 -3
View File
@@ -31,9 +31,10 @@ var (
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot") errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
errFilesFailedVerify = errors.New("files failed verification") errFilesFailedVerify = errors.New("files failed verification")
errDecryptionKeyRequired = errors.New( errDecryptionKeyRequired = errors.New(
"age secret key is missing; set it in the " + "decryption key required\n\n" +
"VAULTIK_AGE_SECRET_KEY environment variable or the " + "Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
"age_secret_key config file field") "age private key file:\n" +
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
// errInvalidAgeSecretKey is returned when the configured key does not // errInvalidAgeSecretKey is returned when the configured key does not
// parse as any age identity. It names the source but never the value, // parse as any age identity. It names the source but never the value,
// which is secret, so the message is safe to print and log. // which is secret, so the message is safe to print and log.
-20
View File
@@ -106,23 +106,3 @@ func TestRestoreIdentitiesMissingKey(t *testing.T) {
_, err := v.restoreIdentities() _, err := v.restoreIdentities()
require.ErrorIs(t, err, errDecryptionKeyRequired) require.ErrorIs(t, err, errDecryptionKeyRequired)
} }
// TestMissingKeyErrorsNameBothSources asserts restore and deep verify
// return the same missing-key error, whose text names both places the key
// can be set: the VAULTIK_AGE_SECRET_KEY environment variable first since
// it is preferred, then the age_secret_key config file field.
func TestMissingKeyErrorsNameBothSources(t *testing.T) {
t.Parallel()
const want = "age secret key is missing; set it in " +
"the VAULTIK_AGE_SECRET_KEY environment variable or the " +
"age_secret_key config file field"
v := &Vaultik{Config: &config.Config{}}
_, err := v.restoreIdentities()
require.EqualError(t, err, want)
err = v.RunDeepVerify("any-snapshot", &VerifyOptions{})
require.EqualError(t, err, want)
}
+42 -7
View File
@@ -1467,7 +1467,8 @@ func (v *Vaultik) RemoveSnapshot(
return result, err return result, err
} }
inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly) snapshotID, remoteKey, inLocalIndex, err := v.findSnapshotToRemove(
snapshotID, opts.LocalOnly)
if err != nil { if err != nil {
return result, err return result, err
} }
@@ -1498,7 +1499,7 @@ func (v *Vaultik) RemoveSnapshot(
} }
result.RemoteRemoved, err = v.removeSnapshotRemote( result.RemoteRemoved, err = v.removeSnapshotRemote(
snapshotID, inLocalIndex, markerErr, opts) snapshotID, remoteKey, inLocalIndex, markerErr, opts)
if err != nil { if err != nil {
return result, err return result, err
} }
@@ -1530,6 +1531,41 @@ func (v *Vaultik) RemoveSnapshot(
return result, nil return result, nil
} }
// findSnapshotToRemove returns the ID of the snapshot that identifier
// names, its remote key, and whether the local index holds it. A remote
// key, or the leading part of one, is resolved on the destination store;
// when the local index holds the snapshot with that key, its ID is
// returned, so it is removed from both.
func (v *Vaultik) findSnapshotToRemove(
identifier string, localOnly bool,
) (string, string, bool, error) {
if !isRemoteKeyOrPrefix(identifier) || localOnly {
inLocalIndex, err := v.checkSnapshotExists(identifier, localOnly)
return identifier, snapshot.RemoteSnapshotKey(identifier),
inLocalIndex, err
}
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
if err != nil {
return "", "", false, err
}
localIDs, err := v.localSnapshotIDs()
if err != nil {
return "", "", false, fmt.Errorf(
"looking up snapshot in local index: %w", err)
}
for _, id := range localIDs {
if snapshot.RemoteSnapshotKey(id) == remoteKey {
return id, remoteKey, true, nil
}
}
return identifier, remoteKey, false, nil
}
// checkSnapshotExists returns errSnapshotNotFound when the local index // checkSnapshotExists returns errSnapshotNotFound when the local index
// has no row for snapshotID and, unless localOnly is set, the // has no row for snapshotID and, unless localOnly is set, the
// destination store has no metadata for it either. inLocalIndex reports // destination store has no metadata for it either. inLocalIndex reports
@@ -1638,8 +1674,8 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
return true return true
} }
// removeSnapshotRemote strips the snapshot's metadata from the // removeSnapshotRemote strips the snapshot's metadata, under remoteKey,
// destination store. It returns true only when the delete finished // from the destination store. It returns true only when the delete finished
// without an error and removed at least one object; under LocalOnly it // without an error and removed at least one object; under LocalOnly it
// does nothing. markerErr is the result of checkDestinationMarker; when // does nothing. markerErr is the result of checkDestinationMarker; when
// it is set, nothing is deleted and it is the failure. When the snapshot // it is set, nothing is deleted and it is the failure. When the snapshot
@@ -1651,7 +1687,7 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
// (`vaultik prune` never removes snapshot metadata that has its // (`vaultik prune` never removes snapshot metadata that has its
// manifest). // manifest).
func (v *Vaultik) removeSnapshotRemote( func (v *Vaultik) removeSnapshotRemote(
snapshotID string, inLocalIndex bool, markerErr error, snapshotID, remoteKey string, inLocalIndex bool, markerErr error,
opts *RemoveOptions, opts *RemoveOptions,
) (bool, error) { ) (bool, error) {
if opts.LocalOnly { if opts.LocalOnly {
@@ -1665,8 +1701,7 @@ func (v *Vaultik) removeSnapshotRemote(
err := markerErr err := markerErr
if err == nil { if err == nil {
deleted, err = v.deleteRemoteSnapshotByKey( deleted, err = v.deleteRemoteSnapshotByKey(remoteKey)
snapshot.RemoteSnapshotKey(snapshotID))
} }
if err != nil && !inLocalIndex { if err != nil && !inLocalIndex {
+5
View File
@@ -163,6 +163,11 @@ func (v *Vaultik) StartOperation(fn func()) func(context.Context) bool {
} }
} }
// CanDecrypt returns true if this Vaultik instance has decryption capabilities
func (v *Vaultik) CanDecrypt() bool {
return v.Config.AgeSecretKey != ""
}
// GetFilesystem returns the filesystem instance used by Vaultik // GetFilesystem returns the filesystem instance used by Vaultik
// //
//nolint:ireturn // afero.Fs is the filesystem abstraction by design //nolint:ireturn // afero.Fs is the filesystem abstraction by design
+9 -3
View File
@@ -22,7 +22,9 @@ import (
// Sentinel errors for snapshot verification failures. // Sentinel errors for snapshot verification failures.
var ( var (
errVerificationFailed = errors.New("verification failed") errVerificationFailed = errors.New("verification failed")
errSecretKeyRequired = errors.New(
"VAULTIK_AGE_SECRET_KEY not set; required for deep verification")
errChunksOutOfOrder = errors.New("chunks out of order") errChunksOutOfOrder = errors.New("chunks out of order")
errChunkHashMismatch = errors.New("chunk hash mismatch") errChunkHashMismatch = errors.New("chunk hash mismatch")
errNegativeChunkLength = errors.New("chunk length is negative") errNegativeChunkLength = errors.New("chunk length is negative")
@@ -104,9 +106,13 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
Mode: "deep", Mode: "deep",
} }
if !v.CanDecrypt() {
return v.deepVerifyFailure(result, opts,
errSecretKeyRequired.Error(), errSecretKeyRequired)
}
// Parse the age secret key once, the same way restore does, and reuse // Parse the age secret key once, the same way restore does, and reuse
// the identities for the database and every blob. A missing key fails // the identities for the database and every blob.
// here, before anything is downloaded.
identities, err := v.restoreIdentities() identities, err := v.restoreIdentities()
if err != nil { if err != nil {
return v.deepVerifyFailure(result, opts, err.Error(), err) return v.deepVerifyFailure(result, opts, err.Error(), err)