Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e084a2746 |
No files matched your search
@@ -554,12 +554,6 @@ count and blob size unknown, `null` in `--json`. A file under `blobs/` whose
|
|||||||
name is not a blob hash is not counted as a blob, since `prune` never deletes
|
name is not a blob hash is not counted as a blob, since `prune` never deletes
|
||||||
it.
|
it.
|
||||||
|
|
||||||
A blob that a manifest lists but the destination does not hold is reported as
|
|
||||||
missing, with a count and size. The report names each snapshot that lists a
|
|
||||||
missing blob, since that snapshot cannot be restored. The command then exits
|
|
||||||
non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and
|
|
||||||
`snapshots_missing_blobs`.
|
|
||||||
|
|
||||||
- `--json`: Output as JSON
|
- `--json`: Output as JSON
|
||||||
|
|
||||||
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
|
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
|
||||||
@@ -608,11 +602,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
|
|||||||
index holds snapshots.
|
index holds snapshots.
|
||||||
|
|
||||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||||
providers. Requires rclone to be configured separately (`rclone config`). The
|
providers. Requires rclone to be configured separately (`rclone config`).
|
||||||
path after the remote name is relative: rclone resolves it on the remote, for
|
|
||||||
example against the login directory on sftp. A second slash makes it absolute,
|
|
||||||
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
|
|
||||||
path; a relative one is refused.
|
|
||||||
|
|
||||||
An upload cut off part-way leaves nothing under the object's name on S3, which
|
An upload cut off part-way leaves nothing under the object's name on S3, which
|
||||||
shows an object only once its upload has completed, and on the local filesystem
|
shows an object only once its upload has completed, and on the local filesystem
|
||||||
|
|||||||
@@ -282,6 +282,13 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
|||||||
storageURLKey, testS3URL,
|
storageURLKey, testS3URL,
|
||||||
"s3.secret_access_key is required for s3:// URLs; " +
|
"s3.secret_access_key is required for s3:// URLs; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key first"},
|
"set s3.access_key_id and s3.secret_access_key first"},
|
||||||
|
// rclone:// URLs that no command could open.
|
||||||
|
{configWithoutRecipients, storageURLKey, "rclone:///backups",
|
||||||
|
"rclone URL missing remote name"},
|
||||||
|
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
|
||||||
|
"rclone URL must not carry parameters in the remote name"},
|
||||||
|
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
|
||||||
|
`unknown query parameter in storage URL: "x"`},
|
||||||
// With its reference filled in, the config loads.
|
// With its reference filled in, the config loads.
|
||||||
{levelReference, "blob_size_limit", "1MB",
|
{levelReference, "blob_size_limit", "1MB",
|
||||||
"blob_size_limit must be at least"},
|
"blob_size_limit must be at least"},
|
||||||
|
|||||||
@@ -75,9 +75,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
- Size of all snapshot metadata (per snapshot and total)
|
- Size of all snapshot metadata (per snapshot and total)
|
||||||
- Count and total size of all blobs
|
- Count and total size of all blobs
|
||||||
- Count and size of referenced blobs (from all manifests)
|
- Count and size of referenced blobs (from all manifests)
|
||||||
- Count and size of orphaned blobs (not referenced by any manifest)
|
- Count and size of orphaned blobs (not referenced by any manifest)`,
|
||||||
- Count and size of referenced blobs missing from remote storage, and the
|
|
||||||
snapshots that list them; the command exits non-zero when any are missing`,
|
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
// Use unified config resolution
|
// Use unified config resolution
|
||||||
|
|||||||
+11
-24
@@ -21,6 +21,7 @@ import (
|
|||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
)
|
)
|
||||||
|
|
||||||
const appName = "vaultik"
|
const appName = "vaultik"
|
||||||
@@ -65,11 +66,6 @@ var (
|
|||||||
"emit (chunk_size times the FastCDC size spread)")
|
"emit (chunk_size times the FastCDC size spread)")
|
||||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
errBadStorageScheme = errors.New(
|
|
||||||
"storage_url must start with s3://, file://, or rclone://")
|
|
||||||
errFileURLNotAbsolute = errors.New(
|
|
||||||
"a file:// storage_url needs an absolute path, " +
|
|
||||||
"as in file:///path/to/backup")
|
|
||||||
errIndexPathNotAbsolute = errors.New(
|
errIndexPathNotAbsolute = errors.New(
|
||||||
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
|
||||||
"as in ~/.local/share/vaultik/index.sqlite")
|
"as in ~/.local/share/vaultik/index.sqlite")
|
||||||
@@ -557,26 +553,20 @@ func (c *Config) validateStorage() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateStorageURL validates URL-based storage configuration. File and
|
// validateStorageURL checks storage_url with the same parser that opens
|
||||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
// the destination, so a value every command would refuse fails at load.
|
||||||
// credential fields. A file URL must name an absolute path: a relative one
|
// File and rclone URLs need no credentials; S3 URLs require the legacy
|
||||||
// would resolve against the working directory, so the destination would
|
// s3.* credential fields.
|
||||||
// move with it.
|
|
||||||
func (c *Config) validateStorageURL() error {
|
func (c *Config) validateStorageURL() error {
|
||||||
switch {
|
parsed, err := storageurl.Parse(c.StorageURL)
|
||||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
if err != nil {
|
||||||
// storage.ParseStorageURL applies the same rule, but config
|
return fmt.Errorf("invalid storage_url: %w", err)
|
||||||
// cannot import storage, which imports config.
|
|
||||||
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
|
||||||
return errFileURLNotAbsolute
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if parsed.Scheme != storageurl.SchemeS3 {
|
||||||
return nil
|
return nil
|
||||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
}
|
||||||
// Rclone storage uses rclone's own config
|
|
||||||
return nil
|
|
||||||
case strings.HasPrefix(c.StorageURL, "s3://"):
|
|
||||||
// S3 storage needs credentials
|
|
||||||
if c.S3.AccessKeyID == "" {
|
if c.S3.AccessKeyID == "" {
|
||||||
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
|
||||||
}
|
}
|
||||||
@@ -586,9 +576,6 @@ func (c *Config) validateStorageURL() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
default:
|
|
||||||
return errBadStorageScheme
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module exports the config module for fx dependency injection.
|
// Module exports the config module for fx dependency injection.
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/chunker"
|
"sneak.berlin/go/vaultik/internal/chunker"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -502,8 +503,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err = Load(path)
|
_, err = Load(path)
|
||||||
if !errors.Is(err, errFileURLNotAbsolute) {
|
if !errors.Is(err, storageurl.ErrRelativeFilePath) {
|
||||||
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "absolute path") {
|
if !strings.Contains(err.Error(), "absolute path") {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileStorer implements Storer using the local filesystem.
|
// FileStorer implements Storer using the local filesystem.
|
||||||
@@ -288,7 +289,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
|
|||||||
// Info returns human-readable storage location information.
|
// Info returns human-readable storage location information.
|
||||||
func (f *FileStorer) Info() Info {
|
func (f *FileStorer) Info() Info {
|
||||||
return Info{
|
return Info{
|
||||||
Type: schemeFile,
|
Type: storageurl.SchemeFile,
|
||||||
Location: f.basePath,
|
Location: f.basePath,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,14 +2,20 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
"sneak.berlin/go/vaultik/internal/s3"
|
"sneak.berlin/go/vaultik/internal/s3"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
|
||||||
|
// has no storage backend.
|
||||||
|
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
||||||
|
|
||||||
// defaultS3Region is used when neither the URL nor the config specify one.
|
// defaultS3Region is used when neither the URL nor the config specify one.
|
||||||
const defaultS3Region = "us-east-1"
|
const defaultS3Region = "us-east-1"
|
||||||
|
|
||||||
@@ -40,19 +46,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
|
|||||||
|
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
||||||
parsed, err := ParseStorageURL(rawURL)
|
parsed, err := storageurl.Parse(rawURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch parsed.Scheme {
|
switch parsed.Scheme {
|
||||||
case schemeFile:
|
case storageurl.SchemeFile:
|
||||||
return NewFileStorer(parsed.Prefix)
|
return NewFileStorer(parsed.Prefix)
|
||||||
|
|
||||||
case schemeS3:
|
case storageurl.SchemeS3:
|
||||||
return storerFromParsedS3URL(parsed, cfg)
|
return storerFromParsedS3URL(parsed, cfg)
|
||||||
|
|
||||||
case schemeRclone:
|
case storageurl.SchemeRclone:
|
||||||
return NewRcloneStorer(
|
return NewRcloneStorer(
|
||||||
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
||||||
|
|
||||||
@@ -65,7 +71,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
|||||||
// filling endpoint protocol and region defaults from the config.
|
// filling endpoint protocol and region defaults from the config.
|
||||||
//
|
//
|
||||||
//nolint:ireturn // factory intentionally returns the Storer interface
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
||||||
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
|
||||||
// Build endpoint URL
|
// Build endpoint URL
|
||||||
endpoint := parsed.Endpoint
|
endpoint := parsed.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -16,6 +15,7 @@ import (
|
|||||||
"github.com/rclone/rclone/fs/config"
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configfile"
|
"github.com/rclone/rclone/fs/config/configfile"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
|
|
||||||
// Import all rclone backends
|
// Import all rclone backends
|
||||||
_ "github.com/rclone/rclone/backend/all"
|
_ "github.com/rclone/rclone/backend/all"
|
||||||
@@ -24,15 +24,6 @@ import (
|
|||||||
// ErrRemoteNotFound is returned when an rclone remote is not configured.
|
// ErrRemoteNotFound is returned when an rclone remote is not configured.
|
||||||
var ErrRemoteNotFound = errors.New("rclone remote not found in config")
|
var ErrRemoteNotFound = errors.New("rclone remote not found in config")
|
||||||
|
|
||||||
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
|
|
||||||
// of type local is relative. rclone would resolve it against the working
|
|
||||||
// directory, so the destination would move with it.
|
|
||||||
var ErrRcloneRelativeLocalPath = errors.New(
|
|
||||||
"the path on an rclone remote of type local must be absolute")
|
|
||||||
|
|
||||||
// rcloneLocalBackend is the type of rclone's local disk backend.
|
|
||||||
const rcloneLocalBackend = "local"
|
|
||||||
|
|
||||||
// RcloneStorer implements Storer using rclone's filesystem abstraction.
|
// RcloneStorer implements Storer using rclone's filesystem abstraction.
|
||||||
// This allows vaultik to use any of rclone's 70+ supported storage providers.
|
// This allows vaultik to use any of rclone's 70+ supported storage providers.
|
||||||
type RcloneStorer struct {
|
type RcloneStorer struct {
|
||||||
@@ -63,19 +54,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
|
|||||||
rclonePath += path
|
rclonePath += path
|
||||||
}
|
}
|
||||||
|
|
||||||
// A remote rclone cannot find is reported by fs.NewFs below. Only the
|
|
||||||
// backend of what fs.ParseRemote returns is needed here.
|
|
||||||
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
|
|
||||||
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
|
|
||||||
example := path
|
|
||||||
if example == "" {
|
|
||||||
example = "path/to/backup"
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w: use rclone://%s//%s",
|
|
||||||
ErrRcloneRelativeLocalPath, remote, example)
|
|
||||||
}
|
|
||||||
|
|
||||||
// On S3, Swift and some other remotes rclone keeps a time of its own in
|
// On S3, Swift and some other remotes rclone keeps a time of its own in
|
||||||
// each object's metadata, which costs one request per object to read.
|
// each object's metadata, which costs one request per object to read.
|
||||||
// rclone's use_server_modtime setting makes ModTime the time the
|
// rclone's use_server_modtime setting makes ModTime the time the
|
||||||
@@ -289,7 +267,7 @@ func (r *RcloneStorer) Info() Info {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Info{
|
return Info{
|
||||||
Type: schemeRclone,
|
Type: storageurl.SchemeRclone,
|
||||||
Location: location,
|
Location: location,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -436,79 +436,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// useRcloneConfig points rclone at a config file holding contents for the
|
|
||||||
// rest of the test.
|
|
||||||
func useRcloneConfig(t *testing.T, contents string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
previous := config.GetConfigPath()
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
|
|
||||||
|
|
||||||
confFile := filepath.Join(t.TempDir(), "rclone.conf")
|
|
||||||
|
|
||||||
err := os.WriteFile(confFile, []byte(contents), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("writing rclone config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("RCLONE_CONFIG", confFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
|
|
||||||
// on a remote of type local is refused, with the absolute form in the
|
|
||||||
// error, and that nothing is created under the working directory. rclone
|
|
||||||
// would otherwise resolve the path against the working directory.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the working directory and the rclone config
|
|
||||||
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
|
|
||||||
useRcloneConfig(t, "[localfs]\ntype = local\n")
|
|
||||||
|
|
||||||
workdir := t.TempDir()
|
|
||||||
t.Chdir(workdir)
|
|
||||||
|
|
||||||
for path, wantInError := range map[string]string{
|
|
||||||
"srv/backup": "rclone://localfs//srv/backup",
|
|
||||||
"": "rclone://localfs//path/to/backup",
|
|
||||||
} {
|
|
||||||
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
|
|
||||||
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
|
|
||||||
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
|
|
||||||
path, err)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), wantInError) {
|
|
||||||
t.Errorf("path %q: error %q does not show %q",
|
|
||||||
path, err, wantInError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(workdir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading working directory: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(entries) != 0 {
|
|
||||||
t.Errorf("working directory holds %d entries, want none", len(entries))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
|
|
||||||
// path is accepted on a remote not of type local, where rclone resolves it
|
|
||||||
// on the remote itself, as sftp does against the login directory.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the process-global rclone config
|
|
||||||
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
|
|
||||||
useRcloneConfig(t, "[mem]\ntype = memory\n")
|
|
||||||
|
|
||||||
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("NewRcloneStorer: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
|
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
|
||||||
// rclone S3 remote takes each object's time from the listing itself. By
|
// rclone S3 remote takes each object's time from the listing itself. By
|
||||||
// default rclone reads the time it keeps in each object's metadata, one
|
// default rclone reads the time it keeps in each object's metadata, one
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
package storage
|
// Package storageurl parses the storage_url config value. The config
|
||||||
|
// package uses it to refuse a bad value when the config loads; the storage
|
||||||
|
// package uses it to open the destination.
|
||||||
|
package storageurl
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -10,9 +13,9 @@ import (
|
|||||||
|
|
||||||
// Storage URL scheme names.
|
// Storage URL scheme names.
|
||||||
const (
|
const (
|
||||||
schemeFile = "file"
|
SchemeFile = "file"
|
||||||
schemeS3 = "s3"
|
SchemeS3 = "s3"
|
||||||
schemeRclone = "rclone"
|
SchemeRclone = "rclone"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for storage URL parsing.
|
// Sentinel errors for storage URL parsing.
|
||||||
@@ -25,7 +28,6 @@ var (
|
|||||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||||
ErrUnsupportedScheme = errors.New(
|
ErrUnsupportedScheme = errors.New(
|
||||||
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||||
ErrUnsupportedStorage = errors.New("unsupported storage scheme")
|
|
||||||
ErrURLCredentials = errors.New(
|
ErrURLCredentials = errors.New(
|
||||||
"storage URL must not carry credentials; " +
|
"storage URL must not carry credentials; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
||||||
@@ -49,12 +51,12 @@ type URL struct {
|
|||||||
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseStorageURL parses a storage URL string.
|
// Parse parses a storage URL string.
|
||||||
// Supported formats:
|
// Supported formats:
|
||||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
||||||
// - file:///absolute/path/to/backup
|
// - file:///absolute/path/to/backup
|
||||||
// - rclone://remote/path/to/backups
|
// - rclone://remote/path/to/backups
|
||||||
func ParseStorageURL(rawURL string) (*URL, error) {
|
func Parse(rawURL string) (*URL, error) {
|
||||||
if rawURL == "" {
|
if rawURL == "" {
|
||||||
return nil, ErrEmptyStorageURL
|
return nil, ErrEmptyStorageURL
|
||||||
}
|
}
|
||||||
@@ -92,7 +94,7 @@ func parseFileURL(rawPath string) (*URL, error) {
|
|||||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &URL{Scheme: schemeFile, Prefix: path}, nil
|
return &URL{Scheme: SchemeFile, Prefix: path}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
@@ -135,7 +137,7 @@ func parseS3URL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: schemeS3,
|
Scheme: SchemeS3,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
Endpoint: query.Get("endpoint"),
|
Endpoint: query.Get("endpoint"),
|
||||||
@@ -189,7 +191,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: schemeRclone,
|
Scheme: SchemeRclone,
|
||||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||||
RcloneRemote: remote,
|
RcloneRemote: remote,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -226,9 +228,9 @@ func wrapParseError(err error) error {
|
|||||||
// String returns a human-readable representation of the storage URL.
|
// String returns a human-readable representation of the storage URL.
|
||||||
func (u *URL) String() string {
|
func (u *URL) String() string {
|
||||||
switch u.Scheme {
|
switch u.Scheme {
|
||||||
case schemeFile:
|
case SchemeFile:
|
||||||
return "file://" + u.Prefix
|
return "file://" + u.Prefix
|
||||||
case schemeS3:
|
case SchemeS3:
|
||||||
endpoint := u.Endpoint
|
endpoint := u.Endpoint
|
||||||
if endpoint == "" {
|
if endpoint == "" {
|
||||||
endpoint = "s3.amazonaws.com"
|
endpoint = "s3.amazonaws.com"
|
||||||
@@ -239,7 +241,7 @@ func (u *URL) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
||||||
case schemeRclone:
|
case SchemeRclone:
|
||||||
if u.Prefix != "" {
|
if u.Prefix != "" {
|
||||||
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package storage_test
|
package storageurl_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -6,12 +6,12 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
"sneak.berlin/go/vaultik/internal/storageurl"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestParseStorageURLValid checks that each supported scheme parses into
|
// TestParseValid checks that each supported scheme parses into the
|
||||||
// the expected fields, since those fields decide which backend is built.
|
// expected fields, since those fields decide which backend is built.
|
||||||
func TestParseStorageURLValid(t *testing.T) {
|
func TestParseValid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -22,27 +22,27 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
raw string
|
raw string
|
||||||
want *storage.URL
|
want *storageurl.URL
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "file absolute path",
|
name: "file absolute path",
|
||||||
raw: "file:///var/backups/vaultik",
|
raw: "file:///var/backups/vaultik",
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file percent-escaped path is decoded",
|
name: "file percent-escaped path is decoded",
|
||||||
raw: "file:///Volumes/my%20stick/backup",
|
raw: "file:///Volumes/my%20stick/backup",
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "file path with a literal space",
|
name: "file path with a literal space",
|
||||||
raw: "file:///Volumes/my stick/backup",
|
raw: "file:///Volumes/my stick/backup",
|
||||||
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 bucket and prefix, ssl defaults on",
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
raw: "s3://mybucket/backups/host",
|
raw: "s3://mybucket/backups/host",
|
||||||
want: &storage.URL{
|
want: &storageurl.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Prefix: "backups/host", UseSSL: true,
|
Prefix: "backups/host", UseSSL: true,
|
||||||
},
|
},
|
||||||
@@ -50,12 +50,12 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 bucket only",
|
name: "s3 bucket only",
|
||||||
raw: "s3://mybucket",
|
raw: "s3://mybucket",
|
||||||
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 with endpoint, region, ssl off",
|
name: "s3 with endpoint, region, ssl off",
|
||||||
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
||||||
want: &storage.URL{
|
want: &storageurl.URL{
|
||||||
Scheme: "s3", Bucket: bucket,
|
Scheme: "s3", Bucket: bucket,
|
||||||
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
||||||
},
|
},
|
||||||
@@ -63,14 +63,14 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "rclone remote and path",
|
name: "rclone remote and path",
|
||||||
raw: "rclone://gdrive/backups/host",
|
raw: "rclone://gdrive/backups/host",
|
||||||
want: &storage.URL{
|
want: &storageurl.URL{
|
||||||
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone remote only",
|
name: "rclone remote only",
|
||||||
raw: "rclone://gdrive",
|
raw: "rclone://gdrive",
|
||||||
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,22 +78,22 @@ func TestParseStorageURLValid(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
got, err := storage.ParseStorageURL(tc.raw)
|
got, err := storageurl.Parse(tc.raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
|
t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
if !reflect.DeepEqual(got, tc.want) {
|
||||||
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
|
// TestParseErrors checks that empty, missing, and unknown-scheme inputs
|
||||||
// inputs fail with the documented sentinel errors instead of parsing to a
|
// fail with the documented sentinel errors instead of parsing to a wrong
|
||||||
// wrong destination.
|
// destination.
|
||||||
func TestParseStorageURLErrors(t *testing.T) {
|
func TestParseErrors(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -101,34 +101,33 @@ func TestParseStorageURLErrors(t *testing.T) {
|
|||||||
raw string
|
raw string
|
||||||
wantErr error
|
wantErr error
|
||||||
}{
|
}{
|
||||||
{"empty url", "", storage.ErrEmptyStorageURL},
|
{"empty url", "", storageurl.ErrEmptyStorageURL},
|
||||||
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
{"file empty path", "file://", storageurl.ErrEmptyFilePath},
|
||||||
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
|
{"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
|
||||||
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
{"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
|
||||||
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
{"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
|
||||||
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
{"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
|
||||||
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
|
{"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
|
||||||
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
|
{"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(tc.raw)
|
_, err := storageurl.Parse(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
t.Errorf("Parse(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLRejectsCredentials checks that a URL carrying
|
// TestParseRejectsCredentials checks that a URL carrying credentials in
|
||||||
// credentials in its userinfo or in an unknown query parameter is
|
// its userinfo or in an unknown query parameter is rejected, and that the
|
||||||
// rejected, and that the error never echoes the secret-bearing URL back
|
// error never echoes the secret-bearing URL back into logs or output.
|
||||||
// into logs or output.
|
func TestParseRejectsCredentials(t *testing.T) {
|
||||||
func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Split so the literals never form a "user:pass@" URL pattern that
|
// Split so the literals never form a "user:pass@" URL pattern that
|
||||||
@@ -147,44 +146,44 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "s3 userinfo",
|
name: "s3 userinfo",
|
||||||
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
|
||||||
wantErr: storage.ErrURLCredentials,
|
wantErr: storageurl.ErrURLCredentials,
|
||||||
secrets: []string{key, secret, "mybucket"},
|
secrets: []string{key, secret, "mybucket"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint",
|
name: "s3 userinfo in endpoint",
|
||||||
raw: "s3://mybucket?endpoint=https://" +
|
raw: "s3://mybucket?endpoint=https://" +
|
||||||
key + ":" + secret + "@minio.example.com",
|
key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storage.ErrURLCredentials,
|
wantErr: storageurl.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 userinfo in endpoint without scheme",
|
name: "s3 userinfo in endpoint without scheme",
|
||||||
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
||||||
wantErr: storage.ErrURLCredentials,
|
wantErr: storageurl.ErrURLCredentials,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 unknown query param",
|
name: "s3 unknown query param",
|
||||||
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
||||||
wantErr: storage.ErrURLUnknownParam,
|
wantErr: storageurl.ErrURLUnknownParam,
|
||||||
secrets: []string{key, secret},
|
secrets: []string{key, secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "s3 misspelt endpoint",
|
name: "s3 misspelt endpoint",
|
||||||
raw: "s3://mybucket?endpiont=minio.example.com",
|
raw: "s3://mybucket?endpiont=minio.example.com",
|
||||||
wantErr: storage.ErrURLUnknownParam,
|
wantErr: storageurl.ErrURLUnknownParam,
|
||||||
secrets: nil,
|
secrets: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone userinfo",
|
name: "rclone userinfo",
|
||||||
raw: "rclone://user:" + secret + "@gdrive/backups",
|
raw: "rclone://user:" + secret + "@gdrive/backups",
|
||||||
wantErr: storage.ErrURLCredentials,
|
wantErr: storageurl.ErrURLCredentials,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "rclone query param",
|
name: "rclone query param",
|
||||||
raw: "rclone://gdrive/backups?token=" + secret,
|
raw: "rclone://gdrive/backups?token=" + secret,
|
||||||
wantErr: storage.ErrURLUnknownParam,
|
wantErr: storageurl.ErrURLUnknownParam,
|
||||||
secrets: []string{secret},
|
secrets: []string{secret},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -193,9 +192,9 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(tc.raw)
|
_, err := storageurl.Parse(tc.raw)
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantErr) {
|
||||||
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
t.Fatalf("Parse(%q) error = %v, want %v",
|
||||||
tc.raw, err, tc.wantErr)
|
tc.raw, err, tc.wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -216,11 +215,11 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
|
// TestParseRejectsRcloneRemoteParams checks that an rclone remote name
|
||||||
// remote name carrying parameters (rclone's "remote,param=value" form,
|
// carrying parameters (rclone's "remote,param=value" form, which can hold
|
||||||
// which can hold a credential) is rejected without echoing the value,
|
// a credential) is rejected without echoing the value, including the URLs
|
||||||
// including the URLs that url.Parse itself refuses.
|
// that url.Parse itself refuses.
|
||||||
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
func TestParseRejectsRcloneRemoteParams(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -249,10 +248,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(tc.raw)
|
_, err := storageurl.Parse(tc.raw)
|
||||||
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
|
if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
|
||||||
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
t.Fatalf("Parse(%q) error = %v, want %v",
|
||||||
tc.raw, err, storage.ErrRcloneRemoteParams)
|
tc.raw, err, storageurl.ErrRcloneRemoteParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, secret := range tc.secrets {
|
for _, secret := range tc.secrets {
|
||||||
@@ -264,10 +263,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
|
// TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
|
||||||
// URL whose query cannot be decoded is rejected, not read with that part
|
// query cannot be decoded is rejected, not read with that part of the
|
||||||
// of the query left out, and that the error does not echo it.
|
// query left out, and that the error does not echo it.
|
||||||
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
|
func TestParseRejectsUndecodableQuery(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -303,14 +302,14 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(tc.raw)
|
_, err := storageurl.Parse(tc.raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
|
t.Fatalf("Parse(%q) returned no error", tc.raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !errors.Is(err, storage.ErrURLBadQuery) {
|
if !errors.Is(err, storageurl.ErrURLBadQuery) {
|
||||||
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
t.Errorf("Parse(%q) error = %v, want %v",
|
||||||
tc.raw, err, storage.ErrURLBadQuery)
|
tc.raw, err, storageurl.ErrURLBadQuery)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, s := range []string{key, secret, "minio.example.com"} {
|
for _, s := range []string{key, secret, "minio.example.com"} {
|
||||||
@@ -322,17 +321,17 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
|
// TestParseFailureHidesURL checks that when url.Parse itself fails, the
|
||||||
// itself fails, the wrapped error carries only the inner cause, not the
|
// wrapped error carries only the inner cause, not the *url.Error whose
|
||||||
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
|
// text embeds the raw (possibly credential-bearing) URL.
|
||||||
func TestParseStorageURLParseFailureHidesURL(t *testing.T) {
|
func TestParseFailureHidesURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const raw = "s3://mybucket/%zz"
|
const raw = "s3://mybucket/%zz"
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(raw)
|
_, err := storageurl.Parse(raw)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("ParseStorageURL(%q) returned no error", raw)
|
t.Fatalf("Parse(%q) returned no error", raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.Contains(err.Error(), "mybucket") {
|
if strings.Contains(err.Error(), "mybucket") {
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ShowInfo displays system and configuration information
|
// ShowInfo displays system and configuration information
|
||||||
@@ -188,9 +187,6 @@ type SnapshotMetadataInfo struct {
|
|||||||
// backup interrupted before its manifest upload leaves a directory
|
// backup interrupted before its manifest upload leaves a directory
|
||||||
// without one, which prune does not treat as a snapshot.
|
// without one, which prune does not treat as a snapshot.
|
||||||
hasManifest bool
|
hasManifest bool
|
||||||
|
|
||||||
// Hashes of the blobs the snapshot's manifest lists.
|
|
||||||
blobHashes []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemoteInfoResult contains all remote storage information
|
// RemoteInfoResult contains all remote storage information
|
||||||
@@ -220,13 +216,6 @@ type RemoteInfoResult struct {
|
|||||||
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
||||||
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
||||||
|
|
||||||
// Blobs that a manifest lists but the destination does not hold,
|
|
||||||
// and the remote key of each snapshot whose manifest lists one.
|
|
||||||
// Those snapshots cannot be restored.
|
|
||||||
MissingBlobCount int `json:"missing_blob_count"`
|
|
||||||
MissingBlobSize int64 `json:"missing_blob_size"`
|
|
||||||
SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"`
|
|
||||||
|
|
||||||
// Remote key of each snapshot whose manifest could not be read
|
// Remote key of each snapshot whose manifest could not be read
|
||||||
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
|
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
|
||||||
|
|
||||||
@@ -294,30 +283,19 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
"snapshots", result.TotalMetadataCount,
|
"snapshots", result.TotalMetadataCount,
|
||||||
"total_blobs", result.TotalBlobCount,
|
"total_blobs", result.TotalBlobCount,
|
||||||
"referenced_blobs", result.ReferencedBlobCount,
|
"referenced_blobs", result.ReferencedBlobCount,
|
||||||
"missing_blobs", result.MissingBlobCount,
|
|
||||||
"unreadable_manifests", len(result.UnreadableManifests))
|
"unreadable_manifests", len(result.UnreadableManifests))
|
||||||
|
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
enc := json.NewEncoder(v.Stdout)
|
enc := json.NewEncoder(v.Stdout)
|
||||||
enc.SetIndent("", " ")
|
enc.SetIndent("", " ")
|
||||||
|
|
||||||
err = enc.Encode(result)
|
return enc.Encode(result)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("encoding JSON: %w", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if showText {
|
if showText {
|
||||||
v.printRemoteInfoTable(result)
|
v.printRemoteInfoTable(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The report is written first, so the exit status adds to it.
|
|
||||||
if result.MissingBlobCount > 0 {
|
|
||||||
return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+
|
|
||||||
"%d snapshot(s) cannot be restored", errProblemsInStorage,
|
|
||||||
result.MissingBlobCount, len(result.SnapshotsMissingBlobs))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -427,7 +405,6 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
|
|||||||
for _, blob := range manifest.Blobs {
|
for _, blob := range manifest.Blobs {
|
||||||
referencedBlobs[blob.Hash] = blob.CompressedSize
|
referencedBlobs[blob.Hash] = blob.CompressedSize
|
||||||
blobsSize += blob.CompressedSize
|
blobsSize += blob.CompressedSize
|
||||||
info.blobHashes = append(info.blobHashes, blob.Hash)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
info.BlobCount = &blobCount
|
info.BlobCount = &blobCount
|
||||||
@@ -485,8 +462,6 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
result.TotalBlobSize += blob.Size
|
result.TotalBlobSize += blob.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
countMissingBlobs(result, referencedBlobs, allBlobs)
|
|
||||||
|
|
||||||
// A blob named only by a manifest that could not be read, or by one
|
// A blob named only by a manifest that could not be read, or by one
|
||||||
// under a skipped name, would be counted as orphaned, so the orphan
|
// under a skipped name, would be counted as orphaned, so the orphan
|
||||||
// figures stay unknown.
|
// figures stay unknown.
|
||||||
@@ -512,33 +487,6 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// countMissingBlobs records in result the referenced blobs that
|
|
||||||
// allBlobs, the listing of the destination, does not hold, and the
|
|
||||||
// snapshots whose manifests list one of them. It reads the snapshots
|
|
||||||
// from result.Snapshots, so populateRemoteInfoResult must run first.
|
|
||||||
func countMissingBlobs(
|
|
||||||
result *RemoteInfoResult, referencedBlobs map[string]int64,
|
|
||||||
allBlobs map[string]storage.ObjectInfo,
|
|
||||||
) {
|
|
||||||
for hash, size := range referencedBlobs {
|
|
||||||
if _, present := allBlobs[hash]; !present {
|
|
||||||
result.MissingBlobCount++
|
|
||||||
result.MissingBlobSize += size
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, info := range result.Snapshots {
|
|
||||||
for _, hash := range info.blobHashes {
|
|
||||||
if _, present := allBlobs[hash]; !present {
|
|
||||||
result.SnapshotsMissingBlobs = append(
|
|
||||||
result.SnapshotsMissingBlobs, info.SnapshotID)
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// printRemoteInfoTable renders the human-readable remote info output
|
// printRemoteInfoTable renders the human-readable remote info output
|
||||||
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
||||||
const (
|
const (
|
||||||
@@ -591,12 +539,6 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
|||||||
"", "", ubytes(result.TotalMetadataSize))
|
"", "", ubytes(result.TotalMetadataSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
v.printBlobStorageSummary(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
// printBlobStorageSummary renders the blob storage section that ends
|
|
||||||
// the human-readable remote info output.
|
|
||||||
func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
|
|
||||||
v.stdoutf("\n=== Blob Storage ===\n")
|
v.stdoutf("\n=== Blob Storage ===\n")
|
||||||
v.stdoutf("Total blobs on remote: %s (%s)\n",
|
v.stdoutf("Total blobs on remote: %s (%s)\n",
|
||||||
humanize.Comma(int64(result.TotalBlobCount)),
|
humanize.Comma(int64(result.TotalBlobCount)),
|
||||||
@@ -604,16 +546,16 @@ func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
|
|||||||
v.stdoutf("Referenced by snapshots: %s (%s)\n",
|
v.stdoutf("Referenced by snapshots: %s (%s)\n",
|
||||||
humanize.Comma(int64(result.ReferencedBlobCount)),
|
humanize.Comma(int64(result.ReferencedBlobCount)),
|
||||||
ubytes(result.ReferencedBlobSize))
|
ubytes(result.ReferencedBlobSize))
|
||||||
v.stdoutf("Missing from remote: %s (%s)\n",
|
|
||||||
humanize.Comma(int64(result.MissingBlobCount)),
|
|
||||||
ubytes(result.MissingBlobSize))
|
|
||||||
|
|
||||||
if result.OrphanedBlobCount == nil {
|
if result.OrphanedBlobCount == nil {
|
||||||
v.stdoutf("Orphaned (unreferenced): unknown "+
|
v.stdoutf("Orphaned (unreferenced): unknown "+
|
||||||
"(%d manifest(s) could not be read, "+
|
"(%d manifest(s) could not be read, "+
|
||||||
"%d manifest(s) under a non-conforming name skipped)\n",
|
"%d manifest(s) under a non-conforming name skipped)\n",
|
||||||
len(result.UnreadableManifests), result.SkippedManifestCount)
|
len(result.UnreadableManifests), result.SkippedManifestCount)
|
||||||
} else {
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
|
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
|
||||||
humanize.Comma(int64(*result.OrphanedBlobCount)),
|
humanize.Comma(int64(*result.OrphanedBlobCount)),
|
||||||
ubytes(*result.OrphanedBlobSize))
|
ubytes(*result.OrphanedBlobSize))
|
||||||
@@ -623,15 +565,6 @@ func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(result.SnapshotsMissingBlobs) > 0 {
|
|
||||||
v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n")
|
|
||||||
|
|
||||||
for _, id := range result.SnapshotsMissingBlobs {
|
|
||||||
v.stdoutf(" %s\n", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ellipsis is appended by truncateString when it shortens a string.
|
// ellipsis is appended by truncateString when it shortens a string.
|
||||||
const ellipsis = "..."
|
const ellipsis = "..."
|
||||||
|
|
||||||
|
|||||||
@@ -83,7 +83,6 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) {
|
|||||||
env := newListEnv(t)
|
env := newListEnv(t)
|
||||||
readableKey := env.addRemote(t, listRemoteID,
|
readableKey := env.addRemote(t, listRemoteID,
|
||||||
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
|
||||||
|
|
||||||
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
|
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
|
||||||
require.NoError(t, env.store.Put(context.Background(),
|
require.NoError(t, env.store.Put(context.Background(),
|
||||||
@@ -271,7 +270,6 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
|||||||
|
|
||||||
env := newListEnv(t)
|
env := newListEnv(t)
|
||||||
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
|
||||||
|
|
||||||
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
|
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
|
||||||
|
|
||||||
@@ -283,61 +281,3 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
|
|||||||
require.NoError(t, env.v.RemoteInfo(false))
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
|
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a
|
|
||||||
// manifest lists but the destination does not hold is counted as
|
|
||||||
// missing, that the snapshot listing it is named, and that remote info
|
|
||||||
// then fails after writing its report.
|
|
||||||
func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newListEnv(t)
|
|
||||||
|
|
||||||
// The first snapshot lists blob A, which is not stored. The second
|
|
||||||
// lists blob B, which is.
|
|
||||||
brokenKey := env.addRemote(t, listRemoteID,
|
|
||||||
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
|
||||||
addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB})
|
|
||||||
addBlob(t, env.store.testStorer, testBlobHashB)
|
|
||||||
|
|
||||||
const wantError = "1 blob(s) that snapshots list are missing; " +
|
|
||||||
"1 snapshot(s) cannot be restored"
|
|
||||||
|
|
||||||
err := env.v.RemoteInfo(false)
|
|
||||||
require.ErrorContains(t, err, wantError)
|
|
||||||
|
|
||||||
text := env.stdout.String()
|
|
||||||
assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)")
|
|
||||||
assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+
|
|
||||||
" "+brokenKey+"\n")
|
|
||||||
assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n")
|
|
||||||
|
|
||||||
env.stdout.Reset()
|
|
||||||
require.ErrorContains(t, env.v.RemoteInfo(true), wantError)
|
|
||||||
|
|
||||||
var doc map[string]any
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
|
|
||||||
assert.InDelta(t, 1, doc["missing_blob_count"], 0)
|
|
||||||
assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0)
|
|
||||||
assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"])
|
|
||||||
|
|
||||||
// --quiet drops the report, not the failure.
|
|
||||||
env.stdout.Reset()
|
|
||||||
env.v.UI.SetQuiet(true)
|
|
||||||
require.ErrorContains(t, env.v.RemoteInfo(false), wantError)
|
|
||||||
assert.Empty(t, env.stdout.String())
|
|
||||||
env.v.UI.SetQuiet(false)
|
|
||||||
|
|
||||||
// With blob A stored again nothing is missing.
|
|
||||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
|
||||||
env.stdout.Reset()
|
|
||||||
require.NoError(t, env.v.RemoteInfo(true))
|
|
||||||
|
|
||||||
doc = nil
|
|
||||||
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
|
|
||||||
assert.InDelta(t, 0, doc["missing_blob_count"], 0)
|
|
||||||
assert.InDelta(t, 0, doc["missing_blob_size"], 0)
|
|
||||||
assert.NotContains(t, doc, "snapshots_missing_blobs")
|
|
||||||
}
|
|
||||||
@@ -349,7 +349,6 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
|
|||||||
env.addRemote(t, listLocalID, start)
|
env.addRemote(t, listLocalID, start)
|
||||||
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
|
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
|
||||||
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
|
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
|
||||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
|
||||||
|
|
||||||
require.NoError(t, env.v.RemoteInfo(false))
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
|
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
|
||||||
|
|||||||
Reference in new issue
Block a user