Compare commits

..
Author SHA1 Message Date
sneak 98a36c6a0c Take a snapshot's start time from the moment in its ID (closes #482)
check / check (push) Waiting to run
CreateSnapshotWithName read the clock once for the ID and again for
started_at. When the second read fell in the next second, the snapshot
shared its started_at with the next run of the same name, and
ListRecent, which orders by started_at, could list the two in either
order. That made TestBackupWithEmptySourcePathDoesNotPrune fail at
random. The start time now comes from the same clock read as the ID, so
successive runs of one name have distinct, increasing start times.

Model: opus-5-5
2026-11-19 12:00:32 +00:00
clawbot 547b9faf36 Add vaultik --version, printing the version command's report (closes #432)
check / check (push) Waiting to run
The flag is defined on the root command rather than enabled through
cobra's Version field: cobra's flag prints a one-line template, not
the full report that `vaultik version` prints.

Model: opus-5-5
2026-11-19 12:48:51 +01:00
clawbot fbcb1b9fc6 Turn off checkptr for SQLite in the race test run (closes #477)
check / check (push) Waiting to run
The internal/vaultik tests timed out under load. Their time goes to
SQLite, which serializes on one allocator lock under -race. checkptr,
which -race turns on, was about a third of the CPU time in SQLite's
translated C code. The test phase now turns it off for modernc.org
packages only; the race detector stays on everywhere and our own
packages keep checkptr.

Model: opus-5-5
2026-11-19 12:32:02 +01:00
sneak 4605eae024 next: integrate accumulated work into main (#203)
check / check (push) Waiting to run
Reviewed-on: #203
2026-11-19 10:27:45 +01:00
clawbot 5b09271751 Report no device number for a zeroed stat (closes #462)
check / check (push) Waiting to run
deviceNumberFromStat returned device 0 and true for a stat whose device
and inode numbers are both zero. afero's unreleased master gives such a
stat for every in-memory file; the pinned v1.15.0 gives none. It now
returns false for it, as its comment says, and a test covers both
in-memory shapes.

Model: opus-5-5
2026-11-19 09:53:18 +01:00
clawbot 9fedcbeb0a Refuse a relative path on an rclone remote of type local (closes #465)
check / check (push) Waiting to run
rclone resolves a relative path on its local backend against the working
directory, so rclone://localfs/srv/backup wrote under whatever directory
vaultik ran from. Opening such a destination now fails with an error that
shows the absolute form, rclone://localfs//srv/backup. Other remotes keep
relative paths, which rclone resolves on the remote. The README says how
to write an absolute rclone path.

Model: opus-5-5
2026-11-19 09:24:16 +01:00
17 changed files with 378 additions and 143 deletions

No files matched your search

+9 -2
View File
@@ -35,9 +35,16 @@ RUN for attempt in 1 2 3; do \
echo "module download failed: giving up after 3 attempts" >&2; \
exit 1
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
# -race also turns on checkptr, which checks every pointer conversion. In
# the SQLite library's translated C code that is about a third of the test
# CPU time, much of it under one lock all SQLite work shares, and the
# tests that use SQLite time out on a busy host. The gcflags turn it off
# for that library only; our own packages keep it.
RUN go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
+6 -1
View File
@@ -183,6 +183,7 @@ vaultik [--config <path>] remote nuke --force
vaultik [--config <path>] database delete [--force]
vaultik completion <bash|zsh|fish|powershell>
vaultik version
vaultik --version
```
### global flags
@@ -602,7 +603,11 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`).
providers. Requires rclone to be configured separately (`rclone config`). The
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem
-7
View File
@@ -282,13 +282,6 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
storageURLKey, testS3URL,
"s3.secret_access_key is required for s3:// URLs; " +
"set s3.access_key_id and s3.secret_access_key first"},
// rclone:// URLs that no command could open.
{configWithoutRecipients, storageURLKey, "rclone:///backups",
"rclone URL missing remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
"rclone URL must not carry parameters in the remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
`unknown query parameter in storage URL: "x"`},
// With its reference filled in, the config loads.
{levelReference, "blob_size_limit", "1MB",
"blob_size_limit must be at least"},
+18 -4
View File
@@ -32,6 +32,8 @@ var rootFlags RootFlags
// It sets up the command structure, global flags, and adds all subcommands.
// This is the main entry point for the CLI command hierarchy.
func NewRootCommand() *cobra.Command {
var showVersion bool
cmd := &cobra.Command{
Use: "vaultik",
Short: "Secure incremental backup tool with asymmetric encryption",
@@ -39,15 +41,27 @@ func NewRootCommand() *cobra.Command {
public keys and uploads to S3-compatible storage. No private keys are needed
on the source system.`,
SilenceUsage: true,
// Bare 'vaultik' (no subcommand): print help. The banner is
// printed once at process startup by Entry, before cobra
// parses arguments, so it appears even when cobra rejects
// args (e.g. "requires at least 2 arg(s)") and on --help.
// Bare 'vaultik' (no subcommand): print help, or with --version
// the report of the version command. The banner is printed once
// at process startup by Entry, before cobra parses arguments, so
// it appears even when cobra rejects args (e.g. "requires at
// least 2 arg(s)") and on --help.
Run: func(cmd *cobra.Command, _ []string) {
if showVersion {
writeVersion(commandUI(cmd))
return
}
_ = cmd.Help()
},
}
// cobra's own --version prints a template, not the version report,
// so vaultik defines the flag itself.
cmd.Flags().BoolVar(&showVersion, "version", false,
"Print version information (same as 'vaultik version')")
// Add global flags
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
+24
View File
@@ -86,6 +86,30 @@ func TestVersionCommandReportLines(t *testing.T) {
}
}
// TestVersionFlagPrintsVersionReport checks that `vaultik --version`
// prints exactly the report of `vaultik version`.
//
//nolint:paralleltest // executes commands that read the global rootFlags
func TestVersionFlagPrintsVersionReport(t *testing.T) {
cmd := cli.NewRootCommand()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs([]string{"--version"})
err := cmd.Execute()
if err != nil {
t.Fatalf("vaultik --version failed: %v", err)
}
want := runVersionCommand(t)
if out.String() != want {
t.Errorf("vaultik --version printed:\n%s\nwant:\n%s", out.String(), want)
}
}
// TestVersionCommandFlagsDevelopmentBuild is the regression test for
// the thing this command exists to prevent: a build that is not a
// release must say so. The notice used to be gated on the version
+33 -20
View File
@@ -21,7 +21,6 @@ import (
"gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
)
const appName = "vaultik"
@@ -64,8 +63,13 @@ var (
errBlobSizeTooSmall = errors.New(
"blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://")
errFileURLNotAbsolute = errors.New(
"a file:// storage_url needs an absolute path, " +
"as in file:///path/to/backup")
errIndexPathNotAbsolute = errors.New(
"index_path and VAULTIK_INDEX_PATH need an absolute path, " +
"as in ~/.local/share/vaultik/index.sqlite")
@@ -553,29 +557,38 @@ func (c *Config) validateStorage() error {
return nil
}
// validateStorageURL checks storage_url with the same parser that opens
// the destination, so a value every command would refuse fails at load.
// File and rclone URLs need no credentials; S3 URLs require the legacy
// s3.* credential fields.
// validateStorageURL validates URL-based storage configuration. File and
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
// credential fields. A file URL must name an absolute path: a relative one
// would resolve against the working directory, so the destination would
// move with it.
func (c *Config) validateStorageURL() error {
parsed, err := storageurl.Parse(c.StorageURL)
if err != nil {
return fmt.Errorf("invalid storage_url: %w", err)
}
switch {
case strings.HasPrefix(c.StorageURL, "file://"):
// storage.ParseStorageURL applies the same rule, but config
// cannot import storage, which imports config.
if !strings.HasPrefix(c.StorageURL, "file:///") {
return errFileURLNotAbsolute
}
if parsed.Scheme != storageurl.SchemeS3 {
return nil
}
case strings.HasPrefix(c.StorageURL, "rclone://"):
// Rclone storage uses rclone's own config
return nil
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
return nil
default:
return errBadStorageScheme
}
return nil
}
// Module exports the config module for fx dependency injection.
+2 -3
View File
@@ -10,7 +10,6 @@ import (
"sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
)
const (
@@ -503,8 +502,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
}
_, err = Load(path)
if !errors.Is(err, storageurl.ErrRelativeFilePath) {
t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
if !errors.Is(err, errFileURLNotAbsolute) {
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
}
if !strings.Contains(err.Error(), "absolute path") {
+35
View File
@@ -4,8 +4,10 @@ package snapshot
import (
"os"
"path/filepath"
"syscall"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -38,3 +40,36 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
}
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+4 -3
View File
@@ -1438,11 +1438,12 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
}
// deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat, as from the
// in-memory test filesystem.
// file info describes, and false when info carries no stat from a real
// filesystem. The in-memory test filesystem gives either no stat or one
// whose device and inode numbers are both zero, which no real file has.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
if !ok || (stat.Dev == 0 && stat.Ino == 0) {
return 0, false
}
+9 -4
View File
@@ -118,17 +118,22 @@ func ShortHostname(hostname string) string {
// hostname_timestamp if name is empty. The timestamp is in whole seconds.
// If the local index already has a snapshot with that ID, from a run of the
// same name that started in the same second, it waits a second and takes a
// new timestamp.
// new timestamp. The snapshot's start time is the moment in its ID, so a
// later run of one name always has a later start time.
func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) {
shortHostname := ShortHostname(hostname)
var snapshotID string
var (
snapshotID string
startedAt time.Time
)
for {
// Build snapshot ID with optional name
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
startedAt = time.Now().UTC()
timestamp := startedAt.Format("2006-01-02T15:04:05Z")
if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -153,7 +158,7 @@ func (sm *SnapshotManager) CreateSnapshotWithName(
Hostname: types.Hostname(hostname),
VaultikVersion: types.Version(version),
VaultikGitRevision: types.GitRevision(gitRevision),
StartedAt: time.Now().UTC(),
StartedAt: startedAt,
CompletedAt: nil, // Not completed yet
FileCount: 0,
ChunkCount: 0,
+52
View File
@@ -492,3 +492,55 @@ func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
t.Fatalf("both creates returned snapshot ID %s", first)
}
}
// Two creates of one snapshot name back to back list newest first. The
// start time is the moment in the ID, so the two cannot share a start
// time even when the clock crosses a second while the first is created.
// See https://git.eeqj.de/sneak/vaultik/issues/482.
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const created = 2
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
sm := &SnapshotManager{repos: repos}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
if err != nil {
t.Fatalf("listing snapshots failed: %v", err)
}
if len(snapshots) != created {
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
}
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
t.Fatalf("start times %v and %v are not in order",
snapshots[1].StartedAt, snapshots[0].StartedAt)
}
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
t.Fatalf("listed %s, %s; want %s, %s",
snapshots[0].ID, snapshots[1].ID, second, first)
}
}
+1 -2
View File
@@ -9,7 +9,6 @@ import (
"strings"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/storageurl"
)
// FileStorer implements Storer using the local filesystem.
@@ -289,7 +288,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
// Info returns human-readable storage location information.
func (f *FileStorer) Info() Info {
return Info{
Type: storageurl.SchemeFile,
Type: schemeFile,
Location: f.basePath,
}
}
+5 -11
View File
@@ -2,20 +2,14 @@ package storage
import (
"context"
"errors"
"fmt"
"strings"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storageurl"
)
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
// has no storage backend.
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
// defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1"
@@ -46,19 +40,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := storageurl.Parse(rawURL)
parsed, err := ParseStorageURL(rawURL)
if err != nil {
return nil, fmt.Errorf("parsing storage URL: %w", err)
}
switch parsed.Scheme {
case storageurl.SchemeFile:
case schemeFile:
return NewFileStorer(parsed.Prefix)
case storageurl.SchemeS3:
case schemeS3:
return storerFromParsedS3URL(parsed, cfg)
case storageurl.SchemeRclone:
case schemeRclone:
return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix)
@@ -71,7 +65,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
// filling endpoint protocol and region defaults from the config.
//
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL
endpoint := parsed.Endpoint
if endpoint == "" {
+24 -2
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
@@ -15,7 +16,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/operations"
"sneak.berlin/go/vaultik/internal/storageurl"
// Import all rclone backends
_ "github.com/rclone/rclone/backend/all"
@@ -24,6 +24,15 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct {
@@ -54,6 +63,19 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path
}
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the
@@ -267,7 +289,7 @@ func (r *RcloneStorer) Info() Info {
}
return Info{
Type: storageurl.SchemeRclone,
Type: schemeRclone,
Location: location,
}
}
+73
View File
@@ -436,6 +436,79 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
}
}
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one
@@ -1,7 +1,4 @@
// Package storageurl parses the storage_url config value. The config
// package uses it to refuse a bad value when the config loads; the storage
// package uses it to open the destination.
package storageurl
package storage
import (
"errors"
@@ -13,9 +10,9 @@ import (
// Storage URL scheme names.
const (
SchemeFile = "file"
SchemeS3 = "s3"
SchemeRclone = "rclone"
schemeFile = "file"
schemeS3 = "s3"
schemeRclone = "rclone"
)
// Sentinel errors for storage URL parsing.
@@ -28,7 +25,8 @@ var (
ErrMissingRemote = errors.New("rclone URL missing remote name")
ErrUnsupportedScheme = errors.New(
"unsupported URL scheme: must start with s3://, file://, or rclone://")
ErrURLCredentials = errors.New(
ErrUnsupportedStorage = errors.New("unsupported storage scheme")
ErrURLCredentials = errors.New(
"storage URL must not carry credentials; " +
"set s3.access_key_id and s3.secret_access_key in the config instead")
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
@@ -51,12 +49,12 @@ type URL struct {
RcloneRemote string // rclone remote name (for rclone:// URLs)
}
// Parse parses a storage URL string.
// ParseStorageURL parses a storage URL string.
// Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true
// - file:///absolute/path/to/backup
// - rclone://remote/path/to/backups
func Parse(rawURL string) (*URL, error) {
func ParseStorageURL(rawURL string) (*URL, error) {
if rawURL == "" {
return nil, ErrEmptyStorageURL
}
@@ -94,7 +92,7 @@ func parseFileURL(rawPath string) (*URL, error) {
return nil, fmt.Errorf("invalid file URL path: %w", err)
}
return &URL{Scheme: SchemeFile, Prefix: path}, nil
return &URL{Scheme: schemeFile, Prefix: path}, nil
}
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
@@ -137,7 +135,7 @@ func parseS3URL(rawURL string) (*URL, error) {
}
return &URL{
Scheme: SchemeS3,
Scheme: schemeS3,
Bucket: bucket,
Prefix: strings.TrimPrefix(u.Path, "/"),
Endpoint: query.Get("endpoint"),
@@ -191,7 +189,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
}
return &URL{
Scheme: SchemeRclone,
Scheme: schemeRclone,
Prefix: strings.TrimPrefix(u.Path, "/"),
RcloneRemote: remote,
}, nil
@@ -228,9 +226,9 @@ func wrapParseError(err error) error {
// String returns a human-readable representation of the storage URL.
func (u *URL) String() string {
switch u.Scheme {
case SchemeFile:
case schemeFile:
return "file://" + u.Prefix
case SchemeS3:
case schemeS3:
endpoint := u.Endpoint
if endpoint == "" {
endpoint = "s3.amazonaws.com"
@@ -241,7 +239,7 @@ func (u *URL) String() string {
}
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case SchemeRclone:
case schemeRclone:
if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
}
@@ -1,4 +1,4 @@
package storageurl_test
package storage_test
import (
"errors"
@@ -6,12 +6,12 @@ import (
"strings"
"testing"
"sneak.berlin/go/vaultik/internal/storageurl"
"sneak.berlin/go/vaultik/internal/storage"
)
// TestParseValid checks that each supported scheme parses into the
// expected fields, since those fields decide which backend is built.
func TestParseValid(t *testing.T) {
// TestParseStorageURLValid checks that each supported scheme parses into
// the expected fields, since those fields decide which backend is built.
func TestParseStorageURLValid(t *testing.T) {
t.Parallel()
const (
@@ -22,27 +22,27 @@ func TestParseValid(t *testing.T) {
cases := []struct {
name string
raw string
want *storageurl.URL
want *storage.URL
}{
{
name: "file absolute path",
raw: "file:///var/backups/vaultik",
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
},
{
name: "file percent-escaped path is decoded",
raw: "file:///Volumes/my%20stick/backup",
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
},
{
name: "file path with a literal space",
raw: "file:///Volumes/my stick/backup",
want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
},
{
name: "s3 bucket and prefix, ssl defaults on",
raw: "s3://mybucket/backups/host",
want: &storageurl.URL{
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Prefix: "backups/host", UseSSL: true,
},
@@ -50,12 +50,12 @@ func TestParseValid(t *testing.T) {
{
name: "s3 bucket only",
raw: "s3://mybucket",
want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
},
{
name: "s3 with endpoint, region, ssl off",
raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false",
want: &storageurl.URL{
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
},
@@ -63,14 +63,14 @@ func TestParseValid(t *testing.T) {
{
name: "rclone remote and path",
raw: "rclone://gdrive/backups/host",
want: &storageurl.URL{
want: &storage.URL{
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
},
},
{
name: "rclone remote only",
raw: "rclone://gdrive",
want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
},
}
@@ -78,22 +78,22 @@ func TestParseValid(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := storageurl.Parse(tc.raw)
got, err := storage.ParseStorageURL(tc.raw)
if err != nil {
t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
}
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
}
})
}
}
// TestParseErrors checks that empty, missing, and unknown-scheme inputs
// fail with the documented sentinel errors instead of parsing to a wrong
// destination.
func TestParseErrors(t *testing.T) {
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
// inputs fail with the documented sentinel errors instead of parsing to a
// wrong destination.
func TestParseStorageURLErrors(t *testing.T) {
t.Parallel()
cases := []struct {
@@ -101,33 +101,34 @@ func TestParseErrors(t *testing.T) {
raw string
wantErr error
}{
{"empty url", "", storageurl.ErrEmptyStorageURL},
{"file empty path", "file://", storageurl.ErrEmptyFilePath},
{"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
{"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
{"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
{"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
{"empty url", "", storage.ErrEmptyStorageURL},
{"file empty path", "file://", storage.ErrEmptyFilePath},
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storageurl.Parse(tc.raw)
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, tc.wantErr) {
t.Errorf("Parse(%q) error = %v, want %v",
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, tc.wantErr)
}
})
}
}
// TestParseRejectsCredentials checks that a URL carrying credentials in
// its userinfo or in an unknown query parameter is rejected, and that the
// error never echoes the secret-bearing URL back into logs or output.
func TestParseRejectsCredentials(t *testing.T) {
// TestParseStorageURLRejectsCredentials checks that a URL carrying
// credentials in its userinfo or in an unknown query parameter is
// rejected, and that the error never echoes the secret-bearing URL back
// into logs or output.
func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Parallel()
// Split so the literals never form a "user:pass@" URL pattern that
@@ -146,44 +147,44 @@ func TestParseRejectsCredentials(t *testing.T) {
{
name: "s3 userinfo",
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
wantErr: storageurl.ErrURLCredentials,
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret, "mybucket"},
},
{
name: "s3 userinfo in endpoint",
raw: "s3://mybucket?endpoint=https://" +
key + ":" + secret + "@minio.example.com",
wantErr: storageurl.ErrURLCredentials,
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{
name: "s3 userinfo in endpoint without scheme",
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
wantErr: storageurl.ErrURLCredentials,
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{
name: "s3 unknown query param",
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
wantErr: storageurl.ErrURLUnknownParam,
wantErr: storage.ErrURLUnknownParam,
secrets: []string{key, secret},
},
{
name: "s3 misspelt endpoint",
raw: "s3://mybucket?endpiont=minio.example.com",
wantErr: storageurl.ErrURLUnknownParam,
wantErr: storage.ErrURLUnknownParam,
secrets: nil,
},
{
name: "rclone userinfo",
raw: "rclone://user:" + secret + "@gdrive/backups",
wantErr: storageurl.ErrURLCredentials,
wantErr: storage.ErrURLCredentials,
secrets: []string{secret},
},
{
name: "rclone query param",
raw: "rclone://gdrive/backups?token=" + secret,
wantErr: storageurl.ErrURLUnknownParam,
wantErr: storage.ErrURLUnknownParam,
secrets: []string{secret},
},
}
@@ -192,9 +193,9 @@ func TestParseRejectsCredentials(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storageurl.Parse(tc.raw)
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, tc.wantErr) {
t.Fatalf("Parse(%q) error = %v, want %v",
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, tc.wantErr)
}
@@ -215,11 +216,11 @@ func TestParseRejectsCredentials(t *testing.T) {
}
}
// TestParseRejectsRcloneRemoteParams checks that an rclone remote name
// carrying parameters (rclone's "remote,param=value" form, which can hold
// a credential) is rejected without echoing the value, including the URLs
// that url.Parse itself refuses.
func TestParseRejectsRcloneRemoteParams(t *testing.T) {
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
// remote name carrying parameters (rclone's "remote,param=value" form,
// which can hold a credential) is rejected without echoing the value,
// including the URLs that url.Parse itself refuses.
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
t.Parallel()
cases := []struct {
@@ -248,10 +249,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storageurl.Parse(tc.raw)
if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
t.Fatalf("Parse(%q) error = %v, want %v",
tc.raw, err, storageurl.ErrRcloneRemoteParams)
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrRcloneRemoteParams)
}
for _, secret := range tc.secrets {
@@ -263,10 +264,10 @@ func TestParseRejectsRcloneRemoteParams(t *testing.T) {
}
}
// TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
// query cannot be decoded is rejected, not read with that part of the
// query left out, and that the error does not echo it.
func TestParseRejectsUndecodableQuery(t *testing.T) {
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
// URL whose query cannot be decoded is rejected, not read with that part
// of the query left out, and that the error does not echo it.
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
t.Parallel()
const (
@@ -302,14 +303,14 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storageurl.Parse(tc.raw)
_, err := storage.ParseStorageURL(tc.raw)
if err == nil {
t.Fatalf("Parse(%q) returned no error", tc.raw)
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
}
if !errors.Is(err, storageurl.ErrURLBadQuery) {
t.Errorf("Parse(%q) error = %v, want %v",
tc.raw, err, storageurl.ErrURLBadQuery)
if !errors.Is(err, storage.ErrURLBadQuery) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrURLBadQuery)
}
for _, s := range []string{key, secret, "minio.example.com"} {
@@ -321,17 +322,17 @@ func TestParseRejectsUndecodableQuery(t *testing.T) {
}
}
// TestParseFailureHidesURL checks that when url.Parse itself fails, the
// wrapped error carries only the inner cause, not the *url.Error whose
// text embeds the raw (possibly credential-bearing) URL.
func TestParseFailureHidesURL(t *testing.T) {
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
// itself fails, the wrapped error carries only the inner cause, not the
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
func TestParseStorageURLParseFailureHidesURL(t *testing.T) {
t.Parallel()
const raw = "s3://mybucket/%zz"
_, err := storageurl.Parse(raw)
_, err := storage.ParseStorageURL(raw)
if err == nil {
t.Fatalf("Parse(%q) returned no error", raw)
t.Fatalf("ParseStorageURL(%q) returned no error", raw)
}
if strings.Contains(err.Error(), "mybucket") {