Vertaa committeja

..
Tekijä SHA1 Viesti Päivämäärä
sneak 4e084a2746 Check storage_url with the storage parser at config load (closes #468)
check / check (push) Successful in 11m20s
config set accepted rclone:// values that every command then refused:
no remote name, parameters in the remote name, a query parameter. The
config only checked the URL's prefix, because config could not import
the parser from storage, which imports config.

The parser moves from internal/storage to a new package,
internal/storageurl, that both import. ParseStorageURL becomes
storageurl.Parse. Loading the config now runs it on every storage_url,
so config set refuses such a value and leaves the file unchanged.

Model: opus-5-5
2026-11-19 07:24:47 +00:00
15 muutettua tiedostoa jossa 172 lisäystä ja 567 poistoa

No files matched your search

+1 -9
Näytä tiedosto
@@ -602,11 +602,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The providers. Requires rclone to be configured separately (`rclone config`).
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
@@ -618,10 +614,6 @@ being written, a killed upload can leave a truncated object under its name,
which a later backup takes for complete. A leftover `.partial` file is ignored which a later backup takes for complete. A leftover `.partial` file is ignored
and can be deleted. and can be deleted.
A blob upload that fails, such as one cut off by a lost connection, is tried
again up to five times, three seconds apart. Each new try is announced with a
warning. If the last try fails too, the backup stops with the upload's error.
A destination shared between hosts or users, each with its own local index, A destination shared between hosts or users, each with its own local index,
relies on `prune_grace_period` to keep one's prune from deleting the blobs of relies on `prune_grace_period` to keep one's prune from deleting the blobs of
another's backup still running, or its directory under `metadata/` that has no another's backup still running, or its directory under `metadata/` that has no
+7
Näytä tiedosto
@@ -282,6 +282,13 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
storageURLKey, testS3URL, storageURLKey, testS3URL,
"s3.secret_access_key is required for s3:// URLs; " + "s3.secret_access_key is required for s3:// URLs; " +
"set s3.access_key_id and s3.secret_access_key first"}, "set s3.access_key_id and s3.secret_access_key first"},
// rclone:// URLs that no command could open.
{configWithoutRecipients, storageURLKey, "rclone:///backups",
"rclone URL missing remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav,user=x/backups",
"rclone URL must not carry parameters in the remote name"},
{configWithoutRecipients, storageURLKey, "rclone://dav/backups?x=1",
`unknown query parameter in storage URL: "x"`},
// With its reference filled in, the config loads. // With its reference filled in, the config loads.
{levelReference, "blob_size_limit", "1MB", {levelReference, "blob_size_limit", "1MB",
"blob_size_limit must be at least"}, "blob_size_limit must be at least"},
+24 -37
Näytä tiedosto
@@ -21,6 +21,7 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
const appName = "vaultik" const appName = "vaultik"
@@ -63,13 +64,8 @@ var (
errBlobSizeTooSmall = errors.New( errBlobSizeTooSmall = errors.New(
"blob_size_limit must be at least the largest chunk the chunker can " + "blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)") "emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19") errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB") errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://")
errFileURLNotAbsolute = errors.New(
"a file:// storage_url needs an absolute path, " +
"as in file:///path/to/backup")
errIndexPathNotAbsolute = errors.New( errIndexPathNotAbsolute = errors.New(
"index_path and VAULTIK_INDEX_PATH need an absolute path, " + "index_path and VAULTIK_INDEX_PATH need an absolute path, " +
"as in ~/.local/share/vaultik/index.sqlite") "as in ~/.local/share/vaultik/index.sqlite")
@@ -557,38 +553,29 @@ func (c *Config) validateStorage() error {
return nil return nil
} }
// validateStorageURL validates URL-based storage configuration. File and // validateStorageURL checks storage_url with the same parser that opens
// rclone URLs need no credentials; S3 URLs require the legacy s3.* // the destination, so a value every command would refuse fails at load.
// credential fields. A file URL must name an absolute path: a relative one // File and rclone URLs need no credentials; S3 URLs require the legacy
// would resolve against the working directory, so the destination would // s3.* credential fields.
// move with it.
func (c *Config) validateStorageURL() error { func (c *Config) validateStorageURL() error {
switch { parsed, err := storageurl.Parse(c.StorageURL)
case strings.HasPrefix(c.StorageURL, "file://"): if err != nil {
// storage.ParseStorageURL applies the same rule, but config return fmt.Errorf("invalid storage_url: %w", err)
// cannot import storage, which imports config.
if !strings.HasPrefix(c.StorageURL, "file:///") {
return errFileURLNotAbsolute
}
return nil
case strings.HasPrefix(c.StorageURL, "rclone://"):
// Rclone storage uses rclone's own config
return nil
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
}
return nil
default:
return errBadStorageScheme
} }
if parsed.Scheme != storageurl.SchemeS3 {
return nil
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
}
return nil
} }
// Module exports the config module for fx dependency injection. // Module exports the config module for fx dependency injection.
+3 -2
Näytä tiedosto
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
const ( const (
@@ -502,8 +503,8 @@ func TestLoadRejectsRelativeFileURL(t *testing.T) {
} }
_, err = Load(path) _, err = Load(path)
if !errors.Is(err, errFileURLNotAbsolute) { if !errors.Is(err, storageurl.ErrRelativeFilePath) {
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err) t.Fatalf("Load() error = %v, want storageurl.ErrRelativeFilePath", err)
} }
if !strings.Contains(err.Error(), "absolute path") { if !strings.Contains(err.Error(), "absolute path") {
@@ -1,154 +0,0 @@
//nolint:testpackage // exercises the unexported uploadBlobIfNeeded
package snapshot
import (
"bytes"
"context"
"io"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blob"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/storage/faultstore"
"sneak.berlin/go/vaultik/internal/ui"
)
// retryTestBlobSize is larger than what a failing faultstore Put reads,
// so a new try that did not rewind the blob would store a short object.
const retryTestBlobSize = 4096
// retryTestBlobKey is where the retry tests upload their blob.
const retryTestBlobKey = "blobs/retry"
// uploadRetryTestBlob uploads a retryTestBlobSize blob through store,
// waiting delay between tries, and returns the blob's bytes, the warnings
// the upload printed and its error.
func uploadRetryTestBlob(
ctx context.Context, store storage.Storer, delay time.Duration,
) ([]byte, int, error) {
data := bytes.Repeat([]byte("0123456789abcdef"), retryTestBlobSize/16)
output := ui.NewWithColor(io.Discard, false)
s := &Scanner{storage: store, ui: output, blobUploadRetryDelay: delay}
_, _, err := s.uploadBlobIfNeeded(ctx, retryTestBlobKey, &blob.WithReader{
FinishedBlob: &blob.FinishedBlob{
Hash: "retry",
Compressed: int64(len(data)),
},
Reader: bytes.NewReader(data),
}, time.Now(), &ScanResult{})
return data, output.WarningCount(), err
}
// A blob upload that fails is tried again from the start of the blob,
// and each failure is a warning. See
// https://git.eeqj.de/sneak/vaultik/issues/476.
func TestFailedBlobUploadIsTriedAgain(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
if puts <= 2 {
return faultstore.PutFail
}
return faultstore.PutNormal
}
data, warnings, err := uploadRetryTestBlob(context.Background(), store, 0)
require.NoError(t, err)
require.Equal(t, 3, puts)
require.Equal(t, 2, warnings)
rc, err := inner.Get(context.Background(), retryTestBlobKey)
require.NoError(t, err)
defer func() { _ = rc.Close() }()
stored, err := io.ReadAll(rc)
require.NoError(t, err)
require.Equal(t, data, stored)
}
// A blob upload that keeps failing is tried blobUploadRetries more times,
// BlobUploadRetryDelay apart, and then the backup stops with its error.
func TestBlobUploadStopsAfterRetries(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
return faultstore.PutFail
}
const delay = 10 * time.Millisecond
start := time.Now()
_, warnings, err := uploadRetryTestBlob(context.Background(), store, delay)
require.ErrorIs(t, err, faultstore.ErrInjectedUpload)
require.Equal(t, 1+blobUploadRetries, puts)
require.Equal(t, blobUploadRetries, warnings)
require.GreaterOrEqual(t, time.Since(start), blobUploadRetries*delay)
}
// An upload stopped by Ctrl-C or SIGTERM is not tried again.
func TestInterruptedBlobUploadIsNotTriedAgain(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
cancel()
return faultstore.PutFail
}
_, warnings, err := uploadRetryTestBlob(ctx, store, 0)
require.Error(t, err)
require.Equal(t, 1, puts)
require.Zero(t, warnings)
}
// Backups wait BlobUploadRetryDelay before each new try.
func TestBackupScannerWaitsBetweenBlobUploadRetries(t *testing.T) {
t.Parallel()
cfg := &config.Config{
AgeRecipients: []string{testAgeRecipient},
ChunkSize: config.Size(1024 * 1024),
BlobSizeLimit: config.Size(4 * 1024 * 1024),
IndexPath: filepath.Join(t.TempDir(), "index.sqlite"),
}
scanner := provideScannerFactory(cfg, nil, nil)(ScannerParams{
Fs: afero.NewMemMapFs(),
})
require.NotNil(t, scanner)
require.Equal(t, BlobUploadRetryDelay, scanner.blobUploadRetryDelay)
}
@@ -4,10 +4,8 @@ package snapshot
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"syscall"
"testing" "testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -40,36 +38,3 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker. // /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev")) assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
} }
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+14 -15
Näytä tiedosto
@@ -54,21 +54,20 @@ func provideScannerFactory(
} }
return NewScanner(ScannerConfig{ return NewScanner(ScannerConfig{
FS: params.Fs, FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(), ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos, Repositories: repos,
Storage: storer, Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(), MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel, CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients, AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress, EnableProgress: params.EnableProgress,
UI: params.UI, UI: params.UI,
Exclude: excludes, Exclude: excludes,
IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()), IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()),
SkipErrors: params.SkipErrors, SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID, PreviousSnapshotID: params.PreviousSnapshotID,
BlobUploadRetryDelay: BlobUploadRetryDelay, DeviceNumber: params.DeviceNumber,
DeviceNumber: params.DeviceNumber,
}) })
} }
} }
+24 -75
Näytä tiedosto
@@ -75,8 +75,6 @@ type Scanner struct {
// skipErrors skips files that cannot be opened or read (logged loudly); // skipErrors skips files that cannot be opened or read (logged loudly);
// packer, database, encryption, and upload errors still abort the run. // packer, database, encryption, and upload errors still abort the run.
skipErrors bool skipErrors bool
// blobUploadRetryDelay is ScannerConfig.BlobUploadRetryDelay.
blobUploadRetryDelay time.Duration
// ui is the user-facing output; never nil (defaults to a discarding writer). // ui is the user-facing output; never nil (defaults to a discarding writer).
ui *ui.Writer ui *ui.Writer
// deviceNumber returns the number of the device that holds a file; // deviceNumber returns the number of the device that holds a file;
@@ -125,16 +123,8 @@ const (
// snapshotAssocBatchSize is how many file IDs are associated with the // snapshotAssocBatchSize is how many file IDs are associated with the
// snapshot per transaction. // snapshot per transaction.
snapshotAssocBatchSize = 1000 snapshotAssocBatchSize = 1000
// blobUploadRetries is how many more times a failed blob upload is
// tried before the backup stops with the error.
blobUploadRetries = 5
) )
// BlobUploadRetryDelay is the wait before each new try of a failed blob
// upload.
const BlobUploadRetryDelay = 3 * time.Second
// ScannerConfig contains configuration for the scanner // ScannerConfig contains configuration for the scanner
type ScannerConfig struct { type ScannerConfig struct {
FS afero.Fs FS afero.Fs
@@ -158,10 +148,6 @@ type ScannerConfig struct {
// PreviousSnapshotID is the most recent completed snapshot of the // PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none. // same name, "" for none.
PreviousSnapshotID string PreviousSnapshotID string
// BlobUploadRetryDelay is the wait before each new try of a failed
// blob upload. Backups use snapshot.BlobUploadRetryDelay; tests leave
// it zero to try again at once.
BlobUploadRetryDelay time.Duration
// DeviceNumber returns the number of the device that holds a file, // DeviceNumber returns the number of the device that holds a file,
// and false when there is none to read. nil reads it from the file's // and false when there is none to read. nil reads it from the file's
// stat; tests replace it, since the in-memory test filesystem has none. // stat; tests replace it, since the in-memory test filesystem has none.
@@ -259,24 +245,23 @@ func NewScanner(cfg ScannerConfig) *Scanner {
} }
return &Scanner{ return &Scanner{
fs: cfg.FS, fs: cfg.FS,
chunker: chunker.NewChunker(cfg.ChunkSize), chunker: chunker.NewChunker(cfg.ChunkSize),
packer: packer, packer: packer,
repos: cfg.Repositories, repos: cfg.Repositories,
storage: cfg.Storage, storage: cfg.Storage,
maxBlobSize: cfg.MaxBlobSize, maxBlobSize: cfg.MaxBlobSize,
compressionLevel: cfg.CompressionLevel, compressionLevel: cfg.CompressionLevel,
ageRecipient: strings.Join(cfg.AgeRecipients, ","), ageRecipient: strings.Join(cfg.AgeRecipients, ","),
exclude: cfg.Exclude, exclude: cfg.Exclude,
compiledExclude: compiledExclude, compiledExclude: compiledExclude,
indexAndLockFiles: indexAndLockFiles, indexAndLockFiles: indexAndLockFiles,
progress: progress, progress: progress,
skipErrors: cfg.SkipErrors, skipErrors: cfg.SkipErrors,
previousSnapshotID: cfg.PreviousSnapshotID, previousSnapshotID: cfg.PreviousSnapshotID,
blobUploadRetryDelay: cfg.BlobUploadRetryDelay, ui: uiw,
ui: uiw, deviceNumber: deviceNumber,
deviceNumber: deviceNumber, pendingChunkHashes: make(map[string]struct{}),
pendingChunkHashes: make(map[string]struct{}),
} }
} }
@@ -1453,12 +1438,11 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
} }
// deviceNumberFromStat returns the number of the device that holds the // deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat from a real // file info describes, and false when info carries no stat, as from the
// filesystem. The in-memory test filesystem gives either no stat or one // in-memory test filesystem.
// whose device and inode numbers are both zero, which no real file has.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) { func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t) stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || (stat.Dev == 0 && stat.Ino == 0) { if !ok {
return 0, false return 0, false
} }
@@ -1891,7 +1875,10 @@ func (s *Scanner) uploadBlobIfNeeded(
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
s.ui.Path(destination)) s.ui.Path(destination))
err = s.putBlob(ctx, blobPath, blobWithReader) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
finishedBlob.Compressed, progressCallback)
if err != nil { if err != nil {
// The record is at info level: the returned error ends the backup // The record is at info level: the returned error ends the backup
// and is printed as its ERROR: line. Only an interrupt cancels // and is printed as its ERROR: line. Only an interrupt cancels
@@ -1935,44 +1922,6 @@ func (s *Scanner) uploadBlobIfNeeded(
return finishedBlob.Compressed, false, nil return finishedBlob.Compressed, false, nil
} }
// putBlob uploads the blob to blobPath. After a failed upload it waits
// s.blobUploadRetryDelay and uploads the whole blob again, up to
// blobUploadRetries times, and then returns the last error. An interrupt
// stops it at once.
func (s *Scanner) putBlob(
ctx context.Context, blobPath string, blobWithReader *blob.WithReader,
) error {
finishedBlob := blobWithReader.FinishedBlob
for retries := 0; ; retries++ {
_, err := blobWithReader.Reader.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("rewinding blob for upload: %w", err)
}
progressCallback := s.makeUploadProgressCallback(
ctx, finishedBlob, time.Now().UTC())
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
finishedBlob.Compressed, progressCallback)
if err == nil || ctx.Err() != nil || retries == blobUploadRetries {
return err
}
log.Info("Blob upload failed, trying again",
"hash", finishedBlob.Hash, "retry", retries+1, "error", err)
s.ui.Warningf("Upload of blob %s failed: %v. Trying again in %s (retry %d of %d).",
s.ui.Hex(finishedBlob.Hash), err,
s.ui.Duration(s.blobUploadRetryDelay), retries+1, blobUploadRetries)
select {
case <-ctx.Done():
return err
case <-time.After(s.blobUploadRetryDelay):
}
}
}
// makeUploadProgressCallback creates a progress callback for blob uploads. // makeUploadProgressCallback creates a progress callback for blob uploads.
// It updates the live progress reporter ~twice/sec for ETAs and prints a // It updates the live progress reporter ~twice/sec for ETAs and prints a
// human-readable status line to s.output at most every 15 seconds. // human-readable status line to s.output at most every 15 seconds.
+2 -1
Näytä tiedosto
@@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
// FileStorer implements Storer using the local filesystem. // FileStorer implements Storer using the local filesystem.
@@ -288,7 +289,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
// Info returns human-readable storage location information. // Info returns human-readable storage location information.
func (f *FileStorer) Info() Info { func (f *FileStorer) Info() Info {
return Info{ return Info{
Type: schemeFile, Type: storageurl.SchemeFile,
Location: f.basePath, Location: f.basePath,
} }
} }
+11 -5
Näytä tiedosto
@@ -2,14 +2,20 @@ package storage
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"strings" "strings"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3" "sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storageurl"
) )
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
// has no storage backend.
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
// defaultS3Region is used when neither the URL nor the config specify one. // defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1" const defaultS3Region = "us-east-1"
@@ -40,19 +46,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
//nolint:ireturn // factory intentionally returns the Storer interface //nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := ParseStorageURL(rawURL) parsed, err := storageurl.Parse(rawURL)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing storage URL: %w", err) return nil, fmt.Errorf("parsing storage URL: %w", err)
} }
switch parsed.Scheme { switch parsed.Scheme {
case schemeFile: case storageurl.SchemeFile:
return NewFileStorer(parsed.Prefix) return NewFileStorer(parsed.Prefix)
case schemeS3: case storageurl.SchemeS3:
return storerFromParsedS3URL(parsed, cfg) return storerFromParsedS3URL(parsed, cfg)
case schemeRclone: case storageurl.SchemeRclone:
return NewRcloneStorer( return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix) context.Background(), parsed.RcloneRemote, parsed.Prefix)
@@ -65,7 +71,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
// filling endpoint protocol and region defaults from the config. // filling endpoint protocol and region defaults from the config.
// //
//nolint:ireturn // factory intentionally returns the Storer interface //nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) { func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL // Build endpoint URL
endpoint := parsed.Endpoint endpoint := parsed.Endpoint
if endpoint == "" { if endpoint == "" {
+2 -24
Näytä tiedosto
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@@ -16,6 +15,7 @@ import (
"github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/operations"
"sneak.berlin/go/vaultik/internal/storageurl"
// Import all rclone backends // Import all rclone backends
_ "github.com/rclone/rclone/backend/all" _ "github.com/rclone/rclone/backend/all"
@@ -24,15 +24,6 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured. // ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config") var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction. // RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers. // This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct { type RcloneStorer struct {
@@ -63,19 +54,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path rclonePath += path
} }
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in // On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read. // each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the // rclone's use_server_modtime setting makes ModTime the time the
@@ -289,7 +267,7 @@ func (r *RcloneStorer) Info() Info {
} }
return Info{ return Info{
Type: schemeRclone, Type: storageurl.SchemeRclone,
Location: location, Location: location,
} }
} }
-73
Näytä tiedosto
@@ -436,79 +436,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
} }
} }
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an // TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By // rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one // default rclone reads the time it keeps in each object's metadata, one
@@ -1,4 +1,7 @@
package storage // Package storageurl parses the storage_url config value. The config
// package uses it to refuse a bad value when the config loads; the storage
// package uses it to open the destination.
package storageurl
import ( import (
"errors" "errors"
@@ -10,9 +13,9 @@ import (
// Storage URL scheme names. // Storage URL scheme names.
const ( const (
schemeFile = "file" SchemeFile = "file"
schemeS3 = "s3" SchemeS3 = "s3"
schemeRclone = "rclone" SchemeRclone = "rclone"
) )
// Sentinel errors for storage URL parsing. // Sentinel errors for storage URL parsing.
@@ -25,8 +28,7 @@ var (
ErrMissingRemote = errors.New("rclone URL missing remote name") ErrMissingRemote = errors.New("rclone URL missing remote name")
ErrUnsupportedScheme = errors.New( ErrUnsupportedScheme = errors.New(
"unsupported URL scheme: must start with s3://, file://, or rclone://") "unsupported URL scheme: must start with s3://, file://, or rclone://")
ErrUnsupportedStorage = errors.New("unsupported storage scheme") ErrURLCredentials = errors.New(
ErrURLCredentials = errors.New(
"storage URL must not carry credentials; " + "storage URL must not carry credentials; " +
"set s3.access_key_id and s3.secret_access_key in the config instead") "set s3.access_key_id and s3.secret_access_key in the config instead")
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL") ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
@@ -49,12 +51,12 @@ type URL struct {
RcloneRemote string // rclone remote name (for rclone:// URLs) RcloneRemote string // rclone remote name (for rclone:// URLs)
} }
// ParseStorageURL parses a storage URL string. // Parse parses a storage URL string.
// Supported formats: // Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true // - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true
// - file:///absolute/path/to/backup // - file:///absolute/path/to/backup
// - rclone://remote/path/to/backups // - rclone://remote/path/to/backups
func ParseStorageURL(rawURL string) (*URL, error) { func Parse(rawURL string) (*URL, error) {
if rawURL == "" { if rawURL == "" {
return nil, ErrEmptyStorageURL return nil, ErrEmptyStorageURL
} }
@@ -92,7 +94,7 @@ func parseFileURL(rawPath string) (*URL, error) {
return nil, fmt.Errorf("invalid file URL path: %w", err) return nil, fmt.Errorf("invalid file URL path: %w", err)
} }
return &URL{Scheme: schemeFile, Prefix: path}, nil return &URL{Scheme: SchemeFile, Prefix: path}, nil
} }
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
@@ -135,7 +137,7 @@ func parseS3URL(rawURL string) (*URL, error) {
} }
return &URL{ return &URL{
Scheme: schemeS3, Scheme: SchemeS3,
Bucket: bucket, Bucket: bucket,
Prefix: strings.TrimPrefix(u.Path, "/"), Prefix: strings.TrimPrefix(u.Path, "/"),
Endpoint: query.Get("endpoint"), Endpoint: query.Get("endpoint"),
@@ -189,7 +191,7 @@ func parseRcloneURL(rawURL string) (*URL, error) {
} }
return &URL{ return &URL{
Scheme: schemeRclone, Scheme: SchemeRclone,
Prefix: strings.TrimPrefix(u.Path, "/"), Prefix: strings.TrimPrefix(u.Path, "/"),
RcloneRemote: remote, RcloneRemote: remote,
}, nil }, nil
@@ -226,9 +228,9 @@ func wrapParseError(err error) error {
// String returns a human-readable representation of the storage URL. // String returns a human-readable representation of the storage URL.
func (u *URL) String() string { func (u *URL) String() string {
switch u.Scheme { switch u.Scheme {
case schemeFile: case SchemeFile:
return "file://" + u.Prefix return "file://" + u.Prefix
case schemeS3: case SchemeS3:
endpoint := u.Endpoint endpoint := u.Endpoint
if endpoint == "" { if endpoint == "" {
endpoint = "s3.amazonaws.com" endpoint = "s3.amazonaws.com"
@@ -239,7 +241,7 @@ func (u *URL) String() string {
} }
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case schemeRclone: case SchemeRclone:
if u.Prefix != "" { if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
} }
@@ -1,4 +1,4 @@
package storage_test package storageurl_test
import ( import (
"errors" "errors"
@@ -6,12 +6,12 @@ import (
"strings" "strings"
"testing" "testing"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storageurl"
) )
// TestParseStorageURLValid checks that each supported scheme parses into // TestParseValid checks that each supported scheme parses into the
// the expected fields, since those fields decide which backend is built. // expected fields, since those fields decide which backend is built.
func TestParseStorageURLValid(t *testing.T) { func TestParseValid(t *testing.T) {
t.Parallel() t.Parallel()
const ( const (
@@ -22,27 +22,27 @@ func TestParseStorageURLValid(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
raw string raw string
want *storage.URL want *storageurl.URL
}{ }{
{ {
name: "file absolute path", name: "file absolute path",
raw: "file:///var/backups/vaultik", raw: "file:///var/backups/vaultik",
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
}, },
{ {
name: "file percent-escaped path is decoded", name: "file percent-escaped path is decoded",
raw: "file:///Volumes/my%20stick/backup", raw: "file:///Volumes/my%20stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
}, },
{ {
name: "file path with a literal space", name: "file path with a literal space",
raw: "file:///Volumes/my stick/backup", raw: "file:///Volumes/my stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"}, want: &storageurl.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
}, },
{ {
name: "s3 bucket and prefix, ssl defaults on", name: "s3 bucket and prefix, ssl defaults on",
raw: "s3://mybucket/backups/host", raw: "s3://mybucket/backups/host",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "s3", Bucket: bucket, Scheme: "s3", Bucket: bucket,
Prefix: "backups/host", UseSSL: true, Prefix: "backups/host", UseSSL: true,
}, },
@@ -50,12 +50,12 @@ func TestParseStorageURLValid(t *testing.T) {
{ {
name: "s3 bucket only", name: "s3 bucket only",
raw: "s3://mybucket", raw: "s3://mybucket",
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true}, want: &storageurl.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
}, },
{ {
name: "s3 with endpoint, region, ssl off", name: "s3 with endpoint, region, ssl off",
raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false", raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "s3", Bucket: bucket, Scheme: "s3", Bucket: bucket,
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false, Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
}, },
@@ -63,14 +63,14 @@ func TestParseStorageURLValid(t *testing.T) {
{ {
name: "rclone remote and path", name: "rclone remote and path",
raw: "rclone://gdrive/backups/host", raw: "rclone://gdrive/backups/host",
want: &storage.URL{ want: &storageurl.URL{
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host", Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
}, },
}, },
{ {
name: "rclone remote only", name: "rclone remote only",
raw: "rclone://gdrive", raw: "rclone://gdrive",
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"}, want: &storageurl.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
}, },
} }
@@ -78,22 +78,22 @@ func TestParseStorageURLValid(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
got, err := storage.ParseStorageURL(tc.raw) got, err := storageurl.Parse(tc.raw)
if err != nil { if err != nil {
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err) t.Fatalf("Parse(%q) returned error: %v", tc.raw, err)
} }
if !reflect.DeepEqual(got, tc.want) { if !reflect.DeepEqual(got, tc.want) {
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want) t.Errorf("Parse(%q) = %+v, want %+v", tc.raw, got, tc.want)
} }
}) })
} }
} }
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme // TestParseErrors checks that empty, missing, and unknown-scheme inputs
// inputs fail with the documented sentinel errors instead of parsing to a // fail with the documented sentinel errors instead of parsing to a wrong
// wrong destination. // destination.
func TestParseStorageURLErrors(t *testing.T) { func TestParseErrors(t *testing.T) {
t.Parallel() t.Parallel()
cases := []struct { cases := []struct {
@@ -101,34 +101,33 @@ func TestParseStorageURLErrors(t *testing.T) {
raw string raw string
wantErr error wantErr error
}{ }{
{"empty url", "", storage.ErrEmptyStorageURL}, {"empty url", "", storageurl.ErrEmptyStorageURL},
{"file empty path", "file://", storage.ErrEmptyFilePath}, {"file empty path", "file://", storageurl.ErrEmptyFilePath},
{"file relative path", "file://dest", storage.ErrRelativeFilePath}, {"file relative path", "file://dest", storageurl.ErrRelativeFilePath},
{"s3 missing bucket", "s3://", storage.ErrMissingBucket}, {"s3 missing bucket", "s3://", storageurl.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket}, {"s3 missing bucket with path", "s3:///justprefix", storageurl.ErrMissingBucket},
{"rclone missing remote", "rclone://", storage.ErrMissingRemote}, {"rclone missing remote", "rclone://", storageurl.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme}, {"unknown scheme", "gs://bucket/x", storageurl.ErrUnsupportedScheme},
{"no scheme", "/local/path", storage.ErrUnsupportedScheme}, {"no scheme", "/local/path", storageurl.ErrUnsupportedScheme},
} }
for _, tc := range cases { for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, tc.wantErr) { if !errors.Is(err, tc.wantErr) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v", t.Errorf("Parse(%q) error = %v, want %v",
tc.raw, err, tc.wantErr) tc.raw, err, tc.wantErr)
} }
}) })
} }
} }
// TestParseStorageURLRejectsCredentials checks that a URL carrying // TestParseRejectsCredentials checks that a URL carrying credentials in
// credentials in its userinfo or in an unknown query parameter is // its userinfo or in an unknown query parameter is rejected, and that the
// rejected, and that the error never echoes the secret-bearing URL back // error never echoes the secret-bearing URL back into logs or output.
// into logs or output. func TestParseRejectsCredentials(t *testing.T) {
func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Parallel() t.Parallel()
// Split so the literals never form a "user:pass@" URL pattern that // Split so the literals never form a "user:pass@" URL pattern that
@@ -147,44 +146,44 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
{ {
name: "s3 userinfo", name: "s3 userinfo",
raw: "s3://" + key + ":" + secret + "@mybucket/prefix", raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret, "mybucket"}, secrets: []string{key, secret, "mybucket"},
}, },
{ {
name: "s3 userinfo in endpoint", name: "s3 userinfo in endpoint",
raw: "s3://mybucket?endpoint=https://" + raw: "s3://mybucket?endpoint=https://" +
key + ":" + secret + "@minio.example.com", key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 userinfo in endpoint without scheme", name: "s3 userinfo in endpoint without scheme",
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com", raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 unknown query param", name: "s3 unknown query param",
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret, raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: []string{key, secret}, secrets: []string{key, secret},
}, },
{ {
name: "s3 misspelt endpoint", name: "s3 misspelt endpoint",
raw: "s3://mybucket?endpiont=minio.example.com", raw: "s3://mybucket?endpiont=minio.example.com",
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: nil, secrets: nil,
}, },
{ {
name: "rclone userinfo", name: "rclone userinfo",
raw: "rclone://user:" + secret + "@gdrive/backups", raw: "rclone://user:" + secret + "@gdrive/backups",
wantErr: storage.ErrURLCredentials, wantErr: storageurl.ErrURLCredentials,
secrets: []string{secret}, secrets: []string{secret},
}, },
{ {
name: "rclone query param", name: "rclone query param",
raw: "rclone://gdrive/backups?token=" + secret, raw: "rclone://gdrive/backups?token=" + secret,
wantErr: storage.ErrURLUnknownParam, wantErr: storageurl.ErrURLUnknownParam,
secrets: []string{secret}, secrets: []string{secret},
}, },
} }
@@ -193,9 +192,9 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, tc.wantErr) { if !errors.Is(err, tc.wantErr) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v", t.Fatalf("Parse(%q) error = %v, want %v",
tc.raw, err, tc.wantErr) tc.raw, err, tc.wantErr)
} }
@@ -216,11 +215,11 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
} }
} }
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone // TestParseRejectsRcloneRemoteParams checks that an rclone remote name
// remote name carrying parameters (rclone's "remote,param=value" form, // carrying parameters (rclone's "remote,param=value" form, which can hold
// which can hold a credential) is rejected without echoing the value, // a credential) is rejected without echoing the value, including the URLs
// including the URLs that url.Parse itself refuses. // that url.Parse itself refuses.
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) { func TestParseRejectsRcloneRemoteParams(t *testing.T) {
t.Parallel() t.Parallel()
cases := []struct { cases := []struct {
@@ -249,10 +248,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if !errors.Is(err, storage.ErrRcloneRemoteParams) { if !errors.Is(err, storageurl.ErrRcloneRemoteParams) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v", t.Fatalf("Parse(%q) error = %v, want %v",
tc.raw, err, storage.ErrRcloneRemoteParams) tc.raw, err, storageurl.ErrRcloneRemoteParams)
} }
for _, secret := range tc.secrets { for _, secret := range tc.secrets {
@@ -264,10 +263,10 @@ func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
} }
} }
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone // TestParseRejectsUndecodableQuery checks that an s3 or rclone URL whose
// URL whose query cannot be decoded is rejected, not read with that part // query cannot be decoded is rejected, not read with that part of the
// of the query left out, and that the error does not echo it. // query left out, and that the error does not echo it.
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) { func TestParseRejectsUndecodableQuery(t *testing.T) {
t.Parallel() t.Parallel()
const ( const (
@@ -303,14 +302,14 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := storage.ParseStorageURL(tc.raw) _, err := storageurl.Parse(tc.raw)
if err == nil { if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw) t.Fatalf("Parse(%q) returned no error", tc.raw)
} }
if !errors.Is(err, storage.ErrURLBadQuery) { if !errors.Is(err, storageurl.ErrURLBadQuery) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v", t.Errorf("Parse(%q) error = %v, want %v",
tc.raw, err, storage.ErrURLBadQuery) tc.raw, err, storageurl.ErrURLBadQuery)
} }
for _, s := range []string{key, secret, "minio.example.com"} { for _, s := range []string{key, secret, "minio.example.com"} {
@@ -322,17 +321,17 @@ func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
} }
} }
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse // TestParseFailureHidesURL checks that when url.Parse itself fails, the
// itself fails, the wrapped error carries only the inner cause, not the // wrapped error carries only the inner cause, not the *url.Error whose
// *url.Error whose text embeds the raw (possibly credential-bearing) URL. // text embeds the raw (possibly credential-bearing) URL.
func TestParseStorageURLParseFailureHidesURL(t *testing.T) { func TestParseFailureHidesURL(t *testing.T) {
t.Parallel() t.Parallel()
const raw = "s3://mybucket/%zz" const raw = "s3://mybucket/%zz"
_, err := storage.ParseStorageURL(raw) _, err := storageurl.Parse(raw)
if err == nil { if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", raw) t.Fatalf("Parse(%q) returned no error", raw)
} }
if strings.Contains(err.Error(), "mybucket") { if strings.Contains(err.Error(), "mybucket") {
@@ -4,7 +4,6 @@ import (
"context" "context"
"maps" "maps"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"github.com/spf13/afero" "github.com/spf13/afero"
@@ -13,7 +12,6 @@ import (
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/storage/faultstore"
"sneak.berlin/go/vaultik/internal/vaultik" "sneak.berlin/go/vaultik/internal/vaultik"
) )
@@ -70,55 +68,3 @@ func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) {
assertRestoredTree(t, fs, restoreDir, files) assertRestoredTree(t, fs, restoreDir, files)
} }
// A backup without --cron whose blob uploads keep failing stops with the
// upload's error, and stops the progress reporter once. See
// https://git.eeqj.de/sneak/vaultik/issues/476.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupWithoutCronStopsWithBlobUploadError(t *testing.T) {
log.Initialize(log.Config{})
const snapshotName = "data"
fs := afero.NewOsFs()
tempDir := t.TempDir()
firstDir := filepath.Join(tempDir, "first")
secondDir := filepath.Join(tempDir, "second")
dbPath := filepath.Join(tempDir, "index.sqlite")
ctx := context.Background()
writeFaultSourceTree(t, fs, firstDir)
writeFaultSourceTree(t, fs, secondDir)
cfg := faultTestConfig()
cfg.IndexPath = dbPath
cfg.ChunkSize = config.Size(faultChunkSize)
cfg.Snapshots = map[string]config.SnapshotConfig{
snapshotName: {Paths: []string{firstDir, secondDir}},
}
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
store := faultstore.New(inner)
store.OnPut = func(key string) faultstore.PutAction {
if strings.HasPrefix(key, "blobs/") {
return faultstore.PutFail
}
return faultstore.PutNormal
}
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
v := newBackupVaultik(ctx, cfg, store, database.NewRepositories(db), db, fs)
err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Snapshots: []string{snapshotName},
})
require.ErrorIs(t, err, faultstore.ErrInjectedUpload)
}