Compare commits

..
Author SHA1 Message Date
sneak cd86db7416 Name the remote and rclone config file in rclone setup errors (closes #467)
check / check (push) Waiting to run
A remote missing from the config and a remote that fails to open now
both name the remote and the rclone config file vaultik read, and say
when that file does not exist.

Model: opus-5-5
2026-11-19 07:30:57 +00:00
3 changed files with 104 additions and 110 deletions

No files matched your search

+1 -5
View File
@@ -602,11 +602,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The providers. Requires rclone to be configured separately (`rclone config`).
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+20 -25
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@@ -24,15 +23,6 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured. // ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config") var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction. // RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers. // This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct { type RcloneStorer struct {
@@ -63,19 +53,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path rclonePath += path
} }
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in // On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read. // each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the // rclone's use_server_modtime setting makes ModTime the time the
@@ -90,10 +67,12 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
// Check for remote not found error // Check for remote not found error
if strings.Contains(err.Error(), "didn't find section in config file") || if strings.Contains(err.Error(), "didn't find section in config file") ||
strings.Contains(err.Error(), "failed to find remote") { strings.Contains(err.Error(), "failed to find remote") {
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote) return nil, fmt.Errorf("%w: %q (%s)",
ErrRemoteNotFound, remote, describeRcloneConfigFile())
} }
return nil, fmt.Errorf("creating rclone filesystem: %w", err) return nil, fmt.Errorf("opening rclone remote %q (%s): %w",
remote, describeRcloneConfigFile(), err)
} }
return &RcloneStorer{ return &RcloneStorer{
@@ -347,6 +326,22 @@ func (r *RcloneStorer) upload(ctx context.Context, key string, data io.Reader) e
return nil return nil
} }
// describeRcloneConfigFile names the rclone config file for an error
// message, and says so when that file does not exist.
func describeRcloneConfigFile() string {
path := config.GetConfigPath()
if path == "" {
return "no config file"
}
_, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return fmt.Sprintf("config file %q, which does not exist", path)
}
return fmt.Sprintf("config file %q", path)
}
// progressReader wraps an io.Reader to track read progress. // progressReader wraps an io.Reader to track read progress.
type progressReader struct { type progressReader struct {
reader io.Reader reader io.Reader
+83 -80
View File
@@ -398,24 +398,100 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
} }
} }
// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only // setRcloneConfigEnv points RCLONE_CONFIG at path for the rest of the test.
// in the file RCLONE_CONFIG names is found and written to, as rclone itself func setRcloneConfigEnv(t *testing.T, path string) {
// would. t.Helper()
func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
// rclone keeps the config file path in a process-global variable, which // rclone keeps the config file path in a process-global variable, which
// NewRcloneStorer leaves pointing at this test's file. // NewRcloneStorer leaves pointing at path.
previous := config.GetConfigPath() previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) }) t.Cleanup(func() { _ = config.SetConfigPath(previous) })
t.Setenv("RCLONE_CONFIG", path)
}
// writeRcloneConfigFile writes contents to a new rclone config file and
// returns its path.
func writeRcloneConfigFile(t *testing.T, contents string) string {
t.Helper()
confFile := filepath.Join(t.TempDir(), "rclone.conf") confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600) err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil { if err != nil {
t.Fatalf("writing rclone config: %v", err) t.Fatalf("writing rclone config: %v", err)
} }
t.Setenv("RCLONE_CONFIG", confFile) return confFile
}
// TestNewRcloneStorerErrorsNameRemoteAndConfigFile checks that each error
// from opening a remote names the remote and the rclone config file, and
// says when that file does not exist.
//
//nolint:paralleltest // sets RCLONE_CONFIG and the process-global rclone config
func TestNewRcloneStorerErrorsNameRemoteAndConfigFile(t *testing.T) {
const doesNotExist = "which does not exist"
missingFile := filepath.Join(t.TempDir(), "missing.conf")
emptyFile := writeRcloneConfigFile(t, "")
badRemoteFile := writeRcloneConfigFile(t,
"[bad]\ntype = vaultik-no-such-backend\n")
tests := []struct {
name string
configFile string
remote string
wantNotFound bool
wantNoSuchFile bool
}{
{"config file missing", missingFile, "dav", true, true},
{"remote not in config file", emptyFile, "dav", true, false},
{"remote fails to open", badRemoteFile, "bad", false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setRcloneConfigEnv(t, tt.configFile)
_, err := storage.NewRcloneStorer(
context.Background(), tt.remote, "path")
if err == nil {
t.Fatal("NewRcloneStorer succeeded, want an error")
}
msg := err.Error()
if errors.Is(err, storage.ErrRemoteNotFound) != tt.wantNotFound {
t.Errorf("error = %v, want ErrRemoteNotFound: %v",
err, tt.wantNotFound)
}
if !strings.Contains(msg, `"`+tt.remote+`"`) {
t.Errorf("error %q does not name remote %q", msg, tt.remote)
}
if !strings.Contains(msg, tt.configFile) {
t.Errorf("error %q does not name config file %q",
msg, tt.configFile)
}
if strings.Contains(msg, doesNotExist) != tt.wantNoSuchFile {
t.Errorf("error %q, want it to say the file does not exist: %v",
msg, tt.wantNoSuchFile)
}
})
}
}
// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
// in the file RCLONE_CONFIG names is found and written to, as rclone itself
// would.
//
//nolint:paralleltest // sets RCLONE_CONFIG and the process-global rclone config
func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
setRcloneConfigEnv(t, writeRcloneConfigFile(t, "[loc]\ntype = local\n"))
dir := t.TempDir() dir := t.TempDir()
ctx := context.Background() ctx := context.Background()
@@ -436,79 +512,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
} }
} }
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an // TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By // rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one // default rclone reads the time it keeps in each object's metadata, one