Vertaa committeja

...
Tekijä SHA1 Viesti Päivämäärä
sneak bf614c58ac Document where vaultik reads the rclone config (closes #469)
check / check (push) Waiting to run
The rclone section now says vaultik reads rclone's own config file,
which needs no rclone tool once it exists, and that RCLONE_CONFIG names
another file. RCLONE_CONFIG joins the environment-variable list, and
restoring on another machine says the new host needs that file.

Model: opus-5-5
2026-11-19 08:39:28 +00:00
clawbot 9fedcbeb0a Refuse a relative path on an rclone remote of type local (closes #465)
check / check (push) Waiting to run
rclone resolves a relative path on its local backend against the working
directory, so rclone://localfs/srv/backup wrote under whatever directory
vaultik ran from. Opening such a destination now fails with an error that
shows the absolute form, rclone://localfs//srv/backup. Other remotes keep
relative paths, which rclone resolves on the remote. The README says how
to write an absolute rclone path.

Model: opus-5-5
2026-11-19 09:24:16 +01:00
3 muutettua tiedostoa jossa 112 lisäystä ja 1 poistoa

No files matched your search

+16 -1
Näytä tiedosto
@@ -127,6 +127,11 @@ vaultik config set s3.secret_access_key "..."
vaultik snapshot list vaultik snapshot list
``` ```
For an rclone destination, the storage credentials are in rclone's config file
instead. Copy that file to the new host, or recreate it with `rclone config`,
and set `storage_url` to the original `rclone://remote/path`. See
[storage backends](#storage-backends).
`snapshot list` reads the destination store without the private key. A snapshot `snapshot list` reads the destination store without the private key. A snapshot
that is not in this host's (empty) local index is shown as remote-only: its row that is not in this host's (empty) local index is shown as remote-only: its row
is identified by `<remote only:...>` rather than by a `hostname_name_timestamp` is identified by `<remote only:...>` rather than by a `hostname_name_timestamp`
@@ -281,6 +286,8 @@ work as written.
the key is not typed into your shell history. the key is not typed into your shell history.
- `VAULTIK_CONFIG`: Path to config file (overridden by `--config`) - `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
- `VAULTIK_INDEX_PATH`: Override local SQLite index path - `VAULTIK_INDEX_PATH`: Override local SQLite index path
- `RCLONE_CONFIG`: Path to the rclone config file used by an `rclone://`
destination (default `~/.config/rclone/rclone.conf`)
- `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the - `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the
run (development/debugging) run (development/debugging)
- `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits - `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits
@@ -602,7 +609,15 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). providers. `remote` is the name of a remote in rclone's config file, which holds
that remote's credentials. vaultik reads the same file the `rclone` tool reads,
by default `~/.config/rclone/rclone.conf`. Create it with `rclone config`, or
copy it from another machine; vaultik does not need the `rclone` tool once the
file exists. The `RCLONE_CONFIG` environment variable names a different file.
The path after the remote name is relative: rclone resolves it on the remote,
for example against the login directory on sftp. A second slash makes it
absolute, as in `rclone://remote//srv/backup`. A remote of type `local` needs an
absolute path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+23
Näytä tiedosto
@@ -8,6 +8,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@@ -23,6 +24,15 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured. // ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config") var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction. // RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers. // This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct { type RcloneStorer struct {
@@ -53,6 +63,19 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path rclonePath += path
} }
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in // On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read. // each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the // rclone's use_server_modtime setting makes ModTime the time the
+73
Näytä tiedosto
@@ -436,6 +436,79 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
} }
} }
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an // TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By // rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one // default rclone reads the time it keeps in each object's metadata, one