Compare commits

..
Author SHA1 Message Date
sneak cd86db7416 Name the remote and rclone config file in rclone setup errors (closes #467)
check / check (push) Failing after 14m33s
A remote missing from the config and a remote that fails to open now
both name the remote and the rclone config file vaultik read, and say
when that file does not exist.

Model: opus-5-5
2026-11-19 07:30:57 +00:00
10 changed files with 117 additions and 262 deletions

No files matched your search

+2 -9
View File
@@ -35,16 +35,9 @@ RUN for attempt in 1 2 3; do \
echo "module download failed: giving up after 3 attempts" >&2; \ echo "module download failed: giving up after 3 attempts" >&2; \
exit 1 exit 1
COPY . . COPY . .
# -race also turns on checkptr, which checks every pointer conversion. In RUN go test -timeout 90s -race -cover ./... || \
# the SQLite library's translated C code that is about a third of the test
# CPU time, much of it under one lock all SQLite work shares, and the
# tests that use SQLite time out on a busy host. The gcflags turn it off
# for that library only; our own packages keep it.
RUN go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \ go test -timeout 90s -race -v ./...; exit 1; }
-v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies # Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run # are what make BuildKit build them first, so this stage cannot run
+1 -6
View File
@@ -183,7 +183,6 @@ vaultik [--config <path>] remote nuke --force
vaultik [--config <path>] database delete [--force] vaultik [--config <path>] database delete [--force]
vaultik completion <bash|zsh|fish|powershell> vaultik completion <bash|zsh|fish|powershell>
vaultik version vaultik version
vaultik --version
``` ```
### global flags ### global flags
@@ -603,11 +602,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The providers. Requires rclone to be configured separately (`rclone config`).
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+4 -18
View File
@@ -32,8 +32,6 @@ var rootFlags RootFlags
// It sets up the command structure, global flags, and adds all subcommands. // It sets up the command structure, global flags, and adds all subcommands.
// This is the main entry point for the CLI command hierarchy. // This is the main entry point for the CLI command hierarchy.
func NewRootCommand() *cobra.Command { func NewRootCommand() *cobra.Command {
var showVersion bool
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "vaultik", Use: "vaultik",
Short: "Secure incremental backup tool with asymmetric encryption", Short: "Secure incremental backup tool with asymmetric encryption",
@@ -41,27 +39,15 @@ func NewRootCommand() *cobra.Command {
public keys and uploads to S3-compatible storage. No private keys are needed public keys and uploads to S3-compatible storage. No private keys are needed
on the source system.`, on the source system.`,
SilenceUsage: true, SilenceUsage: true,
// Bare 'vaultik' (no subcommand): print help, or with --version // Bare 'vaultik' (no subcommand): print help. The banner is
// the report of the version command. The banner is printed once // printed once at process startup by Entry, before cobra
// at process startup by Entry, before cobra parses arguments, so // parses arguments, so it appears even when cobra rejects
// it appears even when cobra rejects args (e.g. "requires at // args (e.g. "requires at least 2 arg(s)") and on --help.
// least 2 arg(s)") and on --help.
Run: func(cmd *cobra.Command, _ []string) { Run: func(cmd *cobra.Command, _ []string) {
if showVersion {
writeVersion(commandUI(cmd))
return
}
_ = cmd.Help() _ = cmd.Help()
}, },
} }
// cobra's own --version prints a template, not the version report,
// so vaultik defines the flag itself.
cmd.Flags().BoolVar(&showVersion, "version", false,
"Print version information (same as 'vaultik version')")
// Add global flags // Add global flags
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)") "Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
-24
View File
@@ -86,30 +86,6 @@ func TestVersionCommandReportLines(t *testing.T) {
} }
} }
// TestVersionFlagPrintsVersionReport checks that `vaultik --version`
// prints exactly the report of `vaultik version`.
//
//nolint:paralleltest // executes commands that read the global rootFlags
func TestVersionFlagPrintsVersionReport(t *testing.T) {
cmd := cli.NewRootCommand()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs([]string{"--version"})
err := cmd.Execute()
if err != nil {
t.Fatalf("vaultik --version failed: %v", err)
}
want := runVersionCommand(t)
if out.String() != want {
t.Errorf("vaultik --version printed:\n%s\nwant:\n%s", out.String(), want)
}
}
// TestVersionCommandFlagsDevelopmentBuild is the regression test for // TestVersionCommandFlagsDevelopmentBuild is the regression test for
// the thing this command exists to prevent: a build that is not a // the thing this command exists to prevent: a build that is not a
// release must say so. The notice used to be gated on the version // release must say so. The notice used to be gated on the version
-35
View File
@@ -4,10 +4,8 @@ package snapshot
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"syscall"
"testing" "testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -40,36 +38,3 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker. // /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev")) assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
} }
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+3 -4
View File
@@ -1438,12 +1438,11 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
} }
// deviceNumberFromStat returns the number of the device that holds the // deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat from a real // file info describes, and false when info carries no stat, as from the
// filesystem. The in-memory test filesystem gives either no stat or one // in-memory test filesystem.
// whose device and inode numbers are both zero, which no real file has.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) { func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t) stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || (stat.Dev == 0 && stat.Ino == 0) { if !ok {
return 0, false return 0, false
} }
+4 -9
View File
@@ -118,22 +118,17 @@ func ShortHostname(hostname string) string {
// hostname_timestamp if name is empty. The timestamp is in whole seconds. // hostname_timestamp if name is empty. The timestamp is in whole seconds.
// If the local index already has a snapshot with that ID, from a run of the // If the local index already has a snapshot with that ID, from a run of the
// same name that started in the same second, it waits a second and takes a // same name that started in the same second, it waits a second and takes a
// new timestamp. The snapshot's start time is the moment in its ID, so a // new timestamp.
// later run of one name always has a later start time.
func (sm *SnapshotManager) CreateSnapshotWithName( func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string, ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) { ) (string, error) {
shortHostname := ShortHostname(hostname) shortHostname := ShortHostname(hostname)
var ( var snapshotID string
snapshotID string
startedAt time.Time
)
for { for {
// Build snapshot ID with optional name // Build snapshot ID with optional name
startedAt = time.Now().UTC() timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
timestamp := startedAt.Format("2006-01-02T15:04:05Z")
if name != "" { if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp) snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -158,7 +153,7 @@ func (sm *SnapshotManager) CreateSnapshotWithName(
Hostname: types.Hostname(hostname), Hostname: types.Hostname(hostname),
VaultikVersion: types.Version(version), VaultikVersion: types.Version(version),
VaultikGitRevision: types.GitRevision(gitRevision), VaultikGitRevision: types.GitRevision(gitRevision),
StartedAt: startedAt, StartedAt: time.Now().UTC(),
CompletedAt: nil, // Not completed yet CompletedAt: nil, // Not completed yet
FileCount: 0, FileCount: 0,
ChunkCount: 0, ChunkCount: 0,
-52
View File
@@ -492,55 +492,3 @@ func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
t.Fatalf("both creates returned snapshot ID %s", first) t.Fatalf("both creates returned snapshot ID %s", first)
} }
} }
// Two creates of one snapshot name back to back list newest first. The
// start time is the moment in the ID, so the two cannot share a start
// time even when the clock crosses a second while the first is created.
// See https://git.eeqj.de/sneak/vaultik/issues/482.
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const created = 2
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
sm := &SnapshotManager{repos: repos}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
if err != nil {
t.Fatalf("listing snapshots failed: %v", err)
}
if len(snapshots) != created {
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
}
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
t.Fatalf("start times %v and %v are not in order",
snapshots[1].StartedAt, snapshots[0].StartedAt)
}
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
t.Fatalf("listed %s, %s; want %s, %s",
snapshots[0].ID, snapshots[1].ID, second, first)
}
}
+20 -25
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@@ -24,15 +23,6 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured. // ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config") var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction. // RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers. // This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct { type RcloneStorer struct {
@@ -63,19 +53,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path rclonePath += path
} }
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in // On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read. // each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the // rclone's use_server_modtime setting makes ModTime the time the
@@ -90,10 +67,12 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
// Check for remote not found error // Check for remote not found error
if strings.Contains(err.Error(), "didn't find section in config file") || if strings.Contains(err.Error(), "didn't find section in config file") ||
strings.Contains(err.Error(), "failed to find remote") { strings.Contains(err.Error(), "failed to find remote") {
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote) return nil, fmt.Errorf("%w: %q (%s)",
ErrRemoteNotFound, remote, describeRcloneConfigFile())
} }
return nil, fmt.Errorf("creating rclone filesystem: %w", err) return nil, fmt.Errorf("opening rclone remote %q (%s): %w",
remote, describeRcloneConfigFile(), err)
} }
return &RcloneStorer{ return &RcloneStorer{
@@ -347,6 +326,22 @@ func (r *RcloneStorer) upload(ctx context.Context, key string, data io.Reader) e
return nil return nil
} }
// describeRcloneConfigFile names the rclone config file for an error
// message, and says so when that file does not exist.
func describeRcloneConfigFile() string {
path := config.GetConfigPath()
if path == "" {
return "no config file"
}
_, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return fmt.Sprintf("config file %q, which does not exist", path)
}
return fmt.Sprintf("config file %q", path)
}
// progressReader wraps an io.Reader to track read progress. // progressReader wraps an io.Reader to track read progress.
type progressReader struct { type progressReader struct {
reader io.Reader reader io.Reader
+83 -80
View File
@@ -398,24 +398,100 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
} }
} }
// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only // setRcloneConfigEnv points RCLONE_CONFIG at path for the rest of the test.
// in the file RCLONE_CONFIG names is found and written to, as rclone itself func setRcloneConfigEnv(t *testing.T, path string) {
// would. t.Helper()
func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
// rclone keeps the config file path in a process-global variable, which // rclone keeps the config file path in a process-global variable, which
// NewRcloneStorer leaves pointing at this test's file. // NewRcloneStorer leaves pointing at path.
previous := config.GetConfigPath() previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) }) t.Cleanup(func() { _ = config.SetConfigPath(previous) })
t.Setenv("RCLONE_CONFIG", path)
}
// writeRcloneConfigFile writes contents to a new rclone config file and
// returns its path.
func writeRcloneConfigFile(t *testing.T, contents string) string {
t.Helper()
confFile := filepath.Join(t.TempDir(), "rclone.conf") confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600) err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil { if err != nil {
t.Fatalf("writing rclone config: %v", err) t.Fatalf("writing rclone config: %v", err)
} }
t.Setenv("RCLONE_CONFIG", confFile) return confFile
}
// TestNewRcloneStorerErrorsNameRemoteAndConfigFile checks that each error
// from opening a remote names the remote and the rclone config file, and
// says when that file does not exist.
//
//nolint:paralleltest // sets RCLONE_CONFIG and the process-global rclone config
func TestNewRcloneStorerErrorsNameRemoteAndConfigFile(t *testing.T) {
const doesNotExist = "which does not exist"
missingFile := filepath.Join(t.TempDir(), "missing.conf")
emptyFile := writeRcloneConfigFile(t, "")
badRemoteFile := writeRcloneConfigFile(t,
"[bad]\ntype = vaultik-no-such-backend\n")
tests := []struct {
name string
configFile string
remote string
wantNotFound bool
wantNoSuchFile bool
}{
{"config file missing", missingFile, "dav", true, true},
{"remote not in config file", emptyFile, "dav", true, false},
{"remote fails to open", badRemoteFile, "bad", false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setRcloneConfigEnv(t, tt.configFile)
_, err := storage.NewRcloneStorer(
context.Background(), tt.remote, "path")
if err == nil {
t.Fatal("NewRcloneStorer succeeded, want an error")
}
msg := err.Error()
if errors.Is(err, storage.ErrRemoteNotFound) != tt.wantNotFound {
t.Errorf("error = %v, want ErrRemoteNotFound: %v",
err, tt.wantNotFound)
}
if !strings.Contains(msg, `"`+tt.remote+`"`) {
t.Errorf("error %q does not name remote %q", msg, tt.remote)
}
if !strings.Contains(msg, tt.configFile) {
t.Errorf("error %q does not name config file %q",
msg, tt.configFile)
}
if strings.Contains(msg, doesNotExist) != tt.wantNoSuchFile {
t.Errorf("error %q, want it to say the file does not exist: %v",
msg, tt.wantNoSuchFile)
}
})
}
}
// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
// in the file RCLONE_CONFIG names is found and written to, as rclone itself
// would.
//
//nolint:paralleltest // sets RCLONE_CONFIG and the process-global rclone config
func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
setRcloneConfigEnv(t, writeRcloneConfigFile(t, "[loc]\ntype = local\n"))
dir := t.TempDir() dir := t.TempDir()
ctx := context.Background() ctx := context.Background()
@@ -436,79 +512,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
} }
} }
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an // TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By // rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one // default rclone reads the time it keeps in each object's metadata, one