Compare commits

..
Author SHA1 Message Date
sneak bf614c58ac Document where vaultik reads the rclone config (closes #469)
check / check (push) Waiting to run
The rclone section now says vaultik reads rclone's own config file,
which needs no rclone tool once it exists, and that RCLONE_CONFIG names
another file. RCLONE_CONFIG joins the environment-variable list, and
restoring on another machine says the new host needs that file.

Model: opus-5-5
2026-11-19 08:39:28 +00:00
8 changed files with 29 additions and 157 deletions

No files matched your search

+2 -9
View File
@@ -35,16 +35,9 @@ RUN for attempt in 1 2 3; do \
echo "module download failed: giving up after 3 attempts" >&2; \ echo "module download failed: giving up after 3 attempts" >&2; \
exit 1 exit 1
COPY . . COPY . .
# -race also turns on checkptr, which checks every pointer conversion. In RUN go test -timeout 90s -race -cover ./... || \
# the SQLite library's translated C code that is about a third of the test
# CPU time, much of it under one lock all SQLite work shares, and the
# tests that use SQLite time out on a busy host. The gcflags turn it off
# for that library only; our own packages keep it.
RUN go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \
-cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -gcflags='modernc.org/...=-d=checkptr=0' \ go test -timeout 90s -race -v ./...; exit 1; }
-v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies # Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run # are what make BuildKit build them first, so this stage cannot run
+16 -6
View File
@@ -127,6 +127,11 @@ vaultik config set s3.secret_access_key "..."
vaultik snapshot list vaultik snapshot list
``` ```
For an rclone destination, the storage credentials are in rclone's config file
instead. Copy that file to the new host, or recreate it with `rclone config`,
and set `storage_url` to the original `rclone://remote/path`. See
[storage backends](#storage-backends).
`snapshot list` reads the destination store without the private key. A snapshot `snapshot list` reads the destination store without the private key. A snapshot
that is not in this host's (empty) local index is shown as remote-only: its row that is not in this host's (empty) local index is shown as remote-only: its row
is identified by `<remote only:...>` rather than by a `hostname_name_timestamp` is identified by `<remote only:...>` rather than by a `hostname_name_timestamp`
@@ -183,7 +188,6 @@ vaultik [--config <path>] remote nuke --force
vaultik [--config <path>] database delete [--force] vaultik [--config <path>] database delete [--force]
vaultik completion <bash|zsh|fish|powershell> vaultik completion <bash|zsh|fish|powershell>
vaultik version vaultik version
vaultik --version
``` ```
### global flags ### global flags
@@ -282,6 +286,8 @@ work as written.
the key is not typed into your shell history. the key is not typed into your shell history.
- `VAULTIK_CONFIG`: Path to config file (overridden by `--config`) - `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
- `VAULTIK_INDEX_PATH`: Override local SQLite index path - `VAULTIK_INDEX_PATH`: Override local SQLite index path
- `RCLONE_CONFIG`: Path to the rclone config file used by an `rclone://`
destination (default `~/.config/rclone/rclone.conf`)
- `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the - `VAULTIK_CPUPROFILE`: Write a CPU profile to this path for the duration of the
run (development/debugging) run (development/debugging)
- `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits - `VAULTIK_MEMPROFILE`: Write a heap profile to this path when the run exits
@@ -603,11 +609,15 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The providers. `remote` is the name of a remote in rclone's config file, which holds
path after the remote name is relative: rclone resolves it on the remote, for that remote's credentials. vaultik reads the same file the `rclone` tool reads,
example against the login directory on sftp. A second slash makes it absolute, by default `~/.config/rclone/rclone.conf`. Create it with `rclone config`, or
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute copy it from another machine; vaultik does not need the `rclone` tool once the
path; a relative one is refused. file exists. The `RCLONE_CONFIG` environment variable names a different file.
The path after the remote name is relative: rclone resolves it on the remote,
for example against the login directory on sftp. A second slash makes it
absolute, as in `rclone://remote//srv/backup`. A remote of type `local` needs an
absolute path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+4 -18
View File
@@ -32,8 +32,6 @@ var rootFlags RootFlags
// It sets up the command structure, global flags, and adds all subcommands. // It sets up the command structure, global flags, and adds all subcommands.
// This is the main entry point for the CLI command hierarchy. // This is the main entry point for the CLI command hierarchy.
func NewRootCommand() *cobra.Command { func NewRootCommand() *cobra.Command {
var showVersion bool
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "vaultik", Use: "vaultik",
Short: "Secure incremental backup tool with asymmetric encryption", Short: "Secure incremental backup tool with asymmetric encryption",
@@ -41,27 +39,15 @@ func NewRootCommand() *cobra.Command {
public keys and uploads to S3-compatible storage. No private keys are needed public keys and uploads to S3-compatible storage. No private keys are needed
on the source system.`, on the source system.`,
SilenceUsage: true, SilenceUsage: true,
// Bare 'vaultik' (no subcommand): print help, or with --version // Bare 'vaultik' (no subcommand): print help. The banner is
// the report of the version command. The banner is printed once // printed once at process startup by Entry, before cobra
// at process startup by Entry, before cobra parses arguments, so // parses arguments, so it appears even when cobra rejects
// it appears even when cobra rejects args (e.g. "requires at // args (e.g. "requires at least 2 arg(s)") and on --help.
// least 2 arg(s)") and on --help.
Run: func(cmd *cobra.Command, _ []string) { Run: func(cmd *cobra.Command, _ []string) {
if showVersion {
writeVersion(commandUI(cmd))
return
}
_ = cmd.Help() _ = cmd.Help()
}, },
} }
// cobra's own --version prints a template, not the version report,
// so vaultik defines the flag itself.
cmd.Flags().BoolVar(&showVersion, "version", false,
"Print version information (same as 'vaultik version')")
// Add global flags // Add global flags
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)") "Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
-24
View File
@@ -86,30 +86,6 @@ func TestVersionCommandReportLines(t *testing.T) {
} }
} }
// TestVersionFlagPrintsVersionReport checks that `vaultik --version`
// prints exactly the report of `vaultik version`.
//
//nolint:paralleltest // executes commands that read the global rootFlags
func TestVersionFlagPrintsVersionReport(t *testing.T) {
cmd := cli.NewRootCommand()
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs([]string{"--version"})
err := cmd.Execute()
if err != nil {
t.Fatalf("vaultik --version failed: %v", err)
}
want := runVersionCommand(t)
if out.String() != want {
t.Errorf("vaultik --version printed:\n%s\nwant:\n%s", out.String(), want)
}
}
// TestVersionCommandFlagsDevelopmentBuild is the regression test for // TestVersionCommandFlagsDevelopmentBuild is the regression test for
// the thing this command exists to prevent: a build that is not a // the thing this command exists to prevent: a build that is not a
// release must say so. The notice used to be gated on the version // release must say so. The notice used to be gated on the version
-35
View File
@@ -4,10 +4,8 @@ package snapshot
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"syscall"
"testing" "testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -40,36 +38,3 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker. // /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev")) assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
} }
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+3 -4
View File
@@ -1438,12 +1438,11 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
} }
// deviceNumberFromStat returns the number of the device that holds the // deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat from a real // file info describes, and false when info carries no stat, as from the
// filesystem. The in-memory test filesystem gives either no stat or one // in-memory test filesystem.
// whose device and inode numbers are both zero, which no real file has.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) { func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t) stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || (stat.Dev == 0 && stat.Ino == 0) { if !ok {
return 0, false return 0, false
} }
+4 -9
View File
@@ -118,22 +118,17 @@ func ShortHostname(hostname string) string {
// hostname_timestamp if name is empty. The timestamp is in whole seconds. // hostname_timestamp if name is empty. The timestamp is in whole seconds.
// If the local index already has a snapshot with that ID, from a run of the // If the local index already has a snapshot with that ID, from a run of the
// same name that started in the same second, it waits a second and takes a // same name that started in the same second, it waits a second and takes a
// new timestamp. The snapshot's start time is the moment in its ID, so a // new timestamp.
// later run of one name always has a later start time.
func (sm *SnapshotManager) CreateSnapshotWithName( func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string, ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) { ) (string, error) {
shortHostname := ShortHostname(hostname) shortHostname := ShortHostname(hostname)
var ( var snapshotID string
snapshotID string
startedAt time.Time
)
for { for {
// Build snapshot ID with optional name // Build snapshot ID with optional name
startedAt = time.Now().UTC() timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
timestamp := startedAt.Format("2006-01-02T15:04:05Z")
if name != "" { if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp) snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -158,7 +153,7 @@ func (sm *SnapshotManager) CreateSnapshotWithName(
Hostname: types.Hostname(hostname), Hostname: types.Hostname(hostname),
VaultikVersion: types.Version(version), VaultikVersion: types.Version(version),
VaultikGitRevision: types.GitRevision(gitRevision), VaultikGitRevision: types.GitRevision(gitRevision),
StartedAt: startedAt, StartedAt: time.Now().UTC(),
CompletedAt: nil, // Not completed yet CompletedAt: nil, // Not completed yet
FileCount: 0, FileCount: 0,
ChunkCount: 0, ChunkCount: 0,
-52
View File
@@ -492,55 +492,3 @@ func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
t.Fatalf("both creates returned snapshot ID %s", first) t.Fatalf("both creates returned snapshot ID %s", first)
} }
} }
// Two creates of one snapshot name back to back list newest first. The
// start time is the moment in the ID, so the two cannot share a start
// time even when the clock crosses a second while the first is created.
// See https://git.eeqj.de/sneak/vaultik/issues/482.
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const created = 2
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
sm := &SnapshotManager{repos: repos}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
if err != nil {
t.Fatalf("listing snapshots failed: %v", err)
}
if len(snapshots) != created {
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
}
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
t.Fatalf("start times %v and %v are not in order",
snapshots[1].StartedAt, snapshots[0].StartedAt)
}
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
t.Fatalf("listed %s, %s; want %s, %s",
snapshots[0].ID, snapshots[1].ID, second, first)
}
}