Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9abe05be61 | ||
|
|
0288735aaf | ||
|
|
35eed79b1a |
@@ -286,7 +286,9 @@ needed.
|
||||
remove orphaned blobs from remote storage. By default keeps only the latest
|
||||
snapshot per name; use `--keep-newer-than` for a rolling window.
|
||||
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`)
|
||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`).
|
||||
Without `--prune`, or with a value that is not a valid duration, the
|
||||
command fails before the backup starts.
|
||||
|
||||
**`snapshot list`**: Show every snapshot known to this host — the union
|
||||
of the local index and the backup destination store — with timestamps
|
||||
|
||||
@@ -22,17 +22,6 @@ the tag exists and is exercised; what is left is merging `next` to
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-13: Made a `file://` destination list through a symlinked
|
||||
directory ([issue #296](https://git.eeqj.de/sneak/vaultik/issues/296)).
|
||||
A `metadata/` or `blobs/` directory that was a symlink to a directory
|
||||
elsewhere was listed as one file, so the destination store listed as
|
||||
empty while backups and restores still worked through the link.
|
||||
`prune` then removed the local snapshot records and deleted every blob,
|
||||
and `remote info` and `remote nuke` found no blobs. The listings now
|
||||
read through the link as `Get` and `Stat` do. A link whose target is
|
||||
missing, such as one to a volume that is not mounted, or a link
|
||||
pointing back up the tree makes them fail.
|
||||
|
||||
- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that
|
||||
is not a blob as an orphaned blob
|
||||
([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray
|
||||
|
||||
@@ -245,6 +245,17 @@ func (p *Packer) Flush(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
||||
// packed, if any, and drops that blob. A caller that stops before Flush
|
||||
// must call it, or the temp file is left behind. Thread-safe.
|
||||
func (p *Packer) DiscardCurrentBlob() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
}
|
||||
|
||||
// FinalizeBlob finalizes the current blob being assembled.
|
||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||
// the content-addressed hash. The finalized blob is either passed to the
|
||||
|
||||
@@ -251,6 +251,12 @@ func NewConfigCommand() *cobra.Command {
|
||||
Use: "config",
|
||||
Short: "Manage the configuration file",
|
||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
cmd.AddCommand(newConfigInitCommand())
|
||||
|
||||
@@ -15,6 +15,12 @@ func NewDatabaseCommand() *cobra.Command {
|
||||
Use: "database",
|
||||
Short: "Manage the local state database",
|
||||
Long: `Commands for managing the local SQLite state database.`,
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
cmd.AddCommand(
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Flags of `snapshot create` used to build argument vectors below.
|
||||
const (
|
||||
flagPrune = "--prune"
|
||||
flagKeepNewerThan = "--keep-newer-than"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup checks that a
|
||||
// --keep-newer-than the post-backup prune could not use stops `snapshot
|
||||
// create` before the backup: the command fails and the destination
|
||||
// store stays empty. It used to be read only by the prune, after the
|
||||
// snapshot was uploaded, and without --prune it was ignored.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
name: "decimal in a day count",
|
||||
args: []string{flagPrune, flagKeepNewerThan, "1.5d"},
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
args: []string{flagPrune, flagKeepNewerThan, "4x"},
|
||||
},
|
||||
{
|
||||
name: "without --prune",
|
||||
args: []string{flagKeepNewerThan, "4w"},
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
code, _, _ := runEntry(t, append(
|
||||
[]string{flagConfig, configPath, cmdSnapshot, cmdCreate},
|
||||
testCase.args...)...)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
|
||||
stored, err := os.ReadDir(storeDir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, stored, "nothing may be uploaded")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreateReportsPostBackupPruneFailure makes the
|
||||
// post-backup prune fail after a good backup, by planting a manifest
|
||||
// that cannot be decoded, and checks that the failure is reported as the
|
||||
// prune's and not as a failed snapshot creation.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
// A remote key is 64 lowercase hex characters. The prune reads every
|
||||
// manifest on the store and fails on one it cannot decode.
|
||||
plantedDir := filepath.Join(storeDir, "metadata", strings.Repeat("a", 64))
|
||||
require.NoError(t, os.MkdirAll(plantedDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(plantedDir, "manifest.json.zst"),
|
||||
[]byte("not a valid manifest"), 0o600))
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "Backup completed, but post-backup prune failed")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2,
|
||||
"the new snapshot is on the store beside the planted one")
|
||||
}
|
||||
|
||||
// writeBackupConfig builds a hermetic config with one small file in its
|
||||
// source directory and an empty destination store directory. Returns the
|
||||
// config path and the store directory.
|
||||
func writeBackupConfig(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
|
||||
// writeHermeticPruneConfig puts the source and store directories
|
||||
// beside the config file.
|
||||
dir := filepath.Dir(configPath)
|
||||
sourceDir := filepath.Join(dir, "source")
|
||||
|
||||
require.NoError(t, os.Mkdir(sourceDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(sourceDir, "file.txt"), []byte("hello"), 0o600))
|
||||
|
||||
return configPath, filepath.Join(dir, "store")
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the command-word constants
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
|
||||
// mistyped subcommand, as a crontab line might carry one. It must fail
|
||||
// with exit 1 and name the word, as `vaultik bogus` does at the top
|
||||
// level, instead of printing the group's help and exiting 0 having done
|
||||
// nothing. The group name alone still prints its help.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||
func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
group string
|
||||
mistyped string
|
||||
}{
|
||||
{group: cmdSnapshot, mistyped: "craete"},
|
||||
{group: "config", mistyped: "inti"},
|
||||
{group: cmdRemote, mistyped: "ifno"},
|
||||
{group: "database", mistyped: "delte"},
|
||||
{group: "completion", mistyped: "nosuchshell"},
|
||||
} {
|
||||
t.Run(testCase.group, func(t *testing.T) {
|
||||
code, stdout, stderr := runEntry(t,
|
||||
testCase.group, testCase.mistyped)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Empty(t, stdout)
|
||||
assert.Contains(t, stderr,
|
||||
`unknown command "`+testCase.mistyped+`"`)
|
||||
|
||||
code, stdout, _ = runEntry(t, testCase.group)
|
||||
|
||||
assert.Equal(t, 0, code)
|
||||
assert.Contains(t, stdout,
|
||||
programName+" "+testCase.group+" [command]")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,12 @@ func NewRemoteCommand() *cobra.Command {
|
||||
Use: "remote",
|
||||
Short: "Remote storage management commands",
|
||||
Long: "Commands for inspecting and managing remote storage",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
|
||||
@@ -74,6 +74,21 @@ on the source system.`,
|
||||
NewDatabaseCommand(),
|
||||
)
|
||||
|
||||
// cobra adds the completion command only when Execute runs, and with
|
||||
// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
|
||||
// printed help and exited 0. Adding it here lets it have a Run. Its
|
||||
// scripts go to os.Stdout as it is now: a later SetOut does not
|
||||
// redirect them.
|
||||
cmd.InitDefaultCompletionCmd()
|
||||
|
||||
for _, sub := range cmd.Commands() {
|
||||
if sub.Name() == "completion" {
|
||||
sub.Run = func(completion *cobra.Command, _ []string) {
|
||||
_ = completion.Help()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,12 @@ func NewSnapshotCommand() *cobra.Command {
|
||||
Use: "snapshot",
|
||||
Short: "Snapshot management commands",
|
||||
Long: "Commands for creating, listing, and managing snapshots",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
@@ -71,6 +77,11 @@ otherwise from the platform config directory (~/.config/vaultik/config.yml
|
||||
on Linux), then /etc/vaultik/config.yml.`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
err := opts.Validate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pass snapshot names from args
|
||||
opts.Snapshots = args
|
||||
// --skip-errors is a global flag on the root command.
|
||||
@@ -97,6 +108,13 @@ on Linux), then /etc/vaultik/config.yml.`,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.CreateSnapshot(opts)
|
||||
}, func(err error) {
|
||||
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
||||
log.Error("Post-backup prune failed", "error", err)
|
||||
ReportErrorf("Backup completed, but %v", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
ReportErrorf("Snapshot creation failed: %v", err)
|
||||
})
|
||||
|
||||
@@ -1295,6 +1295,11 @@ func (s *Scanner) batchAddFilesToSnapshot(
|
||||
func (s *Scanner) processPhase(
|
||||
ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
|
||||
) error {
|
||||
// On an error or cancellation the blob being packed is never
|
||||
// finalized, so its temp file must be removed here. After a
|
||||
// successful flush there is no such blob and this does nothing.
|
||||
defer s.packer.DiscardCurrentBlob()
|
||||
|
||||
// Calculate total bytes to process
|
||||
var totalBytes int64
|
||||
for _, f := range filesToProcess {
|
||||
|
||||
@@ -381,3 +381,60 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
|
||||
// part-way, by a file it cannot read or by a cancellation, removes the
|
||||
// temp file of the blob it was packing.
|
||||
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// a.txt is backed up first and starts a blob; the run stops at b.txt.
|
||||
const target = "/source/b.txt"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
newFs func(cancel context.CancelFunc) afero.Fs
|
||||
}{
|
||||
{
|
||||
name: "file cannot be read",
|
||||
newFs: func(context.CancelFunc) afero.Fs {
|
||||
return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cancelled",
|
||||
newFs: func(cancel context.CancelFunc) afero.Fs {
|
||||
return &cancelOnOpenFs{
|
||||
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
fs := tt.newFs(cancel)
|
||||
writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
|
||||
writeSkipErrorTestFile(t, fs, target, "second file content")
|
||||
|
||||
_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected the scan to stop with an error, got nil")
|
||||
}
|
||||
|
||||
left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
|
||||
if err != nil {
|
||||
t.Fatalf("listing blob temp files: %v", err)
|
||||
}
|
||||
|
||||
if len(left) != 0 {
|
||||
t.Fatalf("expected no blob temp file left, got %v", left)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+10
-45
@@ -120,10 +120,9 @@ func (f *FileStorer) Delete(_ context.Context, key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns all keys with the given prefix, reading through symlinked
|
||||
// directories. It fails when the destination directory is missing or the
|
||||
// prefix is a symlink whose target is missing; a missing prefix under the
|
||||
// destination directory is an empty listing.
|
||||
// List returns all keys with the given prefix. It fails when the
|
||||
// destination directory is missing; a missing prefix under it is an
|
||||
// empty listing.
|
||||
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
var keys []string
|
||||
|
||||
@@ -134,7 +133,8 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
|
||||
basePath := f.fullPath(prefix)
|
||||
|
||||
exists, err := f.prefixExists(basePath)
|
||||
// Check if the prefix exists
|
||||
exists, err := afero.Exists(f.fs, basePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking path: %w", err)
|
||||
}
|
||||
@@ -143,7 +143,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
return keys, nil // Empty list for non-existent prefix
|
||||
}
|
||||
|
||||
err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
|
||||
err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -176,9 +176,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
}
|
||||
|
||||
// ListStream returns a channel of ObjectInfo for large result sets. Like
|
||||
// List, it reads through symlinked directories and sends an error when
|
||||
// the destination directory is missing or the prefix is a symlink whose
|
||||
// target is missing.
|
||||
// List, it sends an error when the destination directory is missing.
|
||||
func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
|
||||
ch := make(chan ObjectInfo)
|
||||
|
||||
@@ -194,7 +192,8 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
|
||||
basePath := f.fullPath(prefix)
|
||||
|
||||
exists, err := f.prefixExists(basePath)
|
||||
// Check if the prefix exists
|
||||
exists, err := afero.Exists(f.fs, basePath)
|
||||
if err != nil {
|
||||
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
|
||||
|
||||
@@ -205,7 +204,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
return // Empty channel for non-existent prefix
|
||||
}
|
||||
|
||||
_ = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
|
||||
_ = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
|
||||
// Check context cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -353,40 +352,6 @@ func (f *FileStorer) writeAtomic(
|
||||
return nil
|
||||
}
|
||||
|
||||
// walk is afero.Walk, except that it reads through a symlinked directory
|
||||
// the way Get and Stat do, where afero.Walk reports the link as one file.
|
||||
// afero.Walk uses Lstat, which does not follow a link, whenever the
|
||||
// filesystem has one; the struct below carries only the afero.Fs methods,
|
||||
// so the walk uses Stat instead. A link back up the tree is followed until
|
||||
// the operating system refuses the path for too many levels of links, and
|
||||
// the walk then fails.
|
||||
func (f *FileStorer) walk(root string, fn filepath.WalkFunc) error {
|
||||
return afero.Walk(struct{ afero.Fs }{f.fs}, root, fn)
|
||||
}
|
||||
|
||||
// prefixExists is afero.Exists, except that it does not follow a symlink
|
||||
// at path. A symlinked prefix whose target is missing, such as a link to
|
||||
// a volume that is not mounted, therefore exists, and walk then fails on
|
||||
// it; with afero.Exists it would list as empty, and prune would delete
|
||||
// every blob.
|
||||
func (f *FileStorer) prefixExists(path string) (bool, error) {
|
||||
lstater, ok := f.fs.(afero.Lstater)
|
||||
if !ok {
|
||||
return afero.Exists(f.fs, path)
|
||||
}
|
||||
|
||||
_, _, err := lstater.LstatIfPossible(path)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
// fullPath returns the full filesystem path for a key.
|
||||
func (f *FileStorer) fullPath(key string) string {
|
||||
return filepath.Join(f.basePath, key)
|
||||
|
||||
@@ -4,10 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
@@ -63,151 +60,3 @@ func TestFileStorerListMissingDestination(t *testing.T) {
|
||||
t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorerListsThroughSymlinkedDirectory checks that List and
|
||||
// ListStream read through a symlinked metadata/ or blobs/ directory, as Get
|
||||
// and Stat do, instead of listing the link as one key. Listing the whole
|
||||
// destination meets the link part-way through the walk; listing its prefix
|
||||
// starts the walk at the link.
|
||||
func TestFileStorerListsThroughSymlinkedDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, dir := range []string{"metadata", "blobs"} {
|
||||
t.Run(dir, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
base := t.TempDir()
|
||||
key := dir + "/aa/bb/object"
|
||||
|
||||
s, err := storage.NewFileStorer(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileStorer: %v", err)
|
||||
}
|
||||
|
||||
err = s.Put(context.Background(), key, strings.NewReader("data"))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
linkPath := filepath.Join(base, dir)
|
||||
movedPath := filepath.Join(t.TempDir(), dir)
|
||||
|
||||
err = os.Rename(linkPath, movedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("moving %s away: %v", dir, err)
|
||||
}
|
||||
|
||||
err = os.Symlink(movedPath, linkPath)
|
||||
if err != nil {
|
||||
t.Fatalf("linking %s: %v", dir, err)
|
||||
}
|
||||
|
||||
want := []string{key}
|
||||
|
||||
for _, prefix := range []string{"", dir + "/"} {
|
||||
got := listSorted(t, s, prefix)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("List(%q) = %v, want %v", prefix, got, want)
|
||||
}
|
||||
|
||||
got = listStreamKeys(t, s, prefix)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("ListStream(%q) = %v, want %v", prefix, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorerListFailsOnSymlinkWithMissingTarget checks that List and
|
||||
// ListStream fail on a symlinked metadata/ or blobs/ directory whose target
|
||||
// is missing, such as a link to a volume that is not mounted, as they do on
|
||||
// a missing destination directory, instead of listing it as empty.
|
||||
func TestFileStorerListFailsOnSymlinkWithMissingTarget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for _, dir := range []string{"metadata", "blobs"} {
|
||||
t.Run(dir, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
base := t.TempDir()
|
||||
|
||||
s, err := storage.NewFileStorer(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileStorer: %v", err)
|
||||
}
|
||||
|
||||
unmounted := filepath.Join(t.TempDir(), "unmounted")
|
||||
|
||||
err = os.Symlink(unmounted, filepath.Join(base, dir))
|
||||
if err != nil {
|
||||
t.Fatalf("Symlink: %v", err)
|
||||
}
|
||||
|
||||
for _, prefix := range []string{"", dir + "/"} {
|
||||
keys, err := s.List(ctx, prefix)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("List(%q) = %v, %v; want a not-exist error",
|
||||
prefix, keys, err)
|
||||
}
|
||||
|
||||
var streamErr error
|
||||
|
||||
for object := range s.ListStream(ctx, prefix) {
|
||||
if object.Err != nil {
|
||||
streamErr = object.Err
|
||||
}
|
||||
}
|
||||
|
||||
if !errors.Is(streamErr, fs.ErrNotExist) {
|
||||
t.Errorf("ListStream(%q) error = %v, want a not-exist error",
|
||||
prefix, streamErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorerListFailsOnSymlinkLoop checks that a symlink pointing back
|
||||
// at the destination makes List and ListStream fail instead of walking it
|
||||
// forever.
|
||||
func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
base := t.TempDir()
|
||||
|
||||
s, err := storage.NewFileStorer(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileStorer: %v", err)
|
||||
}
|
||||
|
||||
err = s.Put(ctx, testBlobKey, strings.NewReader("data"))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
err = os.Symlink(base, filepath.Join(base, "blobs", "loop"))
|
||||
if err != nil {
|
||||
t.Fatalf("Symlink: %v", err)
|
||||
}
|
||||
|
||||
keys, err := s.List(ctx, "blobs/")
|
||||
if err == nil {
|
||||
t.Errorf("List = %d keys and no error, want an error", len(keys))
|
||||
}
|
||||
|
||||
var streamErr error
|
||||
|
||||
for object := range s.ListStream(ctx, "blobs/") {
|
||||
if object.Err != nil {
|
||||
streamErr = object.Err
|
||||
}
|
||||
}
|
||||
|
||||
if streamErr == nil {
|
||||
t.Error("ListStream sent no error, want one")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,14 @@ var (
|
||||
errNoAgeRecipients = errors.New(
|
||||
"creating a snapshot needs at least one public key in " +
|
||||
"age_recipients (generate a keypair with: age-keygen)")
|
||||
errKeepNewerThanNeedsPrune = errors.New("--keep-newer-than requires --prune")
|
||||
)
|
||||
|
||||
// ErrPostBackupPruneFailed wraps a failure of the post-backup prune,
|
||||
// which runs only after every snapshot was created and uploaded, so the
|
||||
// caller can report it apart from a failed backup.
|
||||
var ErrPostBackupPruneFailed = errors.New("post-backup prune failed")
|
||||
|
||||
// listRecentLimit caps how many snapshot rows are fetched from the
|
||||
// local index for listing and purge operations.
|
||||
const listRecentLimit = 10000
|
||||
@@ -45,6 +51,26 @@ type SnapshotCreateOptions struct {
|
||||
Snapshots []string // Optional list of snapshot names to process (empty = all)
|
||||
}
|
||||
|
||||
// Validate fails when KeepNewerThan is set without Prune or is not a
|
||||
// valid duration. Only the post-backup prune reads KeepNewerThan, after
|
||||
// the backup, so the caller checks it before starting one.
|
||||
func (opts *SnapshotCreateOptions) Validate() error {
|
||||
if opts.KeepNewerThan == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !opts.Prune {
|
||||
return errKeepNewerThanNeedsPrune
|
||||
}
|
||||
|
||||
_, err := parseDuration(opts.KeepNewerThan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--keep-newer-than: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSnapshot executes the snapshot creation operation
|
||||
func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
// config.Load accepts an empty list, since listing, verifying and
|
||||
@@ -123,7 +149,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
if opts.Prune {
|
||||
err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("post-backup prune: %w", err)
|
||||
return fmt.Errorf("%w: %w", ErrPostBackupPruneFailed, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestPruneDeletesNothingThroughSymlinkedDirectory checks that prune keeps
|
||||
// every blob and the local snapshot record when the metadata/ or blobs/
|
||||
// directory of a file:// destination is a symlink to a directory elsewhere
|
||||
// (https://git.eeqj.de/sneak/vaultik/issues/296). A backup writes through
|
||||
// such a link, so prune has to list through it too.
|
||||
func TestPruneDeletesNothingThroughSymlinkedDirectory(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
for _, dir := range []string{"metadata", "blobs"} {
|
||||
t.Run(dir, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "store")
|
||||
v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
|
||||
blobFiles := blobFilesIn(t, storeDir)
|
||||
moveBehindSymlink(t, storeDir, dir)
|
||||
|
||||
require.NoError(t, v.Prune(&vaultik.PruneOptions{Force: true}))
|
||||
|
||||
// When blobs/ is the link, these paths run through it.
|
||||
for _, blobFile := range blobFiles {
|
||||
assert.FileExists(t, blobFile, "prune must delete no blob")
|
||||
}
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPruneFailsOnSymlinkedDirectoryWithMissingTarget checks that prune
|
||||
// fails, deleting no blob and keeping the local snapshot record, when the
|
||||
// metadata/ or blobs/ directory of a file:// destination is a symlink
|
||||
// whose target is missing, as a link to a volume that is not mounted is.
|
||||
func TestPruneFailsOnSymlinkedDirectoryWithMissingTarget(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
for _, dir := range []string{"metadata", "blobs"} {
|
||||
t.Run(dir, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "store")
|
||||
v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
|
||||
blobFiles := blobFilesIn(t, storeDir)
|
||||
targetPath := moveBehindSymlink(t, storeDir, dir)
|
||||
|
||||
// Unmount the volume the link points to, and mount it again
|
||||
// once prune has run.
|
||||
unmountedPath := targetPath + "-unmounted"
|
||||
require.NoError(t, os.Rename(targetPath, unmountedPath))
|
||||
|
||||
err := v.Prune(&vaultik.PruneOptions{Force: true})
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
|
||||
require.NoError(t, os.Rename(unmountedPath, targetPath))
|
||||
|
||||
// When blobs/ is the link, these paths run through it.
|
||||
for _, blobFile := range blobFiles {
|
||||
assert.FileExists(t, blobFile, "prune must delete no blob")
|
||||
}
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// blobFilesIn returns the paths of the blob files under storeDir/blobs, and
|
||||
// fails the test when there are none.
|
||||
func blobFilesIn(t *testing.T, storeDir string) []string {
|
||||
t.Helper()
|
||||
|
||||
var blobFiles []string
|
||||
|
||||
err := filepath.WalkDir(filepath.Join(storeDir, "blobs"),
|
||||
func(path string, entry fs.DirEntry, err error) error {
|
||||
if err == nil && !entry.IsDir() {
|
||||
blobFiles = append(blobFiles, path)
|
||||
}
|
||||
|
||||
return err
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, blobFiles)
|
||||
|
||||
return blobFiles
|
||||
}
|
||||
|
||||
// moveBehindSymlink moves storeDir/dir to a directory elsewhere, puts a
|
||||
// symlink to it in its place, and returns where it went.
|
||||
func moveBehindSymlink(t *testing.T, storeDir, dir string) string {
|
||||
t.Helper()
|
||||
|
||||
linkPath := filepath.Join(storeDir, dir)
|
||||
targetPath := filepath.Join(t.TempDir(), dir)
|
||||
require.NoError(t, os.Rename(linkPath, targetPath))
|
||||
require.NoError(t, os.Symlink(targetPath, linkPath))
|
||||
|
||||
return targetPath
|
||||
}
|
||||
Reference in New Issue
Block a user