Compare commits
3
Commits
b1ed100e78
...
62f73397b3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f73397b3 | ||
|
|
05a2c86d65 | ||
|
|
d8f78a1f45 |
No files matched your search
+2
-1
@@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
|
||||
# Average chunk size for content-defined chunking
|
||||
# Smaller chunks = better deduplication but more metadata
|
||||
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
||||
# Minimum 1MB
|
||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
#chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -224,7 +224,7 @@ storage_url: ""
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Accepts: 1MB, 10M, 64KB, etc.
|
||||
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// Words and flags of `snapshot purge` used to build argument vectors
|
||||
// below.
|
||||
const (
|
||||
cmdPurge = "purge"
|
||||
flagOlderThan = "--older-than"
|
||||
flagForce = "--force"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
|
||||
// the local index refuse to delete an old snapshot and checks that
|
||||
// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
|
||||
// the destination store. It used to log the error, delete the metadata
|
||||
// and exit 0.
|
||||
//
|
||||
// `snapshot create --prune` runs the same purge, but cannot be driven
|
||||
// this way: the refusal also breaks the backup's metadata export, which
|
||||
// deletes the other snapshots from its copy of the local index.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
dir := filepath.Dir(configPath)
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
|
||||
// seedStaleSnapshotRecord writes an old snapshot to the local index
|
||||
// only; the manifest planted here puts it on the destination store
|
||||
// too, so the purge selects it rather than dropping it as stale.
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
manifestDir := filepath.Join(dir, "store", "metadata",
|
||||
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
||||
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
||||
|
||||
manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
|
||||
require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, indexPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Conn().ExecContext(ctx, `
|
||||
CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
|
||||
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath,
|
||||
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.FileExists(t, manifestPath,
|
||||
"the snapshot is still in the local index, so its metadata must stay")
|
||||
}
|
||||
@@ -30,13 +30,15 @@ const appName = "vaultik"
|
||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||
|
||||
// Defaults and validation bounds for tunable settings.
|
||||
// Defaults and validation bounds for tunable settings. MB and GB are
|
||||
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
||||
// limits, which are 1024-based (MiB, GiB).
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||
defaultCompressionLevel = 3
|
||||
minChunkSize = 1024 * 1024 // 1MB
|
||||
minChunkSize = 1000 * 1000 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||
|
||||
@@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
||||
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
||||
// (1000-based), so writing a default out in the config file changes nothing.
|
||||
func TestDocumentedSizeDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.ChunkSize != defaultChunkSize {
|
||||
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
||||
cfg.ChunkSize, defaultChunkSize)
|
||||
}
|
||||
|
||||
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
||||
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
||||
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
||||
// every size the README, config.example.yml and the config init template
|
||||
// list as an example is accepted.
|
||||
func TestDocumentedSizeExamples(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutSizes = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
tests := []struct {
|
||||
setting string
|
||||
wantErr error
|
||||
}{
|
||||
{"chunk_size: 1MB", nil},
|
||||
{"chunk_size: 999999", errChunkSizeTooSmall},
|
||||
{"chunk_size: 5M", nil},
|
||||
{"chunk_size: 10M", nil},
|
||||
{"chunk_size: 10MB", nil},
|
||||
{"chunk_size: 64MiB", nil},
|
||||
{"chunk_size: 1GB", nil},
|
||||
// Four times the default chunk_size, the documented minimum.
|
||||
{"blob_size_limit: 40MB", nil},
|
||||
{"blob_size_limit: 500MB", nil},
|
||||
{"blob_size_limit: 1GB", nil},
|
||||
{"blob_size_limit: 1GiB", nil},
|
||||
{"blob_size_limit: 10G", nil},
|
||||
{"blob_size_limit: 10GB", nil},
|
||||
{"s3: {part_size: 5MiB}", nil},
|
||||
{"s3: {part_size: 10MB}", nil},
|
||||
{"s3: {part_size: 16MiB}", nil},
|
||||
{"s3: {part_size: 100MiB}", nil},
|
||||
{"s3: {part_size: 5GiB}", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||
// relative path fails at load. Accepted, it would resolve against the
|
||||
// working directory, so a backup run from another directory would write
|
||||
|
||||
@@ -218,11 +218,7 @@ func (v *Vaultik) finishRestore(
|
||||
v.UI.Duration(result.Duration),
|
||||
)
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
}
|
||||
v.warnIfOwnershipNotPreserved(files, os.Geteuid(), os.Getegid())
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
|
||||
@@ -247,6 +243,28 @@ func (v *Vaultik) finishRestore(
|
||||
return unmatchedErr
|
||||
}
|
||||
|
||||
// warnIfOwnershipNotPreserved warns when restoring files as user uid
|
||||
// and group gid leaves some entry with another owner or group than the
|
||||
// snapshot records. Restore changes ownership only as root (uid 0); any
|
||||
// other user's restore leaves every entry with that user's ids.
|
||||
func (v *Vaultik) warnIfOwnershipNotPreserved(
|
||||
files []*database.File, uid, gid int,
|
||||
) {
|
||||
if uid == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if int(file.UID) != uid || int(file.GID) != gid {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reportUnmatchedPaths warns, by name, about each requested path that
|
||||
// matched nothing in the snapshot, and returns an error counting them,
|
||||
// or nil when there are none.
|
||||
@@ -967,8 +985,9 @@ type restoreSession struct {
|
||||
// root can chown a file to an arbitrary UID/GID — non-root chown
|
||||
// always fails with EPERM. Attempting it anyway produces N
|
||||
// guaranteed-failed syscalls + N noisy debug lines, so we skip
|
||||
// the call entirely as non-root and emit one warning at the end
|
||||
// of the restore explaining that ownership was not preserved.
|
||||
// the call entirely as non-root; warnIfOwnershipNotPreserved warns
|
||||
// once at the end of the restore if that left any owner or group
|
||||
// other than recorded.
|
||||
runningAsRoot bool
|
||||
// directories holds every restored directory, for
|
||||
// applyDirectoryMetadata to finish after the restore loop.
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package vaultik //nolint:testpackage // calls the unexported ownership check
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/316: a
|
||||
// restore by a normal user warns that ownership was not preserved only
|
||||
// when some entry's recorded owner or group differs from the restoring
|
||||
// user's. The restoring user's ids are passed in, because make test runs
|
||||
// as root.
|
||||
|
||||
const (
|
||||
restoringUserID = 1000
|
||||
restoringGroupID = 1000
|
||||
rootUserID = 0
|
||||
otherOwnedPath = "/other"
|
||||
)
|
||||
|
||||
func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ownFile := &database.File{
|
||||
Path: "/own", UID: restoringUserID, GID: restoringGroupID,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
files []*database.File
|
||||
uid int
|
||||
wantWarn bool
|
||||
}{
|
||||
{
|
||||
name: "every owner matches",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: "/own/dir", UID: restoringUserID, GID: restoringGroupID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: false,
|
||||
},
|
||||
{
|
||||
name: "another owner",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: restoringGroupID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
},
|
||||
{
|
||||
name: "another group",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: otherOwnedPath, UID: restoringUserID, GID: otherOwnerID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
},
|
||||
{
|
||||
name: "root restores another owner",
|
||||
files: []*database.File{
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: otherOwnerID},
|
||||
},
|
||||
uid: rootUserID,
|
||||
wantWarn: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v := newContainmentVaultik(context.Background(), afero.NewMemMapFs())
|
||||
v.UI = ui.NewWithColor(&out, false)
|
||||
|
||||
v.warnIfOwnershipNotPreserved(tt.files, tt.uid, restoringGroupID)
|
||||
|
||||
if tt.wantWarn {
|
||||
assert.Contains(t, out.String(), "did not preserve file ownership")
|
||||
} else {
|
||||
assert.Empty(t, out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -669,10 +669,11 @@ func (v *Vaultik) confirmAndExecutePurge(
|
||||
snapshotID := snap.ID.String()
|
||||
log.Info("Deleting snapshot", "id", snapshotID)
|
||||
|
||||
// Stop before the remote delete, so a snapshot still in the local
|
||||
// index keeps its metadata on the destination store.
|
||||
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
||||
if err != nil {
|
||||
log.Error("Failed to delete from local database",
|
||||
"snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
|
||||
}
|
||||
|
||||
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
||||
|
||||
Reference in new issue
Block a user