Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f73397b3 | ||
|
|
05a2c86d65 |
+2
-1
@@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
|
||||
# Average chunk size for content-defined chunking
|
||||
# Smaller chunks = better deduplication but more metadata
|
||||
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
||||
# Minimum 1MB
|
||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
#chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -224,7 +224,7 @@ storage_url: ""
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Accepts: 1MB, 10M, 64KB, etc.
|
||||
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -30,13 +30,15 @@ const appName = "vaultik"
|
||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||
|
||||
// Defaults and validation bounds for tunable settings.
|
||||
// Defaults and validation bounds for tunable settings. MB and GB are
|
||||
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
||||
// limits, which are 1024-based (MiB, GiB).
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||
defaultCompressionLevel = 3
|
||||
minChunkSize = 1024 * 1024 // 1MB
|
||||
minChunkSize = 1000 * 1000 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||
|
||||
@@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
||||
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
||||
// (1000-based), so writing a default out in the config file changes nothing.
|
||||
func TestDocumentedSizeDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.ChunkSize != defaultChunkSize {
|
||||
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
||||
cfg.ChunkSize, defaultChunkSize)
|
||||
}
|
||||
|
||||
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
||||
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
||||
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
||||
// every size the README, config.example.yml and the config init template
|
||||
// list as an example is accepted.
|
||||
func TestDocumentedSizeExamples(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutSizes = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
tests := []struct {
|
||||
setting string
|
||||
wantErr error
|
||||
}{
|
||||
{"chunk_size: 1MB", nil},
|
||||
{"chunk_size: 999999", errChunkSizeTooSmall},
|
||||
{"chunk_size: 5M", nil},
|
||||
{"chunk_size: 10M", nil},
|
||||
{"chunk_size: 10MB", nil},
|
||||
{"chunk_size: 64MiB", nil},
|
||||
{"chunk_size: 1GB", nil},
|
||||
// Four times the default chunk_size, the documented minimum.
|
||||
{"blob_size_limit: 40MB", nil},
|
||||
{"blob_size_limit: 500MB", nil},
|
||||
{"blob_size_limit: 1GB", nil},
|
||||
{"blob_size_limit: 1GiB", nil},
|
||||
{"blob_size_limit: 10G", nil},
|
||||
{"blob_size_limit: 10GB", nil},
|
||||
{"s3: {part_size: 5MiB}", nil},
|
||||
{"s3: {part_size: 10MB}", nil},
|
||||
{"s3: {part_size: 16MiB}", nil},
|
||||
{"s3: {part_size: 100MiB}", nil},
|
||||
{"s3: {part_size: 5GiB}", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||
// relative path fails at load. Accepted, it would resolve against the
|
||||
// working directory, so a backup run from another directory would write
|
||||
|
||||
@@ -218,11 +218,7 @@ func (v *Vaultik) finishRestore(
|
||||
v.UI.Duration(result.Duration),
|
||||
)
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
}
|
||||
v.warnIfOwnershipNotPreserved(files, os.Geteuid(), os.Getegid())
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
|
||||
@@ -247,6 +243,28 @@ func (v *Vaultik) finishRestore(
|
||||
return unmatchedErr
|
||||
}
|
||||
|
||||
// warnIfOwnershipNotPreserved warns when restoring files as user uid
|
||||
// and group gid leaves some entry with another owner or group than the
|
||||
// snapshot records. Restore changes ownership only as root (uid 0); any
|
||||
// other user's restore leaves every entry with that user's ids.
|
||||
func (v *Vaultik) warnIfOwnershipNotPreserved(
|
||||
files []*database.File, uid, gid int,
|
||||
) {
|
||||
if uid == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if int(file.UID) != uid || int(file.GID) != gid {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reportUnmatchedPaths warns, by name, about each requested path that
|
||||
// matched nothing in the snapshot, and returns an error counting them,
|
||||
// or nil when there are none.
|
||||
@@ -967,8 +985,9 @@ type restoreSession struct {
|
||||
// root can chown a file to an arbitrary UID/GID — non-root chown
|
||||
// always fails with EPERM. Attempting it anyway produces N
|
||||
// guaranteed-failed syscalls + N noisy debug lines, so we skip
|
||||
// the call entirely as non-root and emit one warning at the end
|
||||
// of the restore explaining that ownership was not preserved.
|
||||
// the call entirely as non-root; warnIfOwnershipNotPreserved warns
|
||||
// once at the end of the restore if that left any owner or group
|
||||
// other than recorded.
|
||||
runningAsRoot bool
|
||||
// directories holds every restored directory, for
|
||||
// applyDirectoryMetadata to finish after the restore loop.
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package vaultik //nolint:testpackage // calls the unexported ownership check
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/316: a
|
||||
// restore by a normal user warns that ownership was not preserved only
|
||||
// when some entry's recorded owner or group differs from the restoring
|
||||
// user's. The restoring user's ids are passed in, because make test runs
|
||||
// as root.
|
||||
|
||||
const (
|
||||
restoringUserID = 1000
|
||||
restoringGroupID = 1000
|
||||
rootUserID = 0
|
||||
otherOwnedPath = "/other"
|
||||
)
|
||||
|
||||
func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ownFile := &database.File{
|
||||
Path: "/own", UID: restoringUserID, GID: restoringGroupID,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
files []*database.File
|
||||
uid int
|
||||
wantWarn bool
|
||||
}{
|
||||
{
|
||||
name: "every owner matches",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: "/own/dir", UID: restoringUserID, GID: restoringGroupID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: false,
|
||||
},
|
||||
{
|
||||
name: "another owner",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: restoringGroupID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
},
|
||||
{
|
||||
name: "another group",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: otherOwnedPath, UID: restoringUserID, GID: otherOwnerID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
},
|
||||
{
|
||||
name: "root restores another owner",
|
||||
files: []*database.File{
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: otherOwnerID},
|
||||
},
|
||||
uid: rootUserID,
|
||||
wantWarn: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v := newContainmentVaultik(context.Background(), afero.NewMemMapFs())
|
||||
v.UI = ui.NewWithColor(&out, false)
|
||||
|
||||
v.warnIfOwnershipNotPreserved(tt.files, tt.uid, restoringGroupID)
|
||||
|
||||
if tt.wantWarn {
|
||||
assert.Contains(t, out.String(), "did not preserve file ownership")
|
||||
} else {
|
||||
assert.Empty(t, out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user