Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f73397b3 | ||
|
|
05a2c86d65 | ||
|
|
d8f78a1f45 |
No files matched your search
+2
-1
@@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups"
|
|||||||
|
|
||||||
# Average chunk size for content-defined chunking
|
# Average chunk size for content-defined chunking
|
||||||
# Smaller chunks = better deduplication but more metadata
|
# Smaller chunks = better deduplication but more metadata
|
||||||
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
# Minimum 1MB
|
||||||
|
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||||
# Default: 10MB
|
# Default: 10MB
|
||||||
#chunk_size: 10MB
|
#chunk_size: 10MB
|
||||||
|
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ storage_url: ""
|
|||||||
|
|
||||||
# Average chunk size for content-defined chunking (FastCDC).
|
# Average chunk size for content-defined chunking (FastCDC).
|
||||||
# Smaller = better deduplication but more metadata overhead.
|
# Smaller = better deduplication but more metadata overhead.
|
||||||
# Accepts: 1MB, 10M, 64KB, etc.
|
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
||||||
# Default: 10MB
|
# Default: 10MB
|
||||||
# chunk_size: 10MB
|
# chunk_size: 10MB
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Words and flags of `snapshot purge` used to build argument vectors
|
||||||
|
// below.
|
||||||
|
const (
|
||||||
|
cmdPurge = "purge"
|
||||||
|
flagOlderThan = "--older-than"
|
||||||
|
flagForce = "--force"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
|
||||||
|
// the local index refuse to delete an old snapshot and checks that
|
||||||
|
// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
|
||||||
|
// the destination store. It used to log the error, delete the metadata
|
||||||
|
// and exit 0.
|
||||||
|
//
|
||||||
|
// `snapshot create --prune` runs the same purge, but cannot be driven
|
||||||
|
// this way: the refusal also breaks the backup's metadata export, which
|
||||||
|
// deletes the other snapshots from its copy of the local index.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
|
||||||
|
configPath := writeHermeticPruneConfig(t, false)
|
||||||
|
dir := filepath.Dir(configPath)
|
||||||
|
indexPath := filepath.Join(dir, "index.sqlite")
|
||||||
|
|
||||||
|
// seedStaleSnapshotRecord writes an old snapshot to the local index
|
||||||
|
// only; the manifest planted here puts it on the destination store
|
||||||
|
// too, so the purge selects it rather than dropping it as stale.
|
||||||
|
seedStaleSnapshotRecord(t, indexPath)
|
||||||
|
|
||||||
|
manifestDir := filepath.Join(dir, "store", "metadata",
|
||||||
|
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
||||||
|
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
||||||
|
|
||||||
|
manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
|
||||||
|
require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
db, err := database.New(ctx, indexPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = db.Conn().ExecContext(ctx, `
|
||||||
|
CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
code, _, _ := runEntry(t, flagConfig, configPath,
|
||||||
|
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
||||||
|
|
||||||
|
assert.Equal(t, 1, code)
|
||||||
|
assert.FileExists(t, manifestPath,
|
||||||
|
"the snapshot is still in the local index, so its metadata must stay")
|
||||||
|
}
|
||||||
@@ -30,13 +30,15 @@ const appName = "vaultik"
|
|||||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||||
|
|
||||||
// Defaults and validation bounds for tunable settings.
|
// Defaults and validation bounds for tunable settings. MB and GB are
|
||||||
|
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
||||||
|
// limits, which are 1024-based (MiB, GiB).
|
||||||
const (
|
const (
|
||||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
||||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||||
defaultCompressionLevel = 3
|
defaultCompressionLevel = 3
|
||||||
minChunkSize = 1024 * 1024 // 1MB
|
minChunkSize = 1000 * 1000 // 1MB
|
||||||
minCompressionLevel = 1
|
minCompressionLevel = 1
|
||||||
maxCompressionLevel = 19
|
maxCompressionLevel = 19
|
||||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||||
|
|||||||
@@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
||||||
|
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
||||||
|
// (1000-based), so writing a default out in the config file changes nothing.
|
||||||
|
func TestDocumentedSizeDefaults(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.ChunkSize != defaultChunkSize {
|
||||||
|
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
||||||
|
cfg.ChunkSize, defaultChunkSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
||||||
|
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
||||||
|
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
||||||
|
// every size the README, config.example.yml and the config init template
|
||||||
|
// list as an example is accepted.
|
||||||
|
func TestDocumentedSizeExamples(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const withoutSizes = "snapshots:\n" +
|
||||||
|
" test:\n" +
|
||||||
|
" paths: [/tmp/vaultik-test-source]\n" +
|
||||||
|
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
setting string
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{"chunk_size: 1MB", nil},
|
||||||
|
{"chunk_size: 999999", errChunkSizeTooSmall},
|
||||||
|
{"chunk_size: 5M", nil},
|
||||||
|
{"chunk_size: 10M", nil},
|
||||||
|
{"chunk_size: 10MB", nil},
|
||||||
|
{"chunk_size: 64MiB", nil},
|
||||||
|
{"chunk_size: 1GB", nil},
|
||||||
|
// Four times the default chunk_size, the documented minimum.
|
||||||
|
{"blob_size_limit: 40MB", nil},
|
||||||
|
{"blob_size_limit: 500MB", nil},
|
||||||
|
{"blob_size_limit: 1GB", nil},
|
||||||
|
{"blob_size_limit: 1GiB", nil},
|
||||||
|
{"blob_size_limit: 10G", nil},
|
||||||
|
{"blob_size_limit: 10GB", nil},
|
||||||
|
{"s3: {part_size: 5MiB}", nil},
|
||||||
|
{"s3: {part_size: 10MB}", nil},
|
||||||
|
{"s3: {part_size: 16MiB}", nil},
|
||||||
|
{"s3: {part_size: 100MiB}", nil},
|
||||||
|
{"s3: {part_size: 5GiB}", nil},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.setting, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cfg.Validate()
|
||||||
|
if !errors.Is(err, tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||||
// relative path fails at load. Accepted, it would resolve against the
|
// relative path fails at load. Accepted, it would resolve against the
|
||||||
// working directory, so a backup run from another directory would write
|
// working directory, so a backup run from another directory would write
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ const (
|
|||||||
restoringUserID = 1000
|
restoringUserID = 1000
|
||||||
restoringGroupID = 1000
|
restoringGroupID = 1000
|
||||||
rootUserID = 0
|
rootUserID = 0
|
||||||
|
otherOwnedPath = "/other"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRestoreOwnershipWarning(t *testing.T) {
|
func TestRestoreOwnershipWarning(t *testing.T) {
|
||||||
@@ -51,7 +52,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
name: "another owner",
|
name: "another owner",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
ownFile,
|
ownFile,
|
||||||
{Path: "/other", UID: otherOwnerID, GID: restoringGroupID},
|
{Path: otherOwnedPath, UID: otherOwnerID, GID: restoringGroupID},
|
||||||
},
|
},
|
||||||
uid: restoringUserID,
|
uid: restoringUserID,
|
||||||
wantWarn: true,
|
wantWarn: true,
|
||||||
@@ -60,7 +61,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
name: "another group",
|
name: "another group",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
ownFile,
|
ownFile,
|
||||||
{Path: "/other", UID: restoringUserID, GID: otherOwnerID},
|
{Path: otherOwnedPath, UID: restoringUserID, GID: otherOwnerID},
|
||||||
},
|
},
|
||||||
uid: restoringUserID,
|
uid: restoringUserID,
|
||||||
wantWarn: true,
|
wantWarn: true,
|
||||||
@@ -68,7 +69,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "root restores another owner",
|
name: "root restores another owner",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
{Path: "/other", UID: otherOwnerID, GID: otherOwnerID},
|
{Path: otherOwnedPath, UID: otherOwnerID, GID: otherOwnerID},
|
||||||
},
|
},
|
||||||
uid: rootUserID,
|
uid: rootUserID,
|
||||||
wantWarn: false,
|
wantWarn: false,
|
||||||
|
|||||||
@@ -669,10 +669,11 @@ func (v *Vaultik) confirmAndExecutePurge(
|
|||||||
snapshotID := snap.ID.String()
|
snapshotID := snap.ID.String()
|
||||||
log.Info("Deleting snapshot", "id", snapshotID)
|
log.Info("Deleting snapshot", "id", snapshotID)
|
||||||
|
|
||||||
|
// Stop before the remote delete, so a snapshot still in the local
|
||||||
|
// index keeps its metadata on the destination store.
|
||||||
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Failed to delete from local database",
|
return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
|
||||||
"snapshot_id", snapshotID, "error", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
||||||
|
|||||||
Reference in new issue
Block a user