Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1ed100e78 |
No files matched your search
+1
-2
@@ -302,8 +302,7 @@ storage_url: "rclone://myremote/path/to/backups"
|
|||||||
|
|
||||||
# Average chunk size for content-defined chunking
|
# Average chunk size for content-defined chunking
|
||||||
# Smaller chunks = better deduplication but more metadata
|
# Smaller chunks = better deduplication but more metadata
|
||||||
# Minimum 1MB
|
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
||||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
|
||||||
# Default: 10MB
|
# Default: 10MB
|
||||||
#chunk_size: 10MB
|
#chunk_size: 10MB
|
||||||
|
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ storage_url: ""
|
|||||||
|
|
||||||
# Average chunk size for content-defined chunking (FastCDC).
|
# Average chunk size for content-defined chunking (FastCDC).
|
||||||
# Smaller = better deduplication but more metadata overhead.
|
# Smaller = better deduplication but more metadata overhead.
|
||||||
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
# Accepts: 1MB, 10M, 64KB, etc.
|
||||||
# Default: 10MB
|
# Default: 10MB
|
||||||
# chunk_size: 10MB
|
# chunk_size: 10MB
|
||||||
|
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Words and flags of `snapshot purge` used to build argument vectors
|
|
||||||
// below.
|
|
||||||
const (
|
|
||||||
cmdPurge = "purge"
|
|
||||||
flagOlderThan = "--older-than"
|
|
||||||
flagForce = "--force"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
|
|
||||||
// the local index refuse to delete an old snapshot and checks that
|
|
||||||
// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
|
|
||||||
// the destination store. It used to log the error, delete the metadata
|
|
||||||
// and exit 0.
|
|
||||||
//
|
|
||||||
// `snapshot create --prune` runs the same purge, but cannot be driven
|
|
||||||
// this way: the refusal also breaks the backup's metadata export, which
|
|
||||||
// deletes the other snapshots from its copy of the local index.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
|
||||||
func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
|
|
||||||
configPath := writeHermeticPruneConfig(t, false)
|
|
||||||
dir := filepath.Dir(configPath)
|
|
||||||
indexPath := filepath.Join(dir, "index.sqlite")
|
|
||||||
|
|
||||||
// seedStaleSnapshotRecord writes an old snapshot to the local index
|
|
||||||
// only; the manifest planted here puts it on the destination store
|
|
||||||
// too, so the purge selects it rather than dropping it as stale.
|
|
||||||
seedStaleSnapshotRecord(t, indexPath)
|
|
||||||
|
|
||||||
manifestDir := filepath.Join(dir, "store", "metadata",
|
|
||||||
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
|
||||||
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
|
||||||
|
|
||||||
manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
|
|
||||||
require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
db, err := database.New(ctx, indexPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = db.Conn().ExecContext(ctx, `
|
|
||||||
CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
|
|
||||||
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
code, _, _ := runEntry(t, flagConfig, configPath,
|
|
||||||
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
|
||||||
|
|
||||||
assert.Equal(t, 1, code)
|
|
||||||
assert.FileExists(t, manifestPath,
|
|
||||||
"the snapshot is still in the local index, so its metadata must stay")
|
|
||||||
}
|
|
||||||
@@ -30,15 +30,13 @@ const appName = "vaultik"
|
|||||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||||
|
|
||||||
// Defaults and validation bounds for tunable settings. MB and GB are
|
// Defaults and validation bounds for tunable settings.
|
||||||
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
|
||||||
// limits, which are 1024-based (MiB, GiB).
|
|
||||||
const (
|
const (
|
||||||
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||||
defaultCompressionLevel = 3
|
defaultCompressionLevel = 3
|
||||||
minChunkSize = 1000 * 1000 // 1MB
|
minChunkSize = 1024 * 1024 // 1MB
|
||||||
minCompressionLevel = 1
|
minCompressionLevel = 1
|
||||||
maxCompressionLevel = 19
|
maxCompressionLevel = 19
|
||||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||||
|
|||||||
@@ -337,81 +337,6 @@ func TestLoadS3PartSize(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
|
||||||
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
|
||||||
// (1000-based), so writing a default out in the config file changes nothing.
|
|
||||||
func TestDocumentedSizeDefaults(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.ChunkSize != defaultChunkSize {
|
|
||||||
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
|
||||||
cfg.ChunkSize, defaultChunkSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
|
||||||
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
|
||||||
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
|
||||||
// every size the README, config.example.yml and the config init template
|
|
||||||
// list as an example is accepted.
|
|
||||||
func TestDocumentedSizeExamples(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const withoutSizes = "snapshots:\n" +
|
|
||||||
" test:\n" +
|
|
||||||
" paths: [/tmp/vaultik-test-source]\n" +
|
|
||||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
setting string
|
|
||||||
wantErr error
|
|
||||||
}{
|
|
||||||
{"chunk_size: 1MB", nil},
|
|
||||||
{"chunk_size: 999999", errChunkSizeTooSmall},
|
|
||||||
{"chunk_size: 5M", nil},
|
|
||||||
{"chunk_size: 10M", nil},
|
|
||||||
{"chunk_size: 10MB", nil},
|
|
||||||
{"chunk_size: 64MiB", nil},
|
|
||||||
{"chunk_size: 1GB", nil},
|
|
||||||
// Four times the default chunk_size, the documented minimum.
|
|
||||||
{"blob_size_limit: 40MB", nil},
|
|
||||||
{"blob_size_limit: 500MB", nil},
|
|
||||||
{"blob_size_limit: 1GB", nil},
|
|
||||||
{"blob_size_limit: 1GiB", nil},
|
|
||||||
{"blob_size_limit: 10G", nil},
|
|
||||||
{"blob_size_limit: 10GB", nil},
|
|
||||||
{"s3: {part_size: 5MiB}", nil},
|
|
||||||
{"s3: {part_size: 10MB}", nil},
|
|
||||||
{"s3: {part_size: 16MiB}", nil},
|
|
||||||
{"s3: {part_size: 100MiB}", nil},
|
|
||||||
{"s3: {part_size: 5GiB}", nil},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.setting, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cfg.Validate()
|
|
||||||
if !errors.Is(err, tt.wantErr) {
|
|
||||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||||
// relative path fails at load. Accepted, it would resolve against the
|
// relative path fails at load. Accepted, it would resolve against the
|
||||||
// working directory, so a backup run from another directory would write
|
// working directory, so a backup run from another directory would write
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ const (
|
|||||||
restoringUserID = 1000
|
restoringUserID = 1000
|
||||||
restoringGroupID = 1000
|
restoringGroupID = 1000
|
||||||
rootUserID = 0
|
rootUserID = 0
|
||||||
otherOwnedPath = "/other"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRestoreOwnershipWarning(t *testing.T) {
|
func TestRestoreOwnershipWarning(t *testing.T) {
|
||||||
@@ -52,7 +51,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
name: "another owner",
|
name: "another owner",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
ownFile,
|
ownFile,
|
||||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: restoringGroupID},
|
{Path: "/other", UID: otherOwnerID, GID: restoringGroupID},
|
||||||
},
|
},
|
||||||
uid: restoringUserID,
|
uid: restoringUserID,
|
||||||
wantWarn: true,
|
wantWarn: true,
|
||||||
@@ -61,7 +60,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
name: "another group",
|
name: "another group",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
ownFile,
|
ownFile,
|
||||||
{Path: otherOwnedPath, UID: restoringUserID, GID: otherOwnerID},
|
{Path: "/other", UID: restoringUserID, GID: otherOwnerID},
|
||||||
},
|
},
|
||||||
uid: restoringUserID,
|
uid: restoringUserID,
|
||||||
wantWarn: true,
|
wantWarn: true,
|
||||||
@@ -69,7 +68,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "root restores another owner",
|
name: "root restores another owner",
|
||||||
files: []*database.File{
|
files: []*database.File{
|
||||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: otherOwnerID},
|
{Path: "/other", UID: otherOwnerID, GID: otherOwnerID},
|
||||||
},
|
},
|
||||||
uid: rootUserID,
|
uid: rootUserID,
|
||||||
wantWarn: false,
|
wantWarn: false,
|
||||||
|
|||||||
@@ -669,11 +669,10 @@ func (v *Vaultik) confirmAndExecutePurge(
|
|||||||
snapshotID := snap.ID.String()
|
snapshotID := snap.ID.String()
|
||||||
log.Info("Deleting snapshot", "id", snapshotID)
|
log.Info("Deleting snapshot", "id", snapshotID)
|
||||||
|
|
||||||
// Stop before the remote delete, so a snapshot still in the local
|
|
||||||
// index keeps its metadata on the destination store.
|
|
||||||
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
|
log.Error("Failed to delete from local database",
|
||||||
|
"snapshot_id", snapshotID, "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
||||||
|
|||||||
Reference in new issue
Block a user