Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f73397b3 | ||
|
|
05a2c86d65 | ||
|
|
d8f78a1f45 |
No files matched your search
+2
-1
@@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
|
||||
# Average chunk size for content-defined chunking
|
||||
# Smaller chunks = better deduplication but more metadata
|
||||
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
||||
# Minimum 1MB
|
||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
#chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -224,7 +224,7 @@ storage_url: ""
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Accepts: 1MB, 10M, 64KB, etc.
|
||||
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// Words and flags of `snapshot purge` used to build argument vectors
|
||||
// below.
|
||||
const (
|
||||
cmdPurge = "purge"
|
||||
flagOlderThan = "--older-than"
|
||||
flagForce = "--force"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
|
||||
// the local index refuse to delete an old snapshot and checks that
|
||||
// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
|
||||
// the destination store. It used to log the error, delete the metadata
|
||||
// and exit 0.
|
||||
//
|
||||
// `snapshot create --prune` runs the same purge, but cannot be driven
|
||||
// this way: the refusal also breaks the backup's metadata export, which
|
||||
// deletes the other snapshots from its copy of the local index.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
dir := filepath.Dir(configPath)
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
|
||||
// seedStaleSnapshotRecord writes an old snapshot to the local index
|
||||
// only; the manifest planted here puts it on the destination store
|
||||
// too, so the purge selects it rather than dropping it as stale.
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
manifestDir := filepath.Join(dir, "store", "metadata",
|
||||
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
||||
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
||||
|
||||
manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
|
||||
require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, indexPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Conn().ExecContext(ctx, `
|
||||
CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
|
||||
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath,
|
||||
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.FileExists(t, manifestPath,
|
||||
"the snapshot is still in the local index, so its metadata must stay")
|
||||
}
|
||||
@@ -30,13 +30,15 @@ const appName = "vaultik"
|
||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||
|
||||
// Defaults and validation bounds for tunable settings.
|
||||
// Defaults and validation bounds for tunable settings. MB and GB are
|
||||
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
||||
// limits, which are 1024-based (MiB, GiB).
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||
defaultCompressionLevel = 3
|
||||
minChunkSize = 1024 * 1024 // 1MB
|
||||
minChunkSize = 1000 * 1000 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||
|
||||
@@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
||||
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
||||
// (1000-based), so writing a default out in the config file changes nothing.
|
||||
func TestDocumentedSizeDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.ChunkSize != defaultChunkSize {
|
||||
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
||||
cfg.ChunkSize, defaultChunkSize)
|
||||
}
|
||||
|
||||
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
||||
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
||||
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
||||
// every size the README, config.example.yml and the config init template
|
||||
// list as an example is accepted.
|
||||
func TestDocumentedSizeExamples(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutSizes = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
tests := []struct {
|
||||
setting string
|
||||
wantErr error
|
||||
}{
|
||||
{"chunk_size: 1MB", nil},
|
||||
{"chunk_size: 999999", errChunkSizeTooSmall},
|
||||
{"chunk_size: 5M", nil},
|
||||
{"chunk_size: 10M", nil},
|
||||
{"chunk_size: 10MB", nil},
|
||||
{"chunk_size: 64MiB", nil},
|
||||
{"chunk_size: 1GB", nil},
|
||||
// Four times the default chunk_size, the documented minimum.
|
||||
{"blob_size_limit: 40MB", nil},
|
||||
{"blob_size_limit: 500MB", nil},
|
||||
{"blob_size_limit: 1GB", nil},
|
||||
{"blob_size_limit: 1GiB", nil},
|
||||
{"blob_size_limit: 10G", nil},
|
||||
{"blob_size_limit: 10GB", nil},
|
||||
{"s3: {part_size: 5MiB}", nil},
|
||||
{"s3: {part_size: 10MB}", nil},
|
||||
{"s3: {part_size: 16MiB}", nil},
|
||||
{"s3: {part_size: 100MiB}", nil},
|
||||
{"s3: {part_size: 5GiB}", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||
// relative path fails at load. Accepted, it would resolve against the
|
||||
// working directory, so a backup run from another directory would write
|
||||
|
||||
@@ -22,6 +22,7 @@ const (
|
||||
restoringUserID = 1000
|
||||
restoringGroupID = 1000
|
||||
rootUserID = 0
|
||||
otherOwnedPath = "/other"
|
||||
)
|
||||
|
||||
func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
@@ -51,7 +52,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
name: "another owner",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: "/other", UID: otherOwnerID, GID: restoringGroupID},
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: restoringGroupID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
@@ -60,7 +61,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
name: "another group",
|
||||
files: []*database.File{
|
||||
ownFile,
|
||||
{Path: "/other", UID: restoringUserID, GID: otherOwnerID},
|
||||
{Path: otherOwnedPath, UID: restoringUserID, GID: otherOwnerID},
|
||||
},
|
||||
uid: restoringUserID,
|
||||
wantWarn: true,
|
||||
@@ -68,7 +69,7 @@ func TestRestoreOwnershipWarning(t *testing.T) {
|
||||
{
|
||||
name: "root restores another owner",
|
||||
files: []*database.File{
|
||||
{Path: "/other", UID: otherOwnerID, GID: otherOwnerID},
|
||||
{Path: otherOwnedPath, UID: otherOwnerID, GID: otherOwnerID},
|
||||
},
|
||||
uid: rootUserID,
|
||||
wantWarn: false,
|
||||
|
||||
@@ -669,10 +669,11 @@ func (v *Vaultik) confirmAndExecutePurge(
|
||||
snapshotID := snap.ID.String()
|
||||
log.Info("Deleting snapshot", "id", snapshotID)
|
||||
|
||||
// Stop before the remote delete, so a snapshot still in the local
|
||||
// index keeps its metadata on the destination store.
|
||||
err := v.deleteSnapshotFromLocalDB(snapshotID)
|
||||
if err != nil {
|
||||
log.Error("Failed to delete from local database",
|
||||
"snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
|
||||
}
|
||||
|
||||
_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
||||
|
||||
Reference in new issue
Block a user