Compare commits
2
Commits
99d3dbd57d
...
98b981203e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
98b981203e | ||
|
|
3fc8a8f2f4 |
No files matched your search
@@ -22,6 +22,26 @@ the tag exists and is exercised; what is left is merging `next` to
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-07: Made `s3.part_size` set the multipart upload part size
|
||||
([issue #232](https://git.eeqj.de/sneak/vaultik/issues/232)). It was
|
||||
loaded and defaulted but never passed to the S3 client, whose uploader
|
||||
used a fixed 10MiB part. It now reaches the uploader for `storage_url`
|
||||
and for the `s3.*` fields, and a part size S3 refuses, below 5MiB or
|
||||
above 5GiB, fails at config load. The docs gave the default as `5MB`,
|
||||
which the config file reads as 5,000,000 bytes, below the minimum; they
|
||||
now say `5MiB`.
|
||||
|
||||
- 2026-10-07: Made per-name retention work when the hostname contains `_`
|
||||
([issue #230](https://git.eeqj.de/sneak/vaultik/issues/230)). A
|
||||
snapshot ID is `hostname_name_timestamp`, and the name was read as
|
||||
everything between the first and the last `_`, so with
|
||||
`hostname: my_host` the name `home` came out as `host_home`.
|
||||
`snapshot purge --keep-latest --snapshot home` then printed "No
|
||||
snapshots to delete", and `snapshot create --prune` purged nothing
|
||||
without a message. The name is now read using the hostname the
|
||||
`snapshots` table stores with each snapshot, cut at its first `.` as it
|
||||
is in the ID.
|
||||
|
||||
- 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as
|
||||
orphaned when its manifest cannot be read, and stop printing raw
|
||||
names from under `metadata/`
|
||||
|
||||
+4
-4
@@ -287,10 +287,10 @@ storage_url: "rclone://myremote/path/to/backups"
|
||||
# #use_ssl: true
|
||||
#
|
||||
# # Part size for multipart uploads
|
||||
# # Minimum 5MB, affects memory usage during upload
|
||||
# # Supports: 5MB, 10M, 100MiB, etc.
|
||||
# # Default: 5MB
|
||||
# #part_size: 5MB
|
||||
# # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
|
||||
# # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
|
||||
# # Default: 5MiB
|
||||
# #part_size: 5MiB
|
||||
|
||||
# Path to local SQLite index database
|
||||
# This database tracks file state for incremental backups
|
||||
|
||||
@@ -205,7 +205,7 @@ storage_url: ""
|
||||
# access_key_id: YOUR_ACCESS_KEY
|
||||
# secret_access_key: YOUR_SECRET_KEY
|
||||
# # region: us-east-1 # Default: us-east-1
|
||||
# # part_size: 5MB # Multipart upload part size. Default: 5MB
|
||||
# # part_size: 5MiB # Upload part size, 5MiB to 5GiB. Default: 5MiB
|
||||
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
|
||||
|
||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -33,11 +33,14 @@ const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||
defaultCompressionLevel = 3
|
||||
minChunkSize = 1024 * 1024 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||
minS3PartSize = 5 * 1024 * 1024
|
||||
maxS3PartSize = 5 * 1024 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Sentinel validation errors.
|
||||
@@ -55,6 +58,7 @@ var (
|
||||
"blob_size_limit must be at least the largest chunk the chunker can " +
|
||||
"emit (chunk_size times the FastCDC size spread)")
|
||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||
errBadStorageScheme = errors.New(
|
||||
"storage_url must start with s3://, file://, or rclone://")
|
||||
errStorageNotConfigured = errors.New(
|
||||
@@ -332,6 +336,7 @@ func Load(path string) (*Config, error) {
|
||||
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
|
||||
// exceeds the configured limit
|
||||
// - Compression level must be between 1 and 19
|
||||
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
|
||||
//
|
||||
// Returns an error describing the first validation failure encountered.
|
||||
func (c *Config) Validate() error {
|
||||
@@ -376,6 +381,11 @@ func (c *Config) Validate() error {
|
||||
return errBadCompression
|
||||
}
|
||||
|
||||
if c.S3.PartSize.Int64() < minS3PartSize ||
|
||||
c.S3.PartSize.Int64() > maxS3PartSize {
|
||||
return errBadS3PartSize
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -166,6 +166,7 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
||||
ChunkSize: chunkSize,
|
||||
BlobSizeLimit: blobLimit,
|
||||
CompressionLevel: 3,
|
||||
S3: S3Config{PartSize: defaultS3PartSize},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +222,72 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
|
||||
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
|
||||
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
|
||||
func TestValidateS3PartSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
partSize Size
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "5MB is rejected",
|
||||
partSize: 5_000_000,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "one byte below 5MiB is rejected",
|
||||
partSize: minS3PartSize - 1,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "5MiB is accepted",
|
||||
partSize: minS3PartSize,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "5GiB is accepted",
|
||||
partSize: maxS3PartSize,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "one byte above 5GiB is rejected",
|
||||
partSize: maxS3PartSize + 1,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := *base
|
||||
cfg.S3.PartSize = tt.partSize
|
||||
|
||||
err := cfg.Validate()
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, errBadS3PartSize) {
|
||||
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||
@@ -236,6 +303,7 @@ func TestValidateAgeRecipients(t *testing.T) {
|
||||
ChunkSize: Size(10 * 1024 * 1024),
|
||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
|
||||
CompressionLevel: 3,
|
||||
S3: S3Config{PartSize: defaultS3PartSize},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,10 +27,12 @@ type Client struct {
|
||||
bucket string
|
||||
prefix string
|
||||
endpoint string
|
||||
partSize int64
|
||||
}
|
||||
|
||||
// Config contains S3 client configuration.
|
||||
// All fields are required except Prefix, which defaults to an empty string.
|
||||
// All fields are required except Prefix, which defaults to an empty string,
|
||||
// and PartSize, where zero means the SDK default of 5 MiB.
|
||||
// A non-empty Prefix is joined to every key with one "/", whether or not
|
||||
// it ends with one.
|
||||
// The Endpoint field should include the protocol (http:// or https://).
|
||||
@@ -41,6 +43,8 @@ type Config struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Region string
|
||||
// PartSize is the size in bytes of each part of a multipart upload.
|
||||
PartSize int64
|
||||
}
|
||||
|
||||
// nopLogger is a logger that discards all output.
|
||||
@@ -90,6 +94,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
|
||||
bucket: cfg.Bucket,
|
||||
prefix: prefix,
|
||||
endpoint: cfg.Endpoint,
|
||||
partSize: cfg.PartSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -123,12 +128,9 @@ func (c *Client) PutObjectWithProgress(
|
||||
) error {
|
||||
fullKey := c.prefix + key
|
||||
|
||||
// uploadPartSize is 10MB for better progress granularity.
|
||||
const uploadPartSize = 10 * 1024 * 1024
|
||||
|
||||
// Create an uploader with the S3 client
|
||||
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
|
||||
u.PartSize = uploadPartSize
|
||||
u.PartSize = c.partSize
|
||||
})
|
||||
|
||||
// Create a progress reader that tracks upload progress
|
||||
|
||||
@@ -28,6 +28,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
|
||||
AccessKeyID: cfg.S3.AccessKeyID,
|
||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||
Region: cfg.S3.Region,
|
||||
PartSize: cfg.S3.PartSize.Int64(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -105,17 +105,21 @@ func (sm *SnapshotManager) CreateSnapshot(
|
||||
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
|
||||
}
|
||||
|
||||
// ShortHostname returns hostname up to its first dot. A snapshot ID starts
|
||||
// with this form, while the snapshots table stores the full hostname.
|
||||
func ShortHostname(hostname string) string {
|
||||
short, _, _ := strings.Cut(hostname, ".")
|
||||
|
||||
return short
|
||||
}
|
||||
|
||||
// CreateSnapshotWithName creates a new snapshot record with an optional
|
||||
// snapshot name. The snapshot ID format is: hostname_name_timestamp or
|
||||
// hostname_timestamp if name is empty.
|
||||
func (sm *SnapshotManager) CreateSnapshotWithName(
|
||||
ctx context.Context, hostname, name, version, gitRevision string,
|
||||
) (string, error) {
|
||||
// Use short hostname (strip domain if present)
|
||||
shortHostname := hostname
|
||||
if before, _, ok := strings.Cut(hostname, "."); ok {
|
||||
shortHostname = before
|
||||
}
|
||||
shortHostname := ShortHostname(hostname)
|
||||
|
||||
// Build snapshot ID with optional name
|
||||
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||
|
||||
@@ -99,6 +99,7 @@ func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
||||
AccessKeyID: cfg.S3.AccessKeyID,
|
||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||
Region: region,
|
||||
PartSize: cfg.S3.PartSize.Int64(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||
@@ -134,6 +135,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
|
||||
AccessKeyID: cfg.S3.AccessKeyID,
|
||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||
Region: region,
|
||||
PartSize: cfg.S3.PartSize.Int64(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/johannesboyne/gofakes3"
|
||||
@@ -19,6 +22,13 @@ import (
|
||||
// s3TestBucket is the bucket created for each in-process S3 server.
|
||||
const s3TestBucket = "test-bucket"
|
||||
|
||||
// Credentials for the tests that build a storer from a config.Config. The
|
||||
// in-process S3 server accepts any.
|
||||
const (
|
||||
s3TestAccessKeyID = "key"
|
||||
s3TestSecretAccessKey = "secret"
|
||||
)
|
||||
|
||||
// newS3Storer builds an s3:// backend backed by a fresh in-process
|
||||
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
|
||||
// over httptest) that internal/s3 and the not-found regression test use,
|
||||
@@ -128,8 +138,8 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
|
||||
storer, err := storage.NewStorer(&config.Config{
|
||||
StorageURL: storageURL + "?endpoint=" + srv.URL,
|
||||
S3: config.S3Config{
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
AccessKeyID: s3TestAccessKeyID,
|
||||
SecretAccessKey: s3TestSecretAccessKey,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -171,6 +181,90 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestS3UploadUsesConfiguredPartSize checks that s3.part_size reaches the
|
||||
// multipart uploader, through storage_url and through the s3.* fields. An
|
||||
// object three parts long must arrive as three parts; at the SDK's default
|
||||
// of 5 MiB it would arrive as four.
|
||||
func TestS3UploadUsesConfiguredPartSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
partSize = 6 * 1024 * 1024
|
||||
wantParts = 3
|
||||
)
|
||||
|
||||
backend := s3mem.New()
|
||||
|
||||
err := backend.CreateBucket(s3TestBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("create bucket: %v", err)
|
||||
}
|
||||
|
||||
// Every part of a multipart upload is one request with a partNumber.
|
||||
var parts atomic.Int32
|
||||
|
||||
fake := gofakes3.New(backend).Server()
|
||||
srv := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Has("partNumber") {
|
||||
parts.Add(1)
|
||||
}
|
||||
|
||||
fake.ServeHTTP(w, r)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cfg *config.Config
|
||||
}{
|
||||
{
|
||||
name: "storage_url",
|
||||
cfg: &config.Config{
|
||||
StorageURL: "s3://" + s3TestBucket + "?endpoint=" + srv.URL,
|
||||
S3: config.S3Config{
|
||||
AccessKeyID: s3TestAccessKeyID,
|
||||
SecretAccessKey: s3TestSecretAccessKey,
|
||||
PartSize: partSize,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "s3.endpoint",
|
||||
cfg: &config.Config{
|
||||
S3: config.S3Config{
|
||||
Endpoint: srv.URL,
|
||||
Bucket: s3TestBucket,
|
||||
AccessKeyID: s3TestAccessKeyID,
|
||||
SecretAccessKey: s3TestSecretAccessKey,
|
||||
PartSize: partSize,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
parts.Store(0)
|
||||
|
||||
storer, err := storage.NewStorer(tc.cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: NewStorer: %v", tc.name, err)
|
||||
}
|
||||
|
||||
data := bytes.NewReader(make([]byte, wantParts*partSize))
|
||||
|
||||
err = storer.PutWithProgress(
|
||||
context.Background(), "blob", data, data.Size(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: PutWithProgress: %v", tc.name, err)
|
||||
}
|
||||
|
||||
if got := parts.Load(); got != wantParts {
|
||||
t.Errorf("%s: uploaded in %d parts, want %d", tc.name, got, wantParts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listStreamKeys returns the keys ListStream yields under a prefix, and
|
||||
// fails the test on a listing error.
|
||||
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
|
||||
|
||||
+23
-18
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
@@ -46,12 +47,9 @@ const (
|
||||
year = 365 * day
|
||||
)
|
||||
|
||||
// Snapshot IDs split on "_" into hostname, optional name parts, and a
|
||||
// trailing timestamp.
|
||||
const (
|
||||
minSnapshotIDParts = 2
|
||||
minSnapshotIDNameParts = 3
|
||||
)
|
||||
// A snapshot ID split on "_" has at least a hostname and a trailing
|
||||
// timestamp.
|
||||
const minSnapshotIDParts = 2
|
||||
|
||||
// SnapshotInfo contains information about a snapshot.
|
||||
//
|
||||
@@ -121,20 +119,27 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
||||
return timestamp.UTC(), nil
|
||||
}
|
||||
|
||||
// parseSnapshotName extracts the snapshot name from a snapshot ID.
|
||||
// Format: hostname_snapshotname_timestamp — the middle part(s) between hostname
|
||||
// and the RFC3339 timestamp are the snapshot name (may contain underscores).
|
||||
// Returns the snapshot name, or empty string if the ID is malformed.
|
||||
func parseSnapshotName(snapshotID string) string {
|
||||
parts := strings.Split(snapshotID, "_")
|
||||
if len(parts) < minSnapshotIDNameParts {
|
||||
// Format: hostname_timestamp — no snapshot name
|
||||
// parseSnapshotName extracts the snapshot name from a snapshot ID of the
|
||||
// form hostname_name_timestamp, given the hostname stored with that
|
||||
// snapshot. The hostname and the name may both contain underscores, so the
|
||||
// name is what is left after removing the short hostname and its "_" from
|
||||
// the front and the last "_" and the timestamp from the end. Returns "" for
|
||||
// an ID with no name (hostname_timestamp), and for an ID that does not start
|
||||
// with that hostname, which CreateSnapshotWithName never writes.
|
||||
func parseSnapshotName(snapshotID, hostname string) string {
|
||||
prefix := snapshot.ShortHostname(hostname) + "_"
|
||||
|
||||
rest, ok := strings.CutPrefix(snapshotID, prefix)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
// Format: hostname_name_timestamp — middle parts are the name.
|
||||
// The last part is the RFC3339 timestamp, the first part is the hostname,
|
||||
// everything in between is the snapshot name (which may itself contain underscores).
|
||||
return strings.Join(parts[1:len(parts)-1], "_")
|
||||
|
||||
end := strings.LastIndex(rest, "_")
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
// parseDuration parses a duration string with support for human-friendly units:
|
||||
|
||||
@@ -11,33 +11,55 @@ func TestParseSnapshotName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
snapshotID string
|
||||
hostname string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "standard format with name",
|
||||
snapshotID: "myhost_home_2026-01-12T14:41:15Z",
|
||||
hostname: "myhost",
|
||||
want: "home",
|
||||
},
|
||||
{
|
||||
name: "standard format with different name",
|
||||
snapshotID: "server1_system_2026-02-15T09:30:00Z",
|
||||
hostname: "server1",
|
||||
want: "system",
|
||||
},
|
||||
{
|
||||
name: "name with underscores",
|
||||
snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z",
|
||||
hostname: "myhost",
|
||||
want: "my_special_backup",
|
||||
},
|
||||
{
|
||||
name: "hostname with underscores",
|
||||
snapshotID: "my_host_docs_2026-03-01T00:00:00Z",
|
||||
hostname: "my_host",
|
||||
want: "docs",
|
||||
},
|
||||
{
|
||||
name: "stored hostname with domain",
|
||||
snapshotID: "my_host_mail_2026-03-01T00:00:00Z",
|
||||
hostname: "my_host.example.com",
|
||||
want: "mail",
|
||||
},
|
||||
{
|
||||
name: "no name",
|
||||
snapshotID: "my_host_2026-03-01T00:00:00Z",
|
||||
hostname: "my_host",
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := parseSnapshotName(tt.snapshotID)
|
||||
got := parseSnapshotName(tt.snapshotID, tt.hostname)
|
||||
if got != tt.want {
|
||||
t.Errorf("parseSnapshotName(%q) = %q, want %q",
|
||||
tt.snapshotID, got, tt.want)
|
||||
t.Errorf("parseSnapshotName(%q, %q) = %q, want %q",
|
||||
tt.snapshotID, tt.hostname, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func setupConsistencyTest(
|
||||
completedAt := startedAt.Add(5 * time.Minute)
|
||||
snap := &database.Snapshot{
|
||||
ID: types.SnapshotID(id),
|
||||
Hostname: testHostname,
|
||||
Hostname: snapHostname,
|
||||
VaultikVersion: testLabel,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
|
||||
@@ -17,8 +17,10 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// Snapshot IDs reused across the purge tests.
|
||||
// Snapshot IDs reused across the purge tests, and the hostname they were
|
||||
// taken on.
|
||||
const (
|
||||
snapHostname = "testhost"
|
||||
snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z"
|
||||
snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z"
|
||||
snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z"
|
||||
@@ -26,9 +28,12 @@ const (
|
||||
)
|
||||
|
||||
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock
|
||||
// storage pre-populated with the given snapshot IDs. Each snapshot is marked as
|
||||
// completed. Remote metadata stubs are created so syncWithRemote keeps them.
|
||||
func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
|
||||
// storage pre-populated with the given snapshot IDs, all taken on hostname.
|
||||
// Each snapshot is marked as completed. Remote metadata stubs are created so
|
||||
// syncWithRemote keeps them.
|
||||
func setupPurgeTest(
|
||||
t *testing.T, hostname string, snapshotIDs []string,
|
||||
) *vaultik.Vaultik {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -51,7 +56,7 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
|
||||
completedAt := startedAt.Add(5 * time.Minute)
|
||||
snap := &database.Snapshot{
|
||||
ID: types.SnapshotID(id),
|
||||
Hostname: "testhost",
|
||||
Hostname: types.Hostname(hostname),
|
||||
VaultikVersion: testLabel,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
@@ -120,7 +125,7 @@ func TestPurgeKeepLatest_PerName(t *testing.T) {
|
||||
"testhost_system_2026-01-01T04:00:00Z",
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -148,7 +153,7 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) {
|
||||
"testhost_home_2026-01-01T02:00:00Z",
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -176,7 +181,7 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) {
|
||||
"testhost_home_2026-01-01T04:00:00Z",
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -198,7 +203,7 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
v := setupPurgeTest(t, nil)
|
||||
v := setupPurgeTest(t, snapHostname, nil)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -216,7 +221,7 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) {
|
||||
"testhost_system_2026-01-01T01:00:00Z",
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -243,7 +248,7 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
|
||||
snapHomeT0,
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
// Purge only "home" snapshots older than 365 days
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
@@ -277,7 +282,7 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
|
||||
"testhost_home_2026-01-01T06:00:00Z",
|
||||
}
|
||||
|
||||
v := setupPurgeTest(t, snapshotIDs)
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
@@ -291,3 +296,28 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
|
||||
assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z")
|
||||
assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z")
|
||||
}
|
||||
|
||||
// A hostname may contain underscores, so the snapshot name cannot be found
|
||||
// by splitting the ID at them. A purge by name must still select "docs".
|
||||
func TestPurgeKeepLatest_HostnameWithUnderscore(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
system = "my_host_system_2026-01-01T00:00:00Z"
|
||||
docsT1 = "my_host_docs_2026-01-01T01:00:00Z"
|
||||
docsT2 = "my_host_docs_2026-01-01T02:00:00Z"
|
||||
)
|
||||
|
||||
v := setupPurgeTest(t, "my_host", []string{system, docsT1, docsT2})
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
Force: true,
|
||||
Names: []string{"docs"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, []string{system, docsT2},
|
||||
listRemainingSnapshots(t, v))
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// Sentinel errors for snapshot management.
|
||||
@@ -495,19 +496,23 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
nameFilter[n] = struct{}{}
|
||||
}
|
||||
|
||||
// Collect completed snapshots, applying the name filter.
|
||||
// Collect completed snapshots and their names, applying the name filter.
|
||||
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
|
||||
names := make(map[types.SnapshotID]string, len(dbSnapshots))
|
||||
|
||||
for _, s := range dbSnapshots {
|
||||
if s.CompletedAt == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
name := parseSnapshotName(s.ID.String(), s.Hostname.String())
|
||||
if len(nameFilter) > 0 {
|
||||
if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok {
|
||||
if _, ok := nameFilter[name]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
names[s.ID] = name
|
||||
snapshots = append(snapshots, SnapshotInfo{
|
||||
ID: s.ID,
|
||||
Timestamp: s.StartedAt,
|
||||
@@ -520,7 +525,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
|
||||
toDelete, err := selectSnapshotsToPurge(snapshots, opts)
|
||||
toDelete, err := selectSnapshotsToPurge(snapshots, names, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -538,9 +543,11 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
|
||||
// selectSnapshotsToPurge applies the purge retention criteria to the
|
||||
// newest-first sorted snapshot list and returns the deletion
|
||||
// candidates.
|
||||
// candidates. names maps each snapshot's ID to its snapshot name.
|
||||
func selectSnapshotsToPurge(
|
||||
snapshots []SnapshotInfo, opts *SnapshotPurgeOptions,
|
||||
snapshots []SnapshotInfo,
|
||||
names map[types.SnapshotID]string,
|
||||
opts *SnapshotPurgeOptions,
|
||||
) ([]SnapshotInfo, error) {
|
||||
var toDelete []SnapshotInfo
|
||||
|
||||
@@ -551,7 +558,7 @@ func selectSnapshotsToPurge(
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, snap := range snapshots {
|
||||
name := parseSnapshotName(snap.ID.String())
|
||||
name := names[snap.ID]
|
||||
if seen[name] {
|
||||
toDelete = append(toDelete, snap)
|
||||
|
||||
|
||||
Reference in new issue
Block a user