2 Commits
Author SHA1 Message Date
sneak 98b981203e Pass s3.part_size to the multipart uploader (closes #232)
check / check (push) Canceled after 0s
s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, the part sizes S3
accepts. The docs gave the default as 5MB, which the config file reads
as 5,000,000 bytes, below the minimum; they now say 5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.
Trap: at the 5MiB default the uploader's 10,000-part limit caps one
upload at about 48.8GiB, down from about 97.7GiB; blob_size_limit is
not checked against it.

Model: opus-5-5
2026-10-07 10:18:44 +00:00
clawbot 3fc8a8f2f4 Read the snapshot name using the stored hostname (closes #230)
check / check (push) Canceled after 0s
A snapshot ID is hostname_name_timestamp, and purge took the name to be
everything between the first and the last underscore. With a hostname
such as my_host the name home came out as host_home, so
`snapshot purge --keep-latest --snapshot home` found nothing to delete
and `snapshot create --prune` purged nothing without a message. The name
is now read by removing the hostname stored with the snapshot, in the
short form the ID uses, so both may contain underscores. This was chosen
over rejecting underscores in `hostname` when the config loads, which
would also stop restores on such a host.

The purge consistency test stored a hostname that did not match its
snapshot IDs; it now matches, as it always does in production.

Model: opus-5-5
2026-10-07 12:12:08 +02:00
15 changed files with 323 additions and 58 deletions

No files matched your search

+20
View File
@@ -22,6 +22,26 @@ the tag exists and is exercised; what is left is merging `next` to
# Completed Steps # Completed Steps
- 2026-10-07: Made `s3.part_size` set the multipart upload part size
([issue #232](https://git.eeqj.de/sneak/vaultik/issues/232)). It was
loaded and defaulted but never passed to the S3 client, whose uploader
used a fixed 10MiB part. It now reaches the uploader for `storage_url`
and for the `s3.*` fields, and a part size S3 refuses, below 5MiB or
above 5GiB, fails at config load. The docs gave the default as `5MB`,
which the config file reads as 5,000,000 bytes, below the minimum; they
now say `5MiB`.
- 2026-10-07: Made per-name retention work when the hostname contains `_`
([issue #230](https://git.eeqj.de/sneak/vaultik/issues/230)). A
snapshot ID is `hostname_name_timestamp`, and the name was read as
everything between the first and the last `_`, so with
`hostname: my_host` the name `home` came out as `host_home`.
`snapshot purge --keep-latest --snapshot home` then printed "No
snapshots to delete", and `snapshot create --prune` purged nothing
without a message. The name is now read using the hostname the
`snapshots` table stores with each snapshot, cut at its first `.` as it
is in the ID.
- 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as - 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as
orphaned when its manifest cannot be read, and stop printing raw orphaned when its manifest cannot be read, and stop printing raw
names from under `metadata/` names from under `metadata/`
+4 -4
View File
@@ -287,10 +287,10 @@ storage_url: "rclone://myremote/path/to/backups"
# #use_ssl: true # #use_ssl: true
# #
# # Part size for multipart uploads # # Part size for multipart uploads
# # Minimum 5MB, affects memory usage during upload # # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
# # Supports: 5MB, 10M, 100MiB, etc. # # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
# # Default: 5MB # # Default: 5MiB
# #part_size: 5MB # #part_size: 5MiB
# Path to local SQLite index database # Path to local SQLite index database
# This database tracks file state for incremental backups # This database tracks file state for incremental backups
+1 -1
View File
@@ -205,7 +205,7 @@ storage_url: ""
# access_key_id: YOUR_ACCESS_KEY # access_key_id: YOUR_ACCESS_KEY
# secret_access_key: YOUR_SECRET_KEY # secret_access_key: YOUR_SECRET_KEY
# # region: us-east-1 # Default: us-east-1 # # region: us-east-1 # Default: us-east-1
# # part_size: 5MB # Multipart upload part size. Default: 5MB # # part_size: 5MiB # Upload part size, 5MiB to 5GiB. Default: 5MiB
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl. # # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
# ─── OPTIONAL ──────────────────────────────────────────────────────────────── # ─── OPTIONAL ────────────────────────────────────────────────────────────────
+11 -1
View File
@@ -33,11 +33,14 @@ const secretKeyPrefix = "AGE-SECRET-KEY-"
const ( const (
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
defaultCompressionLevel = 3 defaultCompressionLevel = 3
minChunkSize = 1024 * 1024 // 1MB minChunkSize = 1024 * 1024 // 1MB
minCompressionLevel = 1 minCompressionLevel = 1
maxCompressionLevel = 19 maxCompressionLevel = 19
// S3 accepts a multipart upload part from 5MiB to 5GiB.
minS3PartSize = 5 * 1024 * 1024
maxS3PartSize = 5 * 1024 * 1024 * 1024
) )
// Sentinel validation errors. // Sentinel validation errors.
@@ -55,6 +58,7 @@ var (
"blob_size_limit must be at least the largest chunk the chunker can " + "blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)") "emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19") errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New( errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://") "storage_url must start with s3://, file://, or rclone://")
errStorageNotConfigured = errors.New( errStorageNotConfigured = errors.New(
@@ -332,6 +336,7 @@ func Load(path string) (*Config, error) {
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never // (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
// exceeds the configured limit // exceeds the configured limit
// - Compression level must be between 1 and 19 // - Compression level must be between 1 and 19
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
// //
// Returns an error describing the first validation failure encountered. // Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error { func (c *Config) Validate() error {
@@ -376,6 +381,11 @@ func (c *Config) Validate() error {
return errBadCompression return errBadCompression
} }
if c.S3.PartSize.Int64() < minS3PartSize ||
c.S3.PartSize.Int64() > maxS3PartSize {
return errBadS3PartSize
}
return nil return nil
} }
+68
View File
@@ -166,6 +166,7 @@ func TestValidateBlobSizeLimit(t *testing.T) {
ChunkSize: chunkSize, ChunkSize: chunkSize,
BlobSizeLimit: blobLimit, BlobSizeLimit: blobLimit,
CompressionLevel: 3, CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
} }
} }
@@ -221,6 +222,72 @@ func TestValidateBlobSizeLimit(t *testing.T) {
} }
} }
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
func TestValidateS3PartSize(t *testing.T) {
t.Parallel()
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
if err != nil {
t.Fatalf("Failed to load config: %v", err)
}
tests := []struct {
name string
partSize Size
wantErr bool
}{
{
name: "5MB is rejected",
partSize: 5_000_000,
wantErr: true,
},
{
name: "one byte below 5MiB is rejected",
partSize: minS3PartSize - 1,
wantErr: true,
},
{
name: "5MiB is accepted",
partSize: minS3PartSize,
wantErr: false,
},
{
name: "5GiB is accepted",
partSize: maxS3PartSize,
wantErr: false,
},
{
name: "one byte above 5GiB is rejected",
partSize: maxS3PartSize + 1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
cfg := *base
cfg.S3.PartSize = tt.partSize
err := cfg.Validate()
if tt.wantErr {
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
}
return
}
if err != nil {
t.Fatalf("Validate() unexpected error: %v", err)
}
})
}
}
// TestValidateAgeRecipients checks that recipients are parsed at config load // TestValidateAgeRecipients checks that recipients are parsed at config load
// (a bad entry fails immediately, not mid-backup) and that no invalid entry — // (a bad entry fails immediately, not mid-backup) and that no invalid entry —
// least of all a pasted secret key — is echoed in the error. An empty list // least of all a pasted secret key — is echoed in the error. An empty list
@@ -236,6 +303,7 @@ func TestValidateAgeRecipients(t *testing.T) {
ChunkSize: Size(10 * 1024 * 1024), ChunkSize: Size(10 * 1024 * 1024),
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
CompressionLevel: 3, CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
} }
} }
+7 -5
View File
@@ -27,10 +27,12 @@ type Client struct {
bucket string bucket string
prefix string prefix string
endpoint string endpoint string
partSize int64
} }
// Config contains S3 client configuration. // Config contains S3 client configuration.
// All fields are required except Prefix, which defaults to an empty string. // All fields are required except Prefix, which defaults to an empty string,
// and PartSize, where zero means the SDK default of 5 MiB.
// A non-empty Prefix is joined to every key with one "/", whether or not // A non-empty Prefix is joined to every key with one "/", whether or not
// it ends with one. // it ends with one.
// The Endpoint field should include the protocol (http:// or https://). // The Endpoint field should include the protocol (http:// or https://).
@@ -41,6 +43,8 @@ type Config struct {
AccessKeyID string AccessKeyID string
SecretAccessKey string SecretAccessKey string
Region string Region string
// PartSize is the size in bytes of each part of a multipart upload.
PartSize int64
} }
// nopLogger is a logger that discards all output. // nopLogger is a logger that discards all output.
@@ -90,6 +94,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
bucket: cfg.Bucket, bucket: cfg.Bucket,
prefix: prefix, prefix: prefix,
endpoint: cfg.Endpoint, endpoint: cfg.Endpoint,
partSize: cfg.PartSize,
}, nil }, nil
} }
@@ -123,12 +128,9 @@ func (c *Client) PutObjectWithProgress(
) error { ) error {
fullKey := c.prefix + key fullKey := c.prefix + key
// uploadPartSize is 10MB for better progress granularity.
const uploadPartSize = 10 * 1024 * 1024
// Create an uploader with the S3 client // Create an uploader with the S3 client
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
u.PartSize = uploadPartSize u.PartSize = c.partSize
}) })
// Create a progress reader that tracks upload progress // Create a progress reader that tracks upload progress
+1
View File
@@ -28,6 +28,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: cfg.S3.Region, Region: cfg.S3.Region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, err return nil, err
+9 -5
View File
@@ -105,17 +105,21 @@ func (sm *SnapshotManager) CreateSnapshot(
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision) return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
} }
// ShortHostname returns hostname up to its first dot. A snapshot ID starts
// with this form, while the snapshots table stores the full hostname.
func ShortHostname(hostname string) string {
short, _, _ := strings.Cut(hostname, ".")
return short
}
// CreateSnapshotWithName creates a new snapshot record with an optional // CreateSnapshotWithName creates a new snapshot record with an optional
// snapshot name. The snapshot ID format is: hostname_name_timestamp or // snapshot name. The snapshot ID format is: hostname_name_timestamp or
// hostname_timestamp if name is empty. // hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName( func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string, ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) { ) (string, error) {
// Use short hostname (strip domain if present) shortHostname := ShortHostname(hostname)
shortHostname := hostname
if before, _, ok := strings.Cut(hostname, "."); ok {
shortHostname = before
}
// Build snapshot ID with optional name // Build snapshot ID with optional name
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
+2
View File
@@ -99,6 +99,7 @@ func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region, Region: region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
@@ -134,6 +135,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region, Region: region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
+96 -2
View File
@@ -1,11 +1,14 @@
package storage_test package storage_test
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"net/http"
"net/http/httptest" "net/http/httptest"
"slices" "slices"
"strings" "strings"
"sync/atomic"
"testing" "testing"
"github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3"
@@ -19,6 +22,13 @@ import (
// s3TestBucket is the bucket created for each in-process S3 server. // s3TestBucket is the bucket created for each in-process S3 server.
const s3TestBucket = "test-bucket" const s3TestBucket = "test-bucket"
// Credentials for the tests that build a storer from a config.Config. The
// in-process S3 server accepts any.
const (
s3TestAccessKeyID = "key"
s3TestSecretAccessKey = "secret"
)
// newS3Storer builds an s3:// backend backed by a fresh in-process // newS3Storer builds an s3:// backend backed by a fresh in-process
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem // S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
// over httptest) that internal/s3 and the not-found regression test use, // over httptest) that internal/s3 and the not-found regression test use,
@@ -128,8 +138,8 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
storer, err := storage.NewStorer(&config.Config{ storer, err := storage.NewStorer(&config.Config{
StorageURL: storageURL + "?endpoint=" + srv.URL, StorageURL: storageURL + "?endpoint=" + srv.URL,
S3: config.S3Config{ S3: config.S3Config{
AccessKeyID: "key", AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: "secret", SecretAccessKey: s3TestSecretAccessKey,
}, },
}) })
if err != nil { if err != nil {
@@ -171,6 +181,90 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
} }
} }
// TestS3UploadUsesConfiguredPartSize checks that s3.part_size reaches the
// multipart uploader, through storage_url and through the s3.* fields. An
// object three parts long must arrive as three parts; at the SDK's default
// of 5 MiB it would arrive as four.
func TestS3UploadUsesConfiguredPartSize(t *testing.T) {
t.Parallel()
const (
partSize = 6 * 1024 * 1024
wantParts = 3
)
backend := s3mem.New()
err := backend.CreateBucket(s3TestBucket)
if err != nil {
t.Fatalf("create bucket: %v", err)
}
// Every part of a multipart upload is one request with a partNumber.
var parts atomic.Int32
fake := gofakes3.New(backend).Server()
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Has("partNumber") {
parts.Add(1)
}
fake.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
cases := []struct {
name string
cfg *config.Config
}{
{
name: "storage_url",
cfg: &config.Config{
StorageURL: "s3://" + s3TestBucket + "?endpoint=" + srv.URL,
S3: config.S3Config{
AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: s3TestSecretAccessKey,
PartSize: partSize,
},
},
},
{
name: "s3.endpoint",
cfg: &config.Config{
S3: config.S3Config{
Endpoint: srv.URL,
Bucket: s3TestBucket,
AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: s3TestSecretAccessKey,
PartSize: partSize,
},
},
},
}
for _, tc := range cases {
parts.Store(0)
storer, err := storage.NewStorer(tc.cfg)
if err != nil {
t.Fatalf("%s: NewStorer: %v", tc.name, err)
}
data := bytes.NewReader(make([]byte, wantParts*partSize))
err = storer.PutWithProgress(
context.Background(), "blob", data, data.Size(), nil)
if err != nil {
t.Fatalf("%s: PutWithProgress: %v", tc.name, err)
}
if got := parts.Load(); got != wantParts {
t.Errorf("%s: uploaded in %d parts, want %d", tc.name, got, wantParts)
}
}
}
// listStreamKeys returns the keys ListStream yields under a prefix, and // listStreamKeys returns the keys ListStream yields under a prefix, and
// fails the test on a listing error. // fails the test on a listing error.
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string { func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
+23 -18
View File
@@ -9,6 +9,7 @@ import (
"time" "time"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types" "sneak.berlin/go/vaultik/internal/types"
) )
@@ -46,12 +47,9 @@ const (
year = 365 * day year = 365 * day
) )
// Snapshot IDs split on "_" into hostname, optional name parts, and a // A snapshot ID split on "_" has at least a hostname and a trailing
// trailing timestamp. // timestamp.
const ( const minSnapshotIDParts = 2
minSnapshotIDParts = 2
minSnapshotIDNameParts = 3
)
// SnapshotInfo contains information about a snapshot. // SnapshotInfo contains information about a snapshot.
// //
@@ -121,20 +119,27 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
return timestamp.UTC(), nil return timestamp.UTC(), nil
} }
// parseSnapshotName extracts the snapshot name from a snapshot ID. // parseSnapshotName extracts the snapshot name from a snapshot ID of the
// Format: hostname_snapshotname_timestamp — the middle part(s) between hostname // form hostname_name_timestamp, given the hostname stored with that
// and the RFC3339 timestamp are the snapshot name (may contain underscores). // snapshot. The hostname and the name may both contain underscores, so the
// Returns the snapshot name, or empty string if the ID is malformed. // name is what is left after removing the short hostname and its "_" from
func parseSnapshotName(snapshotID string) string { // the front and the last "_" and the timestamp from the end. Returns "" for
parts := strings.Split(snapshotID, "_") // an ID with no name (hostname_timestamp), and for an ID that does not start
if len(parts) < minSnapshotIDNameParts { // with that hostname, which CreateSnapshotWithName never writes.
// Format: hostname_timestamp — no snapshot name func parseSnapshotName(snapshotID, hostname string) string {
prefix := snapshot.ShortHostname(hostname) + "_"
rest, ok := strings.CutPrefix(snapshotID, prefix)
if !ok {
return "" return ""
} }
// Format: hostname_name_timestamp — middle parts are the name.
// The last part is the RFC3339 timestamp, the first part is the hostname, end := strings.LastIndex(rest, "_")
// everything in between is the snapshot name (which may itself contain underscores). if end < 0 {
return strings.Join(parts[1:len(parts)-1], "_") return ""
}
return rest[:end]
} }
// parseDuration parses a duration string with support for human-friendly units: // parseDuration parses a duration string with support for human-friendly units:
+25 -3
View File
@@ -11,33 +11,55 @@ func TestParseSnapshotName(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
snapshotID string snapshotID string
hostname string
want string want string
}{ }{
{ {
name: "standard format with name", name: "standard format with name",
snapshotID: "myhost_home_2026-01-12T14:41:15Z", snapshotID: "myhost_home_2026-01-12T14:41:15Z",
hostname: "myhost",
want: "home", want: "home",
}, },
{ {
name: "standard format with different name", name: "standard format with different name",
snapshotID: "server1_system_2026-02-15T09:30:00Z", snapshotID: "server1_system_2026-02-15T09:30:00Z",
hostname: "server1",
want: "system", want: "system",
}, },
{ {
name: "name with underscores", name: "name with underscores",
snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z", snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z",
hostname: "myhost",
want: "my_special_backup", want: "my_special_backup",
}, },
{
name: "hostname with underscores",
snapshotID: "my_host_docs_2026-03-01T00:00:00Z",
hostname: "my_host",
want: "docs",
},
{
name: "stored hostname with domain",
snapshotID: "my_host_mail_2026-03-01T00:00:00Z",
hostname: "my_host.example.com",
want: "mail",
},
{
name: "no name",
snapshotID: "my_host_2026-03-01T00:00:00Z",
hostname: "my_host",
want: "",
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
got := parseSnapshotName(tt.snapshotID) got := parseSnapshotName(tt.snapshotID, tt.hostname)
if got != tt.want { if got != tt.want {
t.Errorf("parseSnapshotName(%q) = %q, want %q", t.Errorf("parseSnapshotName(%q, %q) = %q, want %q",
tt.snapshotID, got, tt.want) tt.snapshotID, tt.hostname, got, tt.want)
} }
}) })
} }
@@ -42,7 +42,7 @@ func setupConsistencyTest(
completedAt := startedAt.Add(5 * time.Minute) completedAt := startedAt.Add(5 * time.Minute)
snap := &database.Snapshot{ snap := &database.Snapshot{
ID: types.SnapshotID(id), ID: types.SnapshotID(id),
Hostname: testHostname, Hostname: snapHostname,
VaultikVersion: testLabel, VaultikVersion: testLabel,
StartedAt: startedAt, StartedAt: startedAt,
CompletedAt: &completedAt, CompletedAt: &completedAt,
+42 -12
View File
@@ -17,8 +17,10 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik" "sneak.berlin/go/vaultik/internal/vaultik"
) )
// Snapshot IDs reused across the purge tests. // Snapshot IDs reused across the purge tests, and the hostname they were
// taken on.
const ( const (
snapHostname = "testhost"
snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z" snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z"
snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z" snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z"
snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z" snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z"
@@ -26,9 +28,12 @@ const (
) )
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock // setupPurgeTest creates a Vaultik instance with an in-memory database and mock
// storage pre-populated with the given snapshot IDs. Each snapshot is marked as // storage pre-populated with the given snapshot IDs, all taken on hostname.
// completed. Remote metadata stubs are created so syncWithRemote keeps them. // Each snapshot is marked as completed. Remote metadata stubs are created so
func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { // syncWithRemote keeps them.
func setupPurgeTest(
t *testing.T, hostname string, snapshotIDs []string,
) *vaultik.Vaultik {
t.Helper() t.Helper()
ctx := context.Background() ctx := context.Background()
@@ -51,7 +56,7 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
completedAt := startedAt.Add(5 * time.Minute) completedAt := startedAt.Add(5 * time.Minute)
snap := &database.Snapshot{ snap := &database.Snapshot{
ID: types.SnapshotID(id), ID: types.SnapshotID(id),
Hostname: "testhost", Hostname: types.Hostname(hostname),
VaultikVersion: testLabel, VaultikVersion: testLabel,
StartedAt: startedAt, StartedAt: startedAt,
CompletedAt: &completedAt, CompletedAt: &completedAt,
@@ -120,7 +125,7 @@ func TestPurgeKeepLatest_PerName(t *testing.T) {
"testhost_system_2026-01-01T04:00:00Z", "testhost_system_2026-01-01T04:00:00Z",
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -148,7 +153,7 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) {
"testhost_home_2026-01-01T02:00:00Z", "testhost_home_2026-01-01T02:00:00Z",
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -176,7 +181,7 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) {
"testhost_home_2026-01-01T04:00:00Z", "testhost_home_2026-01-01T04:00:00Z",
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -198,7 +203,7 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
v := setupPurgeTest(t, nil) v := setupPurgeTest(t, snapHostname, nil)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -216,7 +221,7 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) {
"testhost_system_2026-01-01T01:00:00Z", "testhost_system_2026-01-01T01:00:00Z",
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -243,7 +248,7 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
snapHomeT0, snapHomeT0,
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
// Purge only "home" snapshots older than 365 days // Purge only "home" snapshots older than 365 days
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
@@ -277,7 +282,7 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
"testhost_home_2026-01-01T06:00:00Z", "testhost_home_2026-01-01T06:00:00Z",
} }
v := setupPurgeTest(t, snapshotIDs) v := setupPurgeTest(t, snapHostname, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -291,3 +296,28 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z") assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z")
assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z") assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z")
} }
// A hostname may contain underscores, so the snapshot name cannot be found
// by splitting the ID at them. A purge by name must still select "docs".
func TestPurgeKeepLatest_HostnameWithUnderscore(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const (
system = "my_host_system_2026-01-01T00:00:00Z"
docsT1 = "my_host_docs_2026-01-01T01:00:00Z"
docsT2 = "my_host_docs_2026-01-01T02:00:00Z"
)
v := setupPurgeTest(t, "my_host", []string{system, docsT1, docsT2})
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true,
Force: true,
Names: []string{"docs"},
})
require.NoError(t, err)
assert.ElementsMatch(t, []string{system, docsT2},
listRemainingSnapshots(t, v))
}
+13 -6
View File
@@ -14,6 +14,7 @@ import (
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types"
) )
// Sentinel errors for snapshot management. // Sentinel errors for snapshot management.
@@ -495,19 +496,23 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
nameFilter[n] = struct{}{} nameFilter[n] = struct{}{}
} }
// Collect completed snapshots, applying the name filter. // Collect completed snapshots and their names, applying the name filter.
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots)) snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
names := make(map[types.SnapshotID]string, len(dbSnapshots))
for _, s := range dbSnapshots { for _, s := range dbSnapshots {
if s.CompletedAt == nil { if s.CompletedAt == nil {
continue continue
} }
name := parseSnapshotName(s.ID.String(), s.Hostname.String())
if len(nameFilter) > 0 { if len(nameFilter) > 0 {
if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok { if _, ok := nameFilter[name]; !ok {
continue continue
} }
} }
names[s.ID] = name
snapshots = append(snapshots, SnapshotInfo{ snapshots = append(snapshots, SnapshotInfo{
ID: s.ID, ID: s.ID,
Timestamp: s.StartedAt, Timestamp: s.StartedAt,
@@ -520,7 +525,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
return snapshots[i].Timestamp.After(snapshots[j].Timestamp) return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
}) })
toDelete, err := selectSnapshotsToPurge(snapshots, opts) toDelete, err := selectSnapshotsToPurge(snapshots, names, opts)
if err != nil { if err != nil {
return err return err
} }
@@ -538,9 +543,11 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
// selectSnapshotsToPurge applies the purge retention criteria to the // selectSnapshotsToPurge applies the purge retention criteria to the
// newest-first sorted snapshot list and returns the deletion // newest-first sorted snapshot list and returns the deletion
// candidates. // candidates. names maps each snapshot's ID to its snapshot name.
func selectSnapshotsToPurge( func selectSnapshotsToPurge(
snapshots []SnapshotInfo, opts *SnapshotPurgeOptions, snapshots []SnapshotInfo,
names map[types.SnapshotID]string,
opts *SnapshotPurgeOptions,
) ([]SnapshotInfo, error) { ) ([]SnapshotInfo, error) {
var toDelete []SnapshotInfo var toDelete []SnapshotInfo
@@ -551,7 +558,7 @@ func selectSnapshotsToPurge(
seen := make(map[string]bool) seen := make(map[string]bool)
for _, snap := range snapshots { for _, snap := range snapshots {
name := parseSnapshotName(snap.ID.String()) name := names[snap.ID]
if seen[name] { if seen[name] {
toDelete = append(toDelete, snap) toDelete = append(toDelete, snap)