Compare commits
2
Commits
99d3dbd57d
...
98b981203e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
98b981203e | ||
|
|
3fc8a8f2f4 |
No files matched your search
@@ -22,6 +22,26 @@ the tag exists and is exercised; what is left is merging `next` to
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-07: Made `s3.part_size` set the multipart upload part size
|
||||||
|
([issue #232](https://git.eeqj.de/sneak/vaultik/issues/232)). It was
|
||||||
|
loaded and defaulted but never passed to the S3 client, whose uploader
|
||||||
|
used a fixed 10MiB part. It now reaches the uploader for `storage_url`
|
||||||
|
and for the `s3.*` fields, and a part size S3 refuses, below 5MiB or
|
||||||
|
above 5GiB, fails at config load. The docs gave the default as `5MB`,
|
||||||
|
which the config file reads as 5,000,000 bytes, below the minimum; they
|
||||||
|
now say `5MiB`.
|
||||||
|
|
||||||
|
- 2026-10-07: Made per-name retention work when the hostname contains `_`
|
||||||
|
([issue #230](https://git.eeqj.de/sneak/vaultik/issues/230)). A
|
||||||
|
snapshot ID is `hostname_name_timestamp`, and the name was read as
|
||||||
|
everything between the first and the last `_`, so with
|
||||||
|
`hostname: my_host` the name `home` came out as `host_home`.
|
||||||
|
`snapshot purge --keep-latest --snapshot home` then printed "No
|
||||||
|
snapshots to delete", and `snapshot create --prune` purged nothing
|
||||||
|
without a message. The name is now read using the hostname the
|
||||||
|
`snapshots` table stores with each snapshot, cut at its first `.` as it
|
||||||
|
is in the ID.
|
||||||
|
|
||||||
- 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as
|
- 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as
|
||||||
orphaned when its manifest cannot be read, and stop printing raw
|
orphaned when its manifest cannot be read, and stop printing raw
|
||||||
names from under `metadata/`
|
names from under `metadata/`
|
||||||
|
|||||||
+4
-4
@@ -287,10 +287,10 @@ storage_url: "rclone://myremote/path/to/backups"
|
|||||||
# #use_ssl: true
|
# #use_ssl: true
|
||||||
#
|
#
|
||||||
# # Part size for multipart uploads
|
# # Part size for multipart uploads
|
||||||
# # Minimum 5MB, affects memory usage during upload
|
# # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
|
||||||
# # Supports: 5MB, 10M, 100MiB, etc.
|
# # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
|
||||||
# # Default: 5MB
|
# # Default: 5MiB
|
||||||
# #part_size: 5MB
|
# #part_size: 5MiB
|
||||||
|
|
||||||
# Path to local SQLite index database
|
# Path to local SQLite index database
|
||||||
# This database tracks file state for incremental backups
|
# This database tracks file state for incremental backups
|
||||||
|
|||||||
@@ -205,7 +205,7 @@ storage_url: ""
|
|||||||
# access_key_id: YOUR_ACCESS_KEY
|
# access_key_id: YOUR_ACCESS_KEY
|
||||||
# secret_access_key: YOUR_SECRET_KEY
|
# secret_access_key: YOUR_SECRET_KEY
|
||||||
# # region: us-east-1 # Default: us-east-1
|
# # region: us-east-1 # Default: us-east-1
|
||||||
# # part_size: 5MB # Multipart upload part size. Default: 5MB
|
# # part_size: 5MiB # Upload part size, 5MiB to 5GiB. Default: 5MiB
|
||||||
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
|
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
|
||||||
|
|
||||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -33,11 +33,14 @@ const secretKeyPrefix = "AGE-SECRET-KEY-"
|
|||||||
const (
|
const (
|
||||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB
|
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||||
defaultCompressionLevel = 3
|
defaultCompressionLevel = 3
|
||||||
minChunkSize = 1024 * 1024 // 1MB
|
minChunkSize = 1024 * 1024 // 1MB
|
||||||
minCompressionLevel = 1
|
minCompressionLevel = 1
|
||||||
maxCompressionLevel = 19
|
maxCompressionLevel = 19
|
||||||
|
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||||
|
minS3PartSize = 5 * 1024 * 1024
|
||||||
|
maxS3PartSize = 5 * 1024 * 1024 * 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel validation errors.
|
// Sentinel validation errors.
|
||||||
@@ -55,6 +58,7 @@ var (
|
|||||||
"blob_size_limit must be at least the largest chunk the chunker can " +
|
"blob_size_limit must be at least the largest chunk the chunker can " +
|
||||||
"emit (chunk_size times the FastCDC size spread)")
|
"emit (chunk_size times the FastCDC size spread)")
|
||||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||||
|
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||||
errBadStorageScheme = errors.New(
|
errBadStorageScheme = errors.New(
|
||||||
"storage_url must start with s3://, file://, or rclone://")
|
"storage_url must start with s3://, file://, or rclone://")
|
||||||
errStorageNotConfigured = errors.New(
|
errStorageNotConfigured = errors.New(
|
||||||
@@ -332,6 +336,7 @@ func Load(path string) (*Config, error) {
|
|||||||
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
|
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
|
||||||
// exceeds the configured limit
|
// exceeds the configured limit
|
||||||
// - Compression level must be between 1 and 19
|
// - Compression level must be between 1 and 19
|
||||||
|
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
|
||||||
//
|
//
|
||||||
// Returns an error describing the first validation failure encountered.
|
// Returns an error describing the first validation failure encountered.
|
||||||
func (c *Config) Validate() error {
|
func (c *Config) Validate() error {
|
||||||
@@ -376,6 +381,11 @@ func (c *Config) Validate() error {
|
|||||||
return errBadCompression
|
return errBadCompression
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if c.S3.PartSize.Int64() < minS3PartSize ||
|
||||||
|
c.S3.PartSize.Int64() > maxS3PartSize {
|
||||||
|
return errBadS3PartSize
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -166,6 +166,7 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
|||||||
ChunkSize: chunkSize,
|
ChunkSize: chunkSize,
|
||||||
BlobSizeLimit: blobLimit,
|
BlobSizeLimit: blobLimit,
|
||||||
CompressionLevel: 3,
|
CompressionLevel: 3,
|
||||||
|
S3: S3Config{PartSize: defaultS3PartSize},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,6 +222,72 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
|
||||||
|
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
|
||||||
|
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
|
||||||
|
func TestValidateS3PartSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to load config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
partSize Size
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "5MB is rejected",
|
||||||
|
partSize: 5_000_000,
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one byte below 5MiB is rejected",
|
||||||
|
partSize: minS3PartSize - 1,
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "5MiB is accepted",
|
||||||
|
partSize: minS3PartSize,
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "5GiB is accepted",
|
||||||
|
partSize: maxS3PartSize,
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one byte above 5GiB is rejected",
|
||||||
|
partSize: maxS3PartSize + 1,
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg := *base
|
||||||
|
cfg.S3.PartSize = tt.partSize
|
||||||
|
|
||||||
|
err := cfg.Validate()
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, errBadS3PartSize) {
|
||||||
|
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||||
@@ -236,6 +303,7 @@ func TestValidateAgeRecipients(t *testing.T) {
|
|||||||
ChunkSize: Size(10 * 1024 * 1024),
|
ChunkSize: Size(10 * 1024 * 1024),
|
||||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
|
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
|
||||||
CompressionLevel: 3,
|
CompressionLevel: 3,
|
||||||
|
S3: S3Config{PartSize: defaultS3PartSize},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,10 +27,12 @@ type Client struct {
|
|||||||
bucket string
|
bucket string
|
||||||
prefix string
|
prefix string
|
||||||
endpoint string
|
endpoint string
|
||||||
|
partSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config contains S3 client configuration.
|
// Config contains S3 client configuration.
|
||||||
// All fields are required except Prefix, which defaults to an empty string.
|
// All fields are required except Prefix, which defaults to an empty string,
|
||||||
|
// and PartSize, where zero means the SDK default of 5 MiB.
|
||||||
// A non-empty Prefix is joined to every key with one "/", whether or not
|
// A non-empty Prefix is joined to every key with one "/", whether or not
|
||||||
// it ends with one.
|
// it ends with one.
|
||||||
// The Endpoint field should include the protocol (http:// or https://).
|
// The Endpoint field should include the protocol (http:// or https://).
|
||||||
@@ -41,6 +43,8 @@ type Config struct {
|
|||||||
AccessKeyID string
|
AccessKeyID string
|
||||||
SecretAccessKey string
|
SecretAccessKey string
|
||||||
Region string
|
Region string
|
||||||
|
// PartSize is the size in bytes of each part of a multipart upload.
|
||||||
|
PartSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// nopLogger is a logger that discards all output.
|
// nopLogger is a logger that discards all output.
|
||||||
@@ -90,6 +94,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
|
|||||||
bucket: cfg.Bucket,
|
bucket: cfg.Bucket,
|
||||||
prefix: prefix,
|
prefix: prefix,
|
||||||
endpoint: cfg.Endpoint,
|
endpoint: cfg.Endpoint,
|
||||||
|
partSize: cfg.PartSize,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,12 +128,9 @@ func (c *Client) PutObjectWithProgress(
|
|||||||
) error {
|
) error {
|
||||||
fullKey := c.prefix + key
|
fullKey := c.prefix + key
|
||||||
|
|
||||||
// uploadPartSize is 10MB for better progress granularity.
|
|
||||||
const uploadPartSize = 10 * 1024 * 1024
|
|
||||||
|
|
||||||
// Create an uploader with the S3 client
|
// Create an uploader with the S3 client
|
||||||
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
|
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
|
||||||
u.PartSize = uploadPartSize
|
u.PartSize = c.partSize
|
||||||
})
|
})
|
||||||
|
|
||||||
// Create a progress reader that tracks upload progress
|
// Create a progress reader that tracks upload progress
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
|
|||||||
AccessKeyID: cfg.S3.AccessKeyID,
|
AccessKeyID: cfg.S3.AccessKeyID,
|
||||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||||
Region: cfg.S3.Region,
|
Region: cfg.S3.Region,
|
||||||
|
PartSize: cfg.S3.PartSize.Int64(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -105,17 +105,21 @@ func (sm *SnapshotManager) CreateSnapshot(
|
|||||||
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
|
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ShortHostname returns hostname up to its first dot. A snapshot ID starts
|
||||||
|
// with this form, while the snapshots table stores the full hostname.
|
||||||
|
func ShortHostname(hostname string) string {
|
||||||
|
short, _, _ := strings.Cut(hostname, ".")
|
||||||
|
|
||||||
|
return short
|
||||||
|
}
|
||||||
|
|
||||||
// CreateSnapshotWithName creates a new snapshot record with an optional
|
// CreateSnapshotWithName creates a new snapshot record with an optional
|
||||||
// snapshot name. The snapshot ID format is: hostname_name_timestamp or
|
// snapshot name. The snapshot ID format is: hostname_name_timestamp or
|
||||||
// hostname_timestamp if name is empty.
|
// hostname_timestamp if name is empty.
|
||||||
func (sm *SnapshotManager) CreateSnapshotWithName(
|
func (sm *SnapshotManager) CreateSnapshotWithName(
|
||||||
ctx context.Context, hostname, name, version, gitRevision string,
|
ctx context.Context, hostname, name, version, gitRevision string,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
// Use short hostname (strip domain if present)
|
shortHostname := ShortHostname(hostname)
|
||||||
shortHostname := hostname
|
|
||||||
if before, _, ok := strings.Cut(hostname, "."); ok {
|
|
||||||
shortHostname = before
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build snapshot ID with optional name
|
// Build snapshot ID with optional name
|
||||||
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||||
|
|||||||
@@ -99,6 +99,7 @@ func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
|||||||
AccessKeyID: cfg.S3.AccessKeyID,
|
AccessKeyID: cfg.S3.AccessKeyID,
|
||||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||||
Region: region,
|
Region: region,
|
||||||
|
PartSize: cfg.S3.PartSize.Int64(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||||
@@ -134,6 +135,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
|
|||||||
AccessKeyID: cfg.S3.AccessKeyID,
|
AccessKeyID: cfg.S3.AccessKeyID,
|
||||||
SecretAccessKey: cfg.S3.SecretAccessKey,
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
||||||
Region: region,
|
Region: region,
|
||||||
|
PartSize: cfg.S3.PartSize.Int64(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
package storage_test
|
package storage_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
"github.com/johannesboyne/gofakes3"
|
||||||
@@ -19,6 +22,13 @@ import (
|
|||||||
// s3TestBucket is the bucket created for each in-process S3 server.
|
// s3TestBucket is the bucket created for each in-process S3 server.
|
||||||
const s3TestBucket = "test-bucket"
|
const s3TestBucket = "test-bucket"
|
||||||
|
|
||||||
|
// Credentials for the tests that build a storer from a config.Config. The
|
||||||
|
// in-process S3 server accepts any.
|
||||||
|
const (
|
||||||
|
s3TestAccessKeyID = "key"
|
||||||
|
s3TestSecretAccessKey = "secret"
|
||||||
|
)
|
||||||
|
|
||||||
// newS3Storer builds an s3:// backend backed by a fresh in-process
|
// newS3Storer builds an s3:// backend backed by a fresh in-process
|
||||||
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
|
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
|
||||||
// over httptest) that internal/s3 and the not-found regression test use,
|
// over httptest) that internal/s3 and the not-found regression test use,
|
||||||
@@ -128,8 +138,8 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
|
|||||||
storer, err := storage.NewStorer(&config.Config{
|
storer, err := storage.NewStorer(&config.Config{
|
||||||
StorageURL: storageURL + "?endpoint=" + srv.URL,
|
StorageURL: storageURL + "?endpoint=" + srv.URL,
|
||||||
S3: config.S3Config{
|
S3: config.S3Config{
|
||||||
AccessKeyID: "key",
|
AccessKeyID: s3TestAccessKeyID,
|
||||||
SecretAccessKey: "secret",
|
SecretAccessKey: s3TestSecretAccessKey,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -171,6 +181,90 @@ func TestS3URLPrefixKeyLayout(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestS3UploadUsesConfiguredPartSize checks that s3.part_size reaches the
|
||||||
|
// multipart uploader, through storage_url and through the s3.* fields. An
|
||||||
|
// object three parts long must arrive as three parts; at the SDK's default
|
||||||
|
// of 5 MiB it would arrive as four.
|
||||||
|
func TestS3UploadUsesConfiguredPartSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
partSize = 6 * 1024 * 1024
|
||||||
|
wantParts = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
backend := s3mem.New()
|
||||||
|
|
||||||
|
err := backend.CreateBucket(s3TestBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create bucket: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every part of a multipart upload is one request with a partNumber.
|
||||||
|
var parts atomic.Int32
|
||||||
|
|
||||||
|
fake := gofakes3.New(backend).Server()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Query().Has("partNumber") {
|
||||||
|
parts.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fake.ServeHTTP(w, r)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
cfg *config.Config
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "storage_url",
|
||||||
|
cfg: &config.Config{
|
||||||
|
StorageURL: "s3://" + s3TestBucket + "?endpoint=" + srv.URL,
|
||||||
|
S3: config.S3Config{
|
||||||
|
AccessKeyID: s3TestAccessKeyID,
|
||||||
|
SecretAccessKey: s3TestSecretAccessKey,
|
||||||
|
PartSize: partSize,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3.endpoint",
|
||||||
|
cfg: &config.Config{
|
||||||
|
S3: config.S3Config{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Bucket: s3TestBucket,
|
||||||
|
AccessKeyID: s3TestAccessKeyID,
|
||||||
|
SecretAccessKey: s3TestSecretAccessKey,
|
||||||
|
PartSize: partSize,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
parts.Store(0)
|
||||||
|
|
||||||
|
storer, err := storage.NewStorer(tc.cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: NewStorer: %v", tc.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.NewReader(make([]byte, wantParts*partSize))
|
||||||
|
|
||||||
|
err = storer.PutWithProgress(
|
||||||
|
context.Background(), "blob", data, data.Size(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: PutWithProgress: %v", tc.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := parts.Load(); got != wantParts {
|
||||||
|
t.Errorf("%s: uploaded in %d parts, want %d", tc.name, got, wantParts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// listStreamKeys returns the keys ListStream yields under a prefix, and
|
// listStreamKeys returns the keys ListStream yields under a prefix, and
|
||||||
// fails the test on a listing error.
|
// fails the test on a listing error.
|
||||||
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
|
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
|
||||||
|
|||||||
+23
-18
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,12 +47,9 @@ const (
|
|||||||
year = 365 * day
|
year = 365 * day
|
||||||
)
|
)
|
||||||
|
|
||||||
// Snapshot IDs split on "_" into hostname, optional name parts, and a
|
// A snapshot ID split on "_" has at least a hostname and a trailing
|
||||||
// trailing timestamp.
|
// timestamp.
|
||||||
const (
|
const minSnapshotIDParts = 2
|
||||||
minSnapshotIDParts = 2
|
|
||||||
minSnapshotIDNameParts = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
// SnapshotInfo contains information about a snapshot.
|
// SnapshotInfo contains information about a snapshot.
|
||||||
//
|
//
|
||||||
@@ -121,20 +119,27 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
|||||||
return timestamp.UTC(), nil
|
return timestamp.UTC(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseSnapshotName extracts the snapshot name from a snapshot ID.
|
// parseSnapshotName extracts the snapshot name from a snapshot ID of the
|
||||||
// Format: hostname_snapshotname_timestamp — the middle part(s) between hostname
|
// form hostname_name_timestamp, given the hostname stored with that
|
||||||
// and the RFC3339 timestamp are the snapshot name (may contain underscores).
|
// snapshot. The hostname and the name may both contain underscores, so the
|
||||||
// Returns the snapshot name, or empty string if the ID is malformed.
|
// name is what is left after removing the short hostname and its "_" from
|
||||||
func parseSnapshotName(snapshotID string) string {
|
// the front and the last "_" and the timestamp from the end. Returns "" for
|
||||||
parts := strings.Split(snapshotID, "_")
|
// an ID with no name (hostname_timestamp), and for an ID that does not start
|
||||||
if len(parts) < minSnapshotIDNameParts {
|
// with that hostname, which CreateSnapshotWithName never writes.
|
||||||
// Format: hostname_timestamp — no snapshot name
|
func parseSnapshotName(snapshotID, hostname string) string {
|
||||||
|
prefix := snapshot.ShortHostname(hostname) + "_"
|
||||||
|
|
||||||
|
rest, ok := strings.CutPrefix(snapshotID, prefix)
|
||||||
|
if !ok {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
// Format: hostname_name_timestamp — middle parts are the name.
|
|
||||||
// The last part is the RFC3339 timestamp, the first part is the hostname,
|
end := strings.LastIndex(rest, "_")
|
||||||
// everything in between is the snapshot name (which may itself contain underscores).
|
if end < 0 {
|
||||||
return strings.Join(parts[1:len(parts)-1], "_")
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return rest[:end]
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseDuration parses a duration string with support for human-friendly units:
|
// parseDuration parses a duration string with support for human-friendly units:
|
||||||
|
|||||||
@@ -11,33 +11,55 @@ func TestParseSnapshotName(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
snapshotID string
|
snapshotID string
|
||||||
|
hostname string
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "standard format with name",
|
name: "standard format with name",
|
||||||
snapshotID: "myhost_home_2026-01-12T14:41:15Z",
|
snapshotID: "myhost_home_2026-01-12T14:41:15Z",
|
||||||
|
hostname: "myhost",
|
||||||
want: "home",
|
want: "home",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "standard format with different name",
|
name: "standard format with different name",
|
||||||
snapshotID: "server1_system_2026-02-15T09:30:00Z",
|
snapshotID: "server1_system_2026-02-15T09:30:00Z",
|
||||||
|
hostname: "server1",
|
||||||
want: "system",
|
want: "system",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "name with underscores",
|
name: "name with underscores",
|
||||||
snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z",
|
snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z",
|
||||||
|
hostname: "myhost",
|
||||||
want: "my_special_backup",
|
want: "my_special_backup",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "hostname with underscores",
|
||||||
|
snapshotID: "my_host_docs_2026-03-01T00:00:00Z",
|
||||||
|
hostname: "my_host",
|
||||||
|
want: "docs",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "stored hostname with domain",
|
||||||
|
snapshotID: "my_host_mail_2026-03-01T00:00:00Z",
|
||||||
|
hostname: "my_host.example.com",
|
||||||
|
want: "mail",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no name",
|
||||||
|
snapshotID: "my_host_2026-03-01T00:00:00Z",
|
||||||
|
hostname: "my_host",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
got := parseSnapshotName(tt.snapshotID)
|
got := parseSnapshotName(tt.snapshotID, tt.hostname)
|
||||||
if got != tt.want {
|
if got != tt.want {
|
||||||
t.Errorf("parseSnapshotName(%q) = %q, want %q",
|
t.Errorf("parseSnapshotName(%q, %q) = %q, want %q",
|
||||||
tt.snapshotID, got, tt.want)
|
tt.snapshotID, tt.hostname, got, tt.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func setupConsistencyTest(
|
|||||||
completedAt := startedAt.Add(5 * time.Minute)
|
completedAt := startedAt.Add(5 * time.Minute)
|
||||||
snap := &database.Snapshot{
|
snap := &database.Snapshot{
|
||||||
ID: types.SnapshotID(id),
|
ID: types.SnapshotID(id),
|
||||||
Hostname: testHostname,
|
Hostname: snapHostname,
|
||||||
VaultikVersion: testLabel,
|
VaultikVersion: testLabel,
|
||||||
StartedAt: startedAt,
|
StartedAt: startedAt,
|
||||||
CompletedAt: &completedAt,
|
CompletedAt: &completedAt,
|
||||||
|
|||||||
@@ -17,8 +17,10 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Snapshot IDs reused across the purge tests.
|
// Snapshot IDs reused across the purge tests, and the hostname they were
|
||||||
|
// taken on.
|
||||||
const (
|
const (
|
||||||
|
snapHostname = "testhost"
|
||||||
snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z"
|
snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z"
|
||||||
snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z"
|
snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z"
|
||||||
snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z"
|
snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z"
|
||||||
@@ -26,9 +28,12 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock
|
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock
|
||||||
// storage pre-populated with the given snapshot IDs. Each snapshot is marked as
|
// storage pre-populated with the given snapshot IDs, all taken on hostname.
|
||||||
// completed. Remote metadata stubs are created so syncWithRemote keeps them.
|
// Each snapshot is marked as completed. Remote metadata stubs are created so
|
||||||
func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
|
// syncWithRemote keeps them.
|
||||||
|
func setupPurgeTest(
|
||||||
|
t *testing.T, hostname string, snapshotIDs []string,
|
||||||
|
) *vaultik.Vaultik {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -51,7 +56,7 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
|
|||||||
completedAt := startedAt.Add(5 * time.Minute)
|
completedAt := startedAt.Add(5 * time.Minute)
|
||||||
snap := &database.Snapshot{
|
snap := &database.Snapshot{
|
||||||
ID: types.SnapshotID(id),
|
ID: types.SnapshotID(id),
|
||||||
Hostname: "testhost",
|
Hostname: types.Hostname(hostname),
|
||||||
VaultikVersion: testLabel,
|
VaultikVersion: testLabel,
|
||||||
StartedAt: startedAt,
|
StartedAt: startedAt,
|
||||||
CompletedAt: &completedAt,
|
CompletedAt: &completedAt,
|
||||||
@@ -120,7 +125,7 @@ func TestPurgeKeepLatest_PerName(t *testing.T) {
|
|||||||
"testhost_system_2026-01-01T04:00:00Z",
|
"testhost_system_2026-01-01T04:00:00Z",
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -148,7 +153,7 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) {
|
|||||||
"testhost_home_2026-01-01T02:00:00Z",
|
"testhost_home_2026-01-01T02:00:00Z",
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -176,7 +181,7 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) {
|
|||||||
"testhost_home_2026-01-01T04:00:00Z",
|
"testhost_home_2026-01-01T04:00:00Z",
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -198,7 +203,7 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) {
|
|||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
v := setupPurgeTest(t, nil)
|
v := setupPurgeTest(t, snapHostname, nil)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -216,7 +221,7 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) {
|
|||||||
"testhost_system_2026-01-01T01:00:00Z",
|
"testhost_system_2026-01-01T01:00:00Z",
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -243,7 +248,7 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
|
|||||||
snapHomeT0,
|
snapHomeT0,
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
// Purge only "home" snapshots older than 365 days
|
// Purge only "home" snapshots older than 365 days
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
@@ -277,7 +282,7 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
|
|||||||
"testhost_home_2026-01-01T06:00:00Z",
|
"testhost_home_2026-01-01T06:00:00Z",
|
||||||
}
|
}
|
||||||
|
|
||||||
v := setupPurgeTest(t, snapshotIDs)
|
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||||
|
|
||||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
KeepLatest: true,
|
KeepLatest: true,
|
||||||
@@ -291,3 +296,28 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
|
|||||||
assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z")
|
assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z")
|
||||||
assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z")
|
assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A hostname may contain underscores, so the snapshot name cannot be found
|
||||||
|
// by splitting the ID at them. A purge by name must still select "docs".
|
||||||
|
func TestPurgeKeepLatest_HostnameWithUnderscore(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
system = "my_host_system_2026-01-01T00:00:00Z"
|
||||||
|
docsT1 = "my_host_docs_2026-01-01T01:00:00Z"
|
||||||
|
docsT2 = "my_host_docs_2026-01-01T02:00:00Z"
|
||||||
|
)
|
||||||
|
|
||||||
|
v := setupPurgeTest(t, "my_host", []string{system, docsT1, docsT2})
|
||||||
|
|
||||||
|
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||||
|
KeepLatest: true,
|
||||||
|
Force: true,
|
||||||
|
Names: []string{"docs"},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, []string{system, docsT2},
|
||||||
|
listRemainingSnapshots(t, v))
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for snapshot management.
|
// Sentinel errors for snapshot management.
|
||||||
@@ -495,19 +496,23 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
|||||||
nameFilter[n] = struct{}{}
|
nameFilter[n] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collect completed snapshots, applying the name filter.
|
// Collect completed snapshots and their names, applying the name filter.
|
||||||
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
|
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
|
||||||
|
names := make(map[types.SnapshotID]string, len(dbSnapshots))
|
||||||
|
|
||||||
for _, s := range dbSnapshots {
|
for _, s := range dbSnapshots {
|
||||||
if s.CompletedAt == nil {
|
if s.CompletedAt == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
name := parseSnapshotName(s.ID.String(), s.Hostname.String())
|
||||||
if len(nameFilter) > 0 {
|
if len(nameFilter) > 0 {
|
||||||
if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok {
|
if _, ok := nameFilter[name]; !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
names[s.ID] = name
|
||||||
snapshots = append(snapshots, SnapshotInfo{
|
snapshots = append(snapshots, SnapshotInfo{
|
||||||
ID: s.ID,
|
ID: s.ID,
|
||||||
Timestamp: s.StartedAt,
|
Timestamp: s.StartedAt,
|
||||||
@@ -520,7 +525,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
|||||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||||
})
|
})
|
||||||
|
|
||||||
toDelete, err := selectSnapshotsToPurge(snapshots, opts)
|
toDelete, err := selectSnapshotsToPurge(snapshots, names, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -538,9 +543,11 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
|||||||
|
|
||||||
// selectSnapshotsToPurge applies the purge retention criteria to the
|
// selectSnapshotsToPurge applies the purge retention criteria to the
|
||||||
// newest-first sorted snapshot list and returns the deletion
|
// newest-first sorted snapshot list and returns the deletion
|
||||||
// candidates.
|
// candidates. names maps each snapshot's ID to its snapshot name.
|
||||||
func selectSnapshotsToPurge(
|
func selectSnapshotsToPurge(
|
||||||
snapshots []SnapshotInfo, opts *SnapshotPurgeOptions,
|
snapshots []SnapshotInfo,
|
||||||
|
names map[types.SnapshotID]string,
|
||||||
|
opts *SnapshotPurgeOptions,
|
||||||
) ([]SnapshotInfo, error) {
|
) ([]SnapshotInfo, error) {
|
||||||
var toDelete []SnapshotInfo
|
var toDelete []SnapshotInfo
|
||||||
|
|
||||||
@@ -551,7 +558,7 @@ func selectSnapshotsToPurge(
|
|||||||
seen := make(map[string]bool)
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
for _, snap := range snapshots {
|
for _, snap := range snapshots {
|
||||||
name := parseSnapshotName(snap.ID.String())
|
name := names[snap.ID]
|
||||||
if seen[name] {
|
if seen[name] {
|
||||||
toDelete = append(toDelete, snap)
|
toDelete = append(toDelete, snap)
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user