sneak
26db096913
- Remove StartTime initialization from globals.New() - Add setupGlobals function in app.go to set StartTime during fx OnStart - Simplify globals package to be just a key/value store - Remove fx dependencies from globals test
307 lines
7.6 KiB
Go
307 lines
7.6 KiB
Go
package s3_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/smithy-go/logging"
|
|
"github.com/johannesboyne/gofakes3"
|
|
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
)
|
|
|
|
const (
|
|
testBucket = "test-bucket"
|
|
testRegion = "us-east-1"
|
|
testAccessKey = "test-access-key"
|
|
testSecretKey = "test-secret-key"
|
|
testEndpoint = "http://localhost:9999"
|
|
)
|
|
|
|
// TestServer represents an in-process S3-compatible test server
|
|
type TestServer struct {
|
|
server *http.Server
|
|
backend gofakes3.Backend
|
|
s3Client *s3.Client
|
|
tempDir string
|
|
logBuf *bytes.Buffer
|
|
}
|
|
|
|
// NewTestServer creates and starts a new test server
|
|
func NewTestServer(t *testing.T) *TestServer {
|
|
// Create temp directory for any file operations
|
|
tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*")
|
|
if err != nil {
|
|
t.Fatalf("failed to create temp dir: %v", err)
|
|
}
|
|
|
|
// Create in-memory backend
|
|
backend := s3mem.New()
|
|
faker := gofakes3.New(backend)
|
|
|
|
// Create HTTP server
|
|
server := &http.Server{
|
|
Addr: "localhost:9999",
|
|
Handler: faker.Server(),
|
|
}
|
|
|
|
// Start server in background
|
|
go func() {
|
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
t.Logf("test server error: %v", err)
|
|
}
|
|
}()
|
|
|
|
// Wait for server to be ready
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
// Create a buffer to capture logs
|
|
logBuf := &bytes.Buffer{}
|
|
|
|
// Create S3 client with custom logger
|
|
cfg, err := config.LoadDefaultConfig(context.Background(),
|
|
config.WithRegion(testRegion),
|
|
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
|
testAccessKey,
|
|
testSecretKey,
|
|
"",
|
|
)),
|
|
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
|
|
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
|
|
// Capture logs to buffer instead of stdout
|
|
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
|
|
time.Now().Format("2006/01/02 15:04:05"),
|
|
string(classification),
|
|
fmt.Sprintf(format, v...))
|
|
})),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to create AWS config: %v", err)
|
|
}
|
|
|
|
s3Client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
|
o.BaseEndpoint = aws.String(testEndpoint)
|
|
o.UsePathStyle = true
|
|
})
|
|
|
|
ts := &TestServer{
|
|
server: server,
|
|
backend: backend,
|
|
s3Client: s3Client,
|
|
tempDir: tempDir,
|
|
logBuf: logBuf,
|
|
}
|
|
|
|
// Register cleanup to show logs on test failure
|
|
t.Cleanup(func() {
|
|
if t.Failed() && logBuf.Len() > 0 {
|
|
t.Logf("S3 SDK Debug Output:\n%s", logBuf.String())
|
|
}
|
|
})
|
|
|
|
// Create test bucket
|
|
_, err = s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
|
Bucket: aws.String(testBucket),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create test bucket: %v", err)
|
|
}
|
|
|
|
return ts
|
|
}
|
|
|
|
// Cleanup shuts down the server and removes temp directory
|
|
func (ts *TestServer) Cleanup() error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
if err := ts.server.Shutdown(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
return os.RemoveAll(ts.tempDir)
|
|
}
|
|
|
|
// Client returns the S3 client configured for the test server
|
|
func (ts *TestServer) Client() *s3.Client {
|
|
return ts.s3Client
|
|
}
|
|
|
|
// TestBasicS3Operations tests basic store and retrieve operations
|
|
func TestBasicS3Operations(t *testing.T) {
|
|
ts := NewTestServer(t)
|
|
defer func() {
|
|
if err := ts.Cleanup(); err != nil {
|
|
t.Errorf("cleanup failed: %v", err)
|
|
}
|
|
}()
|
|
|
|
ctx := context.Background()
|
|
client := ts.Client()
|
|
|
|
// Test data
|
|
testKey := "test/file.txt"
|
|
testData := []byte("Hello, S3 test!")
|
|
|
|
// Put object
|
|
_, err := client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(testKey),
|
|
Body: bytes.NewReader(testData),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to put object: %v", err)
|
|
}
|
|
|
|
// Get object
|
|
result, err := client.GetObject(ctx, &s3.GetObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(testKey),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to get object: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := result.Body.Close(); err != nil {
|
|
t.Errorf("failed to close body: %v", err)
|
|
}
|
|
}()
|
|
|
|
// Read and verify data
|
|
data, err := io.ReadAll(result.Body)
|
|
if err != nil {
|
|
t.Fatalf("failed to read object body: %v", err)
|
|
}
|
|
|
|
if !bytes.Equal(data, testData) {
|
|
t.Errorf("retrieved data mismatch: got %q, want %q", data, testData)
|
|
}
|
|
}
|
|
|
|
// TestBlobOperations tests blob storage patterns for vaultik
|
|
func TestBlobOperations(t *testing.T) {
|
|
ts := NewTestServer(t)
|
|
defer func() {
|
|
if err := ts.Cleanup(); err != nil {
|
|
t.Errorf("cleanup failed: %v", err)
|
|
}
|
|
}()
|
|
|
|
ctx := context.Background()
|
|
client := ts.Client()
|
|
|
|
// Test blob storage with prefix structure
|
|
blobHash := "aabbccddee112233445566778899aabbccddee11"
|
|
blobKey := filepath.Join("blobs", blobHash[:2], blobHash[2:4], blobHash+".zst.age")
|
|
blobData := []byte("compressed and encrypted blob data")
|
|
|
|
// Store blob
|
|
_, err := client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(blobKey),
|
|
Body: bytes.NewReader(blobData),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to store blob: %v", err)
|
|
}
|
|
|
|
// List objects with prefix
|
|
listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(testBucket),
|
|
Prefix: aws.String("blobs/aa/"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to list objects: %v", err)
|
|
}
|
|
|
|
if len(listResult.Contents) != 1 {
|
|
t.Errorf("expected 1 object, got %d", len(listResult.Contents))
|
|
}
|
|
|
|
if listResult.Contents[0].Key != nil && *listResult.Contents[0].Key != blobKey {
|
|
t.Errorf("unexpected key: got %s, want %s", *listResult.Contents[0].Key, blobKey)
|
|
}
|
|
|
|
// Delete blob
|
|
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(blobKey),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to delete blob: %v", err)
|
|
}
|
|
|
|
// Verify deletion
|
|
_, err = client.GetObject(ctx, &s3.GetObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(blobKey),
|
|
})
|
|
if err == nil {
|
|
t.Error("expected error getting deleted object, got nil")
|
|
}
|
|
}
|
|
|
|
// TestMetadataOperations tests metadata storage patterns
|
|
func TestMetadataOperations(t *testing.T) {
|
|
ts := NewTestServer(t)
|
|
defer func() {
|
|
if err := ts.Cleanup(); err != nil {
|
|
t.Errorf("cleanup failed: %v", err)
|
|
}
|
|
}()
|
|
|
|
ctx := context.Background()
|
|
client := ts.Client()
|
|
|
|
// Test metadata storage
|
|
snapshotID := "2024-01-01T12:00:00Z"
|
|
metadataKey := filepath.Join("metadata", snapshotID+".sqlite.age")
|
|
metadataData := []byte("encrypted sqlite database")
|
|
|
|
// Store metadata
|
|
_, err := client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(metadataKey),
|
|
Body: bytes.NewReader(metadataData),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to store metadata: %v", err)
|
|
}
|
|
|
|
// Store manifest
|
|
manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst")
|
|
manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`)
|
|
|
|
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(testBucket),
|
|
Key: aws.String(manifestKey),
|
|
Body: bytes.NewReader(manifestData),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to store manifest: %v", err)
|
|
}
|
|
|
|
// List metadata objects
|
|
listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(testBucket),
|
|
Prefix: aws.String("metadata/"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to list metadata: %v", err)
|
|
}
|
|
|
|
if len(listResult.Contents) != 2 {
|
|
t.Errorf("expected 2 metadata objects, got %d", len(listResult.Contents))
|
|
}
|
|
}
|