package s3_test import ( "bytes" "context" "io" "net/http" "os" "path/filepath" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" ) const ( testBucket = "test-bucket" testRegion = "us-east-1" testAccessKey = "test-access-key" testSecretKey = "test-secret-key" testEndpoint = "http://localhost:9999" ) // TestServer represents an in-process S3-compatible test server type TestServer struct { server *http.Server backend gofakes3.Backend s3Client *s3.Client tempDir string } // NewTestServer creates and starts a new test server func NewTestServer(t *testing.T) *TestServer { // Create temp directory for any file operations tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*") if err != nil { t.Fatalf("failed to create temp dir: %v", err) } // Create in-memory backend backend := s3mem.New() faker := gofakes3.New(backend) // Create HTTP server server := &http.Server{ Addr: "localhost:9999", Handler: faker.Server(), } // Start server in background go func() { if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { t.Logf("test server error: %v", err) } }() // Wait for server to be ready time.Sleep(100 * time.Millisecond) // Create S3 client cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(testRegion), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( testAccessKey, testSecretKey, "", )), config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), ) if err != nil { t.Fatalf("failed to create AWS config: %v", err) } s3Client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(testEndpoint) o.UsePathStyle = true }) ts := &TestServer{ server: server, backend: backend, s3Client: s3Client, tempDir: tempDir, } // Create test bucket _, err = s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ Bucket: aws.String(testBucket), }) if err != nil { t.Fatalf("failed to create test bucket: %v", err) } return ts } // Cleanup shuts down the server and removes temp directory func (ts *TestServer) Cleanup() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := ts.server.Shutdown(ctx); err != nil { return err } return os.RemoveAll(ts.tempDir) } // Client returns the S3 client configured for the test server func (ts *TestServer) Client() *s3.Client { return ts.s3Client } // TestBasicS3Operations tests basic store and retrieve operations func TestBasicS3Operations(t *testing.T) { ts := NewTestServer(t) defer func() { if err := ts.Cleanup(); err != nil { t.Errorf("cleanup failed: %v", err) } }() ctx := context.Background() client := ts.Client() // Test data testKey := "test/file.txt" testData := []byte("Hello, S3 test!") // Put object _, err := client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(testKey), Body: bytes.NewReader(testData), }) if err != nil { t.Fatalf("failed to put object: %v", err) } // Get object result, err := client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(testKey), }) if err != nil { t.Fatalf("failed to get object: %v", err) } defer func() { if err := result.Body.Close(); err != nil { t.Errorf("failed to close body: %v", err) } }() // Read and verify data data, err := io.ReadAll(result.Body) if err != nil { t.Fatalf("failed to read object body: %v", err) } if !bytes.Equal(data, testData) { t.Errorf("retrieved data mismatch: got %q, want %q", data, testData) } } // TestBlobOperations tests blob storage patterns for vaultik func TestBlobOperations(t *testing.T) { ts := NewTestServer(t) defer func() { if err := ts.Cleanup(); err != nil { t.Errorf("cleanup failed: %v", err) } }() ctx := context.Background() client := ts.Client() // Test blob storage with prefix structure blobHash := "aabbccddee112233445566778899aabbccddee11" blobKey := filepath.Join("blobs", blobHash[:2], blobHash[2:4], blobHash+".zst.age") blobData := []byte("compressed and encrypted blob data") // Store blob _, err := client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(blobKey), Body: bytes.NewReader(blobData), }) if err != nil { t.Fatalf("failed to store blob: %v", err) } // List objects with prefix listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(testBucket), Prefix: aws.String("blobs/aa/"), }) if err != nil { t.Fatalf("failed to list objects: %v", err) } if len(listResult.Contents) != 1 { t.Errorf("expected 1 object, got %d", len(listResult.Contents)) } if listResult.Contents[0].Key != nil && *listResult.Contents[0].Key != blobKey { t.Errorf("unexpected key: got %s, want %s", *listResult.Contents[0].Key, blobKey) } // Delete blob _, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(blobKey), }) if err != nil { t.Fatalf("failed to delete blob: %v", err) } // Verify deletion _, err = client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(blobKey), }) if err == nil { t.Error("expected error getting deleted object, got nil") } } // TestMetadataOperations tests metadata storage patterns func TestMetadataOperations(t *testing.T) { ts := NewTestServer(t) defer func() { if err := ts.Cleanup(); err != nil { t.Errorf("cleanup failed: %v", err) } }() ctx := context.Background() client := ts.Client() // Test metadata storage snapshotID := "2024-01-01T12:00:00Z" metadataKey := filepath.Join("metadata", snapshotID+".sqlite.age") metadataData := []byte("encrypted sqlite database") // Store metadata _, err := client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(metadataKey), Body: bytes.NewReader(metadataData), }) if err != nil { t.Fatalf("failed to store metadata: %v", err) } // Store manifest manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst") manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`) _, err = client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), Key: aws.String(manifestKey), Body: bytes.NewReader(manifestData), }) if err != nil { t.Fatalf("failed to store manifest: %v", err) } // List metadata objects listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(testBucket), Prefix: aws.String("metadata/"), }) if err != nil { t.Fatalf("failed to list metadata: %v", err) } if len(listResult.Contents) != 2 { t.Errorf("expected 2 metadata objects, got %d", len(listResult.Contents)) } }