From 36c59cb7b320bffdfa68ca24a5105df22e3ecea9 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 20 Jul 2025 11:09:59 +0200 Subject: [PATCH] Set up S3 testing infrastructure for backup implementation - Add gofakes3 for in-process S3-compatible test server - Create test server that runs on localhost:9999 with temp directory - Implement basic S3 client wrapper with standard operations - Add comprehensive tests for blob and metadata storage patterns - Test cleanup properly removes temporary directories - Use AWS SDK v2 for S3 operations with proper error handling --- go.mod | 36 +++++ go.sum | 129 +++++++++++++++++ internal/cli/snapshot.go | 8 +- internal/s3/client.go | 140 ++++++++++++++++++ internal/s3/client_test.go | 98 +++++++++++++ internal/s3/s3_test.go | 285 +++++++++++++++++++++++++++++++++++++ 6 files changed, 692 insertions(+), 4 deletions(-) create mode 100644 internal/s3/client.go create mode 100644 internal/s3/client_test.go create mode 100644 internal/s3/s3_test.go diff --git a/go.mod b/go.mod index 4a620cf..c17a664 100644 --- a/go.mod +++ b/go.mod @@ -10,18 +10,54 @@ require ( ) require ( + github.com/aws/aws-sdk-go v1.44.256 // indirect + github.com/aws/aws-sdk-go-v2 v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.18 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.71 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.34.1 // indirect + github.com/aws/smithy-go v1.22.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.94 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect + golang.org/x/tools v0.33.0 // indirect modernc.org/libc v1.65.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 7c83a6b..70cba84 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,105 @@ +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.36.6 h1:zJqGjVbRdTPojeCGWn5IR5pbJwSQSBh5RWFTQcEQGdU= +github.com/aws/aws-sdk-go-v2 v1.36.6/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= +github.com/aws/aws-sdk-go-v2/config v1.29.18 h1:x4T1GRPnqKV8HMJOMtNktbpQMl3bIsfx8KbqmveUO2I= +github.com/aws/aws-sdk-go-v2/config v1.29.18/go.mod h1:bvz8oXugIsH8K7HLhBv06vDqnFv3NsGDt2Znpk7zmOU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.71 h1:r2w4mQWnrTMJjOyIsZtGp3R3XGY3nqHn8C26C2lQWgA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.71/go.mod h1:E7VF3acIup4GB5ckzbKFrCK0vTvEQxOxgdq4U3vcMCY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33 h1:D9ixiWSG4lyUBL2DDNK924Px9V/NBVpML90MHqyTADY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33/go.mod h1:caS/m4DI+cij2paz3rtProRBI4s/+TCiWoaWZuQ9010= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 h1:osMWfm/sC/L4tvEdQ65Gri5ZZDCUpuYJZbTTDrsn4I0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37/go.mod h1:ZV2/1fbjOPr4G4v38G3Ww5TBT4+hmsK45s/rxu1fGy0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 h1:v+X21AvTb2wZ+ycg1gx+orkB/9U6L7AOp93R7qYxsxM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37/go.mod h1:G0uM1kyssELxmJ2VZEfG0q2npObR3BAkF3c1VsfVnfs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37 h1:XTZZ0I3SZUHAtBLBU6395ad+VOblE0DwQP6MuaNeics= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37/go.mod h1:Pi6ksbniAWVwu2S8pEzcYPyhUkAcLaufxN7PfAUQjBk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5 h1:M5/B8JUaCI8+9QD+u3S/f4YHpvqE9RpSkV3rf0Iks2w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5/go.mod h1:Bktzci1bwdbpuLiu3AOksiNPMl/LLKmX1TWmqp2xbvs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 h1:vvbXsA2TVO80/KT7ZqCbx934dt6PY+vQ8hZpUZ/cpYg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18/go.mod h1:m2JJHledjBGNMsLOF1g9gbAxprzq3KjC8e4lxtn+eWg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18 h1:OS2e0SKqsU2LiJPqL8u9x41tKc6MMEHrWjLVLn3oysg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18/go.mod h1:+Yrk+MDGzlNGxCXieljNeWpoZTCQUQVL+Jk9hGGJ8qM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1 h1:RkHXU9jP0DptGy7qKI8CBGsUJruWz0v5IgwBa2DwWcU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1/go.mod h1:3xAOf7tdKF+qbb+XpU+EPhNXAdun3Lu1RcDrj8KC24I= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.6 h1:rGtWqkQbPk7Bkwuv3NzpE/scwwL9sC1Ul3tn9x83DUI= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.6/go.mod h1:u4ku9OLv4TO4bCPdxf4fA1upaMaJmP9ZijGk3AAOC6Q= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4 h1:OV/pxyXh+eMA0TExHEC4jyWdumLxNbzz1P0zJoezkJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4/go.mod h1:8Mm5VGYwtm+r305FfPSuc+aFkrypeylGYhFim6XEPoc= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.1 h1:aUrLQwJfZtwv3/ZNG2xRtEen+NqI3iesuacjP51Mv1s= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.1/go.mod h1:3wFBZKoWnX3r+Sm7in79i54fBmNfwhdNdQuscCw7QIk= +github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= +github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668 h1:+Mn8Sj5VzjOTuzyBCxfUnEcS+Iky4/5piUraOC3E5qQ= +github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668/go.mod h1:t6osVdP++3g4v2awHz4+HFccij23BbdT1rX3W7IijqQ= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.94 h1:1ZoksIKPyaSt64AVOyaQvhDOgVC3MfZsWM6mZXRUGtM= +github.com/minio/minio-go/v7 v7.0.94/go.mod h1:71t2CqDt3ThzESgZUlU1rBN54mksGGlkLcFgguDnnAc= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= @@ -34,19 +110,72 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 835df20..63094a4 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -37,7 +37,7 @@ func snapshotListCmd() *cobra.Command { cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix") cmd.Flags().IntVar(&limit, "limit", 10, "Maximum number of snapshots to list") - cmd.MarkFlagRequired("bucket") + _ = cmd.MarkFlagRequired("bucket") return cmd } @@ -61,8 +61,8 @@ func snapshotRmCmd() *cobra.Command { cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix") cmd.Flags().StringVar(&snapshot, "snapshot", "", "Snapshot ID to remove") - cmd.MarkFlagRequired("bucket") - cmd.MarkFlagRequired("snapshot") + _ = cmd.MarkFlagRequired("bucket") + _ = cmd.MarkFlagRequired("snapshot") return cmd } @@ -84,7 +84,7 @@ func snapshotLatestCmd() *cobra.Command { cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") cmd.Flags().StringVar(&prefix, "prefix", "", "S3 prefix") - cmd.MarkFlagRequired("bucket") + _ = cmd.MarkFlagRequired("bucket") return cmd } diff --git a/internal/s3/client.go b/internal/s3/client.go new file mode 100644 index 0000000..c682c8a --- /dev/null +++ b/internal/s3/client.go @@ -0,0 +1,140 @@ +package s3 + +import ( + "context" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Client wraps the AWS S3 client for vaultik operations +type Client struct { + s3Client *s3.Client + bucket string + prefix string +} + +// Config contains S3 client configuration +type Config struct { + Endpoint string + Bucket string + Prefix string + AccessKeyID string + SecretAccessKey string + Region string +} + +// NewClient creates a new S3 client +func NewClient(ctx context.Context, cfg Config) (*Client, error) { + // Create AWS config + awsCfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(cfg.Region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfg.AccessKeyID, + cfg.SecretAccessKey, + "", + )), + ) + if err != nil { + return nil, err + } + + // Configure custom endpoint if provided + s3Opts := func(o *s3.Options) { + if cfg.Endpoint != "" { + o.BaseEndpoint = aws.String(cfg.Endpoint) + o.UsePathStyle = true + } + } + + s3Client := s3.NewFromConfig(awsCfg, s3Opts) + + return &Client{ + s3Client: s3Client, + bucket: cfg.Bucket, + prefix: cfg.Prefix, + }, nil +} + +// PutObject uploads an object to S3 +func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) error { + fullKey := c.prefix + key + _, err := c.s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(fullKey), + Body: data, + }) + return err +} + +// GetObject downloads an object from S3 +func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { + fullKey := c.prefix + key + result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(fullKey), + }) + if err != nil { + return nil, err + } + return result.Body, nil +} + +// DeleteObject removes an object from S3 +func (c *Client) DeleteObject(ctx context.Context, key string) error { + fullKey := c.prefix + key + _, err := c.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(fullKey), + }) + return err +} + +// ListObjects lists objects with the given prefix +func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, error) { + fullPrefix := c.prefix + prefix + + var keys []string + paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucket), + Prefix: aws.String(fullPrefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, obj := range page.Contents { + if obj.Key != nil { + // Remove the base prefix from the key + key := *obj.Key + if len(key) > len(c.prefix) { + key = key[len(c.prefix):] + } + keys = append(keys, key) + } + } + } + + return keys, nil +} + +// HeadObject checks if an object exists +func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) { + fullKey := c.prefix + key + _, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(fullKey), + }) + if err != nil { + // Check if it's a not found error + // TODO: Add proper error type checking + return false, nil + } + return true, nil +} diff --git a/internal/s3/client_test.go b/internal/s3/client_test.go new file mode 100644 index 0000000..db86b74 --- /dev/null +++ b/internal/s3/client_test.go @@ -0,0 +1,98 @@ +package s3_test + +import ( + "bytes" + "context" + "io" + "testing" + + "git.eeqj.de/sneak/vaultik/internal/s3" +) + +func TestClient(t *testing.T) { + ts := NewTestServer(t) + defer func() { + if err := ts.Cleanup(); err != nil { + t.Errorf("cleanup failed: %v", err) + } + }() + + ctx := context.Background() + + // Create client + client, err := s3.NewClient(ctx, s3.Config{ + Endpoint: testEndpoint, + Bucket: testBucket, + Prefix: "test-prefix/", + AccessKeyID: testAccessKey, + SecretAccessKey: testSecretKey, + Region: testRegion, + }) + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + + // Test PutObject + testKey := "foo/bar.txt" + testData := []byte("test data") + err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) + if err != nil { + t.Fatalf("failed to put object: %v", err) + } + + // Test GetObject + reader, err := client.GetObject(ctx, testKey) + if err != nil { + t.Fatalf("failed to get object: %v", err) + } + defer func() { + if err := reader.Close(); err != nil { + t.Errorf("failed to close reader: %v", err) + } + }() + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("failed to read data: %v", err) + } + + if !bytes.Equal(data, testData) { + t.Errorf("data mismatch: got %q, want %q", data, testData) + } + + // Test HeadObject + exists, err := client.HeadObject(ctx, testKey) + if err != nil { + t.Fatalf("failed to head object: %v", err) + } + if !exists { + t.Error("expected object to exist") + } + + // Test ListObjects + keys, err := client.ListObjects(ctx, "foo/") + if err != nil { + t.Fatalf("failed to list objects: %v", err) + } + if len(keys) != 1 { + t.Errorf("expected 1 key, got %d", len(keys)) + } + if keys[0] != testKey { + t.Errorf("unexpected key: got %s, want %s", keys[0], testKey) + } + + // Test DeleteObject + err = client.DeleteObject(ctx, testKey) + if err != nil { + t.Fatalf("failed to delete object: %v", err) + } + + // Verify deletion + exists, err = client.HeadObject(ctx, testKey) + if err != nil { + t.Fatalf("failed to head object after deletion: %v", err) + } + if exists { + t.Error("expected object to not exist after deletion") + } +} diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go new file mode 100644 index 0000000..890a81b --- /dev/null +++ b/internal/s3/s3_test.go @@ -0,0 +1,285 @@ +package s3_test + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" +) + +const ( + testBucket = "test-bucket" + testRegion = "us-east-1" + testAccessKey = "test-access-key" + testSecretKey = "test-secret-key" + testEndpoint = "http://localhost:9999" +) + +// TestServer represents an in-process S3-compatible test server +type TestServer struct { + server *http.Server + backend gofakes3.Backend + s3Client *s3.Client + tempDir string +} + +// NewTestServer creates and starts a new test server +func NewTestServer(t *testing.T) *TestServer { + // Create temp directory for any file operations + tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + + // Create in-memory backend + backend := s3mem.New() + faker := gofakes3.New(backend) + + // Create HTTP server + server := &http.Server{ + Addr: "localhost:9999", + Handler: faker.Server(), + } + + // Start server in background + go func() { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + t.Logf("test server error: %v", err) + } + }() + + // Wait for server to be ready + time.Sleep(100 * time.Millisecond) + + // Create S3 client + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(testRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + testAccessKey, + testSecretKey, + "", + )), + config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), + ) + if err != nil { + t.Fatalf("failed to create AWS config: %v", err) + } + + s3Client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(testEndpoint) + o.UsePathStyle = true + }) + + ts := &TestServer{ + server: server, + backend: backend, + s3Client: s3Client, + tempDir: tempDir, + } + + // Create test bucket + _, err = s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ + Bucket: aws.String(testBucket), + }) + if err != nil { + t.Fatalf("failed to create test bucket: %v", err) + } + + return ts +} + +// Cleanup shuts down the server and removes temp directory +func (ts *TestServer) Cleanup() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := ts.server.Shutdown(ctx); err != nil { + return err + } + + return os.RemoveAll(ts.tempDir) +} + +// Client returns the S3 client configured for the test server +func (ts *TestServer) Client() *s3.Client { + return ts.s3Client +} + +// TestBasicS3Operations tests basic store and retrieve operations +func TestBasicS3Operations(t *testing.T) { + ts := NewTestServer(t) + defer func() { + if err := ts.Cleanup(); err != nil { + t.Errorf("cleanup failed: %v", err) + } + }() + + ctx := context.Background() + client := ts.Client() + + // Test data + testKey := "test/file.txt" + testData := []byte("Hello, S3 test!") + + // Put object + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(testKey), + Body: bytes.NewReader(testData), + }) + if err != nil { + t.Fatalf("failed to put object: %v", err) + } + + // Get object + result, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(testKey), + }) + if err != nil { + t.Fatalf("failed to get object: %v", err) + } + defer func() { + if err := result.Body.Close(); err != nil { + t.Errorf("failed to close body: %v", err) + } + }() + + // Read and verify data + data, err := io.ReadAll(result.Body) + if err != nil { + t.Fatalf("failed to read object body: %v", err) + } + + if !bytes.Equal(data, testData) { + t.Errorf("retrieved data mismatch: got %q, want %q", data, testData) + } +} + +// TestBlobOperations tests blob storage patterns for vaultik +func TestBlobOperations(t *testing.T) { + ts := NewTestServer(t) + defer func() { + if err := ts.Cleanup(); err != nil { + t.Errorf("cleanup failed: %v", err) + } + }() + + ctx := context.Background() + client := ts.Client() + + // Test blob storage with prefix structure + blobHash := "aabbccddee112233445566778899aabbccddee11" + blobKey := filepath.Join("blobs", blobHash[:2], blobHash[2:4], blobHash+".zst.age") + blobData := []byte("compressed and encrypted blob data") + + // Store blob + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(blobKey), + Body: bytes.NewReader(blobData), + }) + if err != nil { + t.Fatalf("failed to store blob: %v", err) + } + + // List objects with prefix + listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(testBucket), + Prefix: aws.String("blobs/aa/"), + }) + if err != nil { + t.Fatalf("failed to list objects: %v", err) + } + + if len(listResult.Contents) != 1 { + t.Errorf("expected 1 object, got %d", len(listResult.Contents)) + } + + if listResult.Contents[0].Key != nil && *listResult.Contents[0].Key != blobKey { + t.Errorf("unexpected key: got %s, want %s", *listResult.Contents[0].Key, blobKey) + } + + // Delete blob + _, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(blobKey), + }) + if err != nil { + t.Fatalf("failed to delete blob: %v", err) + } + + // Verify deletion + _, err = client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(blobKey), + }) + if err == nil { + t.Error("expected error getting deleted object, got nil") + } +} + +// TestMetadataOperations tests metadata storage patterns +func TestMetadataOperations(t *testing.T) { + ts := NewTestServer(t) + defer func() { + if err := ts.Cleanup(); err != nil { + t.Errorf("cleanup failed: %v", err) + } + }() + + ctx := context.Background() + client := ts.Client() + + // Test metadata storage + snapshotID := "2024-01-01T12:00:00Z" + metadataKey := filepath.Join("metadata", snapshotID+".sqlite.age") + metadataData := []byte("encrypted sqlite database") + + // Store metadata + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(metadataKey), + Body: bytes.NewReader(metadataData), + }) + if err != nil { + t.Fatalf("failed to store metadata: %v", err) + } + + // Store manifest + manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst") + manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`) + + _, err = client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(manifestKey), + Body: bytes.NewReader(manifestData), + }) + if err != nil { + t.Fatalf("failed to store manifest: %v", err) + } + + // List metadata objects + listResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(testBucket), + Prefix: aws.String("metadata/"), + }) + if err != nil { + t.Fatalf("failed to list metadata: %v", err) + } + + if len(listResult.Contents) != 2 { + t.Errorf("expected 2 metadata objects, got %d", len(listResult.Contents)) + } +}