Join the S3 prefix to every key with one slash (closes #222)
check / check (push) Successful in 10m54s

The S3 client built each key as prefix + key, and the URL parser keeps
the prefix as written, so s3://bucket/p stored p + "blobs/..." with no
slash while s3://bucket/p/ stored p/blobs/.... A recovery host that wrote
the URL the other way found no snapshots.

NewClient now strips trailing slashes from the prefix and adds one back
when anything is left, giving the README layout for both URL forms; an
empty prefix stays at the bucket root. The s3.prefix config setting
goes through the same client and gets the same join.

A new test writes through each URL shape against an in-process S3
server, checks the key in the bucket, and lists through both List and
ListStream, which every snapshot listing uses.

Model: opus-5-5
This commit is contained in:
2026-10-06 16:44:01 +00:00
parent 5aa5ba5544
commit 4dc14895d4
3 changed files with 128 additions and 1 deletions
+109
View File
@@ -4,11 +4,14 @@ import (
"context"
"errors"
"net/http/httptest"
"slices"
"strings"
"testing"
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storage"
)
@@ -79,3 +82,109 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
t.Errorf("Stat on missing key: got %v, want ErrNotFound", err)
}
}
// TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
// writes: the README's remote storage layout, with the prefix joined to
// each key by one "/". s3://b/p and s3://b/p/ must be the same
// destination, or a host that writes the URL the other way finds no
// snapshots. The listed object is put straight into the bucket, as
// another host would have written it. Both List and ListStream are
// checked: ListStream is what every snapshot listing goes through.
func TestS3URLPrefixKeyLayout(t *testing.T) {
t.Parallel()
const (
blobKey = "blobs/aa/bb/aabbccdd"
listPrefix = "metadata/"
manifestKey = listPrefix + "snap/manifest.json.zst"
manifestBody = "manifest"
)
cases := []struct {
urlPath string // URL path after the bucket name
keyPrefix string // what every key in the bucket must start with
}{
{urlPath: "/p", keyPrefix: "p/"},
{urlPath: "/p/", keyPrefix: "p/"},
{urlPath: "", keyPrefix: ""},
}
for _, tc := range cases {
storageURL := "s3://" + s3TestBucket + tc.urlPath
t.Run(storageURL, func(t *testing.T) {
t.Parallel()
backend := s3mem.New()
err := backend.CreateBucket(s3TestBucket)
if err != nil {
t.Fatalf("create bucket: %v", err)
}
srv := httptest.NewServer(gofakes3.New(backend).Server())
t.Cleanup(srv.Close)
storer, err := storage.NewStorer(&config.Config{
StorageURL: storageURL + "?endpoint=" + srv.URL,
S3: config.S3Config{
AccessKeyID: "key",
SecretAccessKey: "secret",
},
})
if err != nil {
t.Fatalf("NewStorer: %v", err)
}
ctx := context.Background()
err = storer.Put(ctx, blobKey, strings.NewReader("blob"))
if err != nil {
t.Fatalf("Put: %v", err)
}
_, err = backend.HeadObject(s3TestBucket, tc.keyPrefix+blobKey)
if err != nil {
t.Errorf("blob not stored at %q: %v", tc.keyPrefix+blobKey, err)
}
_, err = backend.PutObject(s3TestBucket, tc.keyPrefix+manifestKey,
nil, strings.NewReader(manifestBody), int64(len(manifestBody)))
if err != nil {
t.Fatalf("seed manifest: %v", err)
}
keys, err := storer.List(ctx, listPrefix)
if err != nil {
t.Fatalf("List: %v", err)
}
if !slices.Equal(keys, []string{manifestKey}) {
t.Errorf("List(%q) = %q, want [%q]", listPrefix, keys, manifestKey)
}
streamed := listStreamKeys(t, storer, listPrefix)
if !slices.Equal(streamed, []string{manifestKey}) {
t.Errorf("ListStream(%q) = %q, want [%q]", listPrefix, streamed, manifestKey)
}
})
}
}
// listStreamKeys returns the keys ListStream yields under a prefix, and
// fails the test on a listing error.
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
t.Helper()
var keys []string
for obj := range s.ListStream(context.Background(), prefix) {
if obj.Err != nil {
t.Fatalf("ListStream %q: %v", prefix, obj.Err)
}
keys = append(keys, obj.Key)
}
return keys
}