Make an S3 listing the server cuts short an error (closes #297)
check / check (push) Failing after 3m41s

The SDK paginator takes a listing page marked truncated but carrying no
continuation token for the last page and stops without an error. A
server answering V2 listings V1-style sends such pages, so ListObjects
and ListObjectsStream returned the first page as the whole listing, and
prune treated the blobs of the snapshots on later pages as unreferenced.
Both now fetch each page through nextListPage, which returns
ErrNoContinuationToken for such a page; prune, snapshot list and remote
nuke already handle a listing error.

The test serves a fixed reply from a stub server, since gofakes3 always
sends the token.

Model: opus-5-5
This commit is contained in:
2026-10-13 00:04:07 +00:00
parent c9a519c233
commit 86ec0df2e7
3 changed files with 107 additions and 5 deletions
+29 -2
View File
@@ -18,6 +18,13 @@ import (
"github.com/aws/smithy-go/logging"
)
// ErrNoContinuationToken is returned by ListObjects and ListObjectsStream
// when the server marks a page of a listing truncated but sends no
// continuation token to fetch the rest with, as a server answering V2
// listings V1-style does.
var ErrNoContinuationToken = errors.New(
"server marked the listing truncated but sent no continuation token")
// Client wraps the AWS S3 client for vaultik operations.
// It provides a simplified interface for S3 operations with automatic
// prefix handling and connection management. All operations are performed
@@ -214,7 +221,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
page, err := nextListPage(ctx, paginator)
if err != nil {
return nil, err
}
@@ -235,6 +242,26 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
return keys, nil
}
// nextListPage fetches the next page of a listing. The SDK paginator takes
// a page marked truncated without a continuation token for the last page
// and stops without an error, so such a page is ErrNoContinuationToken
// here instead.
func nextListPage(
ctx context.Context, paginator *s3.ListObjectsV2Paginator,
) (*s3.ListObjectsV2Output, error) {
page, err := paginator.NextPage(ctx)
if err != nil {
return nil, err
}
truncated := aws.ToBool(page.IsTruncated)
if truncated && aws.ToString(page.NextContinuationToken) == "" {
return nil, ErrNoContinuationToken
}
return page, nil
}
// HeadObject checks if an object exists in S3.
// Returns true if the object exists, false otherwise.
// The key is automatically prefixed with the configured prefix.
@@ -300,7 +327,7 @@ func (c *Client) ListObjectsStream(
})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
page, err := nextListPage(ctx, paginator)
if err != nil {
ch <- ObjectInfo{Err: err}