check / check (push) Waiting to run
A killed upload leaves a `.partial` object that only remote nuke removed, and that remote info never showed. The storage interface's DeletePartialUploads becomes ListPartialUploads, which returns each such object's key, size and time; Delete removes it. prune deletes those under metadata/ and blobs/ once older than prune_grace_period, and reports kept, deleted and failed counts. remote info counts them on their own line and in --json. remote nuke lists and deletes them like any other object, and now names any it could not delete. Model: opus-5-5
246 lines
6.9 KiB
Go
246 lines
6.9 KiB
Go
// Package faultstore provides a storage.Storer wrapper that injects
|
|
// faults on demand, so tests can reproduce the failure modes a real
|
|
// backend exhibits: an upload that fails partway, a backend that reports
|
|
// success while storing nothing, reads that return corrupt or truncated
|
|
// bytes, and a delete or a stat the backend refuses. It is the seam
|
|
// called for by the fault-injection tests (sneak/vaultik issue 72) and
|
|
// is meant to be reused by future tests rather than re-implemented per
|
|
// case.
|
|
//
|
|
// The wrapper delegates every method to the inner Storer. Four hooks
|
|
// change that: OnPut decides the fate of each write, OnGet decides how
|
|
// each read's bytes are returned, OnDelete decides whether each delete
|
|
// fails, and OnStat whether each stat fails. All are keyed by the object
|
|
// key, so a test can fault only blobs, only metadata, or a single
|
|
// object.
|
|
package faultstore
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
)
|
|
|
|
// ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
|
|
var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
|
|
|
|
// ErrInjectedDelete is returned by a Delete the OnDelete hook chose to
|
|
// fail.
|
|
var ErrInjectedDelete = errors.New("faultstore: injected delete failure")
|
|
|
|
// ErrInjectedStat is returned by a Stat the OnStat hook chose to fail.
|
|
// It is not storage.ErrNotFound: the backend refused the request, as S3
|
|
// refuses a HEAD to credentials that allow only PUT.
|
|
var ErrInjectedStat = errors.New("faultstore: injected stat failure")
|
|
|
|
// PutAction is the disposition OnPut assigns to a write.
|
|
type PutAction int
|
|
|
|
const (
|
|
// PutNormal writes through to the inner Storer.
|
|
PutNormal PutAction = iota
|
|
// PutFail reads part of the stream, then fails without storing the
|
|
// object — a network upload that dies partway through.
|
|
PutFail
|
|
// PutSwallow reports success but stores nothing — a backend that
|
|
// lies about durability.
|
|
PutSwallow
|
|
)
|
|
|
|
// GetFault is how OnGet chooses to damage a read.
|
|
type GetFault int
|
|
|
|
const (
|
|
// GetNormal returns the stored bytes unchanged.
|
|
GetNormal GetFault = iota
|
|
// GetCorrupt flips a byte so the returned object no longer matches
|
|
// what was stored.
|
|
GetCorrupt
|
|
// GetTruncate returns a short read: the object's bytes cut off
|
|
// before the end.
|
|
GetTruncate
|
|
)
|
|
|
|
// Storer wraps an inner storage.Storer with fault-injection hooks. A
|
|
// zero-valued hook means "no fault": construct with New and set only the
|
|
// hook a test needs.
|
|
type Storer struct {
|
|
inner storage.Storer
|
|
|
|
// OnPut, when set, is consulted before every Put and
|
|
// PutWithProgress with the object key.
|
|
OnPut func(key string) PutAction
|
|
|
|
// OnGet, when set, is consulted for every Get with the object key
|
|
// and damages the returned bytes accordingly.
|
|
OnGet func(key string) GetFault
|
|
|
|
// OnDelete, when set, is consulted before every Delete with the
|
|
// object key. Returning true fails the delete and leaves the object
|
|
// in place.
|
|
OnDelete func(key string) bool
|
|
|
|
// OnStat, when set, is consulted before every Stat with the object
|
|
// key. Returning true fails the stat with ErrInjectedStat.
|
|
OnStat func(key string) bool
|
|
}
|
|
|
|
// New wraps inner. inner must be non-nil.
|
|
func New(inner storage.Storer) *Storer {
|
|
return &Storer{inner: inner}
|
|
}
|
|
|
|
// midStreamBytes is how far a PutFail reads before failing, enough to be
|
|
// past the start of any real blob without depending on the blob's size.
|
|
const midStreamBytes = 512
|
|
|
|
// Put stores data unless OnPut faults the write.
|
|
func (f *Storer) Put(ctx context.Context, key string, data io.Reader) error {
|
|
handled, err := f.injectPut(key, data)
|
|
if handled {
|
|
return err
|
|
}
|
|
|
|
return f.inner.Put(ctx, key, data)
|
|
}
|
|
|
|
// PutWithProgress stores data unless OnPut faults the write.
|
|
func (f *Storer) PutWithProgress(
|
|
ctx context.Context, key string, data io.Reader,
|
|
size int64, progress storage.ProgressCallback,
|
|
) error {
|
|
handled, err := f.injectPut(key, data)
|
|
if handled {
|
|
return err
|
|
}
|
|
|
|
return f.inner.PutWithProgress(ctx, key, data, size, progress)
|
|
}
|
|
|
|
// Get retrieves data, damaging it if OnGet faults the read.
|
|
func (f *Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
|
rc, err := f.inner.Get(ctx, key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fault := GetNormal
|
|
if f.OnGet != nil {
|
|
fault = f.OnGet(key)
|
|
}
|
|
|
|
if fault == GetNormal {
|
|
return rc, nil
|
|
}
|
|
|
|
data, err := io.ReadAll(rc)
|
|
_ = rc.Close()
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return io.NopCloser(bytes.NewReader(damage(fault, data))), nil
|
|
}
|
|
|
|
// damage returns a faulted copy of the stored bytes. GetCorrupt flips a
|
|
// byte in the middle so decryption authentication fails; GetTruncate
|
|
// drops the final byte so the read ends short. Both are no-ops on empty
|
|
// input, which cannot be damaged into something distinguishable.
|
|
func damage(fault GetFault, data []byte) []byte {
|
|
out := make([]byte, len(data))
|
|
copy(out, data)
|
|
|
|
if len(out) == 0 {
|
|
return out
|
|
}
|
|
|
|
switch fault {
|
|
case GetCorrupt:
|
|
out[len(out)/2] ^= 0xff
|
|
case GetTruncate:
|
|
out = out[:len(out)-1]
|
|
case GetNormal:
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
// Stat returns the object's info unless OnStat fails the stat.
|
|
func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
|
if f.OnStat != nil && f.OnStat(key) {
|
|
return nil, fmt.Errorf("%w for %q", ErrInjectedStat, key)
|
|
}
|
|
|
|
return f.inner.Stat(ctx, key)
|
|
}
|
|
|
|
// Delete removes the object unless OnDelete fails the delete.
|
|
func (f *Storer) Delete(ctx context.Context, key string) error {
|
|
if f.OnDelete != nil && f.OnDelete(key) {
|
|
return fmt.Errorf("%w for %q", ErrInjectedDelete, key)
|
|
}
|
|
|
|
return f.inner.Delete(ctx, key)
|
|
}
|
|
|
|
// List delegates unchanged.
|
|
func (f *Storer) List(ctx context.Context, prefix string) ([]string, error) {
|
|
return f.inner.List(ctx, prefix)
|
|
}
|
|
|
|
// ListStream delegates unchanged.
|
|
func (f *Storer) ListStream(
|
|
ctx context.Context, prefix string,
|
|
) <-chan storage.ObjectInfo {
|
|
return f.inner.ListStream(ctx, prefix)
|
|
}
|
|
|
|
// ListPartialUploads delegates unchanged.
|
|
func (f *Storer) ListPartialUploads(
|
|
ctx context.Context, prefix string,
|
|
) ([]storage.ObjectInfo, error) {
|
|
return f.inner.ListPartialUploads(ctx, prefix)
|
|
}
|
|
|
|
// Info delegates unchanged.
|
|
func (f *Storer) Info() storage.Info {
|
|
return f.inner.Info()
|
|
}
|
|
|
|
func (f *Storer) putAction(key string) PutAction {
|
|
if f.OnPut == nil {
|
|
return PutNormal
|
|
}
|
|
|
|
return f.OnPut(key)
|
|
}
|
|
|
|
// injectPut handles the non-normal write dispositions. It reports
|
|
// whether it handled the write and, if so, with what error.
|
|
func (f *Storer) injectPut(key string, data io.Reader) (bool, error) {
|
|
switch f.putAction(key) {
|
|
case PutFail:
|
|
// Consume part of the stream so the failure lands mid-transfer,
|
|
// the way a dropped connection would, then error without
|
|
// storing anything.
|
|
_, _ = io.CopyN(io.Discard, data, midStreamBytes)
|
|
|
|
return true, fmt.Errorf("%w for %q", ErrInjectedUpload, key)
|
|
case PutSwallow:
|
|
// A lying backend still drains the request body, then keeps
|
|
// nothing.
|
|
_, _ = io.Copy(io.Discard, data)
|
|
|
|
return true, nil
|
|
case PutNormal:
|
|
return false, nil
|
|
default:
|
|
return false, nil
|
|
}
|
|
}
|