check / check (push) Waiting to run
A blob upload that failed, such as one cut off by a lost connection, ended the backup at once. It is now tried again from the start of the blob up to five times, 3 seconds apart, with a warning before each retry; an interrupt still stops it at once. The panic in the report, "close of closed channel" from the progress reporter, was already fixed on next by the commit for issue 253; a test now covers a failing upload with the progress reporter on. Model: opus-5-5
155 lines
4.2 KiB
Go
155 lines
4.2 KiB
Go
//nolint:testpackage // exercises the unexported uploadBlobIfNeeded
|
|
package snapshot
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/blob"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
)
|
|
|
|
// retryTestBlobSize is larger than what a failing faultstore Put reads,
|
|
// so a new try that did not rewind the blob would store a short object.
|
|
const retryTestBlobSize = 4096
|
|
|
|
// retryTestBlobKey is where the retry tests upload their blob.
|
|
const retryTestBlobKey = "blobs/retry"
|
|
|
|
// uploadRetryTestBlob uploads a retryTestBlobSize blob through store,
|
|
// waiting delay between tries, and returns the blob's bytes, the warnings
|
|
// the upload printed and its error.
|
|
func uploadRetryTestBlob(
|
|
ctx context.Context, store storage.Storer, delay time.Duration,
|
|
) ([]byte, int, error) {
|
|
data := bytes.Repeat([]byte("0123456789abcdef"), retryTestBlobSize/16)
|
|
output := ui.NewWithColor(io.Discard, false)
|
|
s := &Scanner{storage: store, ui: output, blobUploadRetryDelay: delay}
|
|
|
|
_, _, err := s.uploadBlobIfNeeded(ctx, retryTestBlobKey, &blob.WithReader{
|
|
FinishedBlob: &blob.FinishedBlob{
|
|
Hash: "retry",
|
|
Compressed: int64(len(data)),
|
|
},
|
|
Reader: bytes.NewReader(data),
|
|
}, time.Now(), &ScanResult{})
|
|
|
|
return data, output.WarningCount(), err
|
|
}
|
|
|
|
// A blob upload that fails is tried again from the start of the blob,
|
|
// and each failure is a warning. See
|
|
// https://git.eeqj.de/sneak/vaultik/issues/476.
|
|
func TestFailedBlobUploadIsTriedAgain(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner, err := storage.NewFileStorer(t.TempDir())
|
|
require.NoError(t, err)
|
|
|
|
puts := 0
|
|
store := faultstore.New(inner)
|
|
store.OnPut = func(string) faultstore.PutAction {
|
|
puts++
|
|
if puts <= 2 {
|
|
return faultstore.PutFail
|
|
}
|
|
|
|
return faultstore.PutNormal
|
|
}
|
|
|
|
data, warnings, err := uploadRetryTestBlob(context.Background(), store, 0)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 3, puts)
|
|
require.Equal(t, 2, warnings)
|
|
|
|
rc, err := inner.Get(context.Background(), retryTestBlobKey)
|
|
require.NoError(t, err)
|
|
|
|
defer func() { _ = rc.Close() }()
|
|
|
|
stored, err := io.ReadAll(rc)
|
|
require.NoError(t, err)
|
|
require.Equal(t, data, stored)
|
|
}
|
|
|
|
// A blob upload that keeps failing is tried blobUploadRetries more times,
|
|
// BlobUploadRetryDelay apart, and then the backup stops with its error.
|
|
func TestBlobUploadStopsAfterRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner, err := storage.NewFileStorer(t.TempDir())
|
|
require.NoError(t, err)
|
|
|
|
puts := 0
|
|
store := faultstore.New(inner)
|
|
store.OnPut = func(string) faultstore.PutAction {
|
|
puts++
|
|
|
|
return faultstore.PutFail
|
|
}
|
|
|
|
const delay = 10 * time.Millisecond
|
|
|
|
start := time.Now()
|
|
_, warnings, err := uploadRetryTestBlob(context.Background(), store, delay)
|
|
|
|
require.ErrorIs(t, err, faultstore.ErrInjectedUpload)
|
|
require.Equal(t, 1+blobUploadRetries, puts)
|
|
require.Equal(t, blobUploadRetries, warnings)
|
|
require.GreaterOrEqual(t, time.Since(start), blobUploadRetries*delay)
|
|
}
|
|
|
|
// An upload stopped by Ctrl-C or SIGTERM is not tried again.
|
|
func TestInterruptedBlobUploadIsNotTriedAgain(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
inner, err := storage.NewFileStorer(t.TempDir())
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
|
|
puts := 0
|
|
store := faultstore.New(inner)
|
|
store.OnPut = func(string) faultstore.PutAction {
|
|
puts++
|
|
|
|
cancel()
|
|
|
|
return faultstore.PutFail
|
|
}
|
|
|
|
_, warnings, err := uploadRetryTestBlob(ctx, store, 0)
|
|
|
|
require.Error(t, err)
|
|
require.Equal(t, 1, puts)
|
|
require.Zero(t, warnings)
|
|
}
|
|
|
|
// Backups wait BlobUploadRetryDelay before each new try.
|
|
func TestBackupScannerWaitsBetweenBlobUploadRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cfg := &config.Config{
|
|
AgeRecipients: []string{testAgeRecipient},
|
|
ChunkSize: config.Size(1024 * 1024),
|
|
BlobSizeLimit: config.Size(4 * 1024 * 1024),
|
|
IndexPath: filepath.Join(t.TempDir(), "index.sqlite"),
|
|
}
|
|
|
|
scanner := provideScannerFactory(cfg, nil, nil)(ScannerParams{
|
|
Fs: afero.NewMemMapFs(),
|
|
})
|
|
require.NotNil(t, scanner)
|
|
require.Equal(t, BlobUploadRetryDelay, scanner.blobUploadRetryDelay)
|
|
}
|