//nolint:testpackage // exercises the unexported uploadBlobIfNeeded package snapshot import ( "bytes" "context" "io" "path/filepath" "testing" "time" "github.com/spf13/afero" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/blob" "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage/faultstore" "sneak.berlin/go/vaultik/internal/ui" ) // retryTestBlobSize is larger than what a failing faultstore Put reads, // so a new try that did not rewind the blob would store a short object. const retryTestBlobSize = 4096 // retryTestBlobKey is where the retry tests upload their blob. const retryTestBlobKey = "blobs/retry" // uploadRetryTestBlob uploads a retryTestBlobSize blob through store, // waiting delay between tries, and returns the blob's bytes, the warnings // the upload printed and its error. func uploadRetryTestBlob( ctx context.Context, store storage.Storer, delay time.Duration, ) ([]byte, int, error) { data := bytes.Repeat([]byte("0123456789abcdef"), retryTestBlobSize/16) output := ui.NewWithColor(io.Discard, false) s := &Scanner{storage: store, ui: output, blobUploadRetryDelay: delay} _, _, err := s.uploadBlobIfNeeded(ctx, retryTestBlobKey, &blob.WithReader{ FinishedBlob: &blob.FinishedBlob{ Hash: "retry", Compressed: int64(len(data)), }, Reader: bytes.NewReader(data), }, time.Now(), &ScanResult{}) return data, output.WarningCount(), err } // A blob upload that fails is tried again from the start of the blob, // and each failure is a warning. See // https://git.eeqj.de/sneak/vaultik/issues/476. func TestFailedBlobUploadIsTriedAgain(t *testing.T) { t.Parallel() inner, err := storage.NewFileStorer(t.TempDir()) require.NoError(t, err) puts := 0 store := faultstore.New(inner) store.OnPut = func(string) faultstore.PutAction { puts++ if puts <= 2 { return faultstore.PutFail } return faultstore.PutNormal } data, warnings, err := uploadRetryTestBlob(context.Background(), store, 0) require.NoError(t, err) require.Equal(t, 3, puts) require.Equal(t, 2, warnings) rc, err := inner.Get(context.Background(), retryTestBlobKey) require.NoError(t, err) defer func() { _ = rc.Close() }() stored, err := io.ReadAll(rc) require.NoError(t, err) require.Equal(t, data, stored) } // A blob upload that keeps failing is tried blobUploadRetries more times, // BlobUploadRetryDelay apart, and then the backup stops with its error. func TestBlobUploadStopsAfterRetries(t *testing.T) { t.Parallel() inner, err := storage.NewFileStorer(t.TempDir()) require.NoError(t, err) puts := 0 store := faultstore.New(inner) store.OnPut = func(string) faultstore.PutAction { puts++ return faultstore.PutFail } const delay = 10 * time.Millisecond start := time.Now() _, warnings, err := uploadRetryTestBlob(context.Background(), store, delay) require.ErrorIs(t, err, faultstore.ErrInjectedUpload) require.Equal(t, 1+blobUploadRetries, puts) require.Equal(t, blobUploadRetries, warnings) require.GreaterOrEqual(t, time.Since(start), blobUploadRetries*delay) } // An upload stopped by Ctrl-C or SIGTERM is not tried again. func TestInterruptedBlobUploadIsNotTriedAgain(t *testing.T) { t.Parallel() inner, err := storage.NewFileStorer(t.TempDir()) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) puts := 0 store := faultstore.New(inner) store.OnPut = func(string) faultstore.PutAction { puts++ cancel() return faultstore.PutFail } _, warnings, err := uploadRetryTestBlob(ctx, store, 0) require.Error(t, err) require.Equal(t, 1, puts) require.Zero(t, warnings) } // Backups wait BlobUploadRetryDelay before each new try. func TestBackupScannerWaitsBetweenBlobUploadRetries(t *testing.T) { t.Parallel() cfg := &config.Config{ AgeRecipients: []string{testAgeRecipient}, ChunkSize: config.Size(1024 * 1024), BlobSizeLimit: config.Size(4 * 1024 * 1024), IndexPath: filepath.Join(t.TempDir(), "index.sqlite"), } scanner := provideScannerFactory(cfg, nil, nil)(ScannerParams{ Fs: afero.NewMemMapFs(), }) require.NotNil(t, scanner) require.Equal(t, BlobUploadRetryDelay, scanner.blobUploadRetryDelay) }