diff --git a/README.md b/README.md index 8d9e77e..4edeb59 100644 --- a/README.md +++ b/README.md @@ -618,6 +618,10 @@ being written, a killed upload can leave a truncated object under its name, which a later backup takes for complete. A leftover `.partial` file is ignored and can be deleted. +A blob upload that fails, such as one cut off by a lost connection, is tried +again up to five times, three seconds apart. Each new try is announced with a +warning. If the last try fails too, the backup stops with the upload's error. + A destination shared between hosts or users, each with its own local index, relies on `prune_grace_period` to keep one's prune from deleting the blobs of another's backup still running, or its directory under `metadata/` that has no diff --git a/internal/snapshot/blob_upload_retry_test.go b/internal/snapshot/blob_upload_retry_test.go new file mode 100644 index 0000000..6d3aa20 --- /dev/null +++ b/internal/snapshot/blob_upload_retry_test.go @@ -0,0 +1,154 @@ +//nolint:testpackage // exercises the unexported uploadBlobIfNeeded +package snapshot + +import ( + "bytes" + "context" + "io" + "path/filepath" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blob" + "sneak.berlin/go/vaultik/internal/config" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/storage/faultstore" + "sneak.berlin/go/vaultik/internal/ui" +) + +// retryTestBlobSize is larger than what a failing faultstore Put reads, +// so a new try that did not rewind the blob would store a short object. +const retryTestBlobSize = 4096 + +// retryTestBlobKey is where the retry tests upload their blob. +const retryTestBlobKey = "blobs/retry" + +// uploadRetryTestBlob uploads a retryTestBlobSize blob through store, +// waiting delay between tries, and returns the blob's bytes, the warnings +// the upload printed and its error. +func uploadRetryTestBlob( + ctx context.Context, store storage.Storer, delay time.Duration, +) ([]byte, int, error) { + data := bytes.Repeat([]byte("0123456789abcdef"), retryTestBlobSize/16) + output := ui.NewWithColor(io.Discard, false) + s := &Scanner{storage: store, ui: output, blobUploadRetryDelay: delay} + + _, _, err := s.uploadBlobIfNeeded(ctx, retryTestBlobKey, &blob.WithReader{ + FinishedBlob: &blob.FinishedBlob{ + Hash: "retry", + Compressed: int64(len(data)), + }, + Reader: bytes.NewReader(data), + }, time.Now(), &ScanResult{}) + + return data, output.WarningCount(), err +} + +// A blob upload that fails is tried again from the start of the blob, +// and each failure is a warning. See +// https://git.eeqj.de/sneak/vaultik/issues/476. +func TestFailedBlobUploadIsTriedAgain(t *testing.T) { + t.Parallel() + + inner, err := storage.NewFileStorer(t.TempDir()) + require.NoError(t, err) + + puts := 0 + store := faultstore.New(inner) + store.OnPut = func(string) faultstore.PutAction { + puts++ + if puts <= 2 { + return faultstore.PutFail + } + + return faultstore.PutNormal + } + + data, warnings, err := uploadRetryTestBlob(context.Background(), store, 0) + require.NoError(t, err) + require.Equal(t, 3, puts) + require.Equal(t, 2, warnings) + + rc, err := inner.Get(context.Background(), retryTestBlobKey) + require.NoError(t, err) + + defer func() { _ = rc.Close() }() + + stored, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, data, stored) +} + +// A blob upload that keeps failing is tried blobUploadRetries more times, +// BlobUploadRetryDelay apart, and then the backup stops with its error. +func TestBlobUploadStopsAfterRetries(t *testing.T) { + t.Parallel() + + inner, err := storage.NewFileStorer(t.TempDir()) + require.NoError(t, err) + + puts := 0 + store := faultstore.New(inner) + store.OnPut = func(string) faultstore.PutAction { + puts++ + + return faultstore.PutFail + } + + const delay = 10 * time.Millisecond + + start := time.Now() + _, warnings, err := uploadRetryTestBlob(context.Background(), store, delay) + + require.ErrorIs(t, err, faultstore.ErrInjectedUpload) + require.Equal(t, 1+blobUploadRetries, puts) + require.Equal(t, blobUploadRetries, warnings) + require.GreaterOrEqual(t, time.Since(start), blobUploadRetries*delay) +} + +// An upload stopped by Ctrl-C or SIGTERM is not tried again. +func TestInterruptedBlobUploadIsNotTriedAgain(t *testing.T) { + t.Parallel() + + inner, err := storage.NewFileStorer(t.TempDir()) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + puts := 0 + store := faultstore.New(inner) + store.OnPut = func(string) faultstore.PutAction { + puts++ + + cancel() + + return faultstore.PutFail + } + + _, warnings, err := uploadRetryTestBlob(ctx, store, 0) + + require.Error(t, err) + require.Equal(t, 1, puts) + require.Zero(t, warnings) +} + +// Backups wait BlobUploadRetryDelay before each new try. +func TestBackupScannerWaitsBetweenBlobUploadRetries(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + AgeRecipients: []string{testAgeRecipient}, + ChunkSize: config.Size(1024 * 1024), + BlobSizeLimit: config.Size(4 * 1024 * 1024), + IndexPath: filepath.Join(t.TempDir(), "index.sqlite"), + } + + scanner := provideScannerFactory(cfg, nil, nil)(ScannerParams{ + Fs: afero.NewMemMapFs(), + }) + require.NotNil(t, scanner) + require.Equal(t, BlobUploadRetryDelay, scanner.blobUploadRetryDelay) +} diff --git a/internal/snapshot/module.go b/internal/snapshot/module.go index 9edd50b..64c27aa 100644 --- a/internal/snapshot/module.go +++ b/internal/snapshot/module.go @@ -54,20 +54,21 @@ func provideScannerFactory( } return NewScanner(ScannerConfig{ - FS: params.Fs, - ChunkSize: cfg.ChunkSize.Int64(), - Repositories: repos, - Storage: storer, - MaxBlobSize: cfg.BlobSizeLimit.Int64(), - CompressionLevel: cfg.CompressionLevel, - AgeRecipients: cfg.AgeRecipients, - EnableProgress: params.EnableProgress, - UI: params.UI, - Exclude: excludes, - IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()), - SkipErrors: params.SkipErrors, - PreviousSnapshotID: params.PreviousSnapshotID, - DeviceNumber: params.DeviceNumber, + FS: params.Fs, + ChunkSize: cfg.ChunkSize.Int64(), + Repositories: repos, + Storage: storer, + MaxBlobSize: cfg.BlobSizeLimit.Int64(), + CompressionLevel: cfg.CompressionLevel, + AgeRecipients: cfg.AgeRecipients, + EnableProgress: params.EnableProgress, + UI: params.UI, + Exclude: excludes, + IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()), + SkipErrors: params.SkipErrors, + PreviousSnapshotID: params.PreviousSnapshotID, + BlobUploadRetryDelay: BlobUploadRetryDelay, + DeviceNumber: params.DeviceNumber, }) } } diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index a8b3123..cad9276 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -75,6 +75,8 @@ type Scanner struct { // skipErrors skips files that cannot be opened or read (logged loudly); // packer, database, encryption, and upload errors still abort the run. skipErrors bool + // blobUploadRetryDelay is ScannerConfig.BlobUploadRetryDelay. + blobUploadRetryDelay time.Duration // ui is the user-facing output; never nil (defaults to a discarding writer). ui *ui.Writer // deviceNumber returns the number of the device that holds a file; @@ -123,8 +125,16 @@ const ( // snapshotAssocBatchSize is how many file IDs are associated with the // snapshot per transaction. snapshotAssocBatchSize = 1000 + + // blobUploadRetries is how many more times a failed blob upload is + // tried before the backup stops with the error. + blobUploadRetries = 5 ) +// BlobUploadRetryDelay is the wait before each new try of a failed blob +// upload. +const BlobUploadRetryDelay = 3 * time.Second + // ScannerConfig contains configuration for the scanner type ScannerConfig struct { FS afero.Fs @@ -148,6 +158,10 @@ type ScannerConfig struct { // PreviousSnapshotID is the most recent completed snapshot of the // same name, "" for none. PreviousSnapshotID string + // BlobUploadRetryDelay is the wait before each new try of a failed + // blob upload. Backups use snapshot.BlobUploadRetryDelay; tests leave + // it zero to try again at once. + BlobUploadRetryDelay time.Duration // DeviceNumber returns the number of the device that holds a file, // and false when there is none to read. nil reads it from the file's // stat; tests replace it, since the in-memory test filesystem has none. @@ -245,23 +259,24 @@ func NewScanner(cfg ScannerConfig) *Scanner { } return &Scanner{ - fs: cfg.FS, - chunker: chunker.NewChunker(cfg.ChunkSize), - packer: packer, - repos: cfg.Repositories, - storage: cfg.Storage, - maxBlobSize: cfg.MaxBlobSize, - compressionLevel: cfg.CompressionLevel, - ageRecipient: strings.Join(cfg.AgeRecipients, ","), - exclude: cfg.Exclude, - compiledExclude: compiledExclude, - indexAndLockFiles: indexAndLockFiles, - progress: progress, - skipErrors: cfg.SkipErrors, - previousSnapshotID: cfg.PreviousSnapshotID, - ui: uiw, - deviceNumber: deviceNumber, - pendingChunkHashes: make(map[string]struct{}), + fs: cfg.FS, + chunker: chunker.NewChunker(cfg.ChunkSize), + packer: packer, + repos: cfg.Repositories, + storage: cfg.Storage, + maxBlobSize: cfg.MaxBlobSize, + compressionLevel: cfg.CompressionLevel, + ageRecipient: strings.Join(cfg.AgeRecipients, ","), + exclude: cfg.Exclude, + compiledExclude: compiledExclude, + indexAndLockFiles: indexAndLockFiles, + progress: progress, + skipErrors: cfg.SkipErrors, + previousSnapshotID: cfg.PreviousSnapshotID, + blobUploadRetryDelay: cfg.BlobUploadRetryDelay, + ui: uiw, + deviceNumber: deviceNumber, + pendingChunkHashes: make(map[string]struct{}), } } @@ -1876,10 +1891,7 @@ func (s *Scanner) uploadBlobIfNeeded( s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) - progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - - err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, - finishedBlob.Compressed, progressCallback) + err = s.putBlob(ctx, blobPath, blobWithReader) if err != nil { // The record is at info level: the returned error ends the backup // and is printed as its ERROR: line. Only an interrupt cancels @@ -1923,6 +1935,44 @@ func (s *Scanner) uploadBlobIfNeeded( return finishedBlob.Compressed, false, nil } +// putBlob uploads the blob to blobPath. After a failed upload it waits +// s.blobUploadRetryDelay and uploads the whole blob again, up to +// blobUploadRetries times, and then returns the last error. An interrupt +// stops it at once. +func (s *Scanner) putBlob( + ctx context.Context, blobPath string, blobWithReader *blob.WithReader, +) error { + finishedBlob := blobWithReader.FinishedBlob + + for retries := 0; ; retries++ { + _, err := blobWithReader.Reader.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("rewinding blob for upload: %w", err) + } + + progressCallback := s.makeUploadProgressCallback( + ctx, finishedBlob, time.Now().UTC()) + + err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, + finishedBlob.Compressed, progressCallback) + if err == nil || ctx.Err() != nil || retries == blobUploadRetries { + return err + } + + log.Info("Blob upload failed, trying again", + "hash", finishedBlob.Hash, "retry", retries+1, "error", err) + s.ui.Warningf("Upload of blob %s failed: %v. Trying again in %s (retry %d of %d).", + s.ui.Hex(finishedBlob.Hash), err, + s.ui.Duration(s.blobUploadRetryDelay), retries+1, blobUploadRetries) + + select { + case <-ctx.Done(): + return err + case <-time.After(s.blobUploadRetryDelay): + } + } +} + // makeUploadProgressCallback creates a progress callback for blob uploads. // It updates the live progress reporter ~twice/sec for ETAs and prints a // human-readable status line to s.output at most every 15 seconds. diff --git a/internal/vaultik/two_path_snapshot_test.go b/internal/vaultik/two_path_snapshot_test.go index e6673c9..3068e91 100644 --- a/internal/vaultik/two_path_snapshot_test.go +++ b/internal/vaultik/two_path_snapshot_test.go @@ -4,6 +4,7 @@ import ( "context" "maps" "path/filepath" + "strings" "testing" "github.com/spf13/afero" @@ -12,6 +13,7 @@ import ( "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/storage/faultstore" "sneak.berlin/go/vaultik/internal/vaultik" ) @@ -68,3 +70,55 @@ func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) { assertRestoredTree(t, fs, restoreDir, files) } + +// A backup without --cron whose blob uploads keep failing stops with the +// upload's error, and stops the progress reporter once. See +// https://git.eeqj.de/sneak/vaultik/issues/476. +// +//nolint:paralleltest // installs the global logger via log.Initialize +func TestBackupWithoutCronStopsWithBlobUploadError(t *testing.T) { + log.Initialize(log.Config{}) + + const snapshotName = "data" + + fs := afero.NewOsFs() + tempDir := t.TempDir() + firstDir := filepath.Join(tempDir, "first") + secondDir := filepath.Join(tempDir, "second") + dbPath := filepath.Join(tempDir, "index.sqlite") + + ctx := context.Background() + + writeFaultSourceTree(t, fs, firstDir) + writeFaultSourceTree(t, fs, secondDir) + + cfg := faultTestConfig() + cfg.IndexPath = dbPath + cfg.ChunkSize = config.Size(faultChunkSize) + cfg.Snapshots = map[string]config.SnapshotConfig{ + snapshotName: {Paths: []string{firstDir, secondDir}}, + } + + inner, err := storage.NewFileStorer(t.TempDir()) + require.NoError(t, err) + + store := faultstore.New(inner) + store.OnPut = func(key string) faultstore.PutAction { + if strings.HasPrefix(key, "blobs/") { + return faultstore.PutFail + } + + return faultstore.PutNormal + } + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + v := newBackupVaultik(ctx, cfg, store, database.NewRepositories(db), db, fs) + + err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{ + Snapshots: []string{snapshotName}, + }) + require.ErrorIs(t, err, faultstore.ErrInjectedUpload) +}