970ea9fae8
All checks were successful
check / check (push) Successful in 33s
notify.New accepted an fx.Lifecycle and never used it, so the three dispatch goroutines were untracked. context.WithoutCancel kept a delivery alive past its caller's cancellation but made nothing wait for it: the process could exit while a delivery was still in its retry backoff (up to five attempts, 60s max delay), silently losing exactly the alert most worth keeping. Deliveries are now tracked in a sync.WaitGroup whose counter is incremented on the dispatching goroutine before the worker starts, and notify.New registers an OnStop hook that drains them. The drain is bounded by the context fx passes to OnStop; when it expires with work outstanding, the count is logged at warn level and parked retry backoffs are released via an abandon channel so they stop retrying rather than outliving the drain. Deliveries submitted after the drain has begun are refused and logged, so a stream of new notifications cannot extend shutdown indefinitely. The three near-identical dispatchers now share one tracked dispatch helper. Tests use httptest servers and the existing retry knobs (SetRetryConfig/SetSleepFunc) so nothing waits on a real backoff. README's shutdown claim is reworded to match the bounded semantics.
149 lines
3.3 KiB
Go
149 lines
3.3 KiB
Go
package notify
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"math/rand/v2"
|
|
"time"
|
|
)
|
|
|
|
// Retry defaults.
|
|
const (
|
|
// DefaultMaxRetries is the number of additional attempts
|
|
// after the first failure.
|
|
DefaultMaxRetries = 5
|
|
|
|
// DefaultBaseDelay is the initial delay before the first
|
|
// retry attempt.
|
|
DefaultBaseDelay = 1 * time.Second
|
|
|
|
// DefaultMaxDelay caps the computed backoff delay.
|
|
DefaultMaxDelay = 60 * time.Second
|
|
|
|
// backoffMultiplier is the exponential growth factor.
|
|
backoffMultiplier = 2
|
|
|
|
// jitterFraction controls the ±random spread applied
|
|
// to each delay (0.25 = ±25%).
|
|
jitterFraction = 0.25
|
|
)
|
|
|
|
// RetryConfig holds tuning knobs for the retry loop.
|
|
// Zero values fall back to the package defaults above.
|
|
type RetryConfig struct {
|
|
MaxRetries int
|
|
BaseDelay time.Duration
|
|
MaxDelay time.Duration
|
|
}
|
|
|
|
// defaults returns a copy with zero fields replaced by
|
|
// package defaults.
|
|
func (rc RetryConfig) defaults() RetryConfig {
|
|
if rc.MaxRetries <= 0 {
|
|
rc.MaxRetries = DefaultMaxRetries
|
|
}
|
|
|
|
if rc.BaseDelay <= 0 {
|
|
rc.BaseDelay = DefaultBaseDelay
|
|
}
|
|
|
|
if rc.MaxDelay <= 0 {
|
|
rc.MaxDelay = DefaultMaxDelay
|
|
}
|
|
|
|
return rc
|
|
}
|
|
|
|
// backoff computes the delay for attempt n (0-indexed) with
|
|
// jitter. The raw delay is BaseDelay * 2^n, capped at
|
|
// MaxDelay, then randomised by ±jitterFraction.
|
|
func (rc RetryConfig) backoff(attempt int) time.Duration {
|
|
raw := float64(rc.BaseDelay) *
|
|
math.Pow(backoffMultiplier, float64(attempt))
|
|
|
|
if raw > float64(rc.MaxDelay) {
|
|
raw = float64(rc.MaxDelay)
|
|
}
|
|
|
|
// Apply jitter: uniform in [raw*(1-j), raw*(1+j)].
|
|
lo := raw * (1 - jitterFraction)
|
|
hi := raw * (1 + jitterFraction)
|
|
|
|
jittered := lo + rand.Float64()*(hi-lo) //nolint:gosec // jitter needs no crypto/rand
|
|
|
|
return time.Duration(jittered)
|
|
}
|
|
|
|
// deliverWithRetry calls fn, retrying on error with
|
|
// exponential backoff. It logs every failed attempt and
|
|
// returns the last error if all attempts are exhausted.
|
|
func (svc *Service) deliverWithRetry(
|
|
ctx context.Context,
|
|
endpoint string,
|
|
fn func(context.Context) error,
|
|
) error {
|
|
cfg := svc.retryConfig.defaults()
|
|
|
|
var lastErr error
|
|
|
|
// attempt 0 is the initial call; attempts 1..MaxRetries
|
|
// are retries.
|
|
for attempt := range cfg.MaxRetries + 1 {
|
|
lastErr = fn(ctx)
|
|
if lastErr == nil {
|
|
if attempt > 0 {
|
|
svc.log.Info(
|
|
"notification delivered after retry",
|
|
"endpoint", endpoint,
|
|
"attempt", attempt+1,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Last attempt — don't sleep, just return.
|
|
if attempt == cfg.MaxRetries {
|
|
break
|
|
}
|
|
|
|
delay := cfg.backoff(attempt)
|
|
|
|
svc.log.Warn(
|
|
"notification delivery failed, retrying",
|
|
"endpoint", endpoint,
|
|
"attempt", attempt+1,
|
|
"maxAttempts", cfg.MaxRetries+1,
|
|
"retryIn", delay,
|
|
"error", lastErr,
|
|
)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-svc.abandon:
|
|
// Shutdown drained past its deadline; stop
|
|
// sleeping rather than outlive the process.
|
|
// A nil channel (Service built without a
|
|
// constructor) simply never fires.
|
|
return fmt.Errorf(
|
|
"%w: %s", ErrDeliveryAbandoned, endpoint,
|
|
)
|
|
case <-svc.sleepFunc(delay):
|
|
}
|
|
}
|
|
|
|
return lastErr
|
|
}
|
|
|
|
// sleepFunc returns a channel that closes after d.
|
|
// It is a field-level indirection so tests can override it.
|
|
func (svc *Service) sleepFunc(d time.Duration) <-chan time.Time {
|
|
if svc.sleepFn != nil {
|
|
return svc.sleepFn(d)
|
|
}
|
|
|
|
return time.After(d)
|
|
}
|