notify: drain in-flight deliveries at shutdown (closes #106)

notify.New accepted an fx.Lifecycle and never used it, so the three
dispatch goroutines were untracked. context.WithoutCancel kept a
delivery alive past its caller's cancellation but made nothing wait
for it: the process could exit while a delivery was still in its
retry backoff (up to five attempts, 60s max delay), silently losing
exactly the alert most worth keeping.

Deliveries are now tracked in a sync.WaitGroup whose counter is
incremented on the dispatching goroutine before the worker starts,
and notify.New registers an OnStop hook that drains them. The drain
is bounded by the context fx passes to OnStop; when it expires with
work outstanding, the count is logged at warn level and parked retry
backoffs are released via an abandon channel so they stop retrying
rather than outliving the drain. Deliveries submitted after the
drain has begun are refused and logged, so a stream of new
notifications cannot extend shutdown indefinitely.

The three near-identical dispatchers now share one tracked dispatch
helper. Tests use httptest servers and the existing retry knobs
(SetRetryConfig/SetSleepFunc) so nothing waits on a real backoff.

README's shutdown claim is reworded to match the bounded semantics.

internal/notify/notify.go

@@ -12,6 +12,8 @@ import (
 	"log/slog"
 	"net/http"
 	"net/url"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/fx"
@@ -115,19 +117,41 @@ type Service struct {
 	history              *AlertHistory
 	retryConfig          RetryConfig
 	sleepFn              func(time.Duration) <-chan time.Time
+
+	// Shutdown draining state.  drainMu guards draining and
+	// serialises it against the counter increment in
+	// startDelivery; inFlight tracks the delivery goroutines
+	// themselves and outstanding mirrors its count so a timed
+	// out drain can report how many were abandoned.
+	drainMu     sync.Mutex
+	draining    bool
+	inFlight    sync.WaitGroup
+	outstanding atomic.Int64
+	abandon     chan struct{}
+	abandonOnce sync.Once
+}
+
+// newService builds a Service with the fields every Service
+// needs regardless of how it was constructed.
+func newService(
+	log *slog.Logger,
+	transport http.RoundTripper,
+) *Service {
+	return &Service{
+		log:       log,
+		transport: transport,
+		history:   NewAlertHistory(),
+		abandon:   make(chan struct{}),
+	}
 }
 
 // New creates a new notify Service.
 func New(
-	_ fx.Lifecycle,
+	lifecycle fx.Lifecycle,
 	params Params,
 ) (*Service, error) {
-	svc := &Service{
-		log:       params.Logger.Get(),
-		transport: http.DefaultTransport,
-		config:    params.Config,
-		history:   NewAlertHistory(),
-	}
+	svc := newService(params.Logger.Get(), http.DefaultTransport)
+	svc.config = params.Config
 
 	if params.Config.NtfyTopic != "" {
 		u, err := ValidateWebhookURL(
@@ -168,6 +192,14 @@ func New(
 		svc.mattermostWebhookURL = u
 	}
 
+	lifecycle.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			svc.drain(ctx)
+
+			return nil
+		},
+	})
+
 	return svc, nil
 }
 
@@ -194,6 +226,32 @@ func (svc *Service) SendNotification(
 	svc.dispatchMattermost(ctx, title, message, priority)
 }
 
+// dispatch delivers a notification to one endpoint on a
+// tracked background goroutine.
+//
+// The delivery context is detached from ctx with
+// context.WithoutCancel so that a cancelled caller does not
+// kill a delivery already under way; the shutdown drain, not
+// the caller, decides how long deliveries may keep running.
+func (svc *Service) dispatch(
+	ctx context.Context,
+	endpoint string,
+	send func(context.Context) error,
+) {
+	notifyCtx := context.WithoutCancel(ctx)
+
+	svc.startDelivery(endpoint, func() {
+		err := svc.deliverWithRetry(notifyCtx, endpoint, send)
+		if err != nil {
+			svc.log.Error(
+				"failed to send notification after retries",
+				"endpoint", endpoint,
+				"error", err,
+			)
+		}
+	})
+}
+
 func (svc *Service) dispatchNtfy(
 	ctx context.Context,
 	title, message, priority string,
@@ -202,26 +260,11 @@ func (svc *Service) dispatchNtfy(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "ntfy",
-			func(c context.Context) error {
-				return svc.sendNtfy(
-					c, svc.ntfyURL,
-					title, message, priority,
-				)
-			},
+	svc.dispatch(ctx, "ntfy", func(c context.Context) error {
+		return svc.sendNtfy(
+			c, svc.ntfyURL, title, message, priority,
 		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send ntfy notification "+
-					"after retries",
-				"error", err,
-			)
-		}
-	}()
+	})
 }
 
 func (svc *Service) dispatchSlack(
@@ -232,26 +275,11 @@ func (svc *Service) dispatchSlack(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "slack",
-			func(c context.Context) error {
-				return svc.sendSlack(
-					c, svc.slackWebhookURL,
-					title, message, priority,
-				)
-			},
+	svc.dispatch(ctx, "slack", func(c context.Context) error {
+		return svc.sendSlack(
+			c, svc.slackWebhookURL, title, message, priority,
 		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send slack notification "+
-					"after retries",
-				"error", err,
-			)
-		}
-	}()
+	})
 }
 
 func (svc *Service) dispatchMattermost(
@@ -262,26 +290,15 @@ func (svc *Service) dispatchMattermost(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "mattermost",
-			func(c context.Context) error {
-				return svc.sendSlack(
-					c, svc.mattermostWebhookURL,
-					title, message, priority,
-				)
-			},
-		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send mattermost notification "+
-					"after retries",
-				"error", err,
+	svc.dispatch(
+		ctx, "mattermost",
+		func(c context.Context) error {
+			return svc.sendSlack(
+				c, svc.mattermostWebhookURL,
+				title, message, priority,
 			)
-		}
-	}()
+		},
+	)
 }
 
 func (svc *Service) sendNtfy(