notify: drain in-flight deliveries at shutdown (closes #106)

Shutdown waits for deliveries already in flight instead of dropping them.

Reviewed and green; squashed to next by the dispatcher, dnswatcher having no manager.

Model: opus-5

internal/notify/notify.go

@@ -12,6 +12,8 @@ import (
 	"log/slog"
 	"net/http"
 	"net/url"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/fx"
@@ -115,19 +117,41 @@ type Service struct {
 	history              *AlertHistory
 	retryConfig          RetryConfig
 	sleepFn              func(time.Duration) <-chan time.Time
+
+	// Shutdown draining state.  drainMu guards draining and
+	// serialises it against the counter increment in
+	// startDelivery; inFlight tracks the delivery goroutines
+	// themselves and outstanding mirrors its count so a timed
+	// out drain can report how many were abandoned.
+	drainMu     sync.Mutex
+	draining    bool
+	inFlight    sync.WaitGroup
+	outstanding atomic.Int64
+	abandon     chan struct{}
+	abandonOnce sync.Once
+}
+
+// newService builds a Service with the fields every Service
+// needs regardless of how it was constructed.
+func newService(
+	log *slog.Logger,
+	transport http.RoundTripper,
+) *Service {
+	return &Service{
+		log:       log,
+		transport: transport,
+		history:   NewAlertHistory(),
+		abandon:   make(chan struct{}),
+	}
 }
 
 // New creates a new notify Service.
 func New(
-	_ fx.Lifecycle,
+	lifecycle fx.Lifecycle,
 	params Params,
 ) (*Service, error) {
-	svc := &Service{
-		log:       params.Logger.Get(),
-		transport: http.DefaultTransport,
-		config:    params.Config,
-		history:   NewAlertHistory(),
-	}
+	svc := newService(params.Logger.Get(), http.DefaultTransport)
+	svc.config = params.Config
 
 	if params.Config.NtfyTopic != "" {
 		u, err := ValidateWebhookURL(
@@ -168,6 +192,14 @@ func New(
 		svc.mattermostWebhookURL = u
 	}
 
+	lifecycle.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			svc.drain(ctx)
+
+			return nil
+		},
+	})
+
 	return svc, nil
 }
 
@@ -194,6 +226,32 @@ func (svc *Service) SendNotification(
 	svc.dispatchMattermost(ctx, title, message, priority)
 }
 
+// dispatch delivers a notification to one endpoint on a
+// tracked background goroutine.
+//
+// The delivery context is detached from ctx with
+// context.WithoutCancel so that a cancelled caller does not
+// kill a delivery already under way; the shutdown drain, not
+// the caller, decides how long deliveries may keep running.
+func (svc *Service) dispatch(
+	ctx context.Context,
+	endpoint string,
+	send func(context.Context) error,
+) {
+	notifyCtx := context.WithoutCancel(ctx)
+
+	svc.startDelivery(endpoint, func() {
+		err := svc.deliverWithRetry(notifyCtx, endpoint, send)
+		if err != nil {
+			svc.log.Error(
+				"failed to send notification after retries",
+				"endpoint", endpoint,
+				"error", err,
+			)
+		}
+	})
+}
+
 func (svc *Service) dispatchNtfy(
 	ctx context.Context,
 	title, message, priority string,
@@ -202,26 +260,11 @@ func (svc *Service) dispatchNtfy(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "ntfy",
-			func(c context.Context) error {
-				return svc.sendNtfy(
-					c, svc.ntfyURL,
-					title, message, priority,
-				)
-			},
+	svc.dispatch(ctx, "ntfy", func(c context.Context) error {
+		return svc.sendNtfy(
+			c, svc.ntfyURL, title, message, priority,
 		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send ntfy notification "+
-					"after retries",
-				"error", err,
-			)
-		}
-	}()
+	})
 }
 
 func (svc *Service) dispatchSlack(
@@ -232,26 +275,11 @@ func (svc *Service) dispatchSlack(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "slack",
-			func(c context.Context) error {
-				return svc.sendSlack(
-					c, svc.slackWebhookURL,
-					title, message, priority,
-				)
-			},
+	svc.dispatch(ctx, "slack", func(c context.Context) error {
+		return svc.sendSlack(
+			c, svc.slackWebhookURL, title, message, priority,
 		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send slack notification "+
-					"after retries",
-				"error", err,
-			)
-		}
-	}()
+	})
 }
 
 func (svc *Service) dispatchMattermost(
@@ -262,26 +290,15 @@ func (svc *Service) dispatchMattermost(
 		return
 	}
 
-	go func() {
-		notifyCtx := context.WithoutCancel(ctx)
-
-		err := svc.deliverWithRetry(
-			notifyCtx, "mattermost",
-			func(c context.Context) error {
-				return svc.sendSlack(
-					c, svc.mattermostWebhookURL,
-					title, message, priority,
-				)
-			},
-		)
-		if err != nil {
-			svc.log.Error(
-				"failed to send mattermost notification "+
-					"after retries",
-				"error", err,
+	svc.dispatch(
+		ctx, "mattermost",
+		func(c context.Context) error {
+			return svc.sendSlack(
+				c, svc.mattermostWebhookURL,
+				title, message, priority,
 			)
-		}
-	}()
+		},
+	)
 }
 
 func (svc *Service) sendNtfy(