Expose delivery metrics on /metrics (closes #209) (#224)

internal/delivery/engine.go

@@ -15,6 +15,7 @@ import (
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/lifecycle"
 	"sneak.berlin/go/webhooker/internal/logger"
+	"sneak.berlin/go/webhooker/internal/metrics"
 )
 
 const (
@@ -139,6 +140,12 @@ type Engine struct {
 	retryCh    chan Task
 	workers    int
 
+	// mtr is the delivery metric set. Production wires the
+	// process-wide one; a test can substitute a set registered on
+	// a private registry so its assertions are not disturbed by
+	// deliveries other tests are making at the same time.
+	mtr *metrics.Set
+
 	// targets maps each target type to its implementation.
 	targets map[database.TargetType]Target
 
@@ -164,6 +171,7 @@ func New(
 		deliveryCh: make(chan Task, deliveryChannelSize),
 		retryCh:    make(chan Task, retryChannelSize),
 		workers:    defaultWorkers,
+		mtr:        metrics.Default(),
 	}
 
 	e.initTargets(&http.Client{
@@ -283,6 +291,10 @@ func (e *Engine) start() {
 
 	go e.retrySweep(ctx)
 
+	e.wg.Add(1)
+
+	go e.queueDepthSampler(ctx)
+
 	e.log.Info(
 		"delivery engine started",
 		"workers", e.workers,
@@ -837,8 +849,15 @@ func (e *Engine) failUnretryableRetry(
 		0,
 	)
 
+	// The type is passed rather than assigned onto d: the delivery
+	// is loaded here without its target relation, and populating
+	// d.Target would make GORM's SaveBeforeAssociations upsert the
+	// whole target row — plaintext config, which for a slack target
+	// is the credential — into the per-webhook event database. See
+	// https://git.eeqj.de/sneak/webhooker/issues/206.
 	e.updateDeliveryStatus(
-		webhookDB, d, database.DeliveryStatusFailed,
+		webhookDB, d, target.Type,
+		database.DeliveryStatusFailed,
 	)
 }
 
@@ -859,7 +878,8 @@ func (e *Engine) processDelivery(
 		)
 
 		e.updateDeliveryStatus(
-			webhookDB, d, database.DeliveryStatusFailed,
+			webhookDB, d, d.Target.Type,
+			database.DeliveryStatusFailed,
 		)
 
 		return
@@ -868,6 +888,24 @@ func (e *Engine) processDelivery(
 	target.Deliver(ctx, webhookDB, d, task, e)
 }
 
+// observeAttempt counts one delivery attempt that was actually
+// dispatched to a target, and records how long it took.
+//
+// It is called from the dispatch paths rather than from around
+// Target.Deliver, because Deliver is also entered for deliveries
+// that never reach the wire: a delivery an open circuit breaker
+// refuses sends nothing, records no DeliveryResult, and is
+// rescheduled. Counting those would climb the attempts counter with
+// no traffic behind it and fill the duration histogram with
+// microsecond samples, which would make the delivery-duration
+// quantiles improve during exactly the outage they exist to reveal.
+func (e *Engine) observeAttempt(
+	t database.TargetType, dur time.Duration,
+) {
+	e.mtr.DeliveryAttempted(t)
+	e.mtr.ObserveDeliveryDuration(t, dur)
+}
+
 // recordResult persists a DeliveryResult row describing a
 // single attempt. It is a cross-target helper the targets
 // call.
@@ -901,10 +939,22 @@ func (e *Engine) recordResult(
 }
 
 // updateDeliveryStatus persists a new status for a delivery.
-// It is a cross-target helper the targets call.
+// It is a cross-target helper the targets call, and therefore the
+// single point where a delivery's outcome — delivered, terminally
+// failed, or put back into retry — is counted.
+//
+// The target type is a parameter rather than read off d.Target
+// because one caller — failUnretryableRetry — deliberately holds a
+// delivery loaded without its target relation, and must keep it that
+// way: a populated d.Target makes GORM upsert the target row, config
+// and all, into the per-webhook database.
+//
+// The counter moves only after the row is written, so a transition
+// the database rejected is not claimed as an outcome that happened.
 func (e *Engine) updateDeliveryStatus(
 	webhookDB *gorm.DB,
 	d *database.Delivery,
+	targetType database.TargetType,
 	status database.DeliveryStatus,
 ) {
 	err := webhookDB.Model(d).
@@ -916,7 +966,11 @@ func (e *Engine) updateDeliveryStatus(
 			"status", status,
 			"error", err,
 		)
+
+		return
 	}
+
+	e.mtr.DeliveryStatusChanged(targetType, status)
 }
 
 func truncate(s string, maxLen int) string {