Expose delivery metrics on /metrics (closes #209) (#224)

internal/delivery/target_http.go

@@ -74,6 +74,8 @@ func (c *httpCore) fireAndForget(
 	d *database.Delivery,
 	res attemptResult,
 ) {
+	c.eng.observeAttempt(d.Target.Type, res.elapsed())
+
 	c.eng.recordResult(
 		webhookDB, d, 1, res.success,
 		res.statusCode, res.respBody, res.errMsg,
@@ -82,7 +84,7 @@ func (c *httpCore) fireAndForget(
 
 	if res.success {
 		c.eng.updateDeliveryStatus(
-			webhookDB, d,
+			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
 
@@ -90,7 +92,8 @@ func (c *httpCore) fireAndForget(
 	}
 
 	c.eng.updateDeliveryStatus(
-		webhookDB, d, database.DeliveryStatusFailed,
+		webhookDB, d, d.Target.Type,
+		database.DeliveryStatusFailed,
 	)
 }
 
@@ -107,10 +110,17 @@ func (c *httpCore) withRetry(
 		return
 	}
 
+	// Allow may have moved the breaker to half-open, and the
+	// attempt below may open or close it, so the gauge is
+	// republished on every exit from here.
+	defer c.publishCircuitState(d.Target.Type)
+
 	attemptNum := task.AttemptNum
 
 	res := attempt()
 
+	c.eng.observeAttempt(d.Target.Type, res.elapsed())
+
 	c.eng.recordResult(
 		webhookDB, d, attemptNum, res.success,
 		res.statusCode, res.respBody, res.errMsg,
@@ -121,7 +131,7 @@ func (c *httpCore) withRetry(
 		cb.RecordSuccess()
 
 		c.eng.updateDeliveryStatus(
-			webhookDB, d,
+			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
 
@@ -146,6 +156,8 @@ func (c *httpCore) circuitBreakerBlock(
 		return false
 	}
 
+	defer c.publishCircuitState(d.Target.Type)
+
 	remaining := cb.CooldownRemaining()
 
 	c.eng.log.Info(
@@ -157,7 +169,7 @@ func (c *httpCore) circuitBreakerBlock(
 	)
 
 	c.eng.updateDeliveryStatus(
-		webhookDB, d,
+		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusRetrying,
 	)
 
@@ -177,7 +189,7 @@ func (c *httpCore) handleRetry(
 ) {
 	if attemptNum >= maxRetries {
 		c.eng.updateDeliveryStatus(
-			webhookDB, d,
+			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
 		)
 
@@ -185,7 +197,8 @@ func (c *httpCore) handleRetry(
 	}
 
 	c.eng.updateDeliveryStatus(
-		webhookDB, d, database.DeliveryStatusRetrying,
+		webhookDB, d, d.Target.Type,
+		database.DeliveryStatusRetrying,
 	)
 
 	backoff := calcBackoff(attemptNum)
@@ -215,6 +228,28 @@ func (c *httpCore) getCircuitBreaker(
 	return cb
 }
 
+// publishCircuitState recounts this core's open breakers and
+// publishes the gauge. Each core holds the breakers of exactly one
+// target type, so the recount is over that type's targets alone.
+// Counting rather than adjusting a delta keeps the gauge honest
+// however a breaker changed state.
+func (c *httpCore) publishCircuitState(
+	targetType database.TargetType,
+) {
+	open := 0
+
+	c.circuitBreakers.Range(func(_, val any) bool {
+		cb, ok := val.(*CircuitBreaker)
+		if ok && cb.State() == CircuitOpen {
+			open++
+		}
+
+		return true
+	})
+
+	c.eng.mtr.SetCircuitBreakersOpen(targetType, open)
+}
+
 // remainingBackoff returns how long remains of the backoff
 // window for the last attempt of a recovered retrying
 // delivery. It implements rescheduler.
@@ -302,7 +337,8 @@ func (t *httpTarget) Deliver(
 		)
 
 		t.eng.updateDeliveryStatus(
-			webhookDB, d, database.DeliveryStatusFailed,
+			webhookDB, d, d.Target.Type,
+			database.DeliveryStatusFailed,
 		)
 
 		return