Expose delivery metrics on /metrics (closes #209)

/metrics carried only the inbound HTTP surface, so a destination
failing for an hour, a growing retry backlog and a stuck-open circuit
breaker were all invisible: the receive side stays healthy in each
case because it is.

New internal/metrics registers, on the existing default registry that
the go-http-metrics recorder and the promhttp handler already share:

- webhooker_events_received_total
- webhooker_delivery_attempts_total
- webhooker_deliveries_succeeded_total
- webhooker_deliveries_failed_total
- webhooker_delivery_retries_total
- webhooker_delivery_duration_seconds
- webhooker_deliveries_pending / _retrying
- webhooker_circuit_breakers_open

The route mounting is untouched.

Every delivery metric carries one label, target_type, whose domain is
the four target-type constants; anything outside it collapses to
"unknown" so no series can be minted from a UUID. Target ids, event
ids and entrypoint ids are deliberately not labels.

Instrumentation sits at the points every target type already passes
through: processDelivery for the attempt counter and the duration
histogram, updateDeliveryStatus for the outcome counters. The
queue-depth gauges are counted out of the per-webhook databases by a
30s sampler rather than tracked as deltas, which would need seeding at
startup and would drift on any transition that failed to persist. The
open-breaker gauge is recounted from the target's breaker registry on
every state change.

internal/delivery/engine.go

@@ -15,6 +15,7 @@ import (
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/lifecycle"
 	"sneak.berlin/go/webhooker/internal/logger"
+	"sneak.berlin/go/webhooker/internal/metrics"
 )
 
 const (
@@ -139,6 +140,12 @@ type Engine struct {
 	retryCh    chan Task
 	workers    int
 
+	// mx is the delivery metric set. Production wires the
+	// process-wide one; a test can substitute a set registered on
+	// a private registry so its assertions are not disturbed by
+	// deliveries other tests are making at the same time.
+	mx *metrics.Set
+
 	// targets maps each target type to its implementation.
 	targets map[database.TargetType]Target
 
@@ -164,6 +171,7 @@ func New(
 		deliveryCh: make(chan Task, deliveryChannelSize),
 		retryCh:    make(chan Task, retryChannelSize),
 		workers:    defaultWorkers,
+		mx:         metrics.Default(),
 	}
 
 	e.initTargets(&http.Client{
@@ -283,6 +291,10 @@ func (e *Engine) start() {
 
 	go e.retrySweep(ctx)
 
+	e.wg.Add(1)
+
+	go e.queueDepthSampler(ctx)
+
 	e.log.Info(
 		"delivery engine started",
 		"workers", e.workers,
@@ -826,6 +838,11 @@ func (e *Engine) failUnretryableRetry(
 		target.Type,
 	)
 
+	// The delivery was loaded without its target relation, so
+	// attach it: updateDeliveryStatus reads the type to label the
+	// terminal failure it is about to count.
+	d.Target = *target
+
 	e.recordResult(
 		webhookDB,
 		d,
@@ -844,12 +861,21 @@ func (e *Engine) failUnretryableRetry(
 
 // processDelivery dispatches a delivery to the target that
 // owns its type. Unknown target types fail the delivery.
+//
+// It is also where the attempt counter and the duration histogram
+// are recorded, because it is the one point every target type
+// passes through on every attempt: a target added later is
+// instrumented without touching it, and the duration measured is
+// the whole cost of the attempt rather than whatever each target
+// happens to time for itself.
 func (e *Engine) processDelivery(
 	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	task *Task,
 ) {
+	e.mx.DeliveryAttempted(d.Target.Type)
+
 	target, ok := e.targets[d.Target.Type]
 	if !ok {
 		e.log.Error(
@@ -865,7 +891,13 @@ func (e *Engine) processDelivery(
 		return
 	}
 
+	start := time.Now()
+
 	target.Deliver(ctx, webhookDB, d, task, e)
+
+	e.mx.ObserveDeliveryDuration(
+		d.Target.Type, time.Since(start),
+	)
 }
 
 // recordResult persists a DeliveryResult row describing a
@@ -901,12 +933,16 @@ func (e *Engine) recordResult(
 }
 
 // updateDeliveryStatus persists a new status for a delivery.
-// It is a cross-target helper the targets call.
+// It is a cross-target helper the targets call, and therefore the
+// single point where a delivery's outcome — delivered, terminally
+// failed, or put back into retry — is counted.
 func (e *Engine) updateDeliveryStatus(
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	status database.DeliveryStatus,
 ) {
+	e.mx.DeliveryStatusChanged(d.Target.Type, status)
+
 	err := webhookDB.Model(d).
 		Update("status", status).Error
 	if err != nil {