Expose delivery metrics on /metrics (closes #209)

/metrics carried only the inbound HTTP surface, so a destination
failing for an hour, a growing retry backlog and a stuck-open circuit
breaker were all invisible: the receive side stays healthy in each
case because it is.

New internal/metrics registers, on the existing default registry that
the go-http-metrics recorder and the promhttp handler already share:

- webhooker_events_received_total
- webhooker_delivery_attempts_total
- webhooker_deliveries_succeeded_total
- webhooker_deliveries_failed_total
- webhooker_delivery_retries_total
- webhooker_delivery_duration_seconds
- webhooker_deliveries_pending / _retrying
- webhooker_circuit_breakers_open

The route mounting is untouched.

Every delivery metric carries one label, target_type, whose domain is
the four target-type constants; anything outside it collapses to
"unknown" so no series can be minted from a UUID. Target ids, event
ids and entrypoint ids are deliberately not labels.

Instrumentation sits at the points every target type already passes
through: processDelivery for the attempt counter and the duration
histogram, updateDeliveryStatus for the outcome counters. The
queue-depth gauges are counted out of the per-webhook databases by a
30s sampler rather than tracked as deltas, which would need seeding at
startup and would drift on any transition that failed to persist. The
open-breaker gauge is recounted from the target's breaker registry on
every state change.

internal/metrics/metrics_test.go

@@ -0,0 +1,225 @@
+package metrics_test
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/metrics"
+)
+
+// labelValues returns the target_type label values a metric family
+// currently carries.
+func labelValues(
+	t *testing.T, reg *prometheus.Registry, name string,
+) []string {
+	t.Helper()
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	var values []string
+
+	for _, fam := range families {
+		if fam.GetName() != name {
+			continue
+		}
+
+		for _, m := range fam.GetMetric() {
+			for _, label := range m.GetLabel() {
+				if label.GetName() == "target_type" {
+					values = append(
+						values, label.GetValue(),
+					)
+				}
+			}
+		}
+	}
+
+	return values
+}
+
+func gaugeValue(
+	t *testing.T,
+	reg *prometheus.Registry,
+	name, targetType string,
+) float64 {
+	t.Helper()
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	for _, fam := range families {
+		if fam.GetName() != name {
+			continue
+		}
+
+		for _, m := range fam.GetMetric() {
+			if hasTargetType(m, targetType) {
+				return m.GetGauge().GetValue()
+			}
+		}
+	}
+
+	t.Fatalf(
+		"gauge %s{target_type=%q} not found",
+		name, targetType,
+	)
+
+	return 0
+}
+
+func hasTargetType(m *dto.Metric, targetType string) bool {
+	for _, label := range m.GetLabel() {
+		if label.GetName() == "target_type" &&
+			label.GetValue() == targetType {
+			return true
+		}
+	}
+
+	return false
+}
+
+// TestUnknownTargetTypeCollapses is the cardinality guard: a target
+// type outside the known set must not mint a series of its own, or
+// /metrics grows without bound.
+func TestUnknownTargetTypeCollapses(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	for _, bogus := range []string{
+		"a1b2c3d4-0000-0000-0000-000000000001",
+		"a1b2c3d4-0000-0000-0000-000000000002",
+		"webhook-forwarder",
+	} {
+		set.DeliveryAttempted(database.TargetType(bogus))
+	}
+
+	values := labelValues(
+		t, reg, "webhooker_delivery_attempts_total",
+	)
+
+	assert.ElementsMatch(t,
+		[]string{
+			"http", "database", "log", "slack", "unknown",
+		},
+		values,
+	)
+}
+
+// TestSetQueueDepthsZeroesDrainedTypes proves a queue that has
+// drained reads zero rather than holding its last sample forever.
+func TestSetQueueDepthsZeroesDrainedTypes(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{
+			database.TargetTypeHTTP: 7,
+		},
+		map[database.TargetType]int{
+			database.TargetTypeSlack: 2,
+		},
+	)
+
+	assert.InDelta(t, 7.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "http",
+	), 0)
+	assert.InDelta(t, 2.0, gaugeValue(
+		t, reg, "webhooker_deliveries_retrying", "slack",
+	), 0)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{},
+		map[database.TargetType]int{},
+	)
+
+	assert.InDelta(t, 0.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "http",
+	), 0)
+	assert.InDelta(t, 0.0, gaugeValue(
+		t, reg, "webhooker_deliveries_retrying", "slack",
+	), 0)
+}
+
+// TestKnownSeriesExistBeforeAnyDelivery proves every known target
+// type is published at zero from registration, so an alert rule does
+// not have to cope with a missing series.
+func TestKnownSeriesExistBeforeAnyDelivery(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	metrics.New(reg)
+
+	for _, name := range []string{
+		"webhooker_delivery_attempts_total",
+		"webhooker_deliveries_succeeded_total",
+		"webhooker_deliveries_failed_total",
+		"webhooker_delivery_retries_total",
+		"webhooker_deliveries_pending",
+		"webhooker_deliveries_retrying",
+		"webhooker_circuit_breakers_open",
+	} {
+		assert.ElementsMatch(t,
+			[]string{"http", "database", "log", "slack"},
+			labelValues(t, reg, name),
+			"metric %s", name,
+		)
+	}
+}
+
+// TestDeliveryStatusChangedCounts maps each persisted status onto the
+// counter it moves.
+func TestDeliveryStatusChangedCounts(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusDelivered,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusFailed,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusRetrying,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusPending,
+	)
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	counts := map[string]float64{}
+
+	for _, fam := range families {
+		for _, m := range fam.GetMetric() {
+			if hasTargetType(m, "log") {
+				counts[fam.GetName()] =
+					m.GetCounter().GetValue()
+			}
+		}
+	}
+
+	assert.InDelta(t, 1.0,
+		counts["webhooker_deliveries_succeeded_total"], 0)
+	assert.InDelta(t, 1.0,
+		counts["webhooker_deliveries_failed_total"], 0)
+	assert.InDelta(t, 1.0,
+		counts["webhooker_delivery_retries_total"], 0)
+	assert.InDelta(t, 0.0,
+		counts["webhooker_delivery_attempts_total"], 0)
+}