Expose delivery metrics on /metrics (closes #209) (#224)

internal/metrics/metrics.go

@@ -0,0 +1,344 @@
+// Package metrics defines the Prometheus collectors describing
+// webhooker's delivery pipeline: how many events arrive, how many
+// deliveries are attempted, how they end, how long they take, how
+// deep the queues are, and how many circuit breakers are open.
+//
+// The inbound HTTP metrics come from the go-http-metrics recorder in
+// internal/middleware and land on prometheus.DefaultRegisterer. These
+// collectors register there too, so both surfaces are gathered by the
+// one promhttp handler mounted on the authenticated /metrics route.
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+// namespace prefixes every collector defined here.
+const namespace = "webhooker"
+
+// targetTypeLabel is the only label any delivery metric carries, and
+// cardinality is the whole reason for that.
+//
+// A target type is one of four compile-time constants, so the label
+// domain is bounded by construction. Target ids, event ids and
+// entrypoint ids are not: they are UUIDs minted per operator action
+// or per inbound request, a series is never reclaimed once it exists,
+// and labelling by any of them makes /metrics a memory leak that
+// grows with traffic. normalizeTargetType enforces the bound at every
+// call site — a type the registry does not know collapses into
+// unknownTargetType rather than minting a series of its own.
+const targetTypeLabel = "target_type"
+
+// unknownTargetType is the bucket for a target type outside the known
+// set, so an unrecognised value cannot mint a new series.
+const unknownTargetType = "unknown"
+
+// Delivery duration buckets, exponential from 5ms so the last bucket
+// (about 98s) sits above the 30s outbound HTTP client timeout.
+const (
+	durationBucketStart  = 0.005
+	durationBucketFactor = 3
+	durationBucketCount  = 10
+)
+
+// knownTargetTypes is the fixed label domain: the target types the
+// delivery engine implements.
+//
+//nolint:gochecknoglobals // the label domain, built once per process
+var knownTargetTypes = []database.TargetType{
+	database.TargetTypeHTTP,
+	database.TargetTypeDatabase,
+	database.TargetTypeLog,
+	database.TargetTypeSlack,
+}
+
+// defaultSet is the process-wide metric set, registered on the same
+// registry the HTTP middleware and the /metrics handler already use.
+// It is built on first use rather than in an init so that a test
+// binary that never touches metrics never registers them.
+//
+//nolint:gochecknoglobals // one process-wide registration, by design
+var defaultSet = sync.OnceValue(func() *Set {
+	return New(prometheus.DefaultRegisterer)
+})
+
+// Default returns the process-wide metric set.
+func Default() *Set {
+	return defaultSet()
+}
+
+// Set is one registered group of webhooker's delivery collectors.
+// Production uses the single Default set; tests build their own
+// against a private registry so assertions are not disturbed by
+// deliveries other tests are making concurrently.
+type Set struct {
+	eventsReceived      prometheus.Counter
+	deliveryAttempts    *prometheus.CounterVec
+	deliveriesSucceeded *prometheus.CounterVec
+	deliveriesFailed    *prometheus.CounterVec
+	deliveryRetries     *prometheus.CounterVec
+	deliveryDuration    *prometheus.HistogramVec
+	deliveriesPending   *prometheus.GaugeVec
+	deliveriesRetrying  *prometheus.GaugeVec
+	circuitBreakersOpen *prometheus.GaugeVec
+}
+
+// New registers a full set of delivery collectors on reg and returns
+// it. It panics if reg already holds them, which is the intended
+// behaviour for a duplicate registration.
+func New(reg prometheus.Registerer) *Set {
+	factory := promauto.With(reg)
+
+	s := &Set{
+		eventsReceived: factory.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "events_received_total",
+				Help: "Webhook events received and " +
+					"stored, so the receive and deliver " +
+					"sides can be compared.",
+			},
+		),
+		deliveryDuration: factory.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace: namespace,
+				Name:      "delivery_duration_seconds",
+				Help: "Wall time of a single delivery " +
+					"attempt, by target type.",
+				Buckets: prometheus.ExponentialBuckets(
+					durationBucketStart,
+					durationBucketFactor,
+					durationBucketCount,
+				),
+			},
+			[]string{targetTypeLabel},
+		),
+	}
+
+	s.registerCounters(factory)
+	s.registerGauges(factory)
+	s.initSeries()
+
+	return s
+}
+
+// EventReceived counts one inbound webhook event stored.
+func (s *Set) EventReceived() {
+	s.eventsReceived.Inc()
+}
+
+// DeliveryAttempted counts one delivery attempt dispatched to a
+// target.
+func (s *Set) DeliveryAttempted(t database.TargetType) {
+	s.deliveryAttempts.
+		WithLabelValues(normalizeTargetType(t)).
+		Inc()
+}
+
+// ObserveDeliveryDuration records how long one delivery attempt took.
+func (s *Set) ObserveDeliveryDuration(
+	t database.TargetType, d time.Duration,
+) {
+	s.deliveryDuration.
+		WithLabelValues(normalizeTargetType(t)).
+		Observe(d.Seconds())
+}
+
+// DeliveryStatusChanged counts a delivery's transition into a new
+// status. The mapping from status to counter lives here, next to the
+// collectors, so the engine has a single call for every transition it
+// persists. A move back to pending is not an outcome and counts
+// nothing.
+func (s *Set) DeliveryStatusChanged(
+	t database.TargetType, status database.DeliveryStatus,
+) {
+	label := normalizeTargetType(t)
+
+	switch status {
+	case database.DeliveryStatusDelivered:
+		s.deliveriesSucceeded.WithLabelValues(label).Inc()
+	case database.DeliveryStatusFailed:
+		s.deliveriesFailed.WithLabelValues(label).Inc()
+	case database.DeliveryStatusRetrying:
+		s.deliveryRetries.WithLabelValues(label).Inc()
+	case database.DeliveryStatusPending:
+	}
+}
+
+// SetQueueDepths publishes the pending and retrying queue depths from
+// one sample. Every label in the queue domain is written on every
+// call, so a type whose queue has drained reads zero instead of
+// holding its last value forever.
+func (s *Set) SetQueueDepths(
+	pending, retrying map[database.TargetType]int,
+) {
+	pendingByLabel := foldToLabels(pending)
+	retryingByLabel := foldToLabels(retrying)
+
+	for _, label := range queueDepthLabels() {
+		s.deliveriesPending.WithLabelValues(label).
+			Set(float64(pendingByLabel[label]))
+		s.deliveriesRetrying.WithLabelValues(label).
+			Set(float64(retryingByLabel[label]))
+	}
+}
+
+// queueDepthLabels is the label domain of the two queue-depth gauges:
+// the known target types plus unknown.
+//
+// Unknown is a real bucket here, not a safety net. A delivery queued
+// against a target that has since been deleted carries a target id no
+// longer in the targets table, so the sample resolves it to the empty
+// type; folding it into unknown is what keeps that backlog visible.
+// Dropping it would hide the one queue nobody is watching.
+func queueDepthLabels() []string {
+	labels := make([]string, 0, len(knownTargetTypes)+1)
+
+	for _, t := range knownTargetTypes {
+		labels = append(labels, string(t))
+	}
+
+	return append(labels, unknownTargetType)
+}
+
+// foldToLabels collapses a per-target-type count onto the bounded
+// label domain, summing everything outside the known set into
+// unknown.
+func foldToLabels(
+	counts map[database.TargetType]int,
+) map[string]int {
+	byLabel := make(map[string]int, len(counts))
+
+	for t, n := range counts {
+		byLabel[normalizeTargetType(t)] += n
+	}
+
+	return byLabel
+}
+
+// SetCircuitBreakersOpen publishes how many of a target type's
+// circuit breakers are currently open.
+func (s *Set) SetCircuitBreakersOpen(
+	t database.TargetType, open int,
+) {
+	s.circuitBreakersOpen.
+		WithLabelValues(normalizeTargetType(t)).
+		Set(float64(open))
+}
+
+func (s *Set) registerCounters(factory promauto.Factory) {
+	s.deliveryAttempts = factory.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "delivery_attempts_total",
+			Help: "Delivery attempts dispatched to a " +
+				"target, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+
+	s.deliveriesSucceeded = factory.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "deliveries_succeeded_total",
+			Help: "Deliveries that reached the delivered " +
+				"state, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+
+	s.deliveriesFailed = factory.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "deliveries_failed_total",
+			Help: "Deliveries that failed terminally and " +
+				"will not be retried, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+
+	s.deliveryRetries = factory.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "delivery_retries_total",
+			Help: "Deliveries put back into the retrying " +
+				"state, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+}
+
+func (s *Set) registerGauges(factory promauto.Factory) {
+	s.deliveriesPending = factory.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "deliveries_pending",
+			Help: "Deliveries currently in the pending " +
+				"state, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+
+	s.deliveriesRetrying = factory.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "deliveries_retrying",
+			Help: "Deliveries currently in the retrying " +
+				"state, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+
+	s.circuitBreakersOpen = factory.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "circuit_breakers_open",
+			Help: "Delivery circuit breakers currently " +
+				"open, by target type.",
+		},
+		[]string{targetTypeLabel},
+	)
+}
+
+// initSeries materialises every known-target-type series at zero, so
+// a dashboard and an alert rule see a target type that has not
+// delivered yet rather than a missing series.
+//
+// The queue-depth gauges additionally get their unknown series, which
+// holds deliveries queued against a deleted target. That backlog can
+// predate the process — it is read out of the databases, not counted
+// from transitions — so its series has to exist from the first scrape
+// rather than appearing only once a backlog has already built up.
+func (s *Set) initSeries() {
+	for _, t := range knownTargetTypes {
+		label := string(t)
+
+		s.deliveryAttempts.WithLabelValues(label)
+		s.deliveriesSucceeded.WithLabelValues(label)
+		s.deliveriesFailed.WithLabelValues(label)
+		s.deliveryRetries.WithLabelValues(label)
+		s.deliveriesPending.WithLabelValues(label)
+		s.deliveriesRetrying.WithLabelValues(label)
+		s.circuitBreakersOpen.WithLabelValues(label)
+	}
+
+	s.deliveriesPending.WithLabelValues(unknownTargetType)
+	s.deliveriesRetrying.WithLabelValues(unknownTargetType)
+}
+
+// normalizeTargetType maps a target type onto the bounded label
+// domain, collapsing anything outside it to unknownTargetType.
+func normalizeTargetType(t database.TargetType) string {
+	for _, known := range knownTargetTypes {
+		if t == known {
+			return string(known)
+		}
+	}
+
+	return unknownTargetType
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,285 @@
+package metrics_test
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/metrics"
+)
+
+// knownLabels is the target_type label domain built from the target
+// types the delivery engine implements.
+func knownLabels() []string {
+	return []string{"http", "database", "log", "slack"}
+}
+
+// labelValues returns the target_type label values a metric family
+// currently carries.
+func labelValues(
+	t *testing.T, reg *prometheus.Registry, name string,
+) []string {
+	t.Helper()
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	var values []string
+
+	for _, fam := range families {
+		if fam.GetName() != name {
+			continue
+		}
+
+		for _, m := range fam.GetMetric() {
+			for _, label := range m.GetLabel() {
+				if label.GetName() == "target_type" {
+					values = append(
+						values, label.GetValue(),
+					)
+				}
+			}
+		}
+	}
+
+	return values
+}
+
+func gaugeValue(
+	t *testing.T,
+	reg *prometheus.Registry,
+	name, targetType string,
+) float64 {
+	t.Helper()
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	for _, fam := range families {
+		if fam.GetName() != name {
+			continue
+		}
+
+		for _, m := range fam.GetMetric() {
+			if hasTargetType(m, targetType) {
+				return m.GetGauge().GetValue()
+			}
+		}
+	}
+
+	t.Fatalf(
+		"gauge %s{target_type=%q} not found",
+		name, targetType,
+	)
+
+	return 0
+}
+
+func hasTargetType(m *dto.Metric, targetType string) bool {
+	for _, label := range m.GetLabel() {
+		if label.GetName() == "target_type" &&
+			label.GetValue() == targetType {
+			return true
+		}
+	}
+
+	return false
+}
+
+// TestUnknownTargetTypeCollapses is the cardinality guard: a target
+// type outside the known set must not mint a series of its own, or
+// /metrics grows without bound.
+func TestUnknownTargetTypeCollapses(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	for _, bogus := range []string{
+		"a1b2c3d4-0000-0000-0000-000000000001",
+		"a1b2c3d4-0000-0000-0000-000000000002",
+		"webhook-forwarder",
+	} {
+		set.DeliveryAttempted(database.TargetType(bogus))
+	}
+
+	values := labelValues(
+		t, reg, "webhooker_delivery_attempts_total",
+	)
+
+	assert.ElementsMatch(t,
+		append(knownLabels(), "unknown"),
+		values,
+	)
+}
+
+// TestSetQueueDepthsZeroesDrainedTypes proves a queue that has
+// drained reads zero rather than holding its last sample forever.
+func TestSetQueueDepthsZeroesDrainedTypes(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{
+			database.TargetTypeHTTP: 7,
+		},
+		map[database.TargetType]int{
+			database.TargetTypeSlack: 2,
+		},
+	)
+
+	assert.InDelta(t, 7.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "http",
+	), 0)
+	assert.InDelta(t, 2.0, gaugeValue(
+		t, reg, "webhooker_deliveries_retrying", "slack",
+	), 0)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{},
+		map[database.TargetType]int{},
+	)
+
+	assert.InDelta(t, 0.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "http",
+	), 0)
+	assert.InDelta(t, 0.0, gaugeValue(
+		t, reg, "webhooker_deliveries_retrying", "slack",
+	), 0)
+}
+
+// TestKnownSeriesExistBeforeAnyDelivery proves every known target
+// type is published at zero from registration, so an alert rule does
+// not have to cope with a missing series.
+func TestKnownSeriesExistBeforeAnyDelivery(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	metrics.New(reg)
+
+	for _, name := range []string{
+		"webhooker_delivery_attempts_total",
+		"webhooker_deliveries_succeeded_total",
+		"webhooker_deliveries_failed_total",
+		"webhooker_delivery_retries_total",
+		"webhooker_circuit_breakers_open",
+	} {
+		assert.ElementsMatch(t,
+			knownLabels(),
+			labelValues(t, reg, name),
+			"metric %s", name,
+		)
+	}
+
+	// The queue gauges additionally publish unknown from
+	// registration: a backlog queued against a deleted target lands
+	// there, and it can predate the process, so the series has to
+	// exist before the first sample rather than appearing only once
+	// something is already stuck.
+	for _, name := range []string{
+		"webhooker_deliveries_pending",
+		"webhooker_deliveries_retrying",
+	} {
+		assert.ElementsMatch(t,
+			append(knownLabels(), "unknown"),
+			labelValues(t, reg, name),
+			"metric %s", name,
+		)
+	}
+}
+
+// TestSetQueueDepthsFoldsUnknownTypes proves a queued delivery whose
+// target type is not a known one — a target deleted out from under it
+// resolves to the empty type — is summed into the unknown series
+// instead of being dropped, and that the fold is a sum rather than a
+// last-writer-wins.
+func TestSetQueueDepthsFoldsUnknownTypes(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{
+			database.TargetTypeHTTP:             1,
+			database.TargetType(""):             4,
+			database.TargetType("retired-type"): 3,
+		},
+		map[database.TargetType]int{
+			database.TargetType(""): 2,
+		},
+	)
+
+	assert.InDelta(t, 7.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "unknown",
+	), 0)
+	assert.InDelta(t, 2.0, gaugeValue(
+		t, reg, "webhooker_deliveries_retrying", "unknown",
+	), 0)
+	assert.InDelta(t, 1.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "http",
+	), 0)
+
+	set.SetQueueDepths(
+		map[database.TargetType]int{},
+		map[database.TargetType]int{},
+	)
+
+	assert.InDelta(t, 0.0, gaugeValue(
+		t, reg, "webhooker_deliveries_pending", "unknown",
+	), 0)
+}
+
+// TestDeliveryStatusChangedCounts maps each persisted status onto the
+// counter it moves.
+func TestDeliveryStatusChangedCounts(t *testing.T) {
+	t.Parallel()
+
+	reg := prometheus.NewRegistry()
+	set := metrics.New(reg)
+
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusDelivered,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusFailed,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusRetrying,
+	)
+	set.DeliveryStatusChanged(
+		database.TargetTypeLog,
+		database.DeliveryStatusPending,
+	)
+
+	families, err := reg.Gather()
+	require.NoError(t, err)
+
+	counts := map[string]float64{}
+
+	for _, fam := range families {
+		for _, m := range fam.GetMetric() {
+			if hasTargetType(m, "log") {
+				counts[fam.GetName()] =
+					m.GetCounter().GetValue()
+			}
+		}
+	}
+
+	assert.InDelta(t, 1.0,
+		counts["webhooker_deliveries_succeeded_total"], 0)
+	assert.InDelta(t, 1.0,
+		counts["webhooker_deliveries_failed_total"], 0)
+	assert.InDelta(t, 1.0,
+		counts["webhooker_delivery_retries_total"], 0)
+	assert.InDelta(t, 0.0,
+		counts["webhooker_delivery_attempts_total"], 0)
+}