// Package metrics defines the Prometheus collectors describing // webhooker's delivery pipeline: how many events arrive, how many // deliveries are attempted, how they end, how long they take, how // deep the queues are, and how many circuit breakers are open. // // The inbound HTTP metrics come from the go-http-metrics recorder in // internal/middleware and land on prometheus.DefaultRegisterer. These // collectors register there too, so both surfaces are gathered by the // one promhttp handler mounted on the authenticated /metrics route. package metrics import ( "sync" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "sneak.berlin/go/webhooker/internal/database" ) // namespace prefixes every collector defined here. const namespace = "webhooker" // targetTypeLabel is the only label any delivery metric carries, and // cardinality is the whole reason for that. // // A target type is one of four compile-time constants, so the label // domain is bounded by construction. Target ids, event ids and // entrypoint ids are not: they are UUIDs minted per operator action // or per inbound request, a series is never reclaimed once it exists, // and labelling by any of them makes /metrics a memory leak that // grows with traffic. normalizeTargetType enforces the bound at every // call site — a type the registry does not know collapses into // unknownTargetType rather than minting a series of its own. const targetTypeLabel = "target_type" // unknownTargetType is the bucket for a target type outside the known // set, so an unrecognised value cannot mint a new series. const unknownTargetType = "unknown" // Delivery duration buckets, exponential from 5ms so the last bucket // (about 98s) sits above the 30s outbound HTTP client timeout. const ( durationBucketStart = 0.005 durationBucketFactor = 3 durationBucketCount = 10 ) // knownTargetTypes is the fixed label domain: the target types the // delivery engine implements. // //nolint:gochecknoglobals // the label domain, built once per process var knownTargetTypes = []database.TargetType{ database.TargetTypeHTTP, database.TargetTypeDatabase, database.TargetTypeLog, database.TargetTypeSlack, } // defaultSet is the process-wide metric set, registered on the same // registry the HTTP middleware and the /metrics handler already use. // It is built on first use rather than in an init so that a test // binary that never touches metrics never registers them. // //nolint:gochecknoglobals // one process-wide registration, by design var defaultSet = sync.OnceValue(func() *Set { return New(prometheus.DefaultRegisterer) }) // Default returns the process-wide metric set. func Default() *Set { return defaultSet() } // Set is one registered group of webhooker's delivery collectors. // Production uses the single Default set; tests build their own // against a private registry so assertions are not disturbed by // deliveries other tests are making concurrently. type Set struct { eventsReceived prometheus.Counter deliveryAttempts *prometheus.CounterVec deliveriesSucceeded *prometheus.CounterVec deliveriesFailed *prometheus.CounterVec deliveryRetries *prometheus.CounterVec deliveryDuration *prometheus.HistogramVec deliveriesPending *prometheus.GaugeVec deliveriesRetrying *prometheus.GaugeVec circuitBreakersOpen *prometheus.GaugeVec } // New registers a full set of delivery collectors on reg and returns // it. It panics if reg already holds them, which is the intended // behaviour for a duplicate registration. func New(reg prometheus.Registerer) *Set { factory := promauto.With(reg) s := &Set{ eventsReceived: factory.NewCounter( prometheus.CounterOpts{ Namespace: namespace, Name: "events_received_total", Help: "Webhook events received and " + "stored, so the receive and deliver " + "sides can be compared.", }, ), deliveryDuration: factory.NewHistogramVec( prometheus.HistogramOpts{ Namespace: namespace, Name: "delivery_duration_seconds", Help: "Wall time of a single delivery " + "attempt, by target type.", Buckets: prometheus.ExponentialBuckets( durationBucketStart, durationBucketFactor, durationBucketCount, ), }, []string{targetTypeLabel}, ), } s.registerCounters(factory) s.registerGauges(factory) s.initSeries() return s } // EventReceived counts one inbound webhook event stored. func (s *Set) EventReceived() { s.eventsReceived.Inc() } // DeliveryAttempted counts one delivery attempt dispatched to a // target. func (s *Set) DeliveryAttempted(t database.TargetType) { s.deliveryAttempts. WithLabelValues(normalizeTargetType(t)). Inc() } // ObserveDeliveryDuration records how long one delivery attempt took. func (s *Set) ObserveDeliveryDuration( t database.TargetType, d time.Duration, ) { s.deliveryDuration. WithLabelValues(normalizeTargetType(t)). Observe(d.Seconds()) } // DeliveryStatusChanged counts a delivery's transition into a new // status. The mapping from status to counter lives here, next to the // collectors, so the engine has a single call for every transition it // persists. A move back to pending is not an outcome and counts // nothing. func (s *Set) DeliveryStatusChanged( t database.TargetType, status database.DeliveryStatus, ) { label := normalizeTargetType(t) switch status { case database.DeliveryStatusDelivered: s.deliveriesSucceeded.WithLabelValues(label).Inc() case database.DeliveryStatusFailed: s.deliveriesFailed.WithLabelValues(label).Inc() case database.DeliveryStatusRetrying: s.deliveryRetries.WithLabelValues(label).Inc() case database.DeliveryStatusPending: } } // SetQueueDepths publishes the pending and retrying queue depths from // one sample. Every known target type is written on every call, so a // type whose queue has drained reads zero instead of holding its last // value forever. func (s *Set) SetQueueDepths( pending, retrying map[database.TargetType]int, ) { for _, t := range knownTargetTypes { label := string(t) s.deliveriesPending.WithLabelValues(label). Set(float64(pending[t])) s.deliveriesRetrying.WithLabelValues(label). Set(float64(retrying[t])) } } // SetCircuitBreakersOpen publishes how many of a target type's // circuit breakers are currently open. func (s *Set) SetCircuitBreakersOpen( t database.TargetType, open int, ) { s.circuitBreakersOpen. WithLabelValues(normalizeTargetType(t)). Set(float64(open)) } func (s *Set) registerCounters(factory promauto.Factory) { s.deliveryAttempts = factory.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "delivery_attempts_total", Help: "Delivery attempts dispatched to a " + "target, by target type.", }, []string{targetTypeLabel}, ) s.deliveriesSucceeded = factory.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "deliveries_succeeded_total", Help: "Deliveries that reached the delivered " + "state, by target type.", }, []string{targetTypeLabel}, ) s.deliveriesFailed = factory.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "deliveries_failed_total", Help: "Deliveries that failed terminally and " + "will not be retried, by target type.", }, []string{targetTypeLabel}, ) s.deliveryRetries = factory.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Name: "delivery_retries_total", Help: "Deliveries put back into the retrying " + "state, by target type.", }, []string{targetTypeLabel}, ) } func (s *Set) registerGauges(factory promauto.Factory) { s.deliveriesPending = factory.NewGaugeVec( prometheus.GaugeOpts{ Namespace: namespace, Name: "deliveries_pending", Help: "Deliveries currently in the pending " + "state, by target type.", }, []string{targetTypeLabel}, ) s.deliveriesRetrying = factory.NewGaugeVec( prometheus.GaugeOpts{ Namespace: namespace, Name: "deliveries_retrying", Help: "Deliveries currently in the retrying " + "state, by target type.", }, []string{targetTypeLabel}, ) s.circuitBreakersOpen = factory.NewGaugeVec( prometheus.GaugeOpts{ Namespace: namespace, Name: "circuit_breakers_open", Help: "Delivery circuit breakers currently " + "open, by target type.", }, []string{targetTypeLabel}, ) } // initSeries materialises every known-target-type series at zero, so // a dashboard and an alert rule see a target type that has not // delivered yet rather than a missing series. func (s *Set) initSeries() { for _, t := range knownTargetTypes { label := string(t) s.deliveryAttempts.WithLabelValues(label) s.deliveriesSucceeded.WithLabelValues(label) s.deliveriesFailed.WithLabelValues(label) s.deliveryRetries.WithLabelValues(label) s.deliveriesPending.WithLabelValues(label) s.deliveriesRetrying.WithLabelValues(label) s.circuitBreakersOpen.WithLabelValues(label) } } // normalizeTargetType maps a target type onto the bounded label // domain, collapsing anything outside it to unknownTargetType. func normalizeTargetType(t database.TargetType) string { for _, known := range knownTargetTypes { if t == known { return string(known) } } return unknownTargetType }