Expose delivery metrics on /metrics (closes #209)
All checks were successful
check / check (push) Successful in 4m13s
All checks were successful
check / check (push) Successful in 4m13s
/metrics carried only the inbound HTTP surface, so a destination failing for an hour, a growing retry backlog and a stuck-open circuit breaker were all invisible: the receive side stays healthy in each case because it is. New internal/metrics registers, on the existing default registry that the go-http-metrics recorder and the promhttp handler already share: - webhooker_events_received_total - webhooker_delivery_attempts_total - webhooker_deliveries_succeeded_total - webhooker_deliveries_failed_total - webhooker_delivery_retries_total - webhooker_delivery_duration_seconds - webhooker_deliveries_pending / _retrying - webhooker_circuit_breakers_open The route mounting is untouched. Every delivery metric carries one label, target_type, whose domain is the four target-type constants; anything outside it collapses to "unknown" so no series can be minted from a UUID. Target ids, event ids and entrypoint ids are deliberately not labels. Instrumentation sits at the points every target type already passes through: processDelivery for the attempt counter and the duration histogram, updateDeliveryStatus for the outcome counters. The queue-depth gauges are counted out of the per-webhook databases by a 30s sampler rather than tracked as deltas, which would need seeding at startup and would drift on any transition that failed to persist. The open-breaker gauge is recounted from the target's breaker registry on every state change.
This commit is contained in:
301
internal/metrics/metrics.go
Normal file
301
internal/metrics/metrics.go
Normal file
@@ -0,0 +1,301 @@
|
||||
// Package metrics defines the Prometheus collectors describing
|
||||
// webhooker's delivery pipeline: how many events arrive, how many
|
||||
// deliveries are attempted, how they end, how long they take, how
|
||||
// deep the queues are, and how many circuit breakers are open.
|
||||
//
|
||||
// The inbound HTTP metrics come from the go-http-metrics recorder in
|
||||
// internal/middleware and land on prometheus.DefaultRegisterer. These
|
||||
// collectors register there too, so both surfaces are gathered by the
|
||||
// one promhttp handler mounted on the authenticated /metrics route.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// namespace prefixes every collector defined here.
|
||||
const namespace = "webhooker"
|
||||
|
||||
// targetTypeLabel is the only label any delivery metric carries, and
|
||||
// cardinality is the whole reason for that.
|
||||
//
|
||||
// A target type is one of four compile-time constants, so the label
|
||||
// domain is bounded by construction. Target ids, event ids and
|
||||
// entrypoint ids are not: they are UUIDs minted per operator action
|
||||
// or per inbound request, a series is never reclaimed once it exists,
|
||||
// and labelling by any of them makes /metrics a memory leak that
|
||||
// grows with traffic. normalizeTargetType enforces the bound at every
|
||||
// call site — a type the registry does not know collapses into
|
||||
// unknownTargetType rather than minting a series of its own.
|
||||
const targetTypeLabel = "target_type"
|
||||
|
||||
// unknownTargetType is the bucket for a target type outside the known
|
||||
// set, so an unrecognised value cannot mint a new series.
|
||||
const unknownTargetType = "unknown"
|
||||
|
||||
// Delivery duration buckets, exponential from 5ms so the last bucket
|
||||
// (about 98s) sits above the 30s outbound HTTP client timeout.
|
||||
const (
|
||||
durationBucketStart = 0.005
|
||||
durationBucketFactor = 3
|
||||
durationBucketCount = 10
|
||||
)
|
||||
|
||||
// knownTargetTypes is the fixed label domain: the target types the
|
||||
// delivery engine implements.
|
||||
//
|
||||
//nolint:gochecknoglobals // the label domain, built once per process
|
||||
var knownTargetTypes = []database.TargetType{
|
||||
database.TargetTypeHTTP,
|
||||
database.TargetTypeDatabase,
|
||||
database.TargetTypeLog,
|
||||
database.TargetTypeSlack,
|
||||
}
|
||||
|
||||
// defaultSet is the process-wide metric set, registered on the same
|
||||
// registry the HTTP middleware and the /metrics handler already use.
|
||||
// It is built on first use rather than in an init so that a test
|
||||
// binary that never touches metrics never registers them.
|
||||
//
|
||||
//nolint:gochecknoglobals // one process-wide registration, by design
|
||||
var defaultSet = sync.OnceValue(func() *Set {
|
||||
return New(prometheus.DefaultRegisterer)
|
||||
})
|
||||
|
||||
// Default returns the process-wide metric set.
|
||||
func Default() *Set {
|
||||
return defaultSet()
|
||||
}
|
||||
|
||||
// Set is one registered group of webhooker's delivery collectors.
|
||||
// Production uses the single Default set; tests build their own
|
||||
// against a private registry so assertions are not disturbed by
|
||||
// deliveries other tests are making concurrently.
|
||||
type Set struct {
|
||||
eventsReceived prometheus.Counter
|
||||
deliveryAttempts *prometheus.CounterVec
|
||||
deliveriesSucceeded *prometheus.CounterVec
|
||||
deliveriesFailed *prometheus.CounterVec
|
||||
deliveryRetries *prometheus.CounterVec
|
||||
deliveryDuration *prometheus.HistogramVec
|
||||
deliveriesPending *prometheus.GaugeVec
|
||||
deliveriesRetrying *prometheus.GaugeVec
|
||||
circuitBreakersOpen *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// New registers a full set of delivery collectors on reg and returns
|
||||
// it. It panics if reg already holds them, which is the intended
|
||||
// behaviour for a duplicate registration.
|
||||
func New(reg prometheus.Registerer) *Set {
|
||||
factory := promauto.With(reg)
|
||||
|
||||
s := &Set{
|
||||
eventsReceived: factory.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "events_received_total",
|
||||
Help: "Webhook events received and " +
|
||||
"stored, so the receive and deliver " +
|
||||
"sides can be compared.",
|
||||
},
|
||||
),
|
||||
deliveryDuration: factory.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: namespace,
|
||||
Name: "delivery_duration_seconds",
|
||||
Help: "Wall time of a single delivery " +
|
||||
"attempt, by target type.",
|
||||
Buckets: prometheus.ExponentialBuckets(
|
||||
durationBucketStart,
|
||||
durationBucketFactor,
|
||||
durationBucketCount,
|
||||
),
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
),
|
||||
}
|
||||
|
||||
s.registerCounters(factory)
|
||||
s.registerGauges(factory)
|
||||
s.initSeries()
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// EventReceived counts one inbound webhook event stored.
|
||||
func (s *Set) EventReceived() {
|
||||
s.eventsReceived.Inc()
|
||||
}
|
||||
|
||||
// DeliveryAttempted counts one delivery attempt dispatched to a
|
||||
// target.
|
||||
func (s *Set) DeliveryAttempted(t database.TargetType) {
|
||||
s.deliveryAttempts.
|
||||
WithLabelValues(normalizeTargetType(t)).
|
||||
Inc()
|
||||
}
|
||||
|
||||
// ObserveDeliveryDuration records how long one delivery attempt took.
|
||||
func (s *Set) ObserveDeliveryDuration(
|
||||
t database.TargetType, d time.Duration,
|
||||
) {
|
||||
s.deliveryDuration.
|
||||
WithLabelValues(normalizeTargetType(t)).
|
||||
Observe(d.Seconds())
|
||||
}
|
||||
|
||||
// DeliveryStatusChanged counts a delivery's transition into a new
|
||||
// status. The mapping from status to counter lives here, next to the
|
||||
// collectors, so the engine has a single call for every transition it
|
||||
// persists. A move back to pending is not an outcome and counts
|
||||
// nothing.
|
||||
func (s *Set) DeliveryStatusChanged(
|
||||
t database.TargetType, status database.DeliveryStatus,
|
||||
) {
|
||||
label := normalizeTargetType(t)
|
||||
|
||||
switch status {
|
||||
case database.DeliveryStatusDelivered:
|
||||
s.deliveriesSucceeded.WithLabelValues(label).Inc()
|
||||
case database.DeliveryStatusFailed:
|
||||
s.deliveriesFailed.WithLabelValues(label).Inc()
|
||||
case database.DeliveryStatusRetrying:
|
||||
s.deliveryRetries.WithLabelValues(label).Inc()
|
||||
case database.DeliveryStatusPending:
|
||||
}
|
||||
}
|
||||
|
||||
// SetQueueDepths publishes the pending and retrying queue depths from
|
||||
// one sample. Every known target type is written on every call, so a
|
||||
// type whose queue has drained reads zero instead of holding its last
|
||||
// value forever.
|
||||
func (s *Set) SetQueueDepths(
|
||||
pending, retrying map[database.TargetType]int,
|
||||
) {
|
||||
for _, t := range knownTargetTypes {
|
||||
label := string(t)
|
||||
|
||||
s.deliveriesPending.WithLabelValues(label).
|
||||
Set(float64(pending[t]))
|
||||
s.deliveriesRetrying.WithLabelValues(label).
|
||||
Set(float64(retrying[t]))
|
||||
}
|
||||
}
|
||||
|
||||
// SetCircuitBreakersOpen publishes how many of a target type's
|
||||
// circuit breakers are currently open.
|
||||
func (s *Set) SetCircuitBreakersOpen(
|
||||
t database.TargetType, open int,
|
||||
) {
|
||||
s.circuitBreakersOpen.
|
||||
WithLabelValues(normalizeTargetType(t)).
|
||||
Set(float64(open))
|
||||
}
|
||||
|
||||
func (s *Set) registerCounters(factory promauto.Factory) {
|
||||
s.deliveryAttempts = factory.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "delivery_attempts_total",
|
||||
Help: "Delivery attempts dispatched to a " +
|
||||
"target, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
|
||||
s.deliveriesSucceeded = factory.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "deliveries_succeeded_total",
|
||||
Help: "Deliveries that reached the delivered " +
|
||||
"state, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
|
||||
s.deliveriesFailed = factory.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "deliveries_failed_total",
|
||||
Help: "Deliveries that failed terminally and " +
|
||||
"will not be retried, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
|
||||
s.deliveryRetries = factory.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "delivery_retries_total",
|
||||
Help: "Deliveries put back into the retrying " +
|
||||
"state, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Set) registerGauges(factory promauto.Factory) {
|
||||
s.deliveriesPending = factory.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "deliveries_pending",
|
||||
Help: "Deliveries currently in the pending " +
|
||||
"state, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
|
||||
s.deliveriesRetrying = factory.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "deliveries_retrying",
|
||||
Help: "Deliveries currently in the retrying " +
|
||||
"state, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
|
||||
s.circuitBreakersOpen = factory.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "circuit_breakers_open",
|
||||
Help: "Delivery circuit breakers currently " +
|
||||
"open, by target type.",
|
||||
},
|
||||
[]string{targetTypeLabel},
|
||||
)
|
||||
}
|
||||
|
||||
// initSeries materialises every known-target-type series at zero, so
|
||||
// a dashboard and an alert rule see a target type that has not
|
||||
// delivered yet rather than a missing series.
|
||||
func (s *Set) initSeries() {
|
||||
for _, t := range knownTargetTypes {
|
||||
label := string(t)
|
||||
|
||||
s.deliveryAttempts.WithLabelValues(label)
|
||||
s.deliveriesSucceeded.WithLabelValues(label)
|
||||
s.deliveriesFailed.WithLabelValues(label)
|
||||
s.deliveryRetries.WithLabelValues(label)
|
||||
s.deliveriesPending.WithLabelValues(label)
|
||||
s.deliveriesRetrying.WithLabelValues(label)
|
||||
s.circuitBreakersOpen.WithLabelValues(label)
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeTargetType maps a target type onto the bounded label
|
||||
// domain, collapsing anything outside it to unknownTargetType.
|
||||
func normalizeTargetType(t database.TargetType) string {
|
||||
for _, known := range knownTargetTypes {
|
||||
if t == known {
|
||||
return string(known)
|
||||
}
|
||||
}
|
||||
|
||||
return unknownTargetType
|
||||
}
|
||||
Reference in New Issue
Block a user