webhooker/internal/metrics/metrics_test.go

package metrics_test

import (
	"testing"

	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/webhooker/internal/database"
	"sneak.berlin/go/webhooker/internal/metrics"
)

// knownLabels is the target_type label domain built from the target
// types the delivery engine implements.
func knownLabels() []string {
	return []string{"http", "database", "log", "slack"}
}

// labelValues returns the target_type label values a metric family
// currently carries.
func labelValues(
	t *testing.T, reg *prometheus.Registry, name string,
) []string {
	t.Helper()

	families, err := reg.Gather()
	require.NoError(t, err)

	var values []string

	for _, fam := range families {
		if fam.GetName() != name {
			continue
		}

		for _, m := range fam.GetMetric() {
			for _, label := range m.GetLabel() {
				if label.GetName() == "target_type" {
					values = append(
						values, label.GetValue(),
					)
				}
			}
		}
	}

	return values
}

func gaugeValue(
	t *testing.T,
	reg *prometheus.Registry,
	name, targetType string,
) float64 {
	t.Helper()

	families, err := reg.Gather()
	require.NoError(t, err)

	for _, fam := range families {
		if fam.GetName() != name {
			continue
		}

		for _, m := range fam.GetMetric() {
			if hasTargetType(m, targetType) {
				return m.GetGauge().GetValue()
			}
		}
	}

	t.Fatalf(
		"gauge %s{target_type=%q} not found",
		name, targetType,
	)

	return 0
}

func hasTargetType(m *dto.Metric, targetType string) bool {
	for _, label := range m.GetLabel() {
		if label.GetName() == "target_type" &&
			label.GetValue() == targetType {
			return true
		}
	}

	return false
}

// TestUnknownTargetTypeCollapses is the cardinality guard: a target
// type outside the known set must not mint a series of its own, or
// /metrics grows without bound.
func TestUnknownTargetTypeCollapses(t *testing.T) {
	t.Parallel()

	reg := prometheus.NewRegistry()
	set := metrics.New(reg)

	for _, bogus := range []string{
		"a1b2c3d4-0000-0000-0000-000000000001",
		"a1b2c3d4-0000-0000-0000-000000000002",
		"webhook-forwarder",
	} {
		set.DeliveryAttempted(database.TargetType(bogus))
	}

	values := labelValues(
		t, reg, "webhooker_delivery_attempts_total",
	)

	assert.ElementsMatch(t,
		append(knownLabels(), "unknown"),
		values,
	)
}

// TestSetQueueDepthsZeroesDrainedTypes proves a queue that has
// drained reads zero rather than holding its last sample forever.
func TestSetQueueDepthsZeroesDrainedTypes(t *testing.T) {
	t.Parallel()

	reg := prometheus.NewRegistry()
	set := metrics.New(reg)

	set.SetQueueDepths(
		map[database.TargetType]int{
			database.TargetTypeHTTP: 7,
		},
		map[database.TargetType]int{
			database.TargetTypeSlack: 2,
		},
	)

	assert.InDelta(t, 7.0, gaugeValue(
		t, reg, "webhooker_deliveries_pending", "http",
	), 0)
	assert.InDelta(t, 2.0, gaugeValue(
		t, reg, "webhooker_deliveries_retrying", "slack",
	), 0)

	set.SetQueueDepths(
		map[database.TargetType]int{},
		map[database.TargetType]int{},
	)

	assert.InDelta(t, 0.0, gaugeValue(
		t, reg, "webhooker_deliveries_pending", "http",
	), 0)
	assert.InDelta(t, 0.0, gaugeValue(
		t, reg, "webhooker_deliveries_retrying", "slack",
	), 0)
}

// TestKnownSeriesExistBeforeAnyDelivery proves every known target
// type is published at zero from registration, so an alert rule does
// not have to cope with a missing series.
func TestKnownSeriesExistBeforeAnyDelivery(t *testing.T) {
	t.Parallel()

	reg := prometheus.NewRegistry()
	metrics.New(reg)

	for _, name := range []string{
		"webhooker_delivery_attempts_total",
		"webhooker_deliveries_succeeded_total",
		"webhooker_deliveries_failed_total",
		"webhooker_delivery_retries_total",
		"webhooker_delivery_duration_seconds",
		"webhooker_circuit_breakers_open",
	} {
		assert.ElementsMatch(t,
			knownLabels(),
			labelValues(t, reg, name),
			"metric %s", name,
		)
	}

	// The queue gauges additionally publish unknown from
	// registration: a backlog queued against a deleted target lands
	// there, and it can predate the process, so the series has to
	// exist before the first sample rather than appearing only once
	// something is already stuck.
	for _, name := range []string{
		"webhooker_deliveries_pending",
		"webhooker_deliveries_retrying",
	} {
		assert.ElementsMatch(t,
			append(knownLabels(), "unknown"),
			labelValues(t, reg, name),
			"metric %s", name,
		)
	}
}

// TestSetQueueDepthsFoldsUnknownTypes proves a queued delivery whose
// target type is not a known one — a target deleted out from under it
// resolves to the empty type — is summed into the unknown series
// instead of being dropped, and that the fold is a sum rather than a
// last-writer-wins.
func TestSetQueueDepthsFoldsUnknownTypes(t *testing.T) {
	t.Parallel()

	reg := prometheus.NewRegistry()
	set := metrics.New(reg)

	set.SetQueueDepths(
		map[database.TargetType]int{
			database.TargetTypeHTTP:             1,
			database.TargetType(""):             4,
			database.TargetType("retired-type"): 3,
		},
		map[database.TargetType]int{
			database.TargetType(""): 2,
		},
	)

	assert.InDelta(t, 7.0, gaugeValue(
		t, reg, "webhooker_deliveries_pending", "unknown",
	), 0)
	assert.InDelta(t, 2.0, gaugeValue(
		t, reg, "webhooker_deliveries_retrying", "unknown",
	), 0)
	assert.InDelta(t, 1.0, gaugeValue(
		t, reg, "webhooker_deliveries_pending", "http",
	), 0)

	set.SetQueueDepths(
		map[database.TargetType]int{},
		map[database.TargetType]int{},
	)

	assert.InDelta(t, 0.0, gaugeValue(
		t, reg, "webhooker_deliveries_pending", "unknown",
	), 0)
}

// TestDeliveryStatusChangedCounts maps each persisted status onto the
// counter it moves.
func TestDeliveryStatusChangedCounts(t *testing.T) {
	t.Parallel()

	reg := prometheus.NewRegistry()
	set := metrics.New(reg)

	set.DeliveryStatusChanged(
		database.TargetTypeLog,
		database.DeliveryStatusDelivered,
	)
	set.DeliveryStatusChanged(
		database.TargetTypeLog,
		database.DeliveryStatusFailed,
	)
	set.DeliveryStatusChanged(
		database.TargetTypeLog,
		database.DeliveryStatusRetrying,
	)
	set.DeliveryStatusChanged(
		database.TargetTypeLog,
		database.DeliveryStatusPending,
	)

	families, err := reg.Gather()
	require.NoError(t, err)

	counts := map[string]float64{}

	for _, fam := range families {
		for _, m := range fam.GetMetric() {
			if hasTargetType(m, "log") {
				counts[fam.GetName()] =
					m.GetCounter().GetValue()
			}
		}
	}

	assert.InDelta(t, 1.0,
		counts["webhooker_deliveries_succeeded_total"], 0)
	assert.InDelta(t, 1.0,
		counts["webhooker_deliveries_failed_total"], 0)
	assert.InDelta(t, 1.0,
		counts["webhooker_delivery_retries_total"], 0)
	assert.InDelta(t, 0.0,
		counts["webhooker_delivery_attempts_total"], 0)
}