sneak
b8c8b75e04
All checks were successful
check / check (push) Successful in 4m13s
/metrics carried only the inbound HTTP surface, so a destination failing for an hour, a growing retry backlog and a stuck-open circuit breaker were all invisible: the receive side stays healthy in each case because it is. New internal/metrics registers, on the existing default registry that the go-http-metrics recorder and the promhttp handler already share: - webhooker_events_received_total - webhooker_delivery_attempts_total - webhooker_deliveries_succeeded_total - webhooker_deliveries_failed_total - webhooker_delivery_retries_total - webhooker_delivery_duration_seconds - webhooker_deliveries_pending / _retrying - webhooker_circuit_breakers_open The route mounting is untouched. Every delivery metric carries one label, target_type, whose domain is the four target-type constants; anything outside it collapses to "unknown" so no series can be minted from a UUID. Target ids, event ids and entrypoint ids are deliberately not labels. Instrumentation sits at the points every target type already passes through: processDelivery for the attempt counter and the duration histogram, updateDeliveryStatus for the outcome counters. The queue-depth gauges are counted out of the per-webhook databases by a 30s sampler rather than tracked as deltas, which would need seeding at startup and would drift on any transition that failed to persist. The open-breaker gauge is recounted from the target's breaker registry on every state change.
379 lines
8.8 KiB
Go
379 lines
8.8 KiB
Go
package delivery_test
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
dto "github.com/prometheus/client_model/go"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/webhooker/internal/database"
|
|
"sneak.berlin/go/webhooker/internal/delivery"
|
|
"sneak.berlin/go/webhooker/internal/metrics"
|
|
)
|
|
|
|
// Metric names as exposed on /metrics.
|
|
const (
|
|
mAttempts = "webhooker_delivery_attempts_total"
|
|
mSucceeded = "webhooker_deliveries_succeeded_total"
|
|
mFailed = "webhooker_deliveries_failed_total"
|
|
mRetries = "webhooker_delivery_retries_total"
|
|
mDuration = "webhooker_delivery_duration_seconds"
|
|
mPending = "webhooker_deliveries_pending"
|
|
mRetrying = "webhooker_deliveries_retrying"
|
|
mBreakers = "webhooker_circuit_breakers_open"
|
|
)
|
|
|
|
const (
|
|
mTypeHTTP = "http"
|
|
mTypeLog = "log"
|
|
)
|
|
|
|
// mIsolate gives the setup's engine a metric set registered on a
|
|
// private registry. The process-wide collectors are moved by every
|
|
// other delivery test running in parallel, so exact assertions are
|
|
// only possible against a registry this test owns.
|
|
func mIsolate(
|
|
t *testing.T, s iSetup,
|
|
) *prometheus.Registry {
|
|
t.Helper()
|
|
|
|
reg := prometheus.NewRegistry()
|
|
s.Engine.ExportSetMetrics(metrics.New(reg))
|
|
|
|
return reg
|
|
}
|
|
|
|
// mFind returns the series of the named metric carrying the given
|
|
// target_type label.
|
|
func mFind(
|
|
t *testing.T,
|
|
reg *prometheus.Registry,
|
|
name, targetType string,
|
|
) *dto.Metric {
|
|
t.Helper()
|
|
|
|
families, err := reg.Gather()
|
|
require.NoError(t, err)
|
|
|
|
for _, fam := range families {
|
|
if fam.GetName() != name {
|
|
continue
|
|
}
|
|
|
|
for _, m := range fam.GetMetric() {
|
|
for _, label := range m.GetLabel() {
|
|
if label.GetName() == "target_type" &&
|
|
label.GetValue() == targetType {
|
|
return m
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
t.Fatalf(
|
|
"metric %s{target_type=%q} not found",
|
|
name, targetType,
|
|
)
|
|
|
|
return nil
|
|
}
|
|
|
|
func mCounter(
|
|
t *testing.T,
|
|
reg *prometheus.Registry,
|
|
name, targetType string,
|
|
) float64 {
|
|
t.Helper()
|
|
|
|
return mFind(t, reg, name, targetType).
|
|
GetCounter().GetValue()
|
|
}
|
|
|
|
func mGauge(
|
|
t *testing.T,
|
|
reg *prometheus.Registry,
|
|
name, targetType string,
|
|
) float64 {
|
|
t.Helper()
|
|
|
|
return mFind(t, reg, name, targetType).
|
|
GetGauge().GetValue()
|
|
}
|
|
|
|
func mObservations(
|
|
t *testing.T,
|
|
reg *prometheus.Registry,
|
|
name, targetType string,
|
|
) uint64 {
|
|
t.Helper()
|
|
|
|
return mFind(t, reg, name, targetType).
|
|
GetHistogram().GetSampleCount()
|
|
}
|
|
|
|
// TestDeliveryMetrics_SuccessAndRetryExhaustion drives one delivery
|
|
// that succeeds and one that fails every attempt until its retries
|
|
// are exhausted, and asserts every delivery counter across both.
|
|
func TestDeliveryMetrics_SuccessAndRetryExhaustion(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
reg := mIsolate(t, s)
|
|
|
|
mDeliverOK(t, s)
|
|
|
|
assert.InDelta(t, 1.0,
|
|
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
assert.InDelta(t, 1.0,
|
|
mCounter(t, reg, mSucceeded, mTypeHTTP), 0)
|
|
assert.InDelta(t, 0.0,
|
|
mCounter(t, reg, mFailed, mTypeHTTP), 0)
|
|
assert.InDelta(t, 0.0,
|
|
mCounter(t, reg, mRetries, mTypeHTTP), 0)
|
|
assert.Equal(t, uint64(1),
|
|
mObservations(t, reg, mDuration, mTypeHTTP))
|
|
|
|
mExhaustRetries(t, s)
|
|
|
|
// Two further attempts: the first is retried, the second is
|
|
// the last one allowed and fails the delivery terminally.
|
|
assert.InDelta(t, 3.0,
|
|
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
assert.InDelta(t, 1.0,
|
|
mCounter(t, reg, mSucceeded, mTypeHTTP), 0)
|
|
assert.InDelta(t, 1.0,
|
|
mCounter(t, reg, mRetries, mTypeHTTP), 0)
|
|
assert.InDelta(t, 1.0,
|
|
mCounter(t, reg, mFailed, mTypeHTTP), 0)
|
|
assert.Equal(t, uint64(3),
|
|
mObservations(t, reg, mDuration, mTypeHTTP))
|
|
|
|
// Two consecutive failures are below the trip threshold.
|
|
assert.InDelta(t, 0.0,
|
|
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
|
|
// The label is the target type and nothing finer: two http
|
|
// targets shared one series, and no other type's moved.
|
|
assert.InDelta(t, 0.0,
|
|
mCounter(t, reg, mAttempts, mTypeLog), 0)
|
|
assert.InDelta(t, 0.0,
|
|
mCounter(t, reg, mFailed, mTypeLog), 0)
|
|
}
|
|
|
|
// mDeliverOK delivers one event to a target that answers 200.
|
|
func mDeliverOK(t *testing.T, s iSetup) {
|
|
t.Helper()
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
},
|
|
))
|
|
defer ts.Close()
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"ok":true}`,
|
|
)
|
|
targetID := uuid.New().String()
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
body := event.Body
|
|
task := iTask(
|
|
d, event, s.WebhookID, targetID,
|
|
"metrics-ok", iHTTPConfig(ts.URL), 3, 1, &body,
|
|
)
|
|
|
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
|
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusDelivered,
|
|
)
|
|
}
|
|
|
|
// mExhaustRetries delivers to a target that answers 500 with a
|
|
// two-attempt budget, driving both attempts so the delivery ends
|
|
// terminally failed.
|
|
func mExhaustRetries(t *testing.T, s iSetup) {
|
|
t.Helper()
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
},
|
|
))
|
|
defer ts.Close()
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"ok":false}`,
|
|
)
|
|
targetID := uuid.New().String()
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
body := event.Body
|
|
cfg := iHTTPConfig(ts.URL)
|
|
|
|
first := iTask(
|
|
d, event, s.WebhookID, targetID,
|
|
"metrics-fail", cfg, 2, 1, &body,
|
|
)
|
|
|
|
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
|
|
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusRetrying,
|
|
)
|
|
|
|
// The engine's own scheduler would re-enqueue this after the
|
|
// backoff; driving the second attempt directly keeps the test
|
|
// deterministic and off the wall clock.
|
|
second := iTask(
|
|
d, event, s.WebhookID, targetID,
|
|
"metrics-fail", cfg, 2, 2, &body,
|
|
)
|
|
|
|
s.Engine.ExportProcessRetryTask(
|
|
context.TODO(), &second,
|
|
)
|
|
|
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusFailed,
|
|
)
|
|
}
|
|
|
|
// TestDeliveryMetrics_CircuitBreakerGauge proves the open-breaker
|
|
// gauge follows a breaker that trips.
|
|
func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
reg := mIsolate(t, s)
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
},
|
|
))
|
|
defer ts.Close()
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"trip":true}`,
|
|
)
|
|
targetID := uuid.New().String()
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
body := event.Body
|
|
cfg := iHTTPConfig(ts.URL)
|
|
|
|
// A retry budget above the failure threshold, so the breaker
|
|
// rather than the budget is what stops the delivery.
|
|
maxRetries := delivery.ExportDefaultFailureThreshold + 5
|
|
|
|
first := iTask(
|
|
d, event, s.WebhookID, targetID,
|
|
"metrics-trip", cfg, maxRetries, 1, &body,
|
|
)
|
|
|
|
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
|
|
|
assert.InDelta(t, 0.0,
|
|
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
|
|
for attempt := 2; attempt <= delivery.
|
|
ExportDefaultFailureThreshold; attempt++ {
|
|
task := iTask(
|
|
d, event, s.WebhookID, targetID,
|
|
"metrics-trip", cfg, maxRetries, attempt, &body,
|
|
)
|
|
|
|
s.Engine.ExportProcessRetryTask(
|
|
context.TODO(), &task,
|
|
)
|
|
}
|
|
|
|
assert.InDelta(t, 1.0,
|
|
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
}
|
|
|
|
// TestDeliveryMetrics_QueueDepthGauges proves the sampler publishes
|
|
// the queued deliveries it finds in the per-webhook databases, and
|
|
// that a drained queue reads zero rather than keeping its last
|
|
// value.
|
|
func TestDeliveryMetrics_QueueDepthGauges(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
reg := mIsolate(t, s)
|
|
|
|
iCreateWebhook(
|
|
t, s.MainDB, s.WebhookID, "queue-depth",
|
|
)
|
|
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID, s.WebhookID,
|
|
"queue-depth-target", database.TargetTypeHTTP,
|
|
iHTTPConfig("https://example.com/hook"), 3,
|
|
)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"queued":true}`,
|
|
)
|
|
|
|
pending := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
retrying := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusRetrying,
|
|
)
|
|
|
|
s.Engine.ExportSampleQueueDepths(context.Background())
|
|
|
|
assert.InDelta(t, 2.0,
|
|
mGauge(t, reg, mPending, mTypeHTTP), 0)
|
|
assert.InDelta(t, 1.0,
|
|
mGauge(t, reg, mRetrying, mTypeHTTP), 0)
|
|
assert.InDelta(t, 0.0,
|
|
mGauge(t, reg, mPending, mTypeLog), 0)
|
|
|
|
require.NoError(t, s.WebhookDB.
|
|
Model(&database.Delivery{}).
|
|
Where("id IN ?", []string{pending.ID, retrying.ID}).
|
|
Update(
|
|
"status", database.DeliveryStatusDelivered,
|
|
).Error)
|
|
|
|
s.Engine.ExportSampleQueueDepths(context.Background())
|
|
|
|
assert.InDelta(t, 1.0,
|
|
mGauge(t, reg, mPending, mTypeHTTP), 0)
|
|
assert.InDelta(t, 0.0,
|
|
mGauge(t, reg, mRetrying, mTypeHTTP), 0)
|
|
}
|