package handlers_test import ( "net/http" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/gorm/clause" "sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/delivery" "sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/session" ) // resumesAt is how the pages write when a paused target's deliveries // resume at the end of its breaker's cooldown: the time in UTC, then // how long that is from now. const resumesAt = `\d\d:\d\d:\d\d UTC \(\d+ seconds from now\)` // TestPausedTarget_ShownUntilBreakerCloses takes an http target's // circuit breaker from open through half-open to closed. // // Open, the target's row on the webhook page says its deliveries are // paused and until when, and each retrying delivery says it is waiting // and why in the event log and on the event's page, until the later of // the cooldown's end and the end of its own backoff. Half-open, the row // says deliveries are held while one delivery tests the target, with no // time, and no delivery says it is waiting. Closed, the pages say // neither. The delivered delivery and the log target are shown as // before throughout. func TestPausedTarget_ShownUntilBreakerCloses(t *testing.T) { t.Parallel() var ( h *handlers.Handlers sess *session.Session db *database.Database dbMgr *database.WebhookDBManager breakers *testCircuitBreakers ) app := newTestApp(t, &h, &sess, &db, &dbMgr, &breakers) app.RequireStart() t.Cleanup(app.RequireStop) wh := seedWebhook(t, db) target := seedTarget(t, db, wh.ID, database.TargetTypeHTTP) seedTarget(t, db, wh.ID, database.TargetTypeLog) retrying := seedStoredEvent(t, dbMgr, wh.ID, `{"n":1}`) addDelivery(t, dbMgr, wh.ID, retrying.ID, target.ID, database.DeliveryStatusRetrying) delivered := seedStoredEvent(t, dbMgr, wh.ID, `{"n":2}`) addDelivery(t, dbMgr, wh.ID, delivered.ID, target.ID, database.DeliveryStatusDelivered) // This delivery's 13th attempt failed a minute ago, so its own // backoff ends over an hour from now, long after the cooldown. backedOff := seedStoredEvent(t, dbMgr, wh.ID, `{"n":3}`) backedOffID := addDelivery(t, dbMgr, wh.ID, backedOff.ID, target.ID, database.DeliveryStatusRetrying) failedAt := time.Now().Add(-time.Minute).Truncate(time.Second) addFailedAttempt(t, dbMgr, wh.ID, backedOffID, 13, failedAt) backoffEnds := failedAt.Add(delivery.Backoff(13)).UTC(). Format(time.TimeOnly) + " UTC (1 hour from now)" const waiting = "waiting: target paused after repeated failures, " + "resumes " breakers.Set(target.ID, delivery.CircuitOpen, 30*time.Second) list := targetList(t, renderSourceDetailPage(t, h, sess, wh.ID)) assert.Regexp(t, "t-http http Active Edit Deactivate Delete "+ "Deliveries Paused: after repeated failures, until "+resumesAt, list) assert.Equal(t, 1, strings.Count(list, "Paused")) log := renderSourceLogsPage(t, h, sess, wh.ID) assert.Equal(t, 2, strings.Count(log, "t-http: waiting")) assert.Contains(t, log, "t-http: delivered") assert.Regexp(t, waiting+resumesAt, log) assert.Contains(t, log, waiting+backoffEnds) assert.NotContains(t, log, "retrying") page := eventPage(t, h, sess, wh.ID, retrying.ID) assert.Regexp(t, waiting+resumesAt, page) assert.NotContains(t, page, "retrying") page = eventPage(t, h, sess, wh.ID, backedOff.ID) assert.Contains(t, page, waiting+backoffEnds) assert.NotContains(t, page, "seconds from now") breakers.Set(target.ID, delivery.CircuitHalfOpen, 0) list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID)) assert.Contains(t, list, "t-http http Active Edit Deactivate Delete "+ "Deliveries Paused: held while one delivery tests whether the "+ "target has recovered") assert.NotContains(t, list, "UTC") assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff) breakers.Set(target.ID, delivery.CircuitClosed, 0) list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID)) assert.NotContains(t, list, "Paused") assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff) } // assertRetryingNotWaiting checks that the event log and each event's // page show the http target's delivery of the event as retrying, and // none of them as waiting. func assertRetryingNotWaiting( t *testing.T, h *handlers.Handlers, sess *session.Session, webhookID string, events ...*database.Event, ) { t.Helper() log := renderSourceLogsPage(t, h, sess, webhookID) assert.Equal(t, len(events), strings.Count(log, "t-http: retrying")) assert.NotContains(t, log, "waiting") for _, event := range events { page := eventPage(t, h, sess, webhookID, event.ID) assert.Contains(t, page, ">retrying") assert.NotContains(t, page, "waiting") } } // addDelivery records a delivery of the event to the target, with the // given status, in the webhook's own database, and returns its ID. func addDelivery( t *testing.T, dbMgr *database.WebhookDBManager, webhookID, eventID, targetID string, status database.DeliveryStatus, ) string { t.Helper() webhookDB, err := dbMgr.GetDB(webhookID) require.NoError(t, err) dlv := &database.Delivery{ EventID: eventID, TargetID: targetID, Status: status, } require.NoError(t, webhookDB.Omit(clause.Associations).Create( dlv, ).Error) return dlv.ID } // addFailedAttempt records the delivery's failed attempt attemptNum, // made at the given time. func addFailedAttempt( t *testing.T, dbMgr *database.WebhookDBManager, webhookID, deliveryID string, attemptNum int, at time.Time, ) { t.Helper() webhookDB, err := dbMgr.GetDB(webhookID) require.NoError(t, err) require.NoError(t, webhookDB.Omit(clause.Associations).Create( &database.DeliveryResult{ BaseModel: database.BaseModel{CreatedAt: at}, DeliveryID: deliveryID, AttemptNum: attemptNum, Error: "connection refused", }, ).Error) } // eventPage runs the real event page handler and returns the // rendered HTML. func eventPage( t *testing.T, h *handlers.Handlers, sess *session.Session, webhookID, eventID string, ) string { t.Helper() w := serveEventPage(t, h, sess, webhookID, eventID) require.Equal(t, http.StatusOK, w.Code) return w.Body.String() }