webhooker/internal/handlers/target_paused_test.go

package handlers_test

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gorm.io/gorm/clause"
	"sneak.berlin/go/webhooker/internal/database"
	"sneak.berlin/go/webhooker/internal/delivery"
	"sneak.berlin/go/webhooker/internal/handlers"
	"sneak.berlin/go/webhooker/internal/session"
)

// resumesAt is how the pages write when a paused target's deliveries
// resume at the end of its breaker's cooldown: the time in UTC, then
// how long that is from now.
const resumesAt = `\d\d:\d\d:\d\d UTC \(\d+ seconds from now\)`

// TestPausedTarget_ShownUntilBreakerCloses takes an http target's
// circuit breaker from open through half-open to closed.
//
// Open, the target's row on the webhook page says its deliveries are
// paused and until when, and each retrying delivery says it is waiting
// and why in the event log and on the event's page, until the later of
// the cooldown's end and the end of its own backoff. Half-open, the row
// says deliveries are held while one delivery tests the target, with no
// time, and no delivery says it is waiting. Closed, the pages say
// neither. The delivered delivery and the log target are shown as
// before throughout.
func TestPausedTarget_ShownUntilBreakerCloses(t *testing.T) {
	t.Parallel()

	var (
		h        *handlers.Handlers
		sess     *session.Session
		db       *database.Database
		dbMgr    *database.WebhookDBManager
		breakers *testCircuitBreakers
	)

	app := newTestApp(t, &h, &sess, &db, &dbMgr, &breakers)
	app.RequireStart()

	t.Cleanup(app.RequireStop)

	wh := seedWebhook(t, db)
	target := seedTarget(t, db, wh.ID, database.TargetTypeHTTP)
	seedTarget(t, db, wh.ID, database.TargetTypeLog)

	retrying := seedStoredEvent(t, dbMgr, wh.ID, `{"n":1}`)
	addDelivery(t, dbMgr, wh.ID, retrying.ID, target.ID,
		database.DeliveryStatusRetrying)

	delivered := seedStoredEvent(t, dbMgr, wh.ID, `{"n":2}`)
	addDelivery(t, dbMgr, wh.ID, delivered.ID, target.ID,
		database.DeliveryStatusDelivered)

	// This delivery's 13th attempt failed a minute ago, so its own
	// backoff ends over an hour from now, long after the cooldown.
	backedOff := seedStoredEvent(t, dbMgr, wh.ID, `{"n":3}`)
	backedOffID := addDelivery(t, dbMgr, wh.ID, backedOff.ID, target.ID,
		database.DeliveryStatusRetrying)

	failedAt := time.Now().Add(-time.Minute).Truncate(time.Second)
	addFailedAttempt(t, dbMgr, wh.ID, backedOffID, 13, failedAt)

	backoffEnds := failedAt.Add(delivery.Backoff(13)).UTC().
		Format(time.TimeOnly) + " UTC (1 hour from now)"

	const waiting = "waiting: target paused after repeated failures, " +
		"resumes "

	breakers.Set(target.ID, delivery.CircuitOpen, 30*time.Second)

	list := targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
	assert.Regexp(t, "t-http http Active Edit Deactivate Delete "+
		"Deliveries Paused: after repeated failures, until "+resumesAt,
		list)
	assert.Equal(t, 1, strings.Count(list, "Paused"))

	log := renderSourceLogsPage(t, h, sess, wh.ID)
	assert.Equal(t, 2, strings.Count(log, "t-http: waiting"))
	assert.Contains(t, log, "t-http: delivered")
	assert.Regexp(t, waiting+resumesAt, log)
	assert.Contains(t, log, waiting+backoffEnds)
	assert.NotContains(t, log, "retrying")

	page := eventPage(t, h, sess, wh.ID, retrying.ID)
	assert.Regexp(t, waiting+resumesAt, page)
	assert.NotContains(t, page, "retrying")

	page = eventPage(t, h, sess, wh.ID, backedOff.ID)
	assert.Contains(t, page, waiting+backoffEnds)
	assert.NotContains(t, page, "seconds from now")

	breakers.Set(target.ID, delivery.CircuitHalfOpen, 0)

	list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
	assert.Contains(t, list, "t-http http Active Edit Deactivate Delete "+
		"Deliveries Paused: held while one delivery tests whether the "+
		"target has recovered")
	assert.NotContains(t, list, "UTC")

	assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)

	breakers.Set(target.ID, delivery.CircuitClosed, 0)

	list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
	assert.NotContains(t, list, "Paused")

	assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)
}

// assertRetryingNotWaiting checks that the event log and each event's
// page show the http target's delivery of the event as retrying, and
// none of them as waiting.
func assertRetryingNotWaiting(
	t *testing.T,
	h *handlers.Handlers,
	sess *session.Session,
	webhookID string,
	events ...*database.Event,
) {
	t.Helper()

	log := renderSourceLogsPage(t, h, sess, webhookID)
	assert.Equal(t, len(events), strings.Count(log, "t-http: retrying"))
	assert.NotContains(t, log, "waiting")

	for _, event := range events {
		page := eventPage(t, h, sess, webhookID, event.ID)
		assert.Contains(t, page, ">retrying</span>")
		assert.NotContains(t, page, "waiting")
	}
}

// addDelivery records a delivery of the event to the target, with the
// given status, in the webhook's own database, and returns its ID.
func addDelivery(
	t *testing.T,
	dbMgr *database.WebhookDBManager,
	webhookID, eventID, targetID string,
	status database.DeliveryStatus,
) string {
	t.Helper()

	webhookDB, err := dbMgr.GetDB(webhookID)
	require.NoError(t, err)

	dlv := &database.Delivery{
		EventID:  eventID,
		TargetID: targetID,
		Status:   status,
	}

	require.NoError(t, webhookDB.Omit(clause.Associations).Create(
		dlv,
	).Error)

	return dlv.ID
}

// addFailedAttempt records the delivery's failed attempt attemptNum,
// made at the given time.
func addFailedAttempt(
	t *testing.T,
	dbMgr *database.WebhookDBManager,
	webhookID, deliveryID string,
	attemptNum int,
	at time.Time,
) {
	t.Helper()

	webhookDB, err := dbMgr.GetDB(webhookID)
	require.NoError(t, err)

	require.NoError(t, webhookDB.Omit(clause.Associations).Create(
		&database.DeliveryResult{
			BaseModel:  database.BaseModel{CreatedAt: at},
			DeliveryID: deliveryID,
			AttemptNum: attemptNum,
			Error:      "connection refused",
		},
	).Error)
}

// eventPage runs the real event page handler and returns the
// rendered HTML.
func eventPage(
	t *testing.T,
	h *handlers.Handlers,
	sess *session.Session,
	webhookID, eventID string,
) string {
	t.Helper()

	w := serveEventPage(t, h, sess, webhookID, eventID)
	require.Equal(t, http.StatusOK, w.Code)

	return w.Body.String()
}