rproxy: circuit breaker for asks to waf (closes #392)

Five asks that get no answer in time within five seconds open the
breaker. While open, a request takes
SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION without asking waf, except
one ask a second; the first answer closes it. Slow answers count as
failures. SPEC.md describes it.

The sww test that stops and restarts waf now waits for the breaker to
close before it expects the ban again.

Model: opus-5-5

internal/rproxy/breaker_test.go

@@ -0,0 +1,142 @@
+package rproxy_test
+
+import (
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/rproxy"
+)
+
+// The breaker tests give the breaker each time themselves, counted from
+// time.Now(), and never wait.
+
+// fail notes count failed asks on b, one each step from at.
+func fail(b *rproxy.Breaker, at time.Time, count int, step time.Duration) {
+	for i := range count {
+		b.NoteFailure(at.Add(time.Duration(i) * step))
+	}
+}
+
+// openBreaker returns a breaker opened at now by five failed asks, one a
+// second.
+func openBreaker(now time.Time) *rproxy.Breaker {
+	b := &rproxy.Breaker{}
+	fail(b, now.Add(-4*time.Second), 5, time.Second)
+
+	return b
+}
+
+func TestFourFailedAsksWithinFiveSecondsLeaveTheBreakerClosed(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	b := &rproxy.Breaker{}
+	fail(b, now, 4, time.Second)
+
+	if !b.MayAsk(now.Add(3 * time.Second)) {
+		t.Error("a request may not ask, want it to")
+	}
+}
+
+func TestFiveFailedAsksWithinFiveSecondsOpenTheBreaker(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	b := openBreaker(now)
+
+	if b.MayAsk(now) {
+		t.Error("a request may ask, want it not to")
+	}
+}
+
+func TestFiveFailedAsksOverMoreThanFiveSecondsLeaveTheBreakerClosed(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	now := time.Now()
+	b := &rproxy.Breaker{}
+	fail(b, now, 5, 1300*time.Millisecond)
+
+	if !b.MayAsk(now.Add(6 * time.Second)) {
+		t.Error("a request may not ask, want it to")
+	}
+}
+
+func TestAnAnswerBeforeTheBreakerOpensDoesNotForgetFailures(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	b := &rproxy.Breaker{}
+	fail(b, now, 4, time.Millisecond)
+	b.NoteAnswer()
+	b.NoteFailure(now.Add(time.Second))
+
+	if b.MayAsk(now.Add(time.Second)) {
+		t.Error("a request may ask, want it not to")
+	}
+}
+
+func TestAnOpenBreakerLetsOneRequestAskEachSecond(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	b := openBreaker(now)
+
+	for _, tc := range []struct {
+		after time.Duration
+		want  bool
+	}{
+		{999 * time.Millisecond, false},
+		{time.Second, true},
+		{time.Second, false},
+		{1500 * time.Millisecond, false},
+		{2 * time.Second, true},
+	} {
+		got := b.MayAsk(now.Add(tc.after))
+		if got != tc.want {
+			t.Errorf("may ask %v after it opened: %v, want %v",
+				tc.after, got, tc.want)
+		}
+	}
+}
+
+func TestAFailedAskKeepsTheBreakerOpen(t *testing.T) {
+	t.Parallel()
+
+	opened := time.Now()
+	b := openBreaker(opened)
+	now := opened.Add(time.Second)
+	b.MayAsk(now)
+	b.NoteFailure(now)
+
+	if b.MayAsk(now.Add(999 * time.Millisecond)) {
+		t.Error("a request may ask, want it not to")
+	}
+
+	if !b.MayAsk(now.Add(time.Second)) {
+		t.Error("a request may not ask a second later, want it to")
+	}
+}
+
+func TestOneAnswerClosesTheBreaker(t *testing.T) {
+	t.Parallel()
+
+	opened := time.Now()
+	b := openBreaker(opened)
+	now := opened.Add(time.Second)
+	b.MayAsk(now)
+	b.NoteAnswer()
+
+	if !b.MayAsk(now) {
+		t.Error("a request may not ask, want it to")
+	}
+
+	// The failures before the answer no longer count, so one more does not open
+	// the breaker again.
+	b.NoteFailure(now)
+
+	if !b.MayAsk(now) {
+		t.Error("a request may not ask after one failure, want it to")
+	}
+}