Compare commits

..
Author SHA1 Message Date
sneak 0e8472fc96 coreruleset: time worst-case inspections against reference work (closes #400)
check / check (push) Failing after 3m52s
The worst-case inspection test compared each inspection with a fixed 1
second, so a busy host failed it at random. It now times, on the same
thread and in processor time, a fixed regular-expression job just
before and after each inspection. An inspection fails when it takes
more than 13 times that job, about 1 second on an idle host.

Model: opus-5-5
2026-11-16 08:39:01 +00:00
clawbot 3c6b0d519f rproxy: breaker test no longer needs each failed ask to reach waf (closes #398)
check / check (push) Failing after 2m12s
TestFiveFailedAsksKeepTheNextRequestFromAskingWAF waited for a question
from each of the five failing asks. Under load an ask could time out
before the stand-in for waf saw it, and the test failed with "no
question". It now waits only for each visitor's answer.

Before the sixth request, it checks that rproxy has not written that the
circuit breaker is open, so none of the first five was kept from asking
waf. It checks that the sixth request did not ask by its request ID,
which the app records, so late questions from the failed asks cannot be
mistaken for it.

Model: opus-5-5
2026-11-16 08:58:18 +01:00
2 changed files with 91 additions and 26 deletions

No files matched your search

+63 -22
View File
@@ -5,6 +5,7 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/netip" "net/netip"
"regexp"
"runtime" "runtime"
"slices" "slices"
"strconv" "strconv"
@@ -34,12 +35,45 @@ const (
mostJSONRead = 1 << 10 mostJSONRead = 1 << 10
) )
// inspectionTimeLimit is the most time the Core Rule Set may take on one // The Core Rule Set has 1 second for one request. That is the default of
// request. It is the default of rproxy's SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, 2 // rproxy's SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, 2 seconds, less the default of
// seconds, less the default of SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, 800 // SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, 800 milliseconds, and less 200
// milliseconds, which a question can also wait for GeoJS, and less 200
// milliseconds for waf's other checks. // milliseconds for waf's other checks.
const inspectionTimeLimit = time.Second //
// A busy host slows all work, so the test does not compare an inspection with
// 1 second. It compares it with referenceTime, timed on the same thread just
// before and after. inspectionTimeLimitMultiple is how many times
// referenceTime an inspection may take.
//
// On the host that runs our checks, referenceTime took at least 78
// milliseconds, so 13 times it is about 1 second. With that host fully busy,
// no try of the slowest request took more than 9.5 times referenceTime.
const inspectionTimeLimitMultiple = 13
// referencePattern is what referenceTime matches. Like the Core Rule Set's
// rules, it is a regular expression with alternatives and repeats.
var referencePattern = regexp.MustCompile(
`(?i)(union|select|insert|delete)\s+\w+\s+(from|into)\s+\w+[0-9]+;`,
)
// referenceTime returns the processor time the calling thread takes to do a
// fixed amount of work unrelated to the Core Rule Set, so that a change to the
// rules does not change it. referencePattern matches nowhere in the text, so
// each match reads all of it.
func referenceTime(t *testing.T) time.Duration {
t.Helper()
referenceText := text(4 << 10)
began := threadCPUTime(t)
for range 200 {
if referencePattern.MatchString(referenceText) {
t.Fatal("the reference pattern matched")
}
}
return threadCPUTime(t) - began
}
// text returns n bytes of letters, digits and spaces, which the Core Rule Set // text returns n bytes of letters, digits and spaces, which the Core Rule Set
// takes longer over than one letter repeated. // takes longer over than one letter repeated.
@@ -319,13 +353,16 @@ func threadCPUTime(t *testing.T) time.Duration {
return time.Duration(now.Nano()) return time.Duration(now.Nano())
} }
// inspectionTime returns the processor time crs takes to inspect tc, and // inspectionTime returns the processor time crs takes to inspect tc, and the
// fails t unless it reads all of tc's body and Coraza parses it. Inspect does // mean of referenceTime just before and after. It fails t unless crs reads all
// its work on the calling goroutine, which stays on one thread meanwhile, so // of tc's body and Coraza parses it.
// that thread's processor time is the inspection's. //
// All of it runs on this goroutine, kept on one thread, so the thread's
// processor time measures it. Time other work holds the processor does not
// count.
func inspectionTime( func inspectionTime(
t *testing.T, crs *coreruleset.CoreRuleSet, tc visitorRequest, t *testing.T, crs *coreruleset.CoreRuleSet, tc visitorRequest,
) time.Duration { ) (time.Duration, time.Duration) {
t.Helper() t.Helper()
req := httptest.NewRequestWithContext( req := httptest.NewRequestWithContext(
@@ -338,14 +375,19 @@ func inspectionTime(
} }
runtime.LockOSThread() runtime.LockOSThread()
defer runtime.UnlockOSThread() defer runtime.UnlockOSThread()
before := referenceTime(t)
began := threadCPUTime(t) began := threadCPUTime(t)
result, read, _ := crs.Inspect( result, read, _ := crs.Inspect(
req, netip.MustParseAddr("192.0.2.1"), strings.NewReader(tc.body), req, netip.MustParseAddr("192.0.2.1"), strings.NewReader(tc.body),
) )
took := threadCPUTime(t) - began took := threadCPUTime(t) - began
reference := (before + referenceTime(t)) / 2
if string(read) != tc.body { if string(read) != tc.body {
t.Errorf("read %d bytes of %d", len(read), len(tc.body)) t.Errorf("read %d bytes of %d", len(read), len(tc.body))
} }
@@ -355,15 +397,13 @@ func inspectionTime(
t.Errorf("Coraza could not parse the body") t.Errorf("Coraza could not parse the body")
} }
return took return took, reference
} }
// TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime measures processor // TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime runs one at a time, so
// time, not time on the clock, so that the time other work on the host holds // that the tests running in parallel do not slow it. Other work on the host
// the processor does not count. It runs one at a time, so that the tests // can still slow one try more than the reference work around it, so a request
// running in parallel do not slow it. Other work can still slow one try a // fails only when three tries in a row take too long.
// little, by sharing the processor's caches, so a request fails only when
// three tries in a row take too long.
// //
//nolint:paralleltest // one at a time, as above //nolint:paralleltest // one at a time, as above
func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) { func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
@@ -377,18 +417,19 @@ func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
for _, tc := range worstCaseRequests() { for _, tc := range worstCaseRequests() {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
var took time.Duration var took, reference time.Duration
for range tries { for range tries {
took = inspectionTime(t, crs, tc) took, reference = inspectionTime(t, crs, tc)
if took <= inspectionTimeLimit { if took <= inspectionTimeLimitMultiple*reference {
return return
} }
} }
t.Errorf( t.Errorf(
"took %s on the last of %d tries, over %s", "took %s on the last of %d tries, over %d times the %s "+
took, tries, inspectionTimeLimit, "of the reference work",
took, tries, inspectionTimeLimitMultiple, reference,
) )
}) })
} }
+28 -4
View File
@@ -1424,7 +1424,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
failure(w, r) failure(w, r)
}, },
) )
app := startApp(t, greet) nextID := make(chan string, 1)
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
if answering.Load() {
nextID <- r.Header.Get(requestIDHeader)
}
greet(w, r)
})
addr, out := startRProxy( addr, out := startRProxy(
t, t,
standIn.URL, standIn.URL,
@@ -1434,19 +1441,36 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
}, },
) )
// An ask can time out before it reaches the stand-in, so these
// may leave fewer than five questions, or leave some late.
for range 5 { for range 5 {
wantGreeting(t, get(t, addr, "/")) wantGreeting(t, get(t, addr, "/"))
standIn.question(t) }
// rproxy writes this line before it answers a request it kept
// from asking waf, so no wait is needed.
if out.has("the circuit breaker is open") {
t.Error("the circuit breaker opened before five asks failed")
} }
answering.Store(true) answering.Store(true)
wantGreeting(t, get(t, addr, "/")) wantGreeting(t, get(t, addr, "/"))
out.waitFor(t, "the circuit breaker is open") out.waitFor(t, "the circuit breaker is open")
// Had the next request asked waf, its question would have reached
// the stand-in before the request reached the app.
var id string
select { select {
case <-standIn.questions: case id = <-nextID:
default:
t.Fatal("the app did not get the next request")
}
for len(standIn.questions) > 0 {
asked := <-standIn.questions
if asked.header.Get(requestIDHeader) == id {
t.Error("waf was asked, want it not to be") t.Error("waf was asked, want it not to be")
case <-time.After(10 * pollInterval): }
} }
}) })
} }