check / check (push) Failing after 3m52s
The worst-case inspection test compared each inspection with a fixed 1 second, so a busy host failed it at random. It now times, on the same thread and in processor time, a fixed regular-expression job just before and after each inspection. An inspection fails when it takes more than 13 times that job, about 1 second on an idle host. Model: opus-5-5
437 lines
12 KiB
Go
437 lines
12 KiB
Go
package coreruleset_test
|
|
|
|
import (
|
|
"maps"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"regexp"
|
|
"runtime"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
)
|
|
|
|
// The bounds on what a visitor can send, as SPEC.md gives them.
|
|
const (
|
|
// lineLimit is the longest request line or header line.
|
|
lineLimit = 8 << 10
|
|
// headerLimit is the most bytes rproxy reads of a request line and
|
|
// headers together.
|
|
headerLimit = 32 << 10
|
|
// mostCookies and mostHeaderFields are waf's limits on cookies and header
|
|
// fields.
|
|
mostCookies = 50
|
|
mostHeaderFields = 100
|
|
// mostBodyRead is the largest SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE.
|
|
mostBodyRead = 8 << 10
|
|
// mostJSONRead is the largest JSON body the Core Rule Set reads.
|
|
mostJSONRead = 1 << 10
|
|
)
|
|
|
|
// The Core Rule Set has 1 second for one request. That is the default of
|
|
// rproxy's SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, 2 seconds, less the default of
|
|
// SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, 800 milliseconds, and less 200
|
|
// milliseconds for waf's other checks.
|
|
//
|
|
// A busy host slows all work, so the test does not compare an inspection with
|
|
// 1 second. It compares it with referenceTime, timed on the same thread just
|
|
// before and after. inspectionTimeLimitMultiple is how many times
|
|
// referenceTime an inspection may take.
|
|
//
|
|
// On the host that runs our checks, referenceTime took at least 78
|
|
// milliseconds, so 13 times it is about 1 second. With that host fully busy,
|
|
// no try of the slowest request took more than 9.5 times referenceTime.
|
|
const inspectionTimeLimitMultiple = 13
|
|
|
|
// referencePattern is what referenceTime matches. Like the Core Rule Set's
|
|
// rules, it is a regular expression with alternatives and repeats.
|
|
var referencePattern = regexp.MustCompile(
|
|
`(?i)(union|select|insert|delete)\s+\w+\s+(from|into)\s+\w+[0-9]+;`,
|
|
)
|
|
|
|
// referenceTime returns the processor time the calling thread takes to do a
|
|
// fixed amount of work unrelated to the Core Rule Set, so that a change to the
|
|
// rules does not change it. referencePattern matches nowhere in the text, so
|
|
// each match reads all of it.
|
|
func referenceTime(t *testing.T) time.Duration {
|
|
t.Helper()
|
|
|
|
referenceText := text(4 << 10)
|
|
began := threadCPUTime(t)
|
|
|
|
for range 200 {
|
|
if referencePattern.MatchString(referenceText) {
|
|
t.Fatal("the reference pattern matched")
|
|
}
|
|
}
|
|
|
|
return threadCPUTime(t) - began
|
|
}
|
|
|
|
// text returns n bytes of letters, digits and spaces, which the Core Rule Set
|
|
// takes longer over than one letter repeated.
|
|
func text(n int) string {
|
|
const words = "select 0x2f from tbl_1 where name like abc_9 "
|
|
|
|
return strings.Repeat(words, n/len(words)+1)[:n]
|
|
}
|
|
|
|
// visitorRequest is a request a visitor can send, at the most of one or more
|
|
// of the bounds above.
|
|
type visitorRequest struct {
|
|
name string
|
|
target string
|
|
// header holds the headers besides Host.
|
|
header http.Header
|
|
// contentType is the body's type, "" for no body.
|
|
contentType string
|
|
body string
|
|
}
|
|
|
|
// longestTarget returns a target whose request line, "POST <target>
|
|
// HTTP/1.1", is lineLimit bytes long.
|
|
func longestTarget() string {
|
|
const around = len("POST /?q= HTTP/1.1")
|
|
|
|
return "/?q=" + strings.ReplaceAll(text(lineLimit-around), " ", "+")
|
|
}
|
|
|
|
// cookieLine returns a Cookie value whose line, "Cookie: <value>", is
|
|
// lineLimit bytes long, made of count cookies.
|
|
func cookieLine(count int) string {
|
|
const linePrefix = len("Cookie: ")
|
|
|
|
cookies := make([]string, count)
|
|
for i := range cookies {
|
|
cookies[i] = "c" + strconv.Itoa(i) + "="
|
|
}
|
|
|
|
// Each cookie is followed by "; ", but for the last.
|
|
room := lineLimit - linePrefix - len(strings.Join(cookies, "; "))
|
|
for i := range cookies {
|
|
share := room / count
|
|
if i == 0 {
|
|
share += room % count
|
|
}
|
|
|
|
cookies[i] += strings.ReplaceAll(text(share), " ", "_")
|
|
}
|
|
|
|
return strings.Join(cookies, "; ")
|
|
}
|
|
|
|
// headerValue returns a value whose line, "<name>: <value>", is n bytes long.
|
|
func headerValue(name string, n int) string {
|
|
return text(n - len(name) - len(": "))
|
|
}
|
|
|
|
// formFields returns a form body of mostBodyRead bytes holding count fields.
|
|
func formFields(count int) string {
|
|
fields := make([]string, count)
|
|
for i := range fields {
|
|
fields[i] = "f" + strconv.Itoa(i) + "="
|
|
}
|
|
|
|
room := mostBodyRead - len(strings.Join(fields, "&"))
|
|
for i := range fields {
|
|
share := room / count
|
|
if i == 0 {
|
|
share += room % count
|
|
}
|
|
|
|
fields[i] += strings.ReplaceAll(text(share), " ", "+")
|
|
}
|
|
|
|
return strings.Join(fields, "&")
|
|
}
|
|
|
|
// multipartBody returns a multipart body of mostBodyRead bytes, with the
|
|
// boundary "b", holding fields of about size bytes each.
|
|
func multipartBody(size int) string {
|
|
var body strings.Builder
|
|
|
|
const end = "--b--\r\n"
|
|
|
|
for i := 0; ; i++ {
|
|
head := "--b\r\nContent-Disposition: form-data; name=\"f" +
|
|
strconv.Itoa(i) + "\"\r\n\r\n"
|
|
|
|
room := mostBodyRead - body.Len() - len(
|
|
end,
|
|
) - len(
|
|
head,
|
|
) - len(
|
|
"\r\n",
|
|
)
|
|
if room <= 0 {
|
|
break
|
|
}
|
|
|
|
body.WriteString(head + text(min(size, room)) + "\r\n")
|
|
}
|
|
|
|
body.WriteString(end)
|
|
|
|
return body.String()
|
|
}
|
|
|
|
// jsonArrayOfMostElements returns a JSON array of at most mostJSONRead bytes
|
|
// holding as many elements as fit, each the number 1.
|
|
func jsonArrayOfMostElements() string {
|
|
return "[1" + strings.Repeat(
|
|
",1",
|
|
(mostJSONRead-len("[1]"))/len(",1"),
|
|
) + "]"
|
|
}
|
|
|
|
// jsonOfMostElementsUnderAKey returns a JSON body of mostJSONRead bytes
|
|
// holding as many elements as fit, each the number 1, in an array under a
|
|
// key of 6 bytes. Coraza names each element "json.<key>.<index>", so a
|
|
// longer key makes longer names. 6 bytes is the longest key that keeps the
|
|
// names and values of this body within the 8 KiB Coraza allows.
|
|
func jsonOfMostElementsUnderAKey() string {
|
|
head := `{"` + text(6) + `":[1`
|
|
|
|
return head + strings.Repeat(
|
|
",1",
|
|
(mostJSONRead-len(head+"]}"))/len(",1"),
|
|
) + "]}"
|
|
}
|
|
|
|
// The kinds of body the Core Rule Set reads, and the header that names a
|
|
// request's cookies.
|
|
const (
|
|
formType = "application/x-www-form-urlencoded"
|
|
jsonType = "application/json"
|
|
multipartType = "multipart/form-data; boundary=b"
|
|
cookieHeader = "Cookie"
|
|
)
|
|
|
|
// fullLine is the length of each of the request line and the three or four
|
|
// header lines that together fill rproxy's limit, with room for the line
|
|
// endings and the other headers.
|
|
const fullLine = lineLimit - 64
|
|
|
|
// worstCaseRequests are, for each kind of thing a visitor sends, the requests
|
|
// that keep the Core Rule Set busiest, within the bounds above. The last
|
|
// holds the most of everything at once: Cookie lines keep it busier than
|
|
// other header lines, and the JSON body of the most elements under a key
|
|
// keeps it busier than the other bodies.
|
|
func worstCaseRequests() []visitorRequest {
|
|
fullCookieLine := cookieLine(1)[:fullLine-len("Cookie: ")]
|
|
|
|
return append(
|
|
append(worstCaseHeaders(), worstCaseBodies()...),
|
|
visitorRequest{
|
|
name: "the longest request line, three Cookie lines and body",
|
|
target: longestTarget()[:fullLine-len("POST HTTP/1.1")],
|
|
header: http.Header{cookieHeader: {
|
|
fullCookieLine, fullCookieLine, fullCookieLine,
|
|
}},
|
|
contentType: jsonType, body: jsonOfMostElementsUnderAKey(),
|
|
},
|
|
)
|
|
}
|
|
|
|
// worstCaseHeaders are the requests of worstCaseRequests with no body.
|
|
func worstCaseHeaders() []visitorRequest {
|
|
headerFields := http.Header{}
|
|
|
|
for i := range mostHeaderFields - 1 {
|
|
name := "X-Field-" + strconv.Itoa(i)
|
|
headerFields.Set(
|
|
name,
|
|
headerValue(name, headerLimit/mostHeaderFields),
|
|
)
|
|
}
|
|
|
|
return []visitorRequest{
|
|
{name: "the longest request line", target: longestTarget()},
|
|
{
|
|
name: "the longest request line, as query parameters",
|
|
target: "/?" + strings.Repeat(
|
|
"a=1&",
|
|
(lineLimit-len("POST /? HTTP/1.1"))/len("a=1&"),
|
|
),
|
|
},
|
|
{
|
|
name: "the longest Cookie line, as one cookie",
|
|
target: "/",
|
|
header: http.Header{cookieHeader: {cookieLine(1)}},
|
|
},
|
|
{
|
|
name: "the longest Cookie line, as the most cookies",
|
|
target: "/",
|
|
header: http.Header{cookieHeader: {cookieLine(mostCookies)}},
|
|
},
|
|
{name: "the most header fields", target: "/", header: headerFields},
|
|
{
|
|
name: "the longest header lines",
|
|
target: "/",
|
|
header: http.Header{
|
|
"User-Agent": {headerValue("User-Agent", fullLine)},
|
|
"Referer": {headerValue("Referer", fullLine)},
|
|
"Accept": {headerValue("Accept", fullLine)},
|
|
"X-Long": {headerValue("X-Long", fullLine)},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// worstCaseBodies are the requests of worstCaseRequests whose body holds
|
|
// the most the Core Rule Set reads.
|
|
func worstCaseBodies() []visitorRequest {
|
|
const nested = (mostJSONRead - len(`{"a":1}`)) / len(`{"a":}`)
|
|
|
|
return []visitorRequest{
|
|
{
|
|
name: "a form body of one field", target: "/",
|
|
contentType: formType, body: formFields(1),
|
|
},
|
|
{
|
|
name: "a form body of the most fields Coraza keeps", target: "/",
|
|
contentType: formType, body: formFields(1000),
|
|
},
|
|
{
|
|
name: "a JSON body of one string", target: "/",
|
|
contentType: jsonType,
|
|
body: `{"a":"` + text(
|
|
mostJSONRead-len(`{"a":""}`),
|
|
) + `"}`,
|
|
},
|
|
{
|
|
name: "a JSON body of the most elements", target: "/",
|
|
contentType: jsonType, body: jsonArrayOfMostElements(),
|
|
},
|
|
{
|
|
name: "a JSON body of the most elements under a key", target: "/",
|
|
contentType: jsonType, body: jsonOfMostElementsUnderAKey(),
|
|
},
|
|
{
|
|
name: "a JSON body nested deep", target: "/",
|
|
contentType: jsonType,
|
|
body: strings.Repeat(`{"a":`, nested) + "1" +
|
|
strings.Repeat("}", nested),
|
|
},
|
|
{
|
|
name: "an XML body of many elements", target: "/",
|
|
contentType: "application/xml",
|
|
body: "<a>" + strings.Repeat(
|
|
"<b>x</b>",
|
|
(mostBodyRead-len("<a></a>"))/len("<b>x</b>"),
|
|
) + "</a>",
|
|
},
|
|
{
|
|
name: "a multipart body of one field", target: "/",
|
|
contentType: multipartType, body: multipartBody(mostBodyRead),
|
|
},
|
|
{
|
|
name: "a multipart body of many fields", target: "/",
|
|
contentType: multipartType, body: multipartBody(1),
|
|
},
|
|
}
|
|
}
|
|
|
|
// threadCPUTime returns the processor time the calling thread has used.
|
|
func threadCPUTime(t *testing.T) time.Duration {
|
|
t.Helper()
|
|
|
|
var now unix.Timespec
|
|
|
|
err := unix.ClockGettime(unix.CLOCK_THREAD_CPUTIME_ID, &now)
|
|
if err != nil {
|
|
t.Fatalf("read the thread's processor time: %v", err)
|
|
}
|
|
|
|
return time.Duration(now.Nano())
|
|
}
|
|
|
|
// inspectionTime returns the processor time crs takes to inspect tc, and the
|
|
// mean of referenceTime just before and after. It fails t unless crs reads all
|
|
// of tc's body and Coraza parses it.
|
|
//
|
|
// All of it runs on this goroutine, kept on one thread, so the thread's
|
|
// processor time measures it. Time other work holds the processor does not
|
|
// count.
|
|
func inspectionTime(
|
|
t *testing.T, crs *coreruleset.CoreRuleSet, tc visitorRequest,
|
|
) (time.Duration, time.Duration) {
|
|
t.Helper()
|
|
|
|
req := httptest.NewRequestWithContext(
|
|
t.Context(), http.MethodPost, tc.target, nil,
|
|
)
|
|
maps.Copy(req.Header, tc.header)
|
|
|
|
if tc.contentType != "" {
|
|
req.Header.Set("Content-Type", tc.contentType)
|
|
}
|
|
|
|
runtime.LockOSThread()
|
|
|
|
defer runtime.UnlockOSThread()
|
|
|
|
before := referenceTime(t)
|
|
|
|
began := threadCPUTime(t)
|
|
result, read, _ := crs.Inspect(
|
|
req, netip.MustParseAddr("192.0.2.1"), strings.NewReader(tc.body),
|
|
)
|
|
took := threadCPUTime(t) - began
|
|
|
|
reference := (before + referenceTime(t)) / 2
|
|
|
|
if string(read) != tc.body {
|
|
t.Errorf("read %d bytes of %d", len(read), len(tc.body))
|
|
}
|
|
|
|
// Rule 900440 matches a body Coraza could not parse.
|
|
if slices.Contains(result.RuleIDs, 900440) {
|
|
t.Errorf("Coraza could not parse the body")
|
|
}
|
|
|
|
return took, reference
|
|
}
|
|
|
|
// TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime runs one at a time, so
|
|
// that the tests running in parallel do not slow it. Other work on the host
|
|
// can still slow one try more than the reference work around it, so a request
|
|
// fails only when three tries in a row take too long.
|
|
//
|
|
//nolint:paralleltest // one at a time, as above
|
|
func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
|
|
const tries = 3
|
|
|
|
// The settings that make the Core Rule Set slowest: the highest paranoia
|
|
// level, no rule switched off, and the most of a body read.
|
|
crs := shared(t, coreruleset.Params{
|
|
ParanoiaLevel: 4, BodyInspectionSize: mostBodyRead,
|
|
})
|
|
|
|
for _, tc := range worstCaseRequests() {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
var took, reference time.Duration
|
|
|
|
for range tries {
|
|
took, reference = inspectionTime(t, crs, tc)
|
|
if took <= inspectionTimeLimitMultiple*reference {
|
|
return
|
|
}
|
|
}
|
|
|
|
t.Errorf(
|
|
"took %s on the last of %d tries, over %d times the %s "+
|
|
"of the reference work",
|
|
took, tries, inspectionTimeLimitMultiple, reference,
|
|
)
|
|
})
|
|
}
|
|
}
|