Files
smallwebwaf/internal/waf/besidelookup_test.go
T
clawbot 334a082a42
check / check (push) Waiting to run
sww: lift waf's check time limit and rproxy's wait in tests (closes #418)
Under heavy host load the sww tests got 503 where they expected 200 or
403: waf refused requests whose trap paths, rule files and Core Rule Set
took longer than 500 ms, and rproxy gave up on waf after 2 s.

waf.Params and sww.Params gain BesideLookupTimeLimit; zero keeps the
500 ms limit, so the binary is unchanged. The sww tests' run gives waf
an hour, and startRProxy sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS to
60000, as the rproxy package's tests do. waf's own tests set the limit
through the new field instead of SetBesideLookupTimeLimit, which is gone.

Model: opus-5-5
2026-11-19 08:48:15 +00:00

427 lines
11 KiB
Go

package waf_test
import (
"io"
"maps"
"net/http"
"net/http/httptest"
"net/netip"
"strconv"
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// checksTimeLimit is how long the trap paths, the rule files and the Core Rule
// Set have for a request, as SPEC.md gives it, and checksTimedOut the status of
// a request they have not finished by then.
const (
checksTimeLimit = 500 * time.Millisecond
checksTimedOut = http.StatusServiceUnavailable
)
// heldCoreRuleSet is a stand-in for the Core Rule Set: each inspection takes
// took, then finds result. asked counts the inspections.
type heldCoreRuleSet struct {
took time.Duration
result coreruleset.Result
asked atomic.Int64
}
// Inspect inspects a request as the stand-in does, and reads all of body.
func (c *heldCoreRuleSet) Inspect(
_ *http.Request, _ netip.Addr, body io.Reader,
) (coreruleset.Result, []byte, error) {
c.asked.Add(1)
time.Sleep(c.took)
read, err := io.ReadAll(body)
return c.result, read, err
}
// coreRuleSetMatch returns what the stand-in finds of an attack: rule 942100,
// with a score of 5, the default SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
func coreRuleSetMatch() coreruleset.Result {
return coreruleset.Result{RuleIDs: []int{942100}, Score: 5}
}
// newWAFHoldingTheCoreRuleSet makes a waf, to be asked in a synctest bubble,
// with crs in place of the Core Rule Set, in block mode. GeoJS never answers,
// a setting needs its answer, alerts go to a webhook, and everything else is
// at its default but for env. No request to it may use the network.
func newWAFHoldingTheCoreRuleSet(
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
settings := map[string]string{
tagDeny: denyKP, coreRuleSetMode: block,
alertWebhookURL: alertWebhook,
}
maps.Copy(settings, env)
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
params.CoreRuleSet = crs
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
return waf.New(params)
}, silentGeoJSURL, time.Now, settings)
return server, out, queue
}
// lookupWait is SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS for wait.
func lookupWait(wait time.Duration) string {
return strconv.FormatInt(wait.Milliseconds(), 10)
}
// wantAnswer checks the status of an answer and how long it took.
func wantAnswer(
t *testing.T, status int, took time.Duration,
wantStatus int, wantTook time.Duration, out *output,
) {
t.Helper()
if status != wantStatus || took != wantTook {
t.Errorf(
"answered %d after %s, want %d after %s; output:\n%s",
status, took, wantStatus, wantTook, out.text(),
)
}
}
// wantNoCoreRuleSetTrace checks that what the Core Rule Set found of a request
// left no trace: line has none of its fields, and server has counted no match
// of it and raised no alert for one.
func wantNoCoreRuleSetTrace(
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
) {
t.Helper()
if line.DurationCoreRuleSetMS != nil || line.CoreRuleSetScore != nil ||
line.CoreRuleSetRuleIDs != nil {
t.Errorf(
"log line has Core Rule Set time %v, score %v and rule ids %v, want none",
line.DurationCoreRuleSetMS,
line.CoreRuleSetScore,
line.CoreRuleSetRuleIDs,
)
}
recorder := httptest.NewRecorder()
server.Metrics.ServeHTTP(
recorder,
httptest.NewRequestWithContext(
t.Context(),
http.MethodGet,
waf.MetricsPath,
http.NoBody,
),
)
if strings.Contains(
recorder.Body.String(),
"sww_core_rule_set_matches_total{",
) {
t.Errorf("a Core Rule Set match is counted")
}
for _, alert := range queue.Snapshot().Waiting[alerts.DestinationWebhook] {
if alert.Event == alerts.EventCoreRuleSetMatch {
t.Errorf("alert raised: %+v", alert)
}
}
}
func TestTheLookupAndTheCoreRuleSetTakeTheLongerOfTheirTimesNotTheirSum(
t *testing.T,
) {
t.Parallel()
inspection := checksTimeLimit - time.Millisecond
for _, tc := range []struct {
name string
// wait is how long the request waits for GeoJS.
wait time.Duration
}{
{"the lookup waits longer", defaultLookupWait},
{"the inspection takes longer", inspection / 2},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{took: inspection}
server, out, _ := newWAFHoldingTheCoreRuleSet(
t,
crs,
map[string]string{
asnAndCountryLookupTimeout: lookupWait(tc.wait),
},
)
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
status, took := askDirectly(
t,
server,
func(*http.Request) {},
)
wantAnswer(t, status, took, http.StatusOK,
max(tc.wait, inspection), out)
wantLine(t, out.requestLine(t), http.StatusOK,
requestlog.ActionForward)
})
})
}
}
// firstToRefuse is a case of
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered: a request
// under env, whose inspection by the Core Rule Set takes inspection, refused as
// action after took.
type firstToRefuse struct {
name string
env map[string]string
inspection time.Duration
took time.Duration
action string
}
// firstToRefuseCases are the cases of
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered.
func firstToRefuseCases() []firstToRefuse {
return []firstToRefuse{
{
name: "a trap path, before GeoJS answers",
env: map[string]string{trapPaths: "/", allowedCountries: "DE"},
took: 0, action: requestlog.ActionTemporarilyBanned,
},
{
name: "the Core Rule Set, before GeoJS answers",
env: map[string]string{allowedCountries: "DE"},
inspection: 100 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionBlockedByCoreRuleSet,
},
{
name: "the country list, while the Core Rule Set inspects",
env: map[string]string{
allowedCountries: "DE", asnAndCountryLookupTimeout: "100",
},
inspection: 400 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionCountryDenied,
},
{
name: "the rate limits, while the Core Rule Set inspects",
env: map[string]string{
rateLimitPerMinute: "2", unknownCountryLimitPercent: "50",
asnAndCountryLookupTimeout: "100",
},
inspection: 400 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionClientIsRateLimited,
},
}
}
func TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered(
t *testing.T,
) {
t.Parallel()
const forbidden = http.StatusForbidden
for _, tc := range firstToRefuseCases() {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{
took: tc.inspection,
result: coreRuleSetMatch(),
}
server, out, queue := newWAFHoldingTheCoreRuleSet(t, crs, tc.env)
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Under the rate limits of the last case, the client's next
// request takes it over a limit only at the percentage of an
// unknown country, so it waits for GeoJS.
countClient(t, server, time.Now())
status, took := askDirectly(
t,
server,
func(*http.Request) {},
)
wantAnswer(t, status, took, forbidden, tc.took, out)
line := out.requestLine(t)
wantLine(t, line, forbidden, tc.action)
if tc.action == requestlog.ActionBlockedByCoreRuleSet {
return
}
if tc.action != requestlog.ActionClientIsRateLimited &&
line.Counts != (ratelimit.Counts{}) {
t.Errorf(
"the rate limits counted the request: %+v",
line.Counts,
)
}
// What the Core Rule Set finds once the request is answered is
// dropped.
time.Sleep(tc.inspection)
wantNoCoreRuleSetTrace(t, server, queue, line)
})
})
}
}
func TestARequestRefusedBeforeTheLookupStartsNoInspection(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{took: time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs, nil)
synctest.Test(t, func(t *testing.T) {
banClient(t, server, time.Now())
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusForbidden, 0, out)
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
t.Errorf(
"%d inspections and %v requests to GeoJS, want none",
asked,
requests,
)
}
})
}
func TestAClientInTheAllowNetsIsAllowedAtOnceAndNothingElseStarts(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: time.Second, result: coreRuleSetMatch()}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{allowNets: client, allowedCountries: "DE"})
synctest.Test(t, func(t *testing.T) {
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusOK, 0, out)
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
t.Errorf(
"%d inspections and %v requests to GeoJS, want none",
asked,
requests,
)
}
})
}
func TestARequestTheCoreRuleSetHasNotFinishedWithIn500MillisecondsIsRefused(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{refusalsLimitPerMinute: "1"})
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Were it counted toward the refusal limit, the second refusal would
// ban the client, and the third request would be refused for the ban.
for i := range 3 {
status, took := askDirectly(t, server, func(req *http.Request) {
req.Header.Set(requestIDHeader, "id-"+strconv.Itoa(i))
})
wantAnswer(t, status, took, checksTimedOut, checksTimeLimit, out)
line := out.requestLines(t, i+1)[i]
wantLine(
t,
line,
checksTimedOut,
requestlog.ActionChecksTimedOut,
)
wantHeuristic(t, line, "core_rule_set", `{"time_limit_ms": 500}`)
if line.Offence != "" || line.BanExpires != "" {
t.Errorf(
"log line has offence %q and ban_expires %q, want neither",
line.Offence,
line.BanExpires,
)
}
}
if bans := server.Ledger.Snapshot(); len(bans) != 0 {
t.Errorf("bans %+v, want none", bans)
}
offences := historyOf(t, server, client).Offences
if offences != (ratelimit.Offences{}) {
t.Errorf("history counts the offences %+v, want none", offences)
}
time.Sleep(crs.took)
})
}
func TestObserveModeAllowsARequestTheCoreRuleSetHasNotFinishedWith(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{mode: observe})
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusOK, checksTimeLimit, out)
line := out.requestLine(t)
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
if line.WouldAction != requestlog.ActionChecksTimedOut {
t.Errorf(
"log line has would_action %q, want %q",
line.WouldAction,
requestlog.ActionChecksTimedOut,
)
}
time.Sleep(crs.took)
})
}