check / check (push) Waiting to run
Under heavy host load the sww tests got 503 where they expected 200 or 403: waf refused requests whose trap paths, rule files and Core Rule Set took longer than 500 ms, and rproxy gave up on waf after 2 s. waf.Params and sww.Params gain BesideLookupTimeLimit; zero keeps the 500 ms limit, so the binary is unchanged. The sww tests' run gives waf an hour, and startRProxy sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS to 60000, as the rproxy package's tests do. waf's own tests set the limit through the new field instead of SetBesideLookupTimeLimit, which is gone. Model: opus-5-5
427 lines
11 KiB
Go
427 lines
11 KiB
Go
package waf_test
|
|
|
|
import (
|
|
"io"
|
|
"maps"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
|
)
|
|
|
|
// checksTimeLimit is how long the trap paths, the rule files and the Core Rule
|
|
// Set have for a request, as SPEC.md gives it, and checksTimedOut the status of
|
|
// a request they have not finished by then.
|
|
const (
|
|
checksTimeLimit = 500 * time.Millisecond
|
|
checksTimedOut = http.StatusServiceUnavailable
|
|
)
|
|
|
|
// heldCoreRuleSet is a stand-in for the Core Rule Set: each inspection takes
|
|
// took, then finds result. asked counts the inspections.
|
|
type heldCoreRuleSet struct {
|
|
took time.Duration
|
|
result coreruleset.Result
|
|
asked atomic.Int64
|
|
}
|
|
|
|
// Inspect inspects a request as the stand-in does, and reads all of body.
|
|
func (c *heldCoreRuleSet) Inspect(
|
|
_ *http.Request, _ netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error) {
|
|
c.asked.Add(1)
|
|
time.Sleep(c.took)
|
|
|
|
read, err := io.ReadAll(body)
|
|
|
|
return c.result, read, err
|
|
}
|
|
|
|
// coreRuleSetMatch returns what the stand-in finds of an attack: rule 942100,
|
|
// with a score of 5, the default SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
|
|
func coreRuleSetMatch() coreruleset.Result {
|
|
return coreruleset.Result{RuleIDs: []int{942100}, Score: 5}
|
|
}
|
|
|
|
// newWAFHoldingTheCoreRuleSet makes a waf, to be asked in a synctest bubble,
|
|
// with crs in place of the Core Rule Set, in block mode. GeoJS never answers,
|
|
// a setting needs its answer, alerts go to a webhook, and everything else is
|
|
// at its default but for env. No request to it may use the network.
|
|
func newWAFHoldingTheCoreRuleSet(
|
|
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
|
|
) (*waf.Server, *output, *alerts.Queue) {
|
|
t.Helper()
|
|
|
|
settings := map[string]string{
|
|
tagDeny: denyKP, coreRuleSetMode: block,
|
|
alertWebhookURL: alertWebhook,
|
|
}
|
|
maps.Copy(settings, env)
|
|
|
|
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
|
|
params.CoreRuleSet = crs
|
|
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
|
|
|
|
return waf.New(params)
|
|
}, silentGeoJSURL, time.Now, settings)
|
|
|
|
return server, out, queue
|
|
}
|
|
|
|
// lookupWait is SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS for wait.
|
|
func lookupWait(wait time.Duration) string {
|
|
return strconv.FormatInt(wait.Milliseconds(), 10)
|
|
}
|
|
|
|
// wantAnswer checks the status of an answer and how long it took.
|
|
func wantAnswer(
|
|
t *testing.T, status int, took time.Duration,
|
|
wantStatus int, wantTook time.Duration, out *output,
|
|
) {
|
|
t.Helper()
|
|
|
|
if status != wantStatus || took != wantTook {
|
|
t.Errorf(
|
|
"answered %d after %s, want %d after %s; output:\n%s",
|
|
status, took, wantStatus, wantTook, out.text(),
|
|
)
|
|
}
|
|
}
|
|
|
|
// wantNoCoreRuleSetTrace checks that what the Core Rule Set found of a request
|
|
// left no trace: line has none of its fields, and server has counted no match
|
|
// of it and raised no alert for one.
|
|
func wantNoCoreRuleSetTrace(
|
|
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
|
|
) {
|
|
t.Helper()
|
|
|
|
if line.DurationCoreRuleSetMS != nil || line.CoreRuleSetScore != nil ||
|
|
line.CoreRuleSetRuleIDs != nil {
|
|
t.Errorf(
|
|
"log line has Core Rule Set time %v, score %v and rule ids %v, want none",
|
|
line.DurationCoreRuleSetMS,
|
|
line.CoreRuleSetScore,
|
|
line.CoreRuleSetRuleIDs,
|
|
)
|
|
}
|
|
|
|
recorder := httptest.NewRecorder()
|
|
server.Metrics.ServeHTTP(
|
|
recorder,
|
|
httptest.NewRequestWithContext(
|
|
t.Context(),
|
|
http.MethodGet,
|
|
waf.MetricsPath,
|
|
http.NoBody,
|
|
),
|
|
)
|
|
|
|
if strings.Contains(
|
|
recorder.Body.String(),
|
|
"sww_core_rule_set_matches_total{",
|
|
) {
|
|
t.Errorf("a Core Rule Set match is counted")
|
|
}
|
|
|
|
for _, alert := range queue.Snapshot().Waiting[alerts.DestinationWebhook] {
|
|
if alert.Event == alerts.EventCoreRuleSetMatch {
|
|
t.Errorf("alert raised: %+v", alert)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheLookupAndTheCoreRuleSetTakeTheLongerOfTheirTimesNotTheirSum(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
inspection := checksTimeLimit - time.Millisecond
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
// wait is how long the request waits for GeoJS.
|
|
wait time.Duration
|
|
}{
|
|
{"the lookup waits longer", defaultLookupWait},
|
|
{"the inspection takes longer", inspection / 2},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: inspection}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(
|
|
t,
|
|
crs,
|
|
map[string]string{
|
|
asnAndCountryLookupTimeout: lookupWait(tc.wait),
|
|
},
|
|
)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
status, took := askDirectly(
|
|
t,
|
|
server,
|
|
func(*http.Request) {},
|
|
)
|
|
|
|
wantAnswer(t, status, took, http.StatusOK,
|
|
max(tc.wait, inspection), out)
|
|
wantLine(t, out.requestLine(t), http.StatusOK,
|
|
requestlog.ActionForward)
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
// firstToRefuse is a case of
|
|
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered: a request
|
|
// under env, whose inspection by the Core Rule Set takes inspection, refused as
|
|
// action after took.
|
|
type firstToRefuse struct {
|
|
name string
|
|
env map[string]string
|
|
inspection time.Duration
|
|
took time.Duration
|
|
action string
|
|
}
|
|
|
|
// firstToRefuseCases are the cases of
|
|
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered.
|
|
func firstToRefuseCases() []firstToRefuse {
|
|
return []firstToRefuse{
|
|
{
|
|
name: "a trap path, before GeoJS answers",
|
|
env: map[string]string{trapPaths: "/", allowedCountries: "DE"},
|
|
took: 0, action: requestlog.ActionTemporarilyBanned,
|
|
},
|
|
{
|
|
name: "the Core Rule Set, before GeoJS answers",
|
|
env: map[string]string{allowedCountries: "DE"},
|
|
inspection: 100 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionBlockedByCoreRuleSet,
|
|
},
|
|
{
|
|
name: "the country list, while the Core Rule Set inspects",
|
|
env: map[string]string{
|
|
allowedCountries: "DE", asnAndCountryLookupTimeout: "100",
|
|
},
|
|
inspection: 400 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionCountryDenied,
|
|
},
|
|
{
|
|
name: "the rate limits, while the Core Rule Set inspects",
|
|
env: map[string]string{
|
|
rateLimitPerMinute: "2", unknownCountryLimitPercent: "50",
|
|
asnAndCountryLookupTimeout: "100",
|
|
},
|
|
inspection: 400 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionClientIsRateLimited,
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
const forbidden = http.StatusForbidden
|
|
|
|
for _, tc := range firstToRefuseCases() {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{
|
|
took: tc.inspection,
|
|
result: coreRuleSetMatch(),
|
|
}
|
|
server, out, queue := newWAFHoldingTheCoreRuleSet(t, crs, tc.env)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
// Under the rate limits of the last case, the client's next
|
|
// request takes it over a limit only at the percentage of an
|
|
// unknown country, so it waits for GeoJS.
|
|
countClient(t, server, time.Now())
|
|
|
|
status, took := askDirectly(
|
|
t,
|
|
server,
|
|
func(*http.Request) {},
|
|
)
|
|
|
|
wantAnswer(t, status, took, forbidden, tc.took, out)
|
|
|
|
line := out.requestLine(t)
|
|
wantLine(t, line, forbidden, tc.action)
|
|
|
|
if tc.action == requestlog.ActionBlockedByCoreRuleSet {
|
|
return
|
|
}
|
|
|
|
if tc.action != requestlog.ActionClientIsRateLimited &&
|
|
line.Counts != (ratelimit.Counts{}) {
|
|
t.Errorf(
|
|
"the rate limits counted the request: %+v",
|
|
line.Counts,
|
|
)
|
|
}
|
|
|
|
// What the Core Rule Set finds once the request is answered is
|
|
// dropped.
|
|
time.Sleep(tc.inspection)
|
|
wantNoCoreRuleSetTrace(t, server, queue, line)
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestARequestRefusedBeforeTheLookupStartsNoInspection(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs, nil)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
banClient(t, server, time.Now())
|
|
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusForbidden, 0, out)
|
|
|
|
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
|
|
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
|
|
t.Errorf(
|
|
"%d inspections and %v requests to GeoJS, want none",
|
|
asked,
|
|
requests,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAClientInTheAllowNetsIsAllowedAtOnceAndNothingElseStarts(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: time.Second, result: coreRuleSetMatch()}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{allowNets: client, allowedCountries: "DE"})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusOK, 0, out)
|
|
|
|
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
|
|
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
|
|
t.Errorf(
|
|
"%d inspections and %v requests to GeoJS, want none",
|
|
asked,
|
|
requests,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestARequestTheCoreRuleSetHasNotFinishedWithIn500MillisecondsIsRefused(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{refusalsLimitPerMinute: "1"})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
// Were it counted toward the refusal limit, the second refusal would
|
|
// ban the client, and the third request would be refused for the ban.
|
|
for i := range 3 {
|
|
status, took := askDirectly(t, server, func(req *http.Request) {
|
|
req.Header.Set(requestIDHeader, "id-"+strconv.Itoa(i))
|
|
})
|
|
|
|
wantAnswer(t, status, took, checksTimedOut, checksTimeLimit, out)
|
|
|
|
line := out.requestLines(t, i+1)[i]
|
|
wantLine(
|
|
t,
|
|
line,
|
|
checksTimedOut,
|
|
requestlog.ActionChecksTimedOut,
|
|
)
|
|
wantHeuristic(t, line, "core_rule_set", `{"time_limit_ms": 500}`)
|
|
|
|
if line.Offence != "" || line.BanExpires != "" {
|
|
t.Errorf(
|
|
"log line has offence %q and ban_expires %q, want neither",
|
|
line.Offence,
|
|
line.BanExpires,
|
|
)
|
|
}
|
|
}
|
|
|
|
if bans := server.Ledger.Snapshot(); len(bans) != 0 {
|
|
t.Errorf("bans %+v, want none", bans)
|
|
}
|
|
|
|
offences := historyOf(t, server, client).Offences
|
|
if offences != (ratelimit.Offences{}) {
|
|
t.Errorf("history counts the offences %+v, want none", offences)
|
|
}
|
|
|
|
time.Sleep(crs.took)
|
|
})
|
|
}
|
|
|
|
func TestObserveModeAllowsARequestTheCoreRuleSetHasNotFinishedWith(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{mode: observe})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusOK, checksTimeLimit, out)
|
|
|
|
line := out.requestLine(t)
|
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
|
|
|
if line.WouldAction != requestlog.ActionChecksTimedOut {
|
|
t.Errorf(
|
|
"log line has would_action %q, want %q",
|
|
line.WouldAction,
|
|
requestlog.ActionChecksTimedOut,
|
|
)
|
|
}
|
|
|
|
time.Sleep(crs.took)
|
|
})
|
|
}
|