check / check (push) Failing after 1m51s
While the lookup waits for GeoJS, the trap paths, the rule files and the Core Rule Set run at the same time as the checks after the lookup. The first side to refuse answers the request; what the other side finds leaves no trace. Those three checks have 500 ms together: past it, the request is refused with 503 as checks_timed_out, with no ban and no offence. Tests hold GeoJS and the Core Rule Set with stand-ins under synctest. The test helper lifts the 500 ms limit for tests that run the real Core Rule Set, which the race detector slows past it. Model: opus-5-5
427 lines
11 KiB
Go
427 lines
11 KiB
Go
package waf_test
|
|
|
|
import (
|
|
"io"
|
|
"maps"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
|
)
|
|
|
|
// checksTimeLimit is how long the trap paths, the rule files and the Core Rule
|
|
// Set have for a request, as SPEC.md gives it, and checksTimedOut the status of
|
|
// a request they have not finished by then.
|
|
const (
|
|
checksTimeLimit = 500 * time.Millisecond
|
|
checksTimedOut = http.StatusServiceUnavailable
|
|
)
|
|
|
|
// heldCoreRuleSet is a stand-in for the Core Rule Set: each inspection takes
|
|
// took, then finds result. asked counts the inspections.
|
|
type heldCoreRuleSet struct {
|
|
took time.Duration
|
|
result coreruleset.Result
|
|
asked atomic.Int64
|
|
}
|
|
|
|
// Inspect inspects a request as the stand-in does, and reads all of body.
|
|
func (c *heldCoreRuleSet) Inspect(
|
|
_ *http.Request, _ netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error) {
|
|
c.asked.Add(1)
|
|
time.Sleep(c.took)
|
|
|
|
read, err := io.ReadAll(body)
|
|
|
|
return c.result, read, err
|
|
}
|
|
|
|
// coreRuleSetMatch returns what the stand-in finds of an attack: rule 942100,
|
|
// with a score of 5, the default SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
|
|
func coreRuleSetMatch() coreruleset.Result {
|
|
return coreruleset.Result{RuleIDs: []int{942100}, Score: 5}
|
|
}
|
|
|
|
// newWAFHoldingTheCoreRuleSet makes a waf, to be asked in a synctest bubble,
|
|
// with crs in place of the Core Rule Set, in block mode. GeoJS never answers,
|
|
// a setting needs its answer, alerts go to a webhook, and everything else is
|
|
// at its default but for env. No request to it may use the network.
|
|
func newWAFHoldingTheCoreRuleSet(
|
|
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
|
|
) (*waf.Server, *output, *alerts.Queue) {
|
|
t.Helper()
|
|
|
|
settings := map[string]string{
|
|
tagDeny: denyKP, coreRuleSetMode: block,
|
|
alertWebhookURL: alertWebhook,
|
|
}
|
|
maps.Copy(settings, env)
|
|
|
|
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
|
|
params.CoreRuleSet = crs
|
|
|
|
return waf.New(params)
|
|
}, silentGeoJSURL, time.Now, settings)
|
|
server.SetBesideLookupTimeLimit(waf.BesideLookupTimeLimit)
|
|
|
|
return server, out, queue
|
|
}
|
|
|
|
// lookupWait is SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS for wait.
|
|
func lookupWait(wait time.Duration) string {
|
|
return strconv.FormatInt(wait.Milliseconds(), 10)
|
|
}
|
|
|
|
// wantAnswer checks the status of an answer and how long it took.
|
|
func wantAnswer(
|
|
t *testing.T, status int, took time.Duration,
|
|
wantStatus int, wantTook time.Duration, out *output,
|
|
) {
|
|
t.Helper()
|
|
|
|
if status != wantStatus || took != wantTook {
|
|
t.Errorf(
|
|
"answered %d after %s, want %d after %s; output:\n%s",
|
|
status, took, wantStatus, wantTook, out.text(),
|
|
)
|
|
}
|
|
}
|
|
|
|
// wantNoCoreRuleSetTrace checks that what the Core Rule Set found of a request
|
|
// left no trace: line has none of its fields, and server has counted no match
|
|
// of it and raised no alert for one.
|
|
func wantNoCoreRuleSetTrace(
|
|
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
|
|
) {
|
|
t.Helper()
|
|
|
|
if line.DurationCoreRuleSetMS != nil || line.CoreRuleSetScore != nil ||
|
|
line.CoreRuleSetRuleIDs != nil {
|
|
t.Errorf(
|
|
"log line has Core Rule Set time %v, score %v and rule ids %v, want none",
|
|
line.DurationCoreRuleSetMS,
|
|
line.CoreRuleSetScore,
|
|
line.CoreRuleSetRuleIDs,
|
|
)
|
|
}
|
|
|
|
recorder := httptest.NewRecorder()
|
|
server.Metrics.ServeHTTP(
|
|
recorder,
|
|
httptest.NewRequestWithContext(
|
|
t.Context(),
|
|
http.MethodGet,
|
|
waf.MetricsPath,
|
|
http.NoBody,
|
|
),
|
|
)
|
|
|
|
if strings.Contains(
|
|
recorder.Body.String(),
|
|
"sww_core_rule_set_matches_total{",
|
|
) {
|
|
t.Errorf("a Core Rule Set match is counted")
|
|
}
|
|
|
|
for _, alert := range queue.Snapshot().Waiting[alerts.DestinationWebhook] {
|
|
if alert.Event == alerts.EventCoreRuleSetMatch {
|
|
t.Errorf("alert raised: %+v", alert)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheLookupAndTheCoreRuleSetTakeTheLongerOfTheirTimesNotTheirSum(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
inspection := checksTimeLimit - time.Millisecond
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
// wait is how long the request waits for GeoJS.
|
|
wait time.Duration
|
|
}{
|
|
{"the lookup waits longer", defaultLookupWait},
|
|
{"the inspection takes longer", inspection / 2},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: inspection}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(
|
|
t,
|
|
crs,
|
|
map[string]string{
|
|
asnAndCountryLookupTimeout: lookupWait(tc.wait),
|
|
},
|
|
)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
status, took := askDirectly(
|
|
t,
|
|
server,
|
|
func(*http.Request) {},
|
|
)
|
|
|
|
wantAnswer(t, status, took, http.StatusOK,
|
|
max(tc.wait, inspection), out)
|
|
wantLine(t, out.requestLine(t), http.StatusOK,
|
|
requestlog.ActionForward)
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
// firstToRefuse is a case of
|
|
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered: a request
|
|
// under env, whose inspection by the Core Rule Set takes inspection, refused as
|
|
// action after took.
|
|
type firstToRefuse struct {
|
|
name string
|
|
env map[string]string
|
|
inspection time.Duration
|
|
took time.Duration
|
|
action string
|
|
}
|
|
|
|
// firstToRefuseCases are the cases of
|
|
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered.
|
|
func firstToRefuseCases() []firstToRefuse {
|
|
return []firstToRefuse{
|
|
{
|
|
name: "a trap path, before GeoJS answers",
|
|
env: map[string]string{trapPaths: "/", allowedCountries: "DE"},
|
|
took: 0, action: requestlog.ActionTemporarilyBanned,
|
|
},
|
|
{
|
|
name: "the Core Rule Set, before GeoJS answers",
|
|
env: map[string]string{allowedCountries: "DE"},
|
|
inspection: 100 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionBlockedByCoreRuleSet,
|
|
},
|
|
{
|
|
name: "the country list, while the Core Rule Set inspects",
|
|
env: map[string]string{
|
|
allowedCountries: "DE", asnAndCountryLookupTimeout: "100",
|
|
},
|
|
inspection: 400 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionCountryDenied,
|
|
},
|
|
{
|
|
name: "the rate limits, while the Core Rule Set inspects",
|
|
env: map[string]string{
|
|
rateLimitPerMinute: "2", unknownCountryLimitPercent: "50",
|
|
asnAndCountryLookupTimeout: "100",
|
|
},
|
|
inspection: 400 * time.Millisecond,
|
|
took: 100 * time.Millisecond, action: requestlog.ActionClientIsRateLimited,
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
const forbidden = http.StatusForbidden
|
|
|
|
for _, tc := range firstToRefuseCases() {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{
|
|
took: tc.inspection,
|
|
result: coreRuleSetMatch(),
|
|
}
|
|
server, out, queue := newWAFHoldingTheCoreRuleSet(t, crs, tc.env)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
// Under the rate limits of the last case, the client's next
|
|
// request takes it over a limit only at the percentage of an
|
|
// unknown country, so it waits for GeoJS.
|
|
countClient(t, server, time.Now())
|
|
|
|
status, took := askDirectly(
|
|
t,
|
|
server,
|
|
func(*http.Request) {},
|
|
)
|
|
|
|
wantAnswer(t, status, took, forbidden, tc.took, out)
|
|
|
|
line := out.requestLine(t)
|
|
wantLine(t, line, forbidden, tc.action)
|
|
|
|
if tc.action == requestlog.ActionBlockedByCoreRuleSet {
|
|
return
|
|
}
|
|
|
|
if tc.action != requestlog.ActionClientIsRateLimited &&
|
|
line.Counts != (ratelimit.Counts{}) {
|
|
t.Errorf(
|
|
"the rate limits counted the request: %+v",
|
|
line.Counts,
|
|
)
|
|
}
|
|
|
|
// What the Core Rule Set finds once the request is answered is
|
|
// dropped.
|
|
time.Sleep(tc.inspection)
|
|
wantNoCoreRuleSetTrace(t, server, queue, line)
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestARequestRefusedBeforeTheLookupStartsNoInspection(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs, nil)
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
banClient(t, server, time.Now())
|
|
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusForbidden, 0, out)
|
|
|
|
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
|
|
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
|
|
t.Errorf(
|
|
"%d inspections and %v requests to GeoJS, want none",
|
|
asked,
|
|
requests,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAClientInTheAllowNetsIsAllowedAtOnceAndNothingElseStarts(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: time.Second, result: coreRuleSetMatch()}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{allowNets: client, allowedCountries: "DE"})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusOK, 0, out)
|
|
|
|
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
|
|
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
|
|
t.Errorf(
|
|
"%d inspections and %v requests to GeoJS, want none",
|
|
asked,
|
|
requests,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestARequestTheCoreRuleSetHasNotFinishedWithIn500MillisecondsIsRefused(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{refusalsLimitPerMinute: "1"})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
// Were it counted toward the refusal limit, the second refusal would
|
|
// ban the client, and the third request would be refused for the ban.
|
|
for i := range 3 {
|
|
status, took := askDirectly(t, server, func(req *http.Request) {
|
|
req.Header.Set(requestIDHeader, "id-"+strconv.Itoa(i))
|
|
})
|
|
|
|
wantAnswer(t, status, took, checksTimedOut, checksTimeLimit, out)
|
|
|
|
line := out.requestLines(t, i+1)[i]
|
|
wantLine(
|
|
t,
|
|
line,
|
|
checksTimedOut,
|
|
requestlog.ActionChecksTimedOut,
|
|
)
|
|
wantHeuristic(t, line, "core_rule_set", `{"time_limit_ms": 500}`)
|
|
|
|
if line.Offence != "" || line.BanExpires != "" {
|
|
t.Errorf(
|
|
"log line has offence %q and ban_expires %q, want neither",
|
|
line.Offence,
|
|
line.BanExpires,
|
|
)
|
|
}
|
|
}
|
|
|
|
if bans := server.Ledger.Snapshot(); len(bans) != 0 {
|
|
t.Errorf("bans %+v, want none", bans)
|
|
}
|
|
|
|
offences := historyOf(t, server, client).Offences
|
|
if offences != (ratelimit.Offences{}) {
|
|
t.Errorf("history counts the offences %+v, want none", offences)
|
|
}
|
|
|
|
time.Sleep(crs.took)
|
|
})
|
|
}
|
|
|
|
func TestObserveModeAllowsARequestTheCoreRuleSetHasNotFinishedWith(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
|
|
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
|
|
map[string]string{mode: observe})
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
defer waitForGeoJSRequestsToBeAbandoned()
|
|
|
|
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
|
|
wantAnswer(t, status, took, http.StatusOK, checksTimeLimit, out)
|
|
|
|
line := out.requestLine(t)
|
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
|
|
|
if line.WouldAction != requestlog.ActionChecksTimedOut {
|
|
t.Errorf(
|
|
"log line has would_action %q, want %q",
|
|
line.WouldAction,
|
|
requestlog.ActionChecksTimedOut,
|
|
)
|
|
}
|
|
|
|
time.Sleep(crs.took)
|
|
})
|
|
}
|