Name log fields, metrics and the Core Rule Set package for what they are (closes #192)
check / check (push) Successful in 3m40s

Log field, metric, action and event names now say what they count
and, for timings, the unit. Package internal/waf is internal/coreruleset.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-09 06:46:24 +00:00
1 parent aede434f6a
commit 92ed102595
52 files changed
+654 -607

No files matched your search

+2 -2
View File
@@ -42,7 +42,7 @@ var (
// answerHealth answers a request for the health endpoint.
func (rq *request) answerHealth() {
rq.line.Action = requestlog.ActionAdmin
rq.line.Action = requestlog.ActionOwnEndpoint
// Set here rather than left to Go's server, which would set it only
// after the log line has taken the response's headers.
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
@@ -55,7 +55,7 @@ func (rq *request) answerHealth() {
// refused with 401. An endpoint whose token is unset answers 404, as any other
// path does.
func (rq *request) answerAdmin() {
rq.line.Action = requestlog.ActionAdmin
rq.line.Action = requestlog.ActionOwnEndpoint
token, answer := rq.endpoint()
+2 -2
View File
@@ -356,7 +356,7 @@ func adminEndpoints() []adminEndpoint {
// admin sends a request with method for path, with body, to discriminator's own
// address, with the admin token, and checks that it is answered with status, its
// log line's action admin. It returns the answer.
// log line's action own_endpoint. It returns the answer.
func (s *sender) admin(method, path, body string, status int) answer {
s.t.Helper()
@@ -380,7 +380,7 @@ func (s *sender) adminRequest(
line := s.out.endedLines(s.t, s.sent+1)[s.sent]
s.sent++
wantLine(s.t, line, status, requestlog.ActionAdmin)
wantLine(s.t, line, status, requestlog.ActionOwnEndpoint)
return got
}
+1 -1
View File
@@ -73,7 +73,7 @@ func TestEachScopeAndWindowOverItsThresholdAlertsOncePerCooldown(t *testing.T) {
{anomalyASN, anomaly.ScopeASN, netip.Prefix{}, asnDE, map[string]any{"asn": asnDE}},
{anomalyTotal, anomaly.ScopeTotal, netip.Prefix{}, "the whole service", nil},
{
anomalyWatch, anomaly.ScopeWatch, netip.MustParsePrefix(clientsNet),
anomalyWatch, anomaly.ScopeWatchedNet, netip.MustParsePrefix(clientsNet),
"the named netblock office, " + clientsNet, map[string]any{"name": "office"},
},
} {
+8 -8
View File
@@ -86,8 +86,8 @@ func TestSixteenThousandCookiesAreRefusedWellWithinTheTimeLimit(t *testing.T) {
line := s.out.requestLine(t)
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
if line.DurationWAF != nil {
t.Errorf("the Core Rule Set read the request, for %f ms", *line.DurationWAF)
if line.DurationCoreRuleSetMS != nil {
t.Errorf("the Core Rule Set read the request, for %f ms", *line.DurationCoreRuleSetMS)
}
}
@@ -165,7 +165,7 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
wantDisposition(t, got, http.StatusOK, discriminator.DispositionAllow)
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
if line.DurationWAF == nil {
if line.DurationCoreRuleSetMS == nil {
t.Error("the Core Rule Set did not read a request within the limits")
}
})
@@ -220,9 +220,9 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
line := s.out.requestLine(t)
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
line.DurationWAF != nil {
t.Errorf("would_action %q and duration_waf %v, want %q and none",
line.WouldAction, line.DurationWAF, requestlog.ActionRequestHeaderTooLarge)
line.DurationCoreRuleSetMS != nil {
t.Errorf("would_action %q and duration_core_rule_set_ms %v, want %q and none",
line.WouldAction, line.DurationCoreRuleSetMS, requestlog.ActionRequestHeaderTooLarge)
}
}
@@ -283,8 +283,8 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
line := s.out.requestLine(t)
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
if line.LimitPercent != nil {
t.Errorf("limit percentage %d, want none", *line.LimitPercent)
if line.RateLimitPercent != nil {
t.Errorf("limit percentage %d, want none", *line.RateLimitPercent)
}
// The client is under no ban, and GeoJS places it in a country.
+1 -1
View File
@@ -106,7 +106,7 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) {
case ratelimit.KindBytes:
rq.line.LimitHit = hit.Window + "_bytes" // as counts names the byte totals
case ratelimit.KindRefusals:
rq.line.LimitHit = requestlog.LimitHitErrorBurst
rq.line.LimitHit = requestlog.LimitHitMinuteRefusals
default:
rq.line.LimitHit = hit.Window
}
+21 -13
View File
@@ -79,14 +79,16 @@ func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) {
{http.StatusForbidden, requestlog.ActionClientIsRateLimited},
} {
line := s.get(tc.from, sent.status, sent.action)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
"50 from "+tc.setting)
}
// fromKP, which no setting lists, has the whole limit.
for range 3 {
line := s.get(fromKP, http.StatusOK, requestlog.ActionForward)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
none)
}
})
@@ -119,7 +121,8 @@ func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) {
t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes)
}
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
"50 from "+tc.setting)
// fromKP, which no setting lists, has the whole limit.
@@ -128,7 +131,8 @@ func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) {
t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit)
}
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none)
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting, none)
})
}
}
@@ -177,9 +181,11 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T)
t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit)
}
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.limitPercent)
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
tc.bytesPercent)
})
}
@@ -193,10 +199,10 @@ func TestZeroPercentIsAZeroAllowance(t *testing.T) {
// The first request breaks the limit, and bans the client; the log line
// gives the 0.
line := s.get(unplaced, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
if line.fields["limit_percent"] != float64(0) ||
line.fields["limit_percent_setting"] != unknownLimitPercent {
t.Errorf("log line has limit_percent %v from %v, want 0 from %s",
line.fields["limit_percent"], line.fields["limit_percent_setting"],
if line.fields["rate_limit_percent"] != float64(0) ||
line.fields["rate_limit_percent_setting"] != unknownLimitPercent {
t.Errorf("log line has rate_limit_percent %v from %v, want 0 from %s",
line.fields["rate_limit_percent"], line.fields["rate_limit_percent_setting"],
unknownLimitPercent)
}
@@ -210,7 +216,7 @@ func TestLowestPercentageApplies(t *testing.T) {
name string
env map[string]string
from string
// want is the log line's limit_percent, as percentText gives it.
// want is the log line's rate_limit_percent, as percentText gives it.
want string
}{
{
@@ -251,7 +257,8 @@ func TestLowestPercentageApplies(t *testing.T) {
s.get(tc.from, http.StatusOK, requestlog.ActionForward)
line := s.get(tc.from, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.want)
})
}
@@ -294,7 +301,8 @@ func TestClientWithoutAnAnswerInTimeHasTheUnknownLimitPercent(t *testing.T) {
line := out.requestLine(t)
wantLine(t, line, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
"0 from "+unknownLimitPercent)
})
}
+13 -13
View File
@@ -9,8 +9,8 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// coreRuleSet inspects the request with the Core Rule Set, unless
@@ -18,7 +18,7 @@ import (
// pathExempt decides, and notes the rules it matched and its score in the
// log line, and the rules in the metrics. A score at or over
// SWW_WAF_ANOMALY_THRESHOLD is a match: it adds the tag exploitsignature,
// raises the waf_block alert, and in block mode refuses the request with 403,
// raises the core_rule_set_match alert, and in block mode refuses the request with 403,
// which is an offence its client's history counts. Its figures are the score
// and the threshold.
type coreRuleSet struct{}
@@ -38,12 +38,12 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
result, read, _ := rq.h.coreRuleSet.Inspect(rq.in, rq.client, rq.in.Body)
rq.requestBytes = int64(len(read))
rq.line.DurationWAF = new(requestlog.Milliseconds(time.Since(start)))
rq.line.WAFRuleIDs = result.RuleIDs
rq.line.WAFScore = &result.Score
rq.line.DurationCoreRuleSetMS = new(requestlog.Milliseconds(time.Since(start)))
rq.line.CoreRuleSetRuleIDs = result.RuleIDs
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.WAFMatched(cfg.WAFMode, id)
rq.h.metrics.CoreRuleSetMatched(cfg.WAFMode, id)
}
threshold := cfg.WAFAnomalyThreshold
@@ -52,26 +52,26 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
}
rq.addTag(config.ExploitSignatureTag)
rq.alertWAFBlock(result)
rq.alertCoreRuleSetMatch(result)
if cfg.WAFMode == config.WAFModeDetect {
return nil
}
rq.wafBlocked = true
rq.blockedByCoreRuleSet = true
return &verdict{
action: requestlog.ActionWAFBlocked,
action: requestlog.ActionBlockedByCoreRuleSet,
figures: map[string]any{figureScore: result.Score, "threshold": threshold},
}
}
// alertWAFBlock raises the waf_block alert for the request, which the Core
// Rule Set scored at result, at or over SWW_WAF_ANOMALY_THRESHOLD. Its
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
// which the Core Rule Set scored at result, at or over SWW_WAF_ANOMALY_THRESHOLD. Its
// detail gives the rule ids, the score, the method and the path with the
// query, and, for a request that is not refused for it, the mode: detect,
// or observe in observe mode.
func (rq *request) alertWAFBlock(result waf.Result) {
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
detail := map[string]any{
"rule_ids": result.RuleIDs,
"score": result.Score,
@@ -87,7 +87,7 @@ func (rq *request) alertWAFBlock(result waf.Result) {
}
rq.h.alerts.Raise(alerts.Alert{
Event: alerts.EventWAFBlock,
Event: alerts.EventCoreRuleSetMatch,
Host: rq.site,
Client: rq.client,
Netblock: rq.h.clientGroup(rq.client),
+57 -49
View File
@@ -40,24 +40,30 @@ const (
// SWW_WAF_ANOMALY_THRESHOLD.
const sqlInjection = "/?id=1'%20OR%20'1'='1"
// wantWAF checks the request log line's waf_rule_ids and waf_score, and
// that it has duration_waf, or with no score, that it has none of the
// three: the Core Rule Set did not inspect the request.
func wantWAF(t *testing.T, line logLine, score *int, ruleIDs ...int) {
// wantCoreRuleSet checks the request log line's core_rule_set_rule_ids and
// core_rule_set_score, and that it has duration_core_rule_set_ms, or with no
// score, that it has none of the three: the Core Rule Set did not inspect the
// request.
func wantCoreRuleSet(t *testing.T, line logLine, score *int, ruleIDs ...int) {
t.Helper()
if !slices.Equal(line.WAFRuleIDs, ruleIDs) {
t.Errorf("log line has waf_rule_ids %v, want %v", line.WAFRuleIDs, ruleIDs)
if !slices.Equal(line.CoreRuleSetRuleIDs, ruleIDs) {
t.Errorf("log line has core_rule_set_rule_ids %v, want %v",
line.CoreRuleSetRuleIDs, ruleIDs)
}
switch {
case score == nil && (line.WAFScore != nil || line.DurationWAF != nil):
t.Errorf("log line has waf_score %v and duration_waf %v, want neither",
line.fields["waf_score"], line.fields["duration_waf"])
case score != nil && (line.WAFScore == nil || *line.WAFScore != *score):
t.Errorf("log line has waf_score %v, want %d", line.fields["waf_score"], *score)
case score != nil && line.DurationWAF == nil:
t.Error("log line has no duration_waf")
case score == nil &&
(line.CoreRuleSetScore != nil || line.DurationCoreRuleSetMS != nil):
t.Errorf("log line has core_rule_set_score %v and "+
"duration_core_rule_set_ms %v, want neither",
line.fields["core_rule_set_score"], line.fields["duration_core_rule_set_ms"])
case score != nil &&
(line.CoreRuleSetScore == nil || *line.CoreRuleSetScore != *score):
t.Errorf("log line has core_rule_set_score %v, want %d",
line.fields["core_rule_set_score"], *score)
case score != nil && line.DurationCoreRuleSetMS == nil:
t.Error("log line has no duration_core_rule_set_ms")
}
}
@@ -70,7 +76,7 @@ func TestCoreRuleSetMatchAddsTheExploitSignatureTag(t *testing.T) {
status int
action string
}{
{block, block, http.StatusForbidden, requestlog.ActionWAFBlocked},
{block, block, http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet},
{detect, detect, http.StatusOK, requestlog.ActionForward},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -133,14 +139,14 @@ func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *test
mode, action string
status int
}{
{block, requestlog.ActionWAFBlocked, http.StatusForbidden},
{block, requestlog.ActionBlockedByCoreRuleSet, http.StatusForbidden},
{detect, requestlog.ActionForward, http.StatusOK},
} {
s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode})
line := s.requestWithHeader(client, attack.path, attack.header,
tc.status, tc.action)
wantWAF(t, line, &attack.score, attack.ruleIDs...)
wantCoreRuleSet(t, line, &attack.score, attack.ruleIDs...)
}
})
}
@@ -152,7 +158,7 @@ func TestCoreRuleSetRefusalNamesItsHeuristicScoreAndThreshold(t *testing.T) {
s, _, _ := startWithClock(t, "", map[string]string{wafMode: block})
line := s.request(client, sqlInjection, http.StatusForbidden,
requestlog.ActionWAFBlocked)
requestlog.ActionBlockedByCoreRuleSet)
wantHeuristic(t, line, "core_rule_set", `{"score": 5, "threshold": 5}`)
}
@@ -163,7 +169,7 @@ func TestOrdinaryRequestIsInspectedAndPassed(t *testing.T) {
line := s.request(client, "/owner/repo/src/branch/main/README.md?display=source",
http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, new(0))
wantCoreRuleSet(t, line, new(0))
}
func TestCoreRuleSetIsNotRunWhenOffOrForAnExemptClientPathOrRuleFileRefusal(
@@ -183,26 +189,26 @@ func TestCoreRuleSetIsNotRunWhenOffOrForAnExemptClientPathOrRuleFileRefusal(
// A client in SWW_ALLOW_NETS, and a path SWW_WAF_EXEMPT_PATHS
// exempts, are not inspected.
line := s.request(allowed, sqlInjection, http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, nil)
wantCoreRuleSet(t, line, nil)
line = s.request(client, "/api/v1/repos?id=1'%20OR%20'1'='1", http.StatusOK,
requestlog.ActionForward)
wantWAF(t, line, nil)
wantCoreRuleSet(t, line, nil)
// The prefix is matched as rate limit exempt paths are: a path that
// goes up and out of it is inspected.
line = s.request(client, "/api/../?id=1'%20OR%20'1'='1", http.StatusForbidden,
requestlog.ActionWAFBlocked)
wantWAF(t, line, new(25), 930100, 930110, 942100)
requestlog.ActionBlockedByCoreRuleSet)
wantCoreRuleSet(t, line, new(25), 930100, 930110, 942100)
// A request a rule file refuses is not inspected.
line = s.request(otherClient, "/blocked?id=1'%20OR%20'1'='1", http.StatusForbidden,
requestlog.ActionBlockedByBlockRule)
wantWAF(t, line, nil)
wantCoreRuleSet(t, line, nil)
// With SWW_WAF_MODE off, no request is.
s, _, _ = startWithClock(t, "", map[string]string{wafMode: off})
line = s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, nil)
wantCoreRuleSet(t, line, nil)
}
func TestAnomalyThreshold(t *testing.T) {
@@ -216,13 +222,14 @@ func TestAnomalyThreshold(t *testing.T) {
})
line := s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, new(5), 942100)
wantCoreRuleSet(t, line, new(5), 942100)
}
s, _, _ := startWithClock(t, "", map[string]string{
wafMode: block, wafAnomalyThreshold: "5",
})
s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionWAFBlocked)
s.request(client, sqlInjection, http.StatusForbidden,
requestlog.ActionBlockedByCoreRuleSet)
}
func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) {
@@ -265,7 +272,7 @@ func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) {
s, _, _ := startWithClock(t, "", map[string]string{wafMode: block})
line, _ := s.requestWithBody(request.method, client, request.path,
request.header, body, http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, new(0))
wantCoreRuleSet(t, line, new(0))
// A list given replaces the default.
s, _, _ = startWithClock(t, "", map[string]string{
@@ -273,12 +280,12 @@ func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) {
})
score := 5 * len(request.ruleIDs)
line, _ = s.requestWithBody(request.method, client, request.path,
request.header, body, http.StatusForbidden, requestlog.ActionWAFBlocked)
wantWAF(t, line, &score, request.ruleIDs...)
request.header, body, http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet)
wantCoreRuleSet(t, line, &score, request.ruleIDs...)
// And switches off the rules it lists.
line = s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, new(0))
wantCoreRuleSet(t, line, new(0))
})
}
}
@@ -294,14 +301,14 @@ func TestAttackInAFormBodyIsRefusedOnlyWhileBodiesAreRead(t *testing.T) {
s, _, _ := startWithClock(t, "", map[string]string{wafMode: block})
line, _ := s.requestWithBody(http.MethodPost, client, "/", header, body,
http.StatusOK, requestlog.ActionForward)
wantWAF(t, line, new(0))
wantCoreRuleSet(t, line, new(0))
s, _, _ = startWithClock(t, "", map[string]string{
wafMode: block, wafBodyLimit: sizeLimitSetting,
})
line, _ = s.requestWithBody(http.MethodPost, client, "/", header, body,
http.StatusForbidden, requestlog.ActionWAFBlocked)
wantWAF(t, line, new(5), 942100)
http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet)
wantCoreRuleSet(t, line, new(5), 942100)
}
func TestBodiesReachTheAppAsSentWhileBodiesAreRead(t *testing.T) {
@@ -391,32 +398,33 @@ func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T)
})
for range 2 {
s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionWAFBlocked)
s.request(client, sqlInjection, http.StatusForbidden,
requestlog.ActionBlockedByCoreRuleSet)
}
// The third refusal in a minute breaks the error burst, and bans the
// client.
line := s.request(client, sqlInjection, http.StatusForbidden,
requestlog.ActionWAFBlocked)
if line.LimitHit != requestlog.LimitHitErrorBurst ||
requestlog.ActionBlockedByCoreRuleSet)
if line.LimitHit != requestlog.LimitHitMinuteRefusals ||
line.Offence != requestlog.OffenceLimit {
t.Errorf("log line has limit_hit %q and offence %q, want error_burst and limit",
t.Errorf("log line has limit_hit %q and offence %q, want minute_refusals and limit",
line.LimitHit, line.Offence)
}
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
want := ratelimit.Offences{Limit: 1, WAFBlocked: 3}
want := ratelimit.Offences{Limit: 1, BlockedByCoreRuleSet: 3}
if offences := historyOf(t, server, client).Offences; offences != want {
t.Errorf("history counts the offences %+v, want %+v", offences, want)
}
metrics := s.scrape()
wantMetric(t, metrics,
`sww_waf_matches_total{instance="app",mode="block",rule_id="942100"}`, 3)
`sww_core_rule_set_matches_total{instance="app",mode="block",rule_id="942100"}`, 3)
wantMetric(t, metrics,
`sww_offences_total{instance="app",kind="waf_blocked"}`, 3)
wantMetric(t, metrics, `sww_requests_total{action="waf_blocked",`+
`sww_offences_total{instance="app",kind="blocked_by_core_rule_set"}`, 3)
wantMetric(t, metrics, `sww_requests_total{action="blocked_by_core_rule_set",`+
`instance="app",status_class="4xx"}`, 3)
}
@@ -440,9 +448,9 @@ func TestDetectModeMatchIsNoOffenceAndNotCountedTowardTheErrorBurst(t *testing.T
metrics := s.scrape()
wantMetric(t, metrics,
`sww_waf_matches_total{instance="app",mode="detect",rule_id="942100"}`, 3)
`sww_core_rule_set_matches_total{instance="app",mode="detect",rule_id="942100"}`, 3)
wantNoSeries(t, metrics,
`sww_offences_total{instance="app",kind="waf_blocked"}`)
`sww_offences_total{instance="app",kind="blocked_by_core_rule_set"}`)
}
func TestObserveModeLogsWhatTheCoreRuleSetWouldDo(t *testing.T) {
@@ -451,17 +459,17 @@ func TestObserveModeLogsWhatTheCoreRuleSetWouldDo(t *testing.T) {
s, _, server := startWithClock(t, "", map[string]string{wafMode: block, mode: observe})
line := s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward)
wantWouldAction(t, line, requestlog.ActionWAFBlocked)
wantWAF(t, line, new(5), 942100)
wantWouldAction(t, line, requestlog.ActionBlockedByCoreRuleSet)
wantCoreRuleSet(t, line, new(5), 942100)
// It is an offence as in enforce mode.
want := ratelimit.Offences{WAFBlocked: 1}
want := ratelimit.Offences{BlockedByCoreRuleSet: 1}
if offences := historyOf(t, server, client).Offences; offences != want {
t.Errorf("history counts the offences %+v, want %+v", offences, want)
}
}
func TestCoreRuleSetMatchRaisesTheWAFBlockAlert(t *testing.T) {
func TestCoreRuleSetMatchRaisesTheCoreRuleSetMatchAlert(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
@@ -475,7 +483,7 @@ func TestCoreRuleSetMatchRaisesTheWAFBlockAlert(t *testing.T) {
}{
{
"block", map[string]string{wafMode: block},
http.StatusForbidden, requestlog.ActionWAFBlocked, "",
http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet, "",
},
{
"detect", map[string]string{wafMode: detect},
@@ -510,7 +518,7 @@ func TestCoreRuleSetMatchRaisesTheWAFBlockAlert(t *testing.T) {
wantAlerts(t, queue, alerts.Alert{
Instance: alertInstance,
Time: clk.Now(),
Event: alerts.EventWAFBlock,
Event: alerts.EventCoreRuleSetMatch,
Host: appHost,
Client: netip.MustParseAddr(client),
Netblock: netip.MustParsePrefix(client + "/32"),
+17 -17
View File
@@ -89,12 +89,12 @@ var (
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Report struct {
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotal float64 `json:"duration_total"`
DurationUpstreamTotal float64 `json:"duration_upstream_total"`
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotalMS float64 `json:"duration_total_ms"`
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
}
// New returns the server `sww discriminator` runs. It answers
@@ -208,7 +208,7 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
// The app never sees this prefix. discriminator's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionAdmin
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
}
@@ -405,7 +405,7 @@ func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
for _, name := range []string{
"request_id", "status", "request_bytes", "response_bytes",
"duration_total", "duration_upstream_total",
"duration_total_ms", "duration_upstream_total_ms",
} {
value, there := fields[name]
if !there || string(value) == "null" {
@@ -431,11 +431,11 @@ func (report *Report) checkRanges() error {
return fmt.Errorf("%w: request_bytes", errReportValue)
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
return fmt.Errorf("%w: response_bytes", errReportValue)
case report.DurationTotal < 0 || report.DurationTotal > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total", errReportValue)
case report.DurationUpstreamTotal < 0 ||
report.DurationUpstreamTotal > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total", errReportValue)
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total_ms", errReportValue)
case report.DurationUpstreamTotalMS < 0 ||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
}
return nil
@@ -459,18 +459,18 @@ func (h *handler) finishWith(ctx context.Context, held *decision, report Report)
rq.line.Status = report.Status
rq.line.RequestBytes = report.RequestBytes
rq.line.ResponseBytes = report.ResponseBytes
rq.line.DurationTotal = report.DurationTotal
rq.line.DurationTotalMS = report.DurationTotalMS
if rq.allowed {
rq.line.DurationUpstreamTotal = &report.DurationUpstreamTotal
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.h.metrics.RequestReported(&rq.line,
time.Duration(report.DurationTotal*float64(time.Millisecond)),
time.Duration(report.DurationUpstreamTotal*float64(time.Millisecond)))
time.Duration(report.DurationTotalMS*float64(time.Millisecond)),
time.Duration(report.DurationUpstreamTotalMS*float64(time.Millisecond)))
}
rq.writeLine()
+24 -22
View File
@@ -48,8 +48,8 @@ const (
reportStatus = "status"
reportRequestBytes = "request_bytes"
reportResponseBytes = "response_bytes"
reportDuration = "duration_total"
reportUpstreamDuration = "duration_upstream_total"
reportDuration = "duration_total_ms"
reportUpstreamDuration = "duration_upstream_total_ms"
)
// Discriminator is a discriminator running for one test, on a clock set to midnight.
@@ -693,7 +693,7 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(t *testing.T) {
wantDisposition(t, got, http.StatusNotFound, discriminator.DispositionDeny)
}
wantLine(t, s.out.requestLine(t), http.StatusNotFound, requestlog.ActionAdmin)
wantLine(t, s.out.requestLine(t), http.StatusNotFound, requestlog.ActionOwnEndpoint)
got := s.ask(otherClient, discriminator.MetricsPath)
wantDisposition(t, got, http.StatusForbidden, discriminator.DispositionDeny)
@@ -725,9 +725,9 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheErrorBurst(t *testing.T) {
}
lines := s.out.requestLines(t, 2)
if lines[0].LimitHit != "" || lines[1].LimitHit != requestlog.LimitHitErrorBurst {
if lines[0].LimitHit != "" || lines[1].LimitHit != requestlog.LimitHitMinuteRefusals {
t.Errorf("log lines have limit_hit %q and %q, want none and %q",
lines[0].LimitHit, lines[1].LimitHit, requestlog.LimitHitErrorBurst)
lines[0].LimitHit, lines[1].LimitHit, requestlog.LimitHitMinuteRefusals)
}
}
@@ -744,9 +744,9 @@ func TestDeniedQuestionCountsForTheErrorBurst(t *testing.T) {
}
lines := s.out.requestLines(t, 2)
if lines[0].LimitHit != "" || lines[1].LimitHit != requestlog.LimitHitErrorBurst {
if lines[0].LimitHit != "" || lines[1].LimitHit != requestlog.LimitHitMinuteRefusals {
t.Errorf("log lines have limit_hit %q and %q, want none and %q",
lines[0].LimitHit, lines[1].LimitHit, requestlog.LimitHitErrorBurst)
lines[0].LimitHit, lines[1].LimitHit, requestlog.LimitHitMinuteRefusals)
}
}
@@ -901,7 +901,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
got := s.report(discriminator.Report{
RequestID: denied.Header.Get(requestIDHeader), Status: http.StatusForbidden,
ResponseBytes: 1000, DurationTotal: 2,
ResponseBytes: 1000, DurationTotalMS: 2,
})
wantStatus(t, got, http.StatusNoContent)
@@ -935,7 +935,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
got := s.report(discriminator.Report{
RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotal: 5, DurationUpstreamTotal: 4,
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
})
wantStatus(t, got, http.StatusNoContent)
@@ -951,7 +951,8 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
unscraped := startDiscriminator(t, "", map[string]string{metricsToken: token})
got = unscraped.report(discriminator.Report{
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway, DurationTotal: 5,
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
DurationTotalMS: 5,
})
wantStatus(t, got, http.StatusNoContent)
wantMetric(t, unscraped.metrics(),
@@ -1189,8 +1190,8 @@ func wantReportsFields(t *testing.T, line logLine) {
if line.DiscriminatorLine != requestlog.DiscriminatorLineReport ||
line.Status != http.StatusCreated ||
line.RequestBytes != bodyBytes || line.ResponseBytes != answerBytes ||
line.DurationTotal != 12.5 || line.DurationUpstreamTotal == nil ||
*line.DurationUpstreamTotal != 10.25 {
line.DurationTotalMS != 12.5 || line.DurationUpstreamTotalMS == nil ||
*line.DurationUpstreamTotalMS != 10.25 {
t.Errorf("the report's line lacks the report's fields: %+v", line.Line)
}
}
@@ -1205,8 +1206,8 @@ func wantSameDecision(t *testing.T, reported, decided logLine) {
same.Status = decided.Status
same.RequestBytes = decided.RequestBytes
same.ResponseBytes = decided.ResponseBytes
same.DurationTotal = decided.DurationTotal
same.DurationUpstreamTotal = decided.DurationUpstreamTotal
same.DurationTotalMS = decided.DurationTotalMS
same.DurationUpstreamTotalMS = decided.DurationUpstreamTotalMS
same.Counts.MinuteBytes = decided.Counts.MinuteBytes
same.Counts.HourBytes = decided.Counts.HourBytes
same.Counts.DayBytes = decided.Counts.DayBytes
@@ -1224,7 +1225,7 @@ func wantDecisionsFields(t *testing.T, line logLine, id, query, accept string) {
if line.DiscriminatorLine != requestlog.DiscriminatorLineDecision ||
line.Status != http.StatusOK || line.ResponseBytes != 0 ||
line.DurationUpstreamTotal != nil || line.DurationChecks == nil ||
line.DurationUpstreamTotalMS != nil || line.DurationChecksMS == nil ||
line.RequestID != id || line.Query != query ||
!line.HasAuthorization || !line.HasCookie ||
line.RequestHeaders["accept"] != accept || len(line.RuleIDs) != 1 ||
@@ -1260,7 +1261,7 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(t *testing.T
got := s.report(discriminator.Report{
RequestID: id, Status: http.StatusCreated,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotal: 12.5, DurationUpstreamTotal: 10.25,
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
})
wantStatus(t, got, http.StatusNoContent)
@@ -1305,7 +1306,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
0, requestlog.ActionClientDenied,
},
{"own prefix", nil, discriminator.HealthPath, http.StatusNotFound,
requestlog.ActionAdmin},
requestlog.ActionOwnEndpoint},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
@@ -1317,7 +1318,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
do(t, req)
got := s.report(discriminator.Report{
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotal: 3,
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
})
wantStatus(t, got, http.StatusNoContent)
@@ -1325,8 +1326,8 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
wantLine(t, line, tc.status, tc.action)
if line.DiscriminatorLine != requestlog.DiscriminatorLineReport ||
line.RequestID != id || line.ResponseBytes != 8 || line.DurationTotal != 3 ||
line.DurationUpstreamTotal != nil {
line.RequestID != id || line.ResponseBytes != 8 || line.DurationTotalMS != 3 ||
line.DurationUpstreamTotalMS != nil {
t.Errorf("the report's line is %+v", line.Line)
}
})
@@ -1583,9 +1584,10 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
wantDisposition(t, got, http.StatusForbidden, discriminator.DispositionDeny)
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventWAFBlock ||
if len(waiting) != 1 || waiting[0].Event != alerts.EventCoreRuleSetMatch ||
waiting[0].Host != siteA {
t.Errorf("alerts waiting are %+v, want a waf_block naming %s", waiting, siteA)
t.Errorf("alerts waiting are %+v, want a core_rule_set_match naming %s",
waiting, siteA)
}
}
+2 -2
View File
@@ -91,7 +91,7 @@ func TestMetricsCountTheTraffic(t *testing.T) {
out.endedLines(t, 2)
forward := `{action="forward",instance="app",status_class="2xx"}`
notFound := `{action="admin",instance="app",status_class="4xx"}`
notFound := `{action="own_endpoint",instance="app",status_class="4xx"}`
// The request for the metrics is itself under way.
metrics := scrape(t, server.Addr)
@@ -347,7 +347,7 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
wantMetric(t, metrics, `sww_geojs_requests_total{instance="app"}`, 1)
wantMetric(t, metrics, `sww_geojs_failures_total{instance="app"}`, 1)
wantMetric(t, metrics, `sww_geojs_unanswered_total{instance="app"}`, 1)
wantMetric(t, metrics, `sww_requests_without_geojs_answer_total{instance="app"}`, 1)
}
// keptAnswer returns GeoJS's answer that the client at addr is in
+7 -7
View File
@@ -16,13 +16,13 @@ import (
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// adminPrefix starts the path of each of discriminator's own endpoints. The app never
@@ -65,7 +65,7 @@ type Params struct {
LookupFile *lookup.File
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// while SWW_WAF_MODE is not off, and nil while it is.
CoreRuleSet *waf.CoreRuleSet
CoreRuleSet *coreruleset.CoreRuleSet
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
// normally time.Now in UTC, the time the state files give.
@@ -129,7 +129,7 @@ func newHandler(params Params) *handler {
BytesPerHour: params.Config.BytesLimitPerHour,
BytesPerDay: params.Config.BytesLimitPerDay,
}, params.Config.MaxTrackedClients),
ledger: bans.New(bans.Rules{
ledger: bans.New(bans.Params{
LimitBanDuration: params.Config.LimitBanDuration,
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
MaxBanDuration: params.Config.MaxBanDuration,
@@ -224,15 +224,15 @@ func newReputation(
// SWW_CORE_RULE_SET_EXCLUSIONS_FILE, or nil while SWW_WAF_MODE is off. It
// reads at most QuestionBodyMaxBytes less one byte: a question's body of
// QuestionBodyMaxBytes may be cut short, and the Core Rule Set does not read a
// JSON or XML body over its limit. An exclusions file that waf.New refuses is
// JSON or XML body over its limit. An exclusions file that coreruleset.New refuses is
// an error in every mode. While the mode is off, a set file is loaded with the
// Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.Config) (*waf.CoreRuleSet, error) {
func NewCoreRuleSet(cfg *config.Config) (*coreruleset.CoreRuleSet, error) {
if cfg.WAFMode == config.WAFModeOff && cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := waf.New(waf.Params{
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules,
BodyLimit: min(cfg.WAFBodyLimit, QuestionBodyMaxBytes-1),
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
@@ -266,7 +266,7 @@ type handler struct {
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet *waf.CoreRuleSet
coreRuleSet *coreruleset.CoreRuleSet
alerts *alerts.Queue
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
+2 -2
View File
@@ -29,8 +29,8 @@ func (rateLimits) check(_ context.Context, rq *request, now time.Time) *verdict
}
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting = rq.limitPercent.logged()
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.bytesPercent.logged()
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
rq.limitPercent.percent)
+2 -2
View File
@@ -28,7 +28,7 @@ func (refusalLimit) check(_ context.Context, rq *request, now time.Time) *verdic
return nil
}
if !rq.attack && !rq.blockedByBlockRule && !rq.wafBlocked {
if !rq.attack && !rq.blockedByBlockRule && !rq.blockedByCoreRuleSet {
return nil
}
@@ -42,7 +42,7 @@ func (refusalLimit) check(_ context.Context, rq *request, now time.Time) *verdic
status := rq.out.status
switch rq.line.WouldAction {
case requestlog.ActionBlockedByBlockRule, requestlog.ActionWAFBlocked:
case requestlog.ActionBlockedByBlockRule, requestlog.ActionBlockedByCoreRuleSet:
status = http.StatusForbidden
case requestlog.ActionTemporarilyBanned, requestlog.ActionPermanentlyBanned:
status = cfg.BanResponse
+8 -7
View File
@@ -107,9 +107,9 @@ func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) {
// The fourth is answered as the others were, breaks the error
// burst, and bans the client.
line := tc.refusals[3].send(s, client)
if line.LimitHit != requestlog.LimitHitErrorBurst ||
if line.LimitHit != requestlog.LimitHitMinuteRefusals ||
line.Offence != requestlog.OffenceLimit {
t.Errorf("log line has limit_hit %q and offence %q, want error_burst "+
t.Errorf("log line has limit_hit %q and offence %q, want minute_refusals "+
"and limit", line.LimitHit, line.Offence)
}
@@ -277,7 +277,8 @@ func TestErrorBurstOffOrAtItsDefault(t *testing.T) {
line = blockRule.send(s, client)
}
if broken := line.LimitHit == requestlog.LimitHitErrorBurst; broken != tc.broken {
broken := line.LimitHit == requestlog.LimitHitMinuteRefusals
if broken != tc.broken {
t.Errorf("the 31st refusal broke the error burst: %t, want %t",
broken, tc.broken)
}
@@ -306,8 +307,8 @@ func TestErrorBurstCountsEachClientTheChecksApplyTo(t *testing.T) {
blockRule.send(s, exempt)
line := blockRule.send(s, exempt)
if line.LimitHit != requestlog.LimitHitErrorBurst {
t.Errorf("log line has limit_hit %q, want error_burst", line.LimitHit)
if line.LimitHit != requestlog.LimitHitMinuteRefusals {
t.Errorf("log line has limit_hit %q, want minute_refusals", line.LimitHit)
}
s.get(exempt, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
@@ -367,8 +368,8 @@ func TestObserveModeLogsAndAlertsTheErrorBurst(t *testing.T) {
line := s.request(client, blockedPath, http.StatusOK, requestlog.ActionForward)
wantWouldAction(t, line, requestlog.ActionBlockedByBlockRule)
if line.LimitHit != requestlog.LimitHitErrorBurst || line.BanExpires != "" {
t.Errorf("log line has limit_hit %q and ban_expires %q, want error_burst "+
if line.LimitHit != requestlog.LimitHitMinuteRefusals || line.BanExpires != "" {
t.Errorf("log line has limit_hit %q and ban_expires %q, want minute_refusals "+
"and none", line.LimitHit, line.BanExpires)
}
+30 -20
View File
@@ -54,7 +54,7 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
for _, tc := range []struct {
action string
// statuses and actions are those of a listed client's three
// requests, and percent their limit_percent, as percentText gives it.
// requests, and percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []string
percent string
@@ -90,8 +90,8 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
for i := range 3 {
line := s.get(from, tc.statuses[i], tc.actions[i])
wantReputation(t, line, dropURL)
wantPercent(t, "limit_percent", line.LimitPercent,
line.LimitPercentSetting, tc.percent)
wantPercent(t, "rate_limit_percent", line.RateLimitPercent,
line.RateLimitPercentSetting, tc.percent)
// A request refused for the list is not counted.
counted := line.fields["counts"] != nil
@@ -106,7 +106,8 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
for range 3 {
line := s.get(unplaced, http.StatusOK, forward)
wantReputation(t, line)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
none)
}
@@ -207,7 +208,7 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
for _, tc := range []struct {
action, asnPercent string
// want is the upload's limit_percent and bytes_percent, as
// want is the upload's rate_limit_percent and bytes_limit_percent, as
// percentText gives them, and limitHit its limit_hit.
want, limitHit string
}{
@@ -229,9 +230,11 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
// The upload's 100 bytes are over 49, a quarter of 199, and 99,
// half of it, and within 199.
line := s.uploadFrom(fromDE)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.want)
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
tc.want)
if line.LimitHit != tc.limitHit {
@@ -288,9 +291,11 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T)
loadLists(t, server, map[string][]string{asnURL: {"# by AS number", tc.file}})
line := s.uploadFrom(fromDE)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.limitPercent)
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
tc.bytesPercent)
})
}
@@ -385,7 +390,7 @@ func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
for _, tc := range []struct {
action string
// statuses and actions are those of a listed client's three
// requests, and percent their limit_percent, as percentText gives it.
// requests, and percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []string
percent string
@@ -422,8 +427,8 @@ func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
line := s.get(from, tc.statuses[i], tc.actions[i])
// In the order SWW_DNSBL_ZONES names them.
wantReputation(t, line, listedBy[from]...)
wantPercent(t, "limit_percent", line.LimitPercent,
line.LimitPercentSetting, tc.percent)
wantPercent(t, "rate_limit_percent", line.RateLimitPercent,
line.RateLimitPercentSetting, tc.percent)
// A request refused for the verdict is not counted.
counted := line.fields["counts"] != nil
@@ -438,7 +443,8 @@ func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
for range 3 {
line := s.get(unplaced, http.StatusOK, forward)
wantReputation(t, line)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
none)
}
@@ -546,7 +552,7 @@ func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
for _, tc := range []struct {
blocklistAction, reputationAction string
// want is the request's limit_percent and bytes_percent, as
// want is the request's rate_limit_percent and bytes_limit_percent, as
// percentText gives them.
want string
}{
@@ -571,9 +577,11 @@ func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
// Named by the blocklist, then by the zone.
line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
wantReputation(t, line, dropURL, dnsblZone)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.want)
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
tc.want)
})
}
@@ -819,7 +827,7 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *test
},
{
"the Core Rule Set", sqlInjection, http.StatusForbidden,
requestlog.ActionWAFBlocked, ratelimit.Offences{WAFBlocked: 1},
requestlog.ActionBlockedByCoreRuleSet, ratelimit.Offences{BlockedByCoreRuleSet: 1},
},
{
"a ban rule", probePath, http.StatusForbidden, requestlog.ActionTemporarilyBanned,
@@ -863,7 +871,7 @@ func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *test
for _, tc := range []struct {
action string
// statuses and actions are those of fromDE's three requests, and
// percent their limit_percent, as percentText gives it.
// percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []string
percent string
@@ -896,7 +904,8 @@ func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *test
for i := range 3 {
line := s.get(fromDE, tc.statuses[i], tc.actions[i])
wantReputation(t, line, abuseipdb)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
tc.percent)
// A request refused for the score is not counted.
@@ -910,7 +919,8 @@ func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *test
for range 3 {
line := s.get(fromKP, http.StatusOK, forward)
wantReputation(t, line)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
none)
}
+15 -15
View File
@@ -55,9 +55,9 @@ type decision struct {
limitPercent, bytesPercent percentage
// attack is true for a request that matched a ban rule or asked for a
// trap path, blockedByBlockRule for one a block rule refused, and
// wafBlocked for one the Core Rule Set refused, each an offence its
// client's history counts.
attack, blockedByBlockRule, wafBlocked bool
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
// its client's history counts.
attack, blockedByBlockRule, blockedByCoreRuleSet bool
// reputation is the reputation sources that list the client, for the
// notes of a ban the request makes.
reputation []bans.ReputationHit
@@ -188,7 +188,7 @@ func (rq *request) check(ctx context.Context) *refusal {
return nil
case action == requestlog.ActionBlockedByBlockRule ||
action == requestlog.ActionWAFBlocked:
action == requestlog.ActionBlockedByCoreRuleSet:
return &refusal{status: http.StatusForbidden, action: action}
case action == requestlog.ActionRequestHeaderTooLarge:
return &refusal{status: http.StatusRequestHeaderFieldsTooLarge, action: action}
@@ -262,8 +262,8 @@ func (rq *request) finish() {
}
duration := time.Since(rq.start)
line.DurationTotal = requestlog.Milliseconds(duration)
line.DurationChecks = timing(rq.start, rq.checked)
line.DurationTotalMS = requestlog.Milliseconds(duration)
line.DurationChecksMS = timing(rq.start, rq.checked)
// Counted before the log line is written, so that the metrics count
// every request whose line is out. An allowed request is the exception:
@@ -304,15 +304,15 @@ func timing(start, end time.Time) *float64 {
// when it comes.
func (rq *request) addToHistory() {
request := ratelimit.Request{
Forwarded: rq.allowed,
Refused: rq.refused != nil,
Status: rq.out.status,
RequestBytes: rq.requestBytes,
ResponseBytes: rq.out.bytes,
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
Attack: rq.attack,
BlockedByBlockRule: rq.blockedByBlockRule,
WAFBlocked: rq.wafBlocked,
Forwarded: rq.allowed,
Refused: rq.refused != nil,
Status: rq.out.status,
RequestBytes: rq.requestBytes,
ResponseBytes: rq.out.bytes,
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
Attack: rq.attack,
BlockedByBlockRule: rq.blockedByBlockRule,
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
}
rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request)
+12 -12
View File
@@ -156,9 +156,9 @@ func wantBareLine(t *testing.T, line logLine) {
// withTimings returns want with the timings of line.
func withTimings(line logLine, want requestlog.Line) requestlog.Line {
want.DurationTotal = line.DurationTotal
want.DurationChecks = line.DurationChecks
want.DurationUpstreamTotal = line.DurationUpstreamTotal
want.DurationTotalMS = line.DurationTotalMS
want.DurationChecksMS = line.DurationChecksMS
want.DurationUpstreamTotalMS = line.DurationUpstreamTotalMS
return want
}
@@ -226,10 +226,10 @@ func TestTimingsAreInOrder(t *testing.T) {
// A request passed to the app has every timing; one refused, none of
// the app's; the health check, which runs no check, only the total.
wantTimings(t, forwarded, "duration_total", "duration_checks",
"duration_upstream_total")
wantTimings(t, refused, "duration_total", "duration_checks")
wantTimings(t, health, "duration_total")
wantTimings(t, forwarded, "duration_total_ms", "duration_checks_ms",
"duration_upstream_total_ms")
wantTimings(t, refused, "duration_total_ms", "duration_checks_ms")
wantTimings(t, health, "duration_total_ms")
if t.Failed() {
return
@@ -237,9 +237,9 @@ func TestTimingsAreInOrder(t *testing.T) {
// In whole microseconds, as they are logged, so that the sum below is
// exact.
total := microseconds(forwarded.DurationTotal)
checks := microseconds(*forwarded.DurationChecks)
upstream := microseconds(*forwarded.DurationUpstreamTotal)
total := microseconds(forwarded.DurationTotalMS)
checks := microseconds(*forwarded.DurationChecksMS)
upstream := microseconds(*forwarded.DurationUpstreamTotalMS)
// The checks end before the request is handed to the app.
if checks+upstream > total {
@@ -247,9 +247,9 @@ func TestTimingsAreInOrder(t *testing.T) {
total, checks, upstream)
}
if *refused.DurationChecks > refused.DurationTotal {
if *refused.DurationChecksMS > refused.DurationTotalMS {
t.Errorf("refused request's checks took %v of %v milliseconds",
*refused.DurationChecks, refused.DurationTotal)
*refused.DurationChecksMS, refused.DurationTotalMS)
}
}
+6 -3
View File
@@ -88,7 +88,8 @@ func TestTagLimitPercentLowersALimitTheLowestWinning(t *testing.T) {
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
line := s.get(fromDE, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
wantPercent(t, "rate_limit_percent",
line.RateLimitPercent, line.RateLimitPercentSetting,
"25 from "+tagLimitPercent)
}
@@ -108,7 +109,8 @@ func TestTagLimitPercentLowersAByteLimitByAListTag(t *testing.T) {
t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes)
}
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting,
"50 from "+tagLimitPercent)
// fromKP, which the list does not hold, has the whole limit.
@@ -117,5 +119,6 @@ func TestTagLimitPercentLowersAByteLimitByAListTag(t *testing.T) {
t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit)
}
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none)
wantPercent(t, "bytes_limit_percent",
line.BytesLimitPercent, line.BytesLimitPercentSetting, none)
}