Request tags, with refusing and limits per tag (closes #175)
check / check (push) Successful in 3m36s
check / check (push) Successful in 3m36s
Each request gets sorted tags from the lookup, new address lists, the reputation sources and the rule matches, logged as `tags`. New settings: `SWW_TAG_LISTS`, `SWW_TAG_DENY`, `SWW_TAG_LIMIT_PERCENT`. They replace `SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT` and `SWW_DENIED_COUNTRIES`. New heuristics `tag_lists` and `tag_deny`; the latter runs before the rate limits and again after the Core Rule Set. Model: opus-5-5
This commit is contained in:
1 parent
2349b2a081
commit
60630859af
42 files changed
+1088
-319
No files matched your search
@@ -13,13 +13,14 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// abuseIPDB notes AbuseIPDB, as noteHit does, with the score, when its score
|
||||
// of the client is a hit, and refuses the request when SWW_REPUTATION_ACTION
|
||||
// is deny, as dnsbl does for a zone. While SWW_ABUSEIPDB_KEY is unset it does
|
||||
// nothing. A client without a score is checked in the background, by the
|
||||
// request's address, if its history counts an offence, and the request does
|
||||
// not wait for the answer. The score is then used for each address of the
|
||||
// client. Its figures are the score and SWW_ABUSEIPDB_MIN_SCORE.
|
||||
// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with
|
||||
// the score, when its score of the client is a hit, and refuses the request
|
||||
// when SWW_REPUTATION_ACTION is deny, as dnsbl does for a zone. While
|
||||
// SWW_ABUSEIPDB_KEY is unset it does nothing. A client without a score is
|
||||
// checked in the background, by the request's address, if its history counts
|
||||
// an offence, and the request does not wait for the answer. The score is then
|
||||
// used for each address of the client. Its figures are the score and
|
||||
// SWW_ABUSEIPDB_MIN_SCORE.
|
||||
type abuseIPDB struct{}
|
||||
|
||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||
@@ -39,6 +40,7 @@ func (abuseIPDB) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
}
|
||||
|
||||
rq.abuseIPDBHit = true
|
||||
rq.addTag("abuseipdb")
|
||||
rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
|
||||
"scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE")
|
||||
|
||||
|
||||
@@ -335,7 +335,7 @@ func newDiscriminatorInBubble(
|
||||
) (*discriminator.Server, *output) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{deniedCountries: "KP"}
|
||||
settings := map[string]string{tagDeny: "country=KP"}
|
||||
maps.Copy(settings, env)
|
||||
|
||||
server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now, settings)
|
||||
@@ -542,7 +542,7 @@ func TestDiscriminatorAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
|
||||
|
||||
return discriminator.New(params)
|
||||
}, held, time.Now, map[string]string{
|
||||
deniedCountries: "KP", wafMode: block,
|
||||
tagDeny: "country=KP", wafMode: block,
|
||||
dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(),
|
||||
abuseIPDBKey: accountKey,
|
||||
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
||||
|
||||
@@ -211,7 +211,7 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) {
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "1",
|
||||
banScopeV4Prefix: "24",
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
|
||||
// fromDE's ban covers otherClient, which is refused unasked about.
|
||||
@@ -266,7 +266,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "1",
|
||||
denyNets: denied,
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
}
|
||||
|
||||
if tc.setting != "" {
|
||||
@@ -276,7 +276,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
||||
s, _, _ := startWithClock(t, geojsURL, env)
|
||||
|
||||
s.get(denied, tc.status, requestlog.ActionDenied)
|
||||
s.get(fromKP, tc.status, requestlog.ActionCountryDenied)
|
||||
s.get(fromKP, tc.status, requestlog.ActionDenied)
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
s.get(fromDE, tc.status, requestlog.ActionRateLimited)
|
||||
s.get(fromDE, tc.status, requestlog.ActionBanned)
|
||||
@@ -291,7 +291,7 @@ func TestBanNotes(t *testing.T) {
|
||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "1",
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
start := clk.Now()
|
||||
|
||||
|
||||
@@ -17,30 +17,31 @@ type percentage struct {
|
||||
}
|
||||
|
||||
// biasedThresholdsSet reports whether a biased threshold can lower a
|
||||
// client's limits: one of its lists is not empty,
|
||||
// SWW_UNKNOWN_LIMIT_PERCENT is below 100, or SWW_ASN_LIMIT_PERCENT_URL
|
||||
// is set. The client's lookup is then needed before its request goes on.
|
||||
// client's limits by its lookup: SWW_ASN_BYTES_PERCENT or
|
||||
// SWW_COUNTRY_BYTES_PERCENT is not empty, SWW_UNKNOWN_LIMIT_PERCENT is
|
||||
// below 100, or SWW_ASN_LIMIT_PERCENT_URL is set. The client's lookup is
|
||||
// then needed before its request goes on.
|
||||
func biasedThresholdsSet(cfg *config.Config) bool {
|
||||
return len(cfg.ASNLimitPercent) > 0 || len(cfg.CountryLimitPercent) > 0 ||
|
||||
len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 ||
|
||||
return len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 ||
|
||||
cfg.UnknownLimitPercent < whole || cfg.ASNLimitPercentURL != ""
|
||||
}
|
||||
|
||||
// limitPercentages returns the client's limit percentages, for the rate
|
||||
// limits and for the byte limits, by its AS number and country as looked
|
||||
// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB
|
||||
// that list it. Each is the lowest of those the settings give it, the
|
||||
// first of them in the order below when several are lowest: the
|
||||
// percentage SWW_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
||||
// SWW_ASN_LIMIT_PERCENT_URL names gives it, the one
|
||||
// SWW_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
||||
// country, unless its request skipped the wait for GeoJS,
|
||||
// SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, the
|
||||
// percentage of SWW_BLOCKLIST_ACTION while it is limit, and for a client
|
||||
// a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, the
|
||||
// percentage of SWW_REPUTATION_ACTION while it is limit. For the byte
|
||||
// limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT take the
|
||||
// place of the first three for an AS number or a country they list.
|
||||
// limits and for the byte limits, by its tags, its AS number and country
|
||||
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
||||
// AbuseIPDB that list it. Each is the lowest of those the settings give
|
||||
// it, the first of them in the order below when several are lowest: the
|
||||
// percentage SWW_TAG_LIMIT_PERCENT gives its asn= tag, the one the file
|
||||
// SWW_ASN_LIMIT_PERCENT_URL names gives its AS number, the one
|
||||
// SWW_TAG_LIMIT_PERCENT gives its country= tag, the lowest it gives its
|
||||
// other tags, for a client without a country, unless its request skipped
|
||||
// the wait for GeoJS, SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist
|
||||
// lists, the percentage of SWW_BLOCKLIST_ACTION while it is limit, and
|
||||
// for a client a DNSBL zone's verdict lists, or whose AbuseIPDB score is
|
||||
// a hit, the percentage of SWW_REPUTATION_ACTION while it is limit. For
|
||||
// the byte limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT
|
||||
// take the place of the first three for an AS number or a country they
|
||||
// list.
|
||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg := rq.h.config
|
||||
asn, country := rq.line.ASN, rq.line.Country
|
||||
@@ -65,10 +66,18 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
reputationListed = percentage{cfg.ReputationLimitPercent, "SWW_REPUTATION_ACTION"}
|
||||
}
|
||||
|
||||
asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWW_ASN_LIMIT_PERCENT"),
|
||||
fetched)
|
||||
countryRequests := given(cfg.CountryLimitPercent, country,
|
||||
"SWW_COUNTRY_LIMIT_PERCENT")
|
||||
const tagSetting = "SWW_TAG_LIMIT_PERCENT"
|
||||
|
||||
asnRequests := lowest(given(cfg.TagLimitPercent, asnTag(asn), tagSetting), fetched)
|
||||
countryRequests := given(cfg.TagLimitPercent, countryTag(country), tagSetting)
|
||||
|
||||
otherTags := percentage{percent: whole}
|
||||
|
||||
for _, tag := range rq.line.Tags {
|
||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||
otherTags = lowest(otherTags, given(cfg.TagLimitPercent, tag, tagSetting))
|
||||
}
|
||||
}
|
||||
|
||||
asnBytes, countryBytes := asnRequests, countryRequests
|
||||
if _, listed := cfg.ASNBytesPercent[asn]; listed {
|
||||
@@ -79,12 +88,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
countryBytes = given(cfg.CountryBytesPercent, country, "SWW_COUNTRY_BYTES_PERCENT")
|
||||
}
|
||||
|
||||
return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed),
|
||||
lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed)
|
||||
return lowest(asnRequests, countryRequests, otherTags, unknown, blocklisted,
|
||||
reputationListed),
|
||||
lowest(asnBytes, countryBytes, otherTags, unknown, blocklisted, reputationListed)
|
||||
}
|
||||
|
||||
// given returns the percentage percents, the setting named setting, gives
|
||||
// code, an AS number or a country, or whole when it does not list code.
|
||||
// code, an AS number, a country or a tag, or whole when it does not list code.
|
||||
func given(percents map[string]int64, code, setting string) percentage {
|
||||
percent, listed := percents[code]
|
||||
if !listed {
|
||||
|
||||
@@ -22,8 +22,7 @@ import (
|
||||
|
||||
// The biased thresholds.
|
||||
const (
|
||||
asnLimitPercent = "SWW_ASN_LIMIT_PERCENT"
|
||||
countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT"
|
||||
tagLimitPercent = "SWW_TAG_LIMIT_PERCENT"
|
||||
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
||||
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
||||
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
||||
@@ -35,6 +34,11 @@ const (
|
||||
asnDEHalf = asnDE + ":50"
|
||||
asnDEQuarter = asnDE + ":25"
|
||||
countryDEHalf = "de:50"
|
||||
// The same, as SWW_TAG_LIMIT_PERCENT names fromDE's tags.
|
||||
asnTagDEHalf = "asn=64496:50"
|
||||
asnTagDEQuarter = "asn=64496:25"
|
||||
countryTagDEHalf = "country=de:50"
|
||||
countryTagDEQuarter = "country=de:25"
|
||||
// noCountry is in an AS of its own, AS64500, and in no country.
|
||||
noCountry = "192.0.2.80"
|
||||
// fourAMinute is the rate limit these tests set: half of it is 2
|
||||
@@ -54,8 +58,8 @@ func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
setting, value, from string
|
||||
}{
|
||||
{asnLimitPercent, asnDEHalf, fromDE},
|
||||
{countryLimitPercent, countryDEHalf, fromDE},
|
||||
{tagLimitPercent, asnTagDEHalf, fromDE},
|
||||
{tagLimitPercent, countryTagDEHalf, fromDE},
|
||||
{unknownLimitPercent, "50", unplaced},
|
||||
} {
|
||||
t.Run(tc.setting, func(t *testing.T) {
|
||||
@@ -96,8 +100,8 @@ func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
setting, value, from string
|
||||
}{
|
||||
{asnLimitPercent, asnDEHalf, fromDE},
|
||||
{countryLimitPercent, "DE:50", fromDE},
|
||||
{tagLimitPercent, asnTagDEHalf, fromDE},
|
||||
{tagLimitPercent, "Country=DE:50", fromDE},
|
||||
{unknownLimitPercent, "50", unplaced},
|
||||
{asnBytesPercent, "as64496:50", fromDE},
|
||||
{countryBytesPercent, countryDEHalf, fromDE},
|
||||
@@ -151,13 +155,13 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T)
|
||||
},
|
||||
{
|
||||
"raising the byte limits back",
|
||||
map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"},
|
||||
"50 from " + asnLimitPercent, none, "",
|
||||
map[string]string{tagLimitPercent: asnTagDEHalf, asnBytesPercent: asnDE + ":100"},
|
||||
"50 from " + tagLimitPercent, none, "",
|
||||
},
|
||||
{
|
||||
"raising the byte limits back, by country",
|
||||
map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"},
|
||||
"50 from " + countryLimitPercent, none, "",
|
||||
map[string]string{tagLimitPercent: countryTagDEHalf, countryBytesPercent: "de:100"},
|
||||
"50 from " + tagLimitPercent, none, "",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -184,19 +188,19 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T)
|
||||
func TestZeroPercentIsAZeroAllowance(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"})
|
||||
s, _, _ := startWithLookups(t, map[string]string{unknownLimitPercent: "0"})
|
||||
|
||||
// The first request breaks the limit, and bans the client; the log line
|
||||
// gives the 0.
|
||||
line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
line := s.get(unplaced, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
if line.fields["limit_percent"] != float64(0) ||
|
||||
line.fields["limit_percent_setting"] != asnLimitPercent {
|
||||
line.fields["limit_percent_setting"] != unknownLimitPercent {
|
||||
t.Errorf("log line has limit_percent %v from %v, want 0 from %s",
|
||||
line.fields["limit_percent"], line.fields["limit_percent_setting"],
|
||||
asnLimitPercent)
|
||||
unknownLimitPercent)
|
||||
}
|
||||
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
|
||||
s.get(unplaced, http.StatusForbidden, requestlog.ActionBanned)
|
||||
}
|
||||
|
||||
func TestLowestPercentageApplies(t *testing.T) {
|
||||
@@ -211,29 +215,29 @@ func TestLowestPercentageApplies(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
"the country's",
|
||||
map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"},
|
||||
fromDE, "25 from " + countryLimitPercent,
|
||||
map[string]string{tagLimitPercent: asnTagDEHalf + "," + countryTagDEQuarter},
|
||||
fromDE, "25 from " + tagLimitPercent,
|
||||
},
|
||||
{
|
||||
"the AS number's",
|
||||
map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf},
|
||||
fromDE, "25 from " + asnLimitPercent,
|
||||
map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEHalf},
|
||||
fromDE, "25 from " + tagLimitPercent,
|
||||
},
|
||||
{
|
||||
"the AS number's, the first of two alike",
|
||||
map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"},
|
||||
fromDE, "25 from " + asnLimitPercent,
|
||||
map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEQuarter},
|
||||
fromDE, "25 from " + tagLimitPercent,
|
||||
},
|
||||
{
|
||||
"that for a client without a country",
|
||||
map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"},
|
||||
map[string]string{tagLimitPercent: "asn=64500:50", unknownLimitPercent: "25"},
|
||||
noCountry, "25 from " + unknownLimitPercent,
|
||||
},
|
||||
{
|
||||
// SWW_UNKNOWN_LIMIT_PERCENT is left at its default, 100.
|
||||
"the AS number's, for a client without a country",
|
||||
map[string]string{asnLimitPercent: "AS64500:25"},
|
||||
noCountry, "25 from " + asnLimitPercent,
|
||||
map[string]string{tagLimitPercent: "asn=64500:25"},
|
||||
noCountry, "25 from " + tagLimitPercent,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -304,8 +308,12 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
|
||||
setting, value string
|
||||
waits bool
|
||||
}{
|
||||
{asnLimitPercent, asnDEHalf, true},
|
||||
{countryLimitPercent, countryDEHalf, true},
|
||||
{tagLimitPercent, asnTagDEHalf, true},
|
||||
{tagLimitPercent, countryTagDEHalf, true},
|
||||
{tagDeny, denyKP, true},
|
||||
// A tag the lookup does not give needs no answer.
|
||||
{tagLimitPercent, "vpn:50", false},
|
||||
{tagDeny, "vpn", false},
|
||||
{asnBytesPercent, asnDEHalf, true},
|
||||
{countryBytesPercent, countryDEHalf, true},
|
||||
{unknownLimitPercent, "99", true},
|
||||
@@ -363,15 +371,15 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T
|
||||
{
|
||||
// A quarter of 12 requests a minute is 3: the fourth breaks it.
|
||||
"a rate limit",
|
||||
map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter},
|
||||
map[string]string{rateLimitPerMinute: "12", tagLimitPercent: asnTagDEQuarter},
|
||||
3, http.StatusForbidden, requestlog.ActionRateLimited,
|
||||
"requests per minute over the limit of 3", "25 from " + asnLimitPercent,
|
||||
"requests per minute over the limit of 3", "25 from " + tagLimitPercent,
|
||||
},
|
||||
{
|
||||
// The byte limits' percentage, not the rate limits'.
|
||||
"a byte limit",
|
||||
map[string]string{
|
||||
bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter,
|
||||
bytesLimitPerMinute: twoUploads, tagLimitPercent: asnTagDEQuarter,
|
||||
asnBytesPercent: asnDEHalf,
|
||||
},
|
||||
0, http.StatusOK, requestlog.ActionForward,
|
||||
@@ -487,7 +495,7 @@ func wantPercent(t *testing.T, what string, percent *int64, setting, want string
|
||||
}
|
||||
|
||||
// percentText gives a limit percentage and the setting that gave it as
|
||||
// text, such as "50 from SWW_ASN_LIMIT_PERCENT", or none when both are
|
||||
// text, such as "50 from SWW_TAG_LIMIT_PERCENT", or none when both are
|
||||
// left out.
|
||||
func percentText(percent *int64, setting string) string {
|
||||
switch {
|
||||
|
||||
@@ -17,9 +17,10 @@ import (
|
||||
// SWW_WAF_MODE is off or SWW_WAF_EXEMPT_PATHS exempts its path, as
|
||||
// pathExempt decides, and notes the rules it matched and its score in the
|
||||
// log line, and the rules in the metrics. A score at or over
|
||||
// SWW_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
|
||||
// and in block mode refuses the request with 403, which is an offence its
|
||||
// client's history counts. Its figures are the score and the threshold.
|
||||
// SWW_WAF_ANOMALY_THRESHOLD is a match: it adds the tag exploitsignature,
|
||||
// raises the waf_block alert, and in block mode refuses the request with 403,
|
||||
// which is an offence its client's history counts. Its figures are the score
|
||||
// and the threshold.
|
||||
type coreRuleSet struct{}
|
||||
|
||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||
@@ -50,6 +51,7 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.addTag(config.ExploitSignatureTag)
|
||||
rq.alertWAFBlock(result)
|
||||
|
||||
if cfg.WAFMode == config.WAFModeDetect {
|
||||
|
||||
@@ -61,6 +61,41 @@ func wantWAF(t *testing.T, line logLine, score *int, ruleIDs ...int) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreRuleSetMatchAddsTheExploitSignatureTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mode string
|
||||
status int
|
||||
action string
|
||||
}{
|
||||
{block, block, http.StatusForbidden, requestlog.ActionWAFBlocked},
|
||||
{detect, detect, http.StatusOK, requestlog.ActionForward},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode})
|
||||
|
||||
wantTags(t, s.request(client, sqlInjection, tc.status, tc.action),
|
||||
"exploitsignature")
|
||||
wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectedAttackIsRefusedByTagDenyLast(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{
|
||||
wafMode: detect, tagDeny: "exploitsignature",
|
||||
})
|
||||
|
||||
line := s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`)
|
||||
}
|
||||
|
||||
func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package discriminator
|
||||
|
||||
// The heuristic countries. It reads SWW_DENIED_COUNTRIES and
|
||||
// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
// The heuristic countries. It reads SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,32 +10,26 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// countries refuses a request by the client's country as it was looked up. A
|
||||
// client without a country, or whose country cannot be found, is refused only
|
||||
// by SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. Its figures are the country and the
|
||||
// setting that refused it.
|
||||
// countries refuses a request when SWW_EXCLUSIVELY_ALLOWED_COUNTRIES is set and
|
||||
// does not hold the client's country as it was looked up. A client without a
|
||||
// country, or whose country cannot be found, is refused. Its figures are the
|
||||
// country and the setting that refused it.
|
||||
type countries struct{}
|
||||
|
||||
func (countries) name() string { return "countries" }
|
||||
|
||||
func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
denied := rq.h.config.DeniedCountries
|
||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
||||
country := rq.line.Country
|
||||
|
||||
var setting string
|
||||
|
||||
switch {
|
||||
case slices.Contains(denied, country):
|
||||
setting = "SWW_DENIED_COUNTRIES"
|
||||
case len(allowed) > 0 && !slices.Contains(allowed, country):
|
||||
setting = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
default:
|
||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionCountryDenied,
|
||||
figures: map[string]any{"country": country, "setting": setting},
|
||||
action: requestlog.ActionCountryDenied,
|
||||
figures: map[string]any{
|
||||
"country": country, "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -33,13 +33,13 @@ func TestCountryLists(t *testing.T) {
|
||||
env map[string]string
|
||||
refused []string
|
||||
}{
|
||||
{"denied", map[string]string{deniedCountries: "kp"}, []string{fromKP}},
|
||||
{
|
||||
"exclusively allowed", map[string]string{allowedCountries: "DE"},
|
||||
[]string{fromKP, unplaced},
|
||||
},
|
||||
{
|
||||
"both", map[string]string{deniedCountries: "kp", allowedCountries: "de,fr"},
|
||||
"with a country in SWW_TAG_DENY",
|
||||
map[string]string{tagDeny: denyKP, allowedCountries: "de,fr"},
|
||||
[]string{fromKP, unplaced},
|
||||
},
|
||||
} {
|
||||
@@ -95,12 +95,12 @@ func TestCountryRefusalNamesItsHeuristicCountryAndSetting(t *testing.T) {
|
||||
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
s, _, _ := startWithClock(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h", deniedCountries: "kp", allowedCountries: "de",
|
||||
lookupTimeout: "1h", allowedCountries: "de",
|
||||
})
|
||||
|
||||
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||
wantHeuristic(t, line, "countries",
|
||||
`{"country": "KP", "setting": "SWW_DENIED_COUNTRIES"}`)
|
||||
`{"country": "KP", "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"}`)
|
||||
|
||||
line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||
wantHeuristic(t, line, "countries",
|
||||
@@ -117,9 +117,9 @@ func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
|
||||
})
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
deniedCountries: "kp",
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
allowedCountries: "de",
|
||||
})
|
||||
|
||||
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
||||
@@ -202,7 +202,7 @@ func TestPrivateAddressIsNeverLookedUp(t *testing.T) {
|
||||
env map[string]string
|
||||
}{
|
||||
{"no setting needs the lookup", nil},
|
||||
{"a country list is set", map[string]string{deniedCountries: "kp"}},
|
||||
{"a tag setting needs it", map[string]string{tagDeny: denyKP}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -13,9 +13,9 @@ import (
|
||||
)
|
||||
|
||||
// crowdSec refuses a request when a decision of the CrowdSec decision list on
|
||||
// the client is in force at now. It then notes the list, as noteHit does, and
|
||||
// bans the client until that decision ends. Its figures are the decision's
|
||||
// scenario and when it ends.
|
||||
// the client is in force at now. It then adds the tag crowdsec, notes the
|
||||
// list, as noteHit does, and bans the client until that decision ends. Its
|
||||
// figures are the decision's scenario and when it ends.
|
||||
type crowdSec struct{}
|
||||
|
||||
func (crowdSec) name() string { return "crowdsec" }
|
||||
@@ -26,6 +26,7 @@ func (crowdSec) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.addTag("crowdsec")
|
||||
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
||||
"listed by the CrowdSec decision list")
|
||||
rq.banForCrowdSec(now, decision)
|
||||
|
||||
@@ -108,6 +108,7 @@ func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) {
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+
|
||||
`"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`)
|
||||
wantTags(t, line, "crowdsec")
|
||||
}
|
||||
|
||||
func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
|
||||
|
||||
@@ -414,7 +414,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||
|
||||
geojsURL, asked, release := startHeldGeoJS(t)
|
||||
s := startDiscriminator(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h", deniedCountries: "kp", bytesLimitPerMinute: byteLimit,
|
||||
lookupTimeout: "1h", tagDeny: denyKP, bytesLimitPerMinute: byteLimit,
|
||||
})
|
||||
|
||||
first := s.question(siteA, client, "/")
|
||||
@@ -765,7 +765,7 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
|
||||
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
s := startDiscriminator(t, geojsURL, map[string]string{
|
||||
addLookupHeaders: tc.setting, lookupTimeout: "1h", deniedCountries: "kp",
|
||||
addLookupHeaders: tc.setting, lookupTimeout: "1h", tagDeny: denyKP,
|
||||
})
|
||||
|
||||
got := s.ask(fromDE, "/")
|
||||
|
||||
@@ -11,7 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
|
||||
// does, and refuses the request when SWW_REPUTATION_ACTION is deny. Being
|
||||
// does, adds the tag dnsbl=<zone> for each, the zone's key masked, and refuses
|
||||
// the request when SWW_REPUTATION_ACTION is deny. Being
|
||||
// limit, it lowers the client's limits instead (see limitPercentages), and
|
||||
// being log, it does nothing more. A zone without a verdict on the client is
|
||||
// asked about it in the background, and the request does not wait for the
|
||||
@@ -25,6 +26,10 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
for _, zone := range listedBy {
|
||||
rq.addTag("dnsbl=" + zone)
|
||||
}
|
||||
|
||||
if !rq.dnsblListed || rq.h.config.ReputationAction != deny {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -49,18 +49,23 @@ func beforeLookup() []heuristic {
|
||||
}
|
||||
|
||||
// afterLookup are the heuristics that run once the lookup has been made.
|
||||
// tag_deny runs twice: before the rate limits, on the tags known by then, and
|
||||
// last, on the tag rule_files and core_rule_set can add.
|
||||
func afterLookup() []heuristic {
|
||||
return []heuristic{
|
||||
tagLists{},
|
||||
countries{},
|
||||
blocklists{},
|
||||
crowdSec{},
|
||||
dnsbl{},
|
||||
abuseIPDB{},
|
||||
tagDeny{},
|
||||
rateLimits{},
|
||||
trapPaths{},
|
||||
ruleFiles{},
|
||||
headerLimits{},
|
||||
coreRuleSet{},
|
||||
tagDeny{},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "2",
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
start := clk.Now()
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
@@ -19,7 +20,8 @@ const (
|
||||
)
|
||||
|
||||
// lookUp looks up the client's AS number and country, in the lookup
|
||||
// database or through GeoJS, and notes them for the log line, unless
|
||||
// database or through GeoJS, and notes them for the log line and as the
|
||||
// request's asn= and country= tags, unless
|
||||
// SWW_LOOKUP_SOURCE is off or the client is on a private, loopback or
|
||||
// link-local address, which no lookup can place. The lookup database
|
||||
// answers at once. With GeoJS, while a setting needs the answer, such as a
|
||||
@@ -47,6 +49,34 @@ func (rq *request) lookUp(ctx context.Context, now time.Time) {
|
||||
rq.line.ASN = rq.lookupAnswer.ASN
|
||||
rq.line.ASName = rq.lookupAnswer.ASName
|
||||
rq.line.Country = rq.lookupAnswer.Country
|
||||
|
||||
if tag := asnTag(rq.line.ASN); tag != "" {
|
||||
rq.addTag(tag)
|
||||
}
|
||||
|
||||
if tag := countryTag(rq.line.Country); tag != "" {
|
||||
rq.addTag(tag)
|
||||
}
|
||||
}
|
||||
|
||||
// asnTag returns the tag of asn, an AS number as the lookup gives it, such as
|
||||
// asn=64496 for AS64496, or "" for no AS number.
|
||||
func asnTag(asn string) string {
|
||||
if asn == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
return "asn=" + strings.TrimPrefix(asn, "AS")
|
||||
}
|
||||
|
||||
// countryTag returns the tag of country, a code as the lookup gives it, such
|
||||
// as country=de for DE, or "" for no country.
|
||||
func countryTag(country string) string {
|
||||
if country == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
return "country=" + strings.ToLower(country)
|
||||
}
|
||||
|
||||
// addLookup adds answer, an answer about a client from the lookup
|
||||
|
||||
@@ -94,9 +94,9 @@ func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
|
||||
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 58, 0, time.UTC)}
|
||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now,
|
||||
map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
deniedCountries: "kp",
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
|
||||
requestFromDE := func() *http.Request {
|
||||
|
||||
@@ -195,10 +195,10 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
||||
_, _ = io.WriteString(w, "hello")
|
||||
})
|
||||
env := map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
metricsToken: token,
|
||||
metricsTopN: "2",
|
||||
deniedCountries: "kp",
|
||||
trustedProxies: trustLocalhost,
|
||||
metricsToken: token,
|
||||
metricsTopN: "2",
|
||||
allowedCountries: "de,fr",
|
||||
}
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
|
||||
@@ -322,13 +322,13 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||
addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
|
||||
map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
metricsToken: token,
|
||||
deniedCountries: "kp",
|
||||
trustedProxies: trustLocalhost,
|
||||
metricsToken: token,
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
|
||||
// GeoJS fails, so the client counts as coming from an unknown country,
|
||||
// which SWW_DENIED_COUNTRIES does not refuse.
|
||||
// GeoJS fails, so the client has no country= tag, and SWW_TAG_DENY does
|
||||
// not refuse it.
|
||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||
req.Header.Set(forwardedFor, fromDE)
|
||||
wantStatus(t, do(t, req), http.StatusOK)
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
||||
lookupTimeout: "1h",
|
||||
rateLimitPerMinute: "1",
|
||||
denyNets: denied,
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
}
|
||||
|
||||
if tc.setting != "" {
|
||||
@@ -71,7 +71,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
||||
for _, sent := range []struct{ from, refusal string }{
|
||||
{denied, requestlog.ActionDenied},
|
||||
{banned, requestlog.ActionBanned},
|
||||
{fromKP, requestlog.ActionCountryDenied},
|
||||
{fromKP, requestlog.ActionDenied},
|
||||
{fromDE, requestlog.ActionRateLimited},
|
||||
} {
|
||||
if !tc.observe {
|
||||
|
||||
@@ -155,11 +155,12 @@ func newHandler(params Params) *handler {
|
||||
h.geojs = lookup.New(lookup.Params{
|
||||
URL: params.GeoJSURL,
|
||||
Timeout: params.Config.LookupTimeout,
|
||||
// The country lists, the headers and the biased thresholds act on
|
||||
// the answer before the request goes on.
|
||||
Wait: len(params.Config.DeniedCountries) > 0 ||
|
||||
len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||
params.Config.AddLookupHeaders || biasedThresholdsSet(params.Config),
|
||||
// The country list, the headers, the tag settings that name an asn=
|
||||
// or country= tag and the biased thresholds act on the answer before
|
||||
// the request goes on.
|
||||
Wait: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
|
||||
biasedThresholdsSet(params.Config),
|
||||
Answered: h.addLookup,
|
||||
Now: params.Now,
|
||||
ProcessLog: params.ProcessLog,
|
||||
@@ -183,6 +184,7 @@ func newReputation(
|
||||
lists := reputation.New(reputation.Params{
|
||||
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
||||
TagLists: cfg.TagLists,
|
||||
CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey,
|
||||
Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||
})
|
||||
|
||||
@@ -69,7 +69,9 @@ const (
|
||||
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
||||
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
||||
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
||||
deniedCountries = "SWW_DENIED_COUNTRIES"
|
||||
tagDeny = "SWW_TAG_DENY"
|
||||
// denyKP is the value of SWW_TAG_DENY that refuses fromKP's country.
|
||||
denyKP = "country=kp"
|
||||
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
banResponse = "SWW_BAN_RESPONSE"
|
||||
limitBanDuration = "SWW_LIMIT_BAN_DURATION"
|
||||
|
||||
@@ -136,7 +136,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
|
||||
rateLimitPerMinute: "1",
|
||||
rateLimitExemptPaths: "/assets/,/favicon.ico",
|
||||
denyNets: denied,
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
|
||||
// The answers are kept before the requests, so that none waits for
|
||||
@@ -167,7 +167,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
|
||||
s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned)
|
||||
s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied)
|
||||
s.request(fromKP, "/assets/app.js",
|
||||
http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||
http.StatusForbidden, requestlog.ActionDenied)
|
||||
}
|
||||
|
||||
func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
|
||||
|
||||
@@ -191,7 +191,7 @@ func TestErrorBurstIsNotLoweredForAClientWithLowerLimits(t *testing.T) {
|
||||
lookupTimeout: "1h",
|
||||
errorBurstThreshold: "2",
|
||||
rulesDir: writeRules(t, testRules),
|
||||
countryLimitPercent: countryDEHalf,
|
||||
tagLimitPercent: countryTagDEHalf,
|
||||
})
|
||||
|
||||
// Half of the threshold would be one, which the second refusal is over.
|
||||
|
||||
@@ -172,7 +172,7 @@ func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, queue := startWithLookups(t, map[string]string{
|
||||
blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE,
|
||||
blocklistURLs: dropURL, allowedCountries: "de", allowNets: fromDE,
|
||||
})
|
||||
loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})
|
||||
|
||||
@@ -211,18 +211,18 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
|
||||
// percentText gives them, and limitHit its limit_hit.
|
||||
want, limitHit string
|
||||
}{
|
||||
{limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes},
|
||||
{limitHalf, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes},
|
||||
{limitQuarter, asnTagDEHalf, "25 from " + blocklistAction, minuteBytes},
|
||||
// The AS number's, the first of two alike.
|
||||
{limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{actionLog, asnDE + ":100", none, ""},
|
||||
{limitQuarter, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes},
|
||||
{actionLog, "asn=64511:25", none, ""},
|
||||
} {
|
||||
t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
|
||||
blocklistAction: tc.action, asnLimitPercent: tc.asnPercent,
|
||||
blocklistAction: tc.action, tagLimitPercent: tc.asnPercent,
|
||||
})
|
||||
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
||||
|
||||
@@ -246,7 +246,7 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T)
|
||||
|
||||
const (
|
||||
fromURL = "25 from " + asnLimitPercentURL
|
||||
fromSetting = "25 from " + asnLimitPercent
|
||||
fromSetting = "25 from " + tagLimitPercent
|
||||
)
|
||||
|
||||
for _, tc := range []struct {
|
||||
@@ -260,16 +260,16 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T)
|
||||
{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
|
||||
{
|
||||
"the file's, lower than the setting's",
|
||||
map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL,
|
||||
map[string]string{tagLimitPercent: asnTagDEHalf}, asnDEQuarter, fromURL, fromURL,
|
||||
},
|
||||
{
|
||||
"the setting's, lower than the file's",
|
||||
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf,
|
||||
map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEHalf,
|
||||
fromSetting, fromSetting,
|
||||
},
|
||||
{
|
||||
"the setting's, the first of two alike",
|
||||
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter,
|
||||
map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEQuarter,
|
||||
fromSetting, fromSetting,
|
||||
},
|
||||
{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
|
||||
@@ -482,6 +482,46 @@ func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
load func(*discriminator.Server)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
"dnsbl",
|
||||
map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionLog,
|
||||
},
|
||||
func(server *discriminator.Server) {
|
||||
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
|
||||
},
|
||||
[]string{"asn=64496", "country=de", "dnsbl=" + dnsblZone},
|
||||
},
|
||||
{
|
||||
abuseipdb,
|
||||
map[string]string{abuseIPDBKey: accountKey, reputationAction: actionLog},
|
||||
func(server *discriminator.Server) {
|
||||
loadScores(server, map[string]int64{fromDE: 80})
|
||||
},
|
||||
[]string{abuseipdb, "asn=64496", "country=de"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, tc.env)
|
||||
tc.load(server)
|
||||
|
||||
wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
tc.want...)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -138,6 +138,17 @@ func (h *handler) startRequest(
|
||||
return rq
|
||||
}
|
||||
|
||||
// addTag adds tag, in lower case, to the request's tags, which stay sorted and
|
||||
// hold each tag once.
|
||||
func (rq *request) addTag(tag string) {
|
||||
tag = strings.ToLower(tag)
|
||||
|
||||
at, found := slices.BinarySearch(rq.line.Tags, tag)
|
||||
if !found {
|
||||
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag)
|
||||
}
|
||||
}
|
||||
|
||||
// requestHeaders returns the headers of r that names lists, by name in
|
||||
// lower case, each with its values joined by ", ". Authorization, Cookie
|
||||
// and Set-Cookie are never among them, whatever names says.
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// ruleFiles checks the request against the rules of the rule files at now,
|
||||
// notes the ids of those it matches in the log line, and refuses the request
|
||||
// notes the ids of those it matches in the log line, adds the tag
|
||||
// exploitsignature when it matches any, and refuses the request
|
||||
// when the last rule it matches refuses it: with ActionRuleBlocked for a block
|
||||
// rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock
|
||||
// for a clear sign of attack, or in observe mode raises the alert for the ban
|
||||
@@ -35,6 +37,8 @@ func (ruleFiles) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.addTag(config.ExploitSignatureTag)
|
||||
|
||||
// Only the last rule matched can refuse the request.
|
||||
last := matched[len(matched)-1]
|
||||
figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action}
|
||||
|
||||
@@ -93,6 +93,20 @@ func TestEachRuleAction(t *testing.T) {
|
||||
s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
|
||||
}
|
||||
|
||||
func TestEachRuleActionAddsTheExploitSignatureTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{
|
||||
rulesDir: writeRules(t, "noted path log ^/noted$\n"+testRules),
|
||||
})
|
||||
|
||||
wantTags(t, s.request(client, "/noted", http.StatusOK, requestlog.ActionForward),
|
||||
"exploitsignature")
|
||||
wantTags(t, s.request(client, "/blocked", http.StatusForbidden,
|
||||
requestlog.ActionRuleBlocked), "exploitsignature")
|
||||
wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward))
|
||||
}
|
||||
|
||||
func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestAllowNetsSkipEveryCheck(t *testing.T) {
|
||||
trustedProxies: trustLocalhost,
|
||||
allowNets: "198.51.100.0/24",
|
||||
denyNets: fromKP,
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
rateLimitPerMinute: "1",
|
||||
})
|
||||
|
||||
@@ -83,9 +83,9 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
|
||||
})
|
||||
geojsURL, asked := startGeoJS(t)
|
||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
denyNets: "203.0.113.0/24",
|
||||
deniedCountries: "kp",
|
||||
trustedProxies: trustLocalhost,
|
||||
denyNets: "203.0.113.0/24",
|
||||
tagDeny: denyKP,
|
||||
})
|
||||
|
||||
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
||||
@@ -142,7 +142,7 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) {
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
rateLimitExemptNets: listedAddr + "," + fromKP,
|
||||
deniedCountries: "kp",
|
||||
tagDeny: denyKP,
|
||||
rateLimitPerMinute: "1",
|
||||
})
|
||||
|
||||
@@ -154,7 +154,7 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) {
|
||||
{listedAddr, http.StatusOK, requestlog.ActionForward},
|
||||
{unlistedAddr, http.StatusOK, requestlog.ActionForward},
|
||||
{unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited},
|
||||
{fromKP, http.StatusForbidden, requestlog.ActionCountryDenied},
|
||||
{fromKP, http.StatusForbidden, requestlog.ActionDenied},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package discriminator
|
||||
|
||||
// The heuristic tag_deny. It reads SWW_TAG_DENY.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// tagDeny refuses a request that carries a tag in SWW_TAG_DENY. Its figure is
|
||||
// the first such tag, in sorted order.
|
||||
type tagDeny struct{}
|
||||
|
||||
func (tagDeny) name() string { return "tag_deny" }
|
||||
|
||||
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, tag := range rq.line.Tags {
|
||||
if slices.Contains(rq.h.config.TagDeny, tag) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionDenied,
|
||||
figures: map[string]any{"tag": tag},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package discriminator_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
func TestTagDenyRefusesALookupTagAndNamesTheTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithLookups(t, map[string]string{tagDeny: "Country=KP"})
|
||||
|
||||
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, "tag_deny", `{"tag": "country=kp"}`)
|
||||
wantTags(t, line, "asn=64511", "country=kp")
|
||||
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestTagDenyRefusesAnASNTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithLookups(t, map[string]string{tagDeny: "asn=64511"})
|
||||
|
||||
wantHeuristic(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied),
|
||||
"tag_deny", `{"tag": "asn=64511"}`)
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestTagDenyRefusesAListTagBeforeTheRateLimitsCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
tagLists: "torexit=" + torList, tagDeny: "torexit", rateLimitPerMinute: "1",
|
||||
})
|
||||
loadLists(t, server, map[string][]string{torList: {fromDE}})
|
||||
|
||||
// Each is refused for its tag: none is counted to break the limit of one.
|
||||
for range 3 {
|
||||
wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied),
|
||||
"tag_deny", `{"tag": "torexit"}`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagDenyRefusesAnExploitSignatureOfARuleThatOnlyLogs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{
|
||||
rulesDir: writeRules(t, "noted path log ^/\n"), tagDeny: "exploitsignature",
|
||||
})
|
||||
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`)
|
||||
wantTags(t, line, "exploitsignature")
|
||||
}
|
||||
|
||||
func TestTagLimitPercentLowersALimitTheLowestWinning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
rateLimitPerMinute: fourAMinute,
|
||||
tagLists: "vpn=" + vpnList + ",tor=" + torList,
|
||||
tagLimitPercent: "vpn:50,tor:25",
|
||||
})
|
||||
loadLists(t, server, map[string][]string{vpnList: {fromDE}, torList: {fromDE}})
|
||||
|
||||
// A quarter of 4 requests a minute: the second breaks the limit.
|
||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||
"25 from "+tagLimitPercent)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package discriminator
|
||||
|
||||
// The heuristic tag_lists. It reads the address lists of SWW_TAG_LISTS through
|
||||
// the reputation lists.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// tagLists adds to the request one tag for each list of SWW_TAG_LISTS that
|
||||
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
||||
// first so that every refusal's log line carries these tags.
|
||||
type tagLists struct{}
|
||||
|
||||
func (tagLists) name() string { return "tag_lists" }
|
||||
|
||||
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, name := range rq.h.lists.TagsOf(rq.client) {
|
||||
rq.addTag(name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package discriminator_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// tagLists is SWW_TAG_LISTS, and vpnList and torList lists it names.
|
||||
const (
|
||||
tagLists = "SWW_TAG_LISTS"
|
||||
vpnList = "https://lists.example/vpn.txt"
|
||||
torList = "https://lists.example/torexit.txt"
|
||||
)
|
||||
|
||||
// wantTags checks the log line's tags.
|
||||
func wantTags(t *testing.T, line logLine, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
got, _ := line.fields["tags"].([]any)
|
||||
|
||||
tags := make([]string, 0, len(got))
|
||||
for _, tag := range got {
|
||||
text, _ := tag.(string)
|
||||
tags = append(tags, text)
|
||||
}
|
||||
|
||||
if !slices.Equal(tags, want) {
|
||||
t.Errorf("log line has tags %v, want %v", tags, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupAndTagListsGiveTheRequestItsTagsSorted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
tagLists: "vpn=mullvad=" + vpnList + ",TorExit=" + torList,
|
||||
})
|
||||
loadLists(t, server, map[string][]string{
|
||||
vpnList: {fromDE}, torList: {fromDE + "/32", fromKP},
|
||||
})
|
||||
|
||||
wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
"asn=64496", "country=de", "torexit", "vpn=mullvad")
|
||||
wantTags(t, s.get(fromKP, http.StatusOK, requestlog.ActionForward),
|
||||
"asn=64511", "country=kp", "torexit")
|
||||
}
|
||||
|
||||
func TestRequestWithoutATagLeavesTheFieldOut(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// With no lookup, a client is not placed and no setting gives it a tag.
|
||||
s, _, _ := startWithClock(t, "", map[string]string{lookupSource: "off"})
|
||||
|
||||
line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
if _, present := line.fields["tags"]; present {
|
||||
t.Errorf("log line has tags %v, want none", line.fields["tags"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagListsTagEvenARequestTheCountryListRefuses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
tagLists: "vpn=" + vpnList, allowedCountries: "de",
|
||||
})
|
||||
loadLists(t, server, map[string][]string{vpnList: {fromKP}})
|
||||
|
||||
wantTags(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied),
|
||||
"asn=64511", "country=kp", "vpn")
|
||||
}
|
||||
Reference in new issue
Block a user