Request tags, with refusing and limits per tag (closes #175)
check / check (push) Successful in 3m36s
check / check (push) Successful in 3m36s
Each request gets sorted tags from the lookup, new address lists, the reputation sources and the rule matches, logged as `tags`. New settings: `SWW_TAG_LISTS`, `SWW_TAG_DENY`, `SWW_TAG_LIMIT_PERCENT`. They replace `SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT` and `SWW_DENIED_COUNTRIES`. New heuristics `tag_lists` and `tag_deny`; the latter runs before the rate limits and again after the Core Rule Set. Model: opus-5-5
This commit is contained in:
sneak committed
1 parent
2349b2a081
commit
60630859af
42 files changed
+1088
-319
No files matched your search
+123
-49
@@ -42,10 +42,10 @@ const (
|
||||
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
||||
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
||||
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
||||
deniedCountries = "SWW_DENIED_COUNTRIES"
|
||||
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
asnLimitPercent = "SWW_ASN_LIMIT_PERCENT"
|
||||
countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT"
|
||||
tagLists = "SWW_TAG_LISTS"
|
||||
tagDeny = "SWW_TAG_DENY"
|
||||
tagLimitPercent = "SWW_TAG_LIMIT_PERCENT"
|
||||
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
||||
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
||||
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
||||
@@ -274,9 +274,13 @@ func TestDefaults(t *testing.T) {
|
||||
wantNetblocks(t, cfg.AllowNets)
|
||||
wantNetblocks(t, cfg.RateLimitExemptNets)
|
||||
wantNetblocks(t, cfg.DenyNets)
|
||||
wantCountries(t, deniedCountries, cfg.DeniedCountries)
|
||||
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries)
|
||||
|
||||
if len(cfg.TagLists) != 0 || len(cfg.TagDeny) != 0 || len(cfg.TagLimitPercent) != 0 {
|
||||
t.Errorf("tag settings %v, %v and %v by default, want all empty",
|
||||
cfg.TagLists, cfg.TagDeny, cfg.TagLimitPercent)
|
||||
}
|
||||
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil || hostname == "" || cfg.InstanceName != hostname {
|
||||
t.Errorf("%s is %q, want the host's name %q (%v)", instanceName,
|
||||
@@ -327,7 +331,6 @@ func TestValuesAsSet(t *testing.T) {
|
||||
rateLimitPerMinute: "60",
|
||||
rateLimitPerHour: "600",
|
||||
rateLimitPerDay: "6000",
|
||||
deniedCountries: "cn, RU,kp,Xk",
|
||||
allowedCountries: "de",
|
||||
banResponse: "429",
|
||||
limitBanDuration: "15m",
|
||||
@@ -370,7 +373,6 @@ func TestValuesAsSet(t *testing.T) {
|
||||
wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
|
||||
wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
|
||||
wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
|
||||
wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK")
|
||||
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
|
||||
}
|
||||
|
||||
@@ -1101,7 +1103,7 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := config.FromEnvironment(environment{
|
||||
deniedCountries: "cn,ru",
|
||||
tagDeny: "country=cn,country=ru",
|
||||
allowedCountries: "de,RU",
|
||||
}.lookupEnv)
|
||||
if err == nil {
|
||||
@@ -1176,12 +1178,11 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, value := range map[string]string{
|
||||
deniedCountries: "kp",
|
||||
allowedCountries: "de",
|
||||
addLookupHeaders: enabled,
|
||||
tagDeny: "country=kp",
|
||||
tagLimitPercent: "asn=64496:50",
|
||||
asnLimitPercentURL: asnURL,
|
||||
asnLimitPercent: "AS64496:50",
|
||||
countryLimitPercent: "cn:25",
|
||||
asnBytesPercent: "AS64496:50",
|
||||
countryBytesPercent: "cn:25",
|
||||
unknownLimitPercent: "99",
|
||||
@@ -1204,12 +1205,12 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
|
||||
}
|
||||
|
||||
// Set empty, the lists need nothing looked up, and nor does
|
||||
// SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly
|
||||
// threshold per AS number that is off, any other anomaly threshold, or
|
||||
// a blocklist.
|
||||
// SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, a tag the
|
||||
// lookup does not give, an anomaly threshold per AS number that is off,
|
||||
// any other anomaly threshold, or a blocklist.
|
||||
env := environment{
|
||||
lookupSource: off, deniedCountries: "", allowedCountries: "",
|
||||
asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "",
|
||||
lookupSource: off, allowedCountries: "", asnBytesPercent: "",
|
||||
tagDeny: torExitTag, tagLimitPercent: "vpn:50",
|
||||
countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "",
|
||||
blocklistURLs: dropURL,
|
||||
}
|
||||
@@ -1355,34 +1356,96 @@ func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// torExitTag is a tag these tests name.
|
||||
const torExitTag = "torexit"
|
||||
|
||||
func TestTagSettingsAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const torURL = "https://example.com/tor.txt"
|
||||
|
||||
// Names and tags in either case, an AS number with leading zeros, and a
|
||||
// name holding a =.
|
||||
cfg := fromEnvironment(t, environment{
|
||||
tagLists: "TorExit=" + torURL +
|
||||
", vpn=mullvad=https://example.com/mullvad.txt",
|
||||
tagDeny: "TorExit, Country=KP, asn=014061",
|
||||
tagLimitPercent: "vpn:50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||
})
|
||||
|
||||
wantLists := []config.TagList{
|
||||
{Name: torExitTag, URL: torURL},
|
||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||
}
|
||||
if !slices.Equal(cfg.TagLists, wantLists) {
|
||||
t.Errorf("%s gave %+v, want %+v", tagLists, cfg.TagLists, wantLists)
|
||||
}
|
||||
|
||||
wantDeny := []string{torExitTag, "country=kp", "asn=14061"}
|
||||
if !slices.Equal(cfg.TagDeny, wantDeny) {
|
||||
t.Errorf("%s gave %v, want %v", tagDeny, cfg.TagDeny, wantDeny)
|
||||
}
|
||||
|
||||
wantPercents := map[string]int64{
|
||||
"vpn": 50, "asn=14061": 25, "country=cn": 1, "dnsbl=zen.example": 99,
|
||||
}
|
||||
if !maps.Equal(cfg.TagLimitPercent, wantPercents) {
|
||||
t.Errorf("%s gave %v, want %v", tagLimitPercent, cfg.TagLimitPercent, wantPercents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const lapi = "http://172.17.0.1:8080"
|
||||
|
||||
for _, tc := range []struct {
|
||||
name, listURL string
|
||||
env environment
|
||||
}{
|
||||
{"a blocklist", dropURL, environment{blocklistURLs: dropURL}},
|
||||
{"the file of AS:percent lines", asnURL, environment{asnLimitPercentURL: asnURL}},
|
||||
{
|
||||
"the CrowdSec decision list", lapi + "/v1/decisions",
|
||||
environment{"SWW_CROWDSEC_LAPI_URL": lapi, "SWW_CROWDSEC_LAPI_KEY": "key"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tc.env[tagLists] = "dup=" + tc.listURL
|
||||
|
||||
want := tagLists + `: "` + tc.listURL + `" is in SWW_BLOCKLIST_URLS, is ` +
|
||||
"SWW_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too"
|
||||
|
||||
_, err := config.FromEnvironment(tc.env.lookupEnv)
|
||||
if err == nil || err.Error() != want {
|
||||
t.Errorf("error %v, want %s", err, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBiasedThresholdsAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := fromEnvironment(t, environment{})
|
||||
if len(cfg.ASNLimitPercent) != 0 || len(cfg.CountryLimitPercent) != 0 ||
|
||||
len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 ||
|
||||
if len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 ||
|
||||
cfg.UnknownLimitPercent != 100 {
|
||||
t.Errorf("biased thresholds %v, %v, %v, %v and %d by default, "+
|
||||
"want four empty lists and 100", cfg.ASNLimitPercent, cfg.CountryLimitPercent,
|
||||
t.Errorf("biased thresholds %v, %v and %d by default, "+
|
||||
"want two empty lists and 100",
|
||||
cfg.ASNBytesPercent, cfg.CountryBytesPercent, cfg.UnknownLimitPercent)
|
||||
}
|
||||
|
||||
// AS numbers and countries in either case, an AS number with leading
|
||||
// zeros, 0 and 100.
|
||||
cfg = fromEnvironment(t, environment{
|
||||
asnLimitPercent: "AS14061:50, as16276:0,AS045102:100",
|
||||
countryLimitPercent: "cn:25,RU:50",
|
||||
asnBytesPercent: "as16276:75",
|
||||
countryBytesPercent: "ru:10",
|
||||
unknownLimitPercent: "0",
|
||||
})
|
||||
|
||||
for name, tc := range map[string]struct{ got, want map[string]int64 }{
|
||||
asnLimitPercent: {
|
||||
cfg.ASNLimitPercent,
|
||||
map[string]int64{"AS14061": 50, "AS16276": 0, "AS45102": 100},
|
||||
},
|
||||
countryLimitPercent: {cfg.CountryLimitPercent, map[string]int64{"CN": 25, "RU": 50}},
|
||||
asnBytesPercent: {cfg.ASNBytesPercent, map[string]int64{"AS16276": 75}},
|
||||
countryBytesPercent: {cfg.CountryBytesPercent, map[string]int64{"RU": 10}},
|
||||
} {
|
||||
@@ -1406,16 +1469,16 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
||||
)
|
||||
|
||||
for _, tc := range []struct{ name, value, want string }{
|
||||
{asnLimitPercent, "14061:50", `"14061"` + notASN},
|
||||
{asnLimitPercent, "AS4294967296:50", `"AS4294967296"` + notASN},
|
||||
{asnLimitPercent, "AS14061", `"AS14061"` + notItem},
|
||||
{asnLimitPercent, "AS14061:101", `"101"` + notPercent},
|
||||
{asnLimitPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`},
|
||||
{asnBytesPercent, "14061:50", `"14061"` + notASN},
|
||||
{asnBytesPercent, "AS4294967296:50", `"AS4294967296"` + notASN},
|
||||
{asnBytesPercent, "AS14061", `"AS14061"` + notItem},
|
||||
{asnBytesPercent, "AS14061:101", `"101"` + notPercent},
|
||||
{asnBytesPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`},
|
||||
{
|
||||
countryLimitPercent, "nk:25",
|
||||
countryBytesPercent, "nk:25",
|
||||
`"nk" is not a two-letter country code such as de or kp`,
|
||||
},
|
||||
{countryLimitPercent, "cn:25,CN:50", `"CN" is listed twice`},
|
||||
{countryBytesPercent, "cn:25,CN:50", `"CN" is listed twice`},
|
||||
{asnBytesPercent, "AS14061:-1", `"-1"` + notPercent},
|
||||
{countryBytesPercent, "cn:50%", `"50%"` + notPercent},
|
||||
{unknownLimitPercent, "101", `"101"` + notPercent},
|
||||
@@ -2003,23 +2066,34 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
|
||||
{lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
|
||||
{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
|
||||
{addLookupHeaders, "yes"},
|
||||
{deniedCountries, "nk"},
|
||||
{deniedCountries, "kp,,ir"},
|
||||
{deniedCountries, "prk"},
|
||||
{deniedCountries, "408"},
|
||||
{deniedCountries, "k"},
|
||||
{deniedCountries, "eu"},
|
||||
{deniedCountries, "un"},
|
||||
{deniedCountries, "su"},
|
||||
{allowedCountries, "nk"},
|
||||
{allowedCountries, "kp,,ir"},
|
||||
{allowedCountries, "prk"},
|
||||
{allowedCountries, "408"},
|
||||
{allowedCountries, "k"},
|
||||
{allowedCountries, "eu"},
|
||||
{allowedCountries, "un"},
|
||||
{allowedCountries, "su"},
|
||||
{allowedCountries, "ac"},
|
||||
{allowedCountries, "uk"},
|
||||
{allowedCountries, "zz"},
|
||||
{allowedCountries, "de,germany"},
|
||||
{asnLimitPercent, "AS14061:50,,AS16276:50"}, {asnLimitPercent, "ASX:50"},
|
||||
{asnLimitPercent, "AS14061:"}, {asnLimitPercent, "AS14061 :50"},
|
||||
{asnLimitPercent, "AS14061:1.5"}, {asnLimitPercent, "AS-1:50"},
|
||||
{countryLimitPercent, "cn"}, {countryLimitPercent, "cn:"},
|
||||
{countryLimitPercent, "cn:25:50"}, {countryLimitPercent, "china:25"},
|
||||
{asnBytesPercent, "AS14061:50,,AS16276:50"}, {asnBytesPercent, "ASX:50"},
|
||||
{asnBytesPercent, "AS14061:"}, {asnBytesPercent, "AS14061 :50"},
|
||||
{asnBytesPercent, "AS14061:1.5"}, {asnBytesPercent, "AS-1:50"},
|
||||
{countryBytesPercent, "cn"}, {countryBytesPercent, "cn:"},
|
||||
{countryBytesPercent, "cn:25:50"}, {countryBytesPercent, "china:25"},
|
||||
{tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"},
|
||||
{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"},
|
||||
{tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, {tagLimitPercent, "vpn:"},
|
||||
{tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
|
||||
{tagLimitPercent, "vpn:1.5"}, {tagLimitPercent, "vpn:50,VPN:25"},
|
||||
{tagLimitPercent, "exploitsignature:50"}, {tagLimitPercent, "asn=abc:50"},
|
||||
{tagLists, torExitTag}, {tagLists, "=https://example.com/a.txt"},
|
||||
{tagLists, "a,b=https://example.com/a.txt"},
|
||||
{tagLists, "a=ftp://example.com/a.txt"},
|
||||
{tagLists, "a=https://example.com/a.txt,A=https://example.com/b.txt"},
|
||||
{tagLists, "a=https://example.com/a.txt,b=https://example.com/a.txt"},
|
||||
{asnBytesPercent, "AS14061:101"}, {countryBytesPercent, "su:50"},
|
||||
{unknownLimitPercent, ""}, {unknownLimitPercent, "-1"},
|
||||
{unknownLimitPercent, "50%"},
|
||||
@@ -2286,10 +2360,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
lookupDBPath: "",
|
||||
lookupTimeout: "800ms",
|
||||
addLookupHeaders: "false",
|
||||
deniedCountries: "",
|
||||
allowedCountries: "",
|
||||
asnLimitPercent: "",
|
||||
countryLimitPercent: "",
|
||||
tagLists: "",
|
||||
tagDeny: "",
|
||||
tagLimitPercent: "",
|
||||
asnBytesPercent: "",
|
||||
countryBytesPercent: "",
|
||||
unknownLimitPercent: "100",
|
||||
|
||||
Reference in new issue
Block a user