sneak/smallwebwaf
1
0
Fork 0
Code Issues 44 Pull Requests 2 Actions 49 Packages Projects Releases Wiki Activity

Blocklists and an AS percentage file fetched by URL (closes #29)
check / check (push) Waiting to run
Details

Browse Source 
SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, even one cut off by a stop, which a restart waits on
as a running instance does, and its last good copy, whole, used while a
fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed
client; the log line names the lists, each raises reputation_hit, and a
failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched
the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-07 16:51:48 +00:00
parent 82e20e0cb5
commit 271aaafccc
19 changed files with 2588 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+155 -27
View File
@@ -131,12 +131,27 @@ type Config struct {
// percentages from 0 to 100, by AS number, written as AS64496, or by
// country, a two-letter code in capitals, as the lookup gives them.
// UnknownLimitPercent is the percentage of every limit a client without
// a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT).
// a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is
// where a file of AS:percent lines is fetched from, whose percentages
// count as those of ASNLimitPercent do (SWWAF_ASN_LIMIT_PERCENT_URL), ""
// while it is unset.
ASNLimitPercent map[string]int64
CountryLimitPercent map[string]int64
ASNBytesPercent map[string]int64
CountryBytesPercent map[string]int64
UnknownLimitPercent int64
ASNLimitPercentURL string
// BlocklistURLs are where the blocklists are fetched from
// (SWWAF_BLOCKLIST_URLS). Each list, and ASNLimitPercentURL's, is
// fetched again BlocklistRefresh after it was last fetched or tried
// (SWWAF_BLOCKLIST_REFRESH), which is never less than an hour.
// BlocklistAction is what is done with a client a blocklist lists
// (SWWAF_BLOCKLIST_ACTION): deny, limit or log; for limit,
// BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string
BlocklistRefresh time.Duration
BlocklistAction string
BlocklistLimitPercent int64
// BanResponse is the status a refused client is answered with, 403
// or 429, or 0 to close the connection without an answer
// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -344,6 +359,13 @@ var (
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
errNotPercent = errors.New("is not a percentage, a whole number from 0 to 100")
errListedTwice = errors.New("is listed twice")
errNotListURL = errors.New(
"is not an http or https URL without a user or a fragment, " +
"such as https://www.spamhaus.org/drop/drop.txt")
errInBlocklistURLs = errors.New("is in SWWAF_BLOCKLIST_URLS too")
errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h")
errNotAction = errors.New(
"is not deny, limit:<percent> such as limit:25, or log")
)
// FromEnvironment reads the settings with lookupEnv, normally
@@ -388,11 +410,14 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""),
ExclusivelyAllowedCountries: env.countries(
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", parseASN),
ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", ParseASN),
CountryLimitPercent: env.percents("SWWAF_COUNTRY_LIMIT_PERCENT", parseCountry),
ASNBytesPercent: env.percents("SWWAF_ASN_BYTES_PERCENT", parseASN),
ASNBytesPercent: env.percents("SWWAF_ASN_BYTES_PERCENT", ParseASN),
CountryBytesPercent: env.percents("SWWAF_COUNTRY_BYTES_PERCENT", parseCountry),
UnknownLimitPercent: env.percent("SWWAF_UNKNOWN_LIMIT_PERCENT", "100"),
ASNLimitPercentURL: env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"),
BlocklistURLs: env.listURLs("SWWAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"),
BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"),
LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
@@ -435,10 +460,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
cfg.InstanceName, cfg.LogRemoteURL != nil)
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWWAF_BLOCKLIST_ACTION", "deny")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg)
env.checkCountriesAndLookups(cfg)
env.checkASNLimitPercentURL(cfg)
if env.err != nil {
return nil, env.err
@@ -664,12 +692,66 @@ func (e *environment) percents(
// percent reads a setting that is a percentage, from 0 to 100.
func (e *environment) percent(name, defaultValue string) int64 {
percent, err := parsePercent(e.value(name, defaultValue))
percent, err := ParsePercent(e.value(name, defaultValue))
e.check(name, err)
return percent
}
// listURL reads a setting that is the URL a list is fetched from, "" while
// it is unset or empty.
func (e *environment) listURL(name string) string {
value := e.value(name, "")
if value != "" && !isListURL(value) {
e.check(name, fmt.Errorf("%q %w", value, errNotListURL))
}
return value
}
// listURLs reads a setting that is a list of the URLs lists are fetched
// from. It is empty by default.
func (e *environment) listURLs(name string) []string {
urls, err := parseListURLs(e.value(name, ""))
e.check(name, err)
return urls
}
// refresh reads the setting that is how long after a list was last
// fetched or tried it is fetched again: a duration of an hour or more,
// since the Spamhaus lists may be fetched no more often, which cannot be
// off.
func (e *environment) refresh(name, defaultValue string) time.Duration {
value := e.value(name, defaultValue)
duration, err := parseDuration(value)
if err != nil || duration < time.Hour {
e.check(name, fmt.Errorf("%q %w", value, errNotAnHourOrMore))
}
return duration
}
// action reads a setting that is what is done with a client a list names:
// deny, log, or limit:<percent>, which it returns as limit and the
// percentage.
func (e *environment) action(name, defaultValue string) (string, int64) {
value := e.value(name, defaultValue)
if value == "deny" || value == "log" {
return value, 0
}
percentText, isLimit := strings.CutPrefix(value, "limit:")
percent, err := ParsePercent(percentText)
if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
}
return "limit", percent
}
// lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off.
func (e *environment) lookupSource(name, defaultValue string) string {
@@ -698,8 +780,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) {
// checkCountriesAndLookups refuses a country on both country lists, and,
// while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, the biased thresholds,
// of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it
// lowers a limit, and the anomaly thresholds per AS number.
// SWWAF_ASN_LIMIT_PERCENT_URL among them, of which
// SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a
// limit, and the anomaly thresholds per AS number.
func (e *environment) checkCountriesAndLookups(cfg *Config) {
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.DeniedCountries, country) {
@@ -724,6 +807,7 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
{"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0},
{"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0},
{"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100},
{"SWWAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{"SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWWAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -736,6 +820,15 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
}
}
// checkASNLimitPercentURL refuses SWWAF_ASN_LIMIT_PERCENT_URL naming a
// blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *Config) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check("SWWAF_ASN_LIMIT_PERCENT_URL",
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
}
}
// headerNames reads a setting that is a list of header names, and
// returns them in lower case.
func (e *environment) headerNames(name, defaultValue string) []string {
@@ -1187,7 +1280,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
netblocks := make([]netip.Prefix, 0, len(items))
for _, item := range items {
netblock, err := parseNetblock(item)
netblock, err := ParseNetblock(item)
if err != nil {
return nil, err
}
@@ -1198,9 +1291,10 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
return netblocks, nil
}
// parseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128.
func parseNetblock(value string) (netip.Prefix, error) {
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
@@ -1237,7 +1331,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
}
netblock, err := parseNetblock(strings.TrimSpace(netblockText))
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
if err != nil {
return nil, err
}
@@ -1337,10 +1431,11 @@ func parseCountry(value string) (string, error) {
return country, nil
}
// parseASN reads an AS number such as AS64496, in either case, and
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros.
func parseASN(value string) (string, error) {
// without leading zeros. The file SWWAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
@@ -1376,7 +1471,7 @@ func parsePercents(
return nil, err
}
percent, err := parsePercent(percentText)
percent, err := ParsePercent(percentText)
if err != nil {
return nil, err
}
@@ -1391,8 +1486,9 @@ func parsePercents(
return percents, nil
}
// parsePercent reads a percentage, a whole number from 0 to 100.
func parsePercent(value string) (int64, error) {
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWWAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
@@ -1548,16 +1644,7 @@ func parseWebhookURL(value string) (*url.URL, string, error) {
}
webhook, err := url.Parse(value)
if err != nil {
return nil, "", errNotWebhookURL
}
port, err := strconv.ParseUint(webhook.Port(), 10, 16)
valid := (webhook.Scheme == "http" || webhook.Scheme == "https") &&
webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) &&
webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == ""
if !valid {
if err != nil || !isHTTPURL(webhook) {
return nil, "", errNotWebhookURL
}
@@ -1569,6 +1656,47 @@ func parseWebhookURL(value string) (*url.URL, string, error) {
return webhook, logged, nil
}
// isHTTPURL reports whether u is http or https, with a host, and an
// optional port from 1 to 65535, path and query, without a user or a
// fragment.
func isHTTPURL(u *url.URL) bool {
port, err := strconv.ParseUint(u.Port(), 10, 16)
return (u.Scheme == "http" || u.Scheme == "https") && u.Hostname() != "" &&
(u.Port() == "" || (err == nil && port != 0)) &&
u.User == nil && u.Opaque == "" && u.Fragment == ""
}
// isListURL reports whether value is a URL a list can be fetched from, as
// isHTTPURL says.
func isListURL(value string) bool {
u, err := url.Parse(value)
return err == nil && isHTTPURL(u)
}
// parseListURLs reads a comma-separated list of the URLs lists are fetched
// from. A URL listed twice is an error: it would be fetched twice as
// often.
func parseListURLs(value string) ([]string, error) {
urls, err := parseList(value)
if err != nil {
return nil, err
}
for i, listURL := range urls {
if !isListURL(listURL) {
return nil, fmt.Errorf("%q %w", listURL, errNotListURL)
}
if slices.Contains(urls[:i], listURL) {
return nil, fmt.Errorf("%q %w", listURL, errListedTwice)
}
}
return urls, nil
}
// parseWebhookHeaders reads a comma-separated list of headers, each its
// name, :, and its value, and returns them, and how the log shows them,
// with each value as ********. An error names the item by its place in
internal/config/config_test.go
+123 -2
View File
@@ -57,6 +57,10 @@ const (
asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT"
countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT"
unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT"
asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
blocklistURLs = "SWWAF_BLOCKLIST_URLS"
blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH"
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
banResponse = "SWWAF_BAN_RESPONSE"
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
@@ -953,6 +957,7 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
deniedCountries: "kp",
allowedCountries: "de",
addLookupHeaders: enabled,
asnLimitPercentURL: asnURL,
asnLimitPercent: "AS64496:50",
countryLimitPercent: "cn:25",
asnBytesPercent: "AS64496:50",
@@ -978,11 +983,13 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
// Set empty, the lists need nothing looked up, and nor does
// SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly
// threshold per AS number that is off, or any other anomaly threshold.
// threshold per AS number that is off, any other anomaly threshold, or
// a blocklist.
env := environment{
lookupSource: off, deniedCountries: "", allowedCountries: "",
asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "",
countryBytesPercent: "", unknownLimitPercent: "100",
countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "",
blocklistURLs: dropURL,
}
for _, name := range anomalyThresholds() {
env[name] = "1000"
@@ -1205,6 +1212,116 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) {
}
}
// dropURL and torURL are blocklists, and asnURL a file of AS:percent
// lines.
const (
dropURL = "https://www.spamhaus.org/drop/drop.txt"
torURL = "https://lists.example/tor-exits.txt"
asnURL = "https://lists.example/asn.txt"
)
// The actions of SWWAF_BLOCKLIST_ACTION, as Config gives them.
const (
actionDeny = "deny"
actionLimit = "limit"
actionLog = "log"
)
func TestReputationSettingsAsSet(t *testing.T) {
t.Parallel()
cfg := fromEnvironment(t, environment{})
if len(cfg.BlocklistURLs) != 0 || cfg.BlocklistRefresh != 24*time.Hour ||
cfg.BlocklistAction != actionDeny || cfg.ASNLimitPercentURL != "" {
t.Errorf("%s, %s, %s and %s gave %v, %s, %s and %q by default, "+
"want none, 24h, deny and none", blocklistURLs, blocklistRefresh,
blocklistAction, asnLimitPercentURL, cfg.BlocklistURLs, cfg.BlocklistRefresh,
cfg.BlocklistAction, cfg.ASNLimitPercentURL)
}
for _, tc := range []struct {
value, action string
percent int64
}{
{actionDeny, actionDeny, 0},
{actionLog, actionLog, 0},
{"limit:25", actionLimit, 25},
{"limit:0", actionLimit, 0},
} {
// An hour, the shortest refresh allowed.
cfg := fromEnvironment(t, environment{
blocklistURLs: dropURL + ", " + torURL, blocklistRefresh: "1h",
blocklistAction: tc.value, asnLimitPercentURL: asnURL,
})
if !slices.Equal(cfg.BlocklistURLs, []string{dropURL, torURL}) ||
cfg.BlocklistRefresh != time.Hour || cfg.BlocklistAction != tc.action ||
cfg.BlocklistLimitPercent != tc.percent || cfg.ASNLimitPercentURL != asnURL {
t.Errorf("%s=%s gave %v, %s, %s, %d and %s", blocklistAction, tc.value,
cfg.BlocklistURLs, cfg.BlocklistRefresh, cfg.BlocklistAction,
cfg.BlocklistLimitPercent, cfg.ASNLimitPercentURL)
}
}
}
func TestInvalidReputationSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
t.Parallel()
const (
notURL = " is not an http or https URL without a user or a fragment, " +
"such as https://www.spamhaus.org/drop/drop.txt"
notAnHour = " is not a duration of 1h or more, such as 24h"
notAction = " is not deny, limit:<percent> such as limit:25, or log"
)
for _, tc := range []struct{ name, value, want string }{
{
blocklistURLs, "ftp://lists.example/drop.txt",
`"ftp://lists.example/drop.txt"` + notURL,
},
{blocklistURLs, "lists.example/drop.txt", `"lists.example/drop.txt"` + notURL},
{
blocklistURLs, "https://me:secret@lists.example/drop.txt",
`"https://me:secret@lists.example/drop.txt"` + notURL,
},
{
blocklistURLs, dropURL + "," + torURL + "," + dropURL,
`"` + dropURL + `" is listed twice`,
},
{asnLimitPercentURL, asnURL + "#top", `"` + asnURL + `#top"` + notURL},
{blocklistRefresh, "59m", `"59m"` + notAnHour},
{blocklistRefresh, off, `"off"` + notAnHour},
{blocklistRefresh, "a day", `"a day"` + notAnHour},
{blocklistAction, "block", `"block"` + notAction},
{blocklistAction, actionLimit, `"limit"` + notAction},
{blocklistAction, "limit:101", `"limit:101"` + notAction},
} {
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
t.Parallel()
_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
want := tc.name + ": " + tc.want
if err == nil || err.Error() != want {
t.Errorf("error %v, want %s", err, want)
}
})
}
}
func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) {
t.Parallel()
_, err := config.FromEnvironment(environment{
blocklistURLs: dropURL + "," + asnURL, asnLimitPercentURL: asnURL,
}.lookupEnv)
want := asnLimitPercentURL + `: "` + asnURL + `" is in SWWAF_BLOCKLIST_URLS too`
if err == nil || err.Error() != want {
t.Errorf("error %v, want %s", err, want)
}
}
func TestSizesAndOff(t *testing.T) {
t.Parallel()
@@ -1628,6 +1745,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
asnBytesPercent: "",
countryBytesPercent: "",
unknownLimitPercent: "100",
asnLimitPercentURL: "",
blocklistURLs: "",
blocklistRefresh: "24h",
blocklistAction: actionDeny,
banResponse: "403",
limitBanDuration: "1h",
limitBanRepeatWindow: "24h",