Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands for its IPv4 address or netblock. reputation.json keeps each list's last try, failed or not, which a restart waits on as a running instance does, and its last good copy, whole, used while a fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log line names the lists, each raises reputation_hit, and a failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning. Judgement call: a failed fetch is retried after the refresh, not sooner. Not done: ban notes do not name the lists yet. Model: opus-5-5
2026-10-07 16:13:40 +00:00
parent 82e20e0cb5
commit a61597d39c
19 changed files with 2553 additions and 322 deletions
@@ -131,12 +131,27 @@ type Config struct {
 	// percentages from 0 to 100, by AS number, written as AS64496, or by
 	// country, a two-letter code in capitals, as the lookup gives them.
 	// UnknownLimitPercent is the percentage of every limit a client without
-	// a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT).
+	// a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is
+	// where a file of AS:percent lines is fetched from, whose percentages
+	// count as those of ASNLimitPercent do (SWWAF_ASN_LIMIT_PERCENT_URL), ""
+	// while it is unset.
 	ASNLimitPercent     map[string]int64
 	CountryLimitPercent map[string]int64
 	ASNBytesPercent     map[string]int64
 	CountryBytesPercent map[string]int64
 	UnknownLimitPercent int64
+	ASNLimitPercentURL  string
+	// BlocklistURLs are where the blocklists are fetched from
+	// (SWWAF_BLOCKLIST_URLS). Each list, and ASNLimitPercentURL's, is
+	// fetched again BlocklistRefresh after it was last fetched or tried
+	// (SWWAF_BLOCKLIST_REFRESH), which is never less than an hour.
+	// BlocklistAction is what is done with a client a blocklist lists
+	// (SWWAF_BLOCKLIST_ACTION): deny, limit or log; for limit,
+	// BlocklistLimitPercent is the percentage of every limit it gets.
+	BlocklistURLs         []string
+	BlocklistRefresh      time.Duration
+	BlocklistAction       string
+	BlocklistLimitPercent int64
 	// BanResponse is the status a refused client is answered with, 403
 	// or 429, or 0 to close the connection without an answer
 	// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -344,6 +359,13 @@ var (
 		"is not a code, : and a percentage, such as AS64496:50 or cn:25")
 	errNotPercent  = errors.New("is not a percentage, a whole number from 0 to 100")
 	errListedTwice = errors.New("is listed twice")
+	errNotListURL  = errors.New(
+		"is not an http or https URL without a user or a fragment, " +
+			"such as https://www.spamhaus.org/drop/drop.txt")
+	errInBlocklistURLs = errors.New("is in SWWAF_BLOCKLIST_URLS too")
+	errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h")
+	errNotAction       = errors.New(
+		"is not deny, limit:<percent> such as limit:25, or log")
 )

 // FromEnvironment reads the settings with lookupEnv, normally
@@ -388,11 +410,14 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		DeniedCountries:         env.countries("SWWAF_DENIED_COUNTRIES", ""),
 		ExclusivelyAllowedCountries: env.countries(
 			"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
-		ASNLimitPercent:      env.percents("SWWAF_ASN_LIMIT_PERCENT", parseASN),
+		ASNLimitPercent:      env.percents("SWWAF_ASN_LIMIT_PERCENT", ParseASN),
 		CountryLimitPercent:  env.percents("SWWAF_COUNTRY_LIMIT_PERCENT", parseCountry),
-		ASNBytesPercent:      env.percents("SWWAF_ASN_BYTES_PERCENT", parseASN),
+		ASNBytesPercent:      env.percents("SWWAF_ASN_BYTES_PERCENT", ParseASN),
 		CountryBytesPercent:  env.percents("SWWAF_COUNTRY_BYTES_PERCENT", parseCountry),
 		UnknownLimitPercent:  env.percent("SWWAF_UNKNOWN_LIMIT_PERCENT", "100"),
+		ASNLimitPercentURL:   env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"),
+		BlocklistURLs:        env.listURLs("SWWAF_BLOCKLIST_URLS"),
+		BlocklistRefresh:     env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"),
 		BanResponse:          env.banResponse("SWWAF_BAN_RESPONSE", "403"),
 		LimitBanDuration:     env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
 		LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
@@ -435,10 +460,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {

 	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
 		cfg.InstanceName, cfg.LogRemoteURL != nil)
+	cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
+		"SWWAF_BLOCKLIST_ACTION", "deny")

 	env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
 	env.checkLookupDBPath(cfg)
 	env.checkCountriesAndLookups(cfg)
+	env.checkASNLimitPercentURL(cfg)

 	if env.err != nil {
 		return nil, env.err
@@ -664,12 +692,66 @@ func (e *environment) percents(

 // percent reads a setting that is a percentage, from 0 to 100.
 func (e *environment) percent(name, defaultValue string) int64 {
-	percent, err := parsePercent(e.value(name, defaultValue))
+	percent, err := ParsePercent(e.value(name, defaultValue))
 	e.check(name, err)

 	return percent
 }

+// listURL reads a setting that is the URL a list is fetched from, "" while
+// it is unset or empty.
+func (e *environment) listURL(name string) string {
+	value := e.value(name, "")
+	if value != "" && !isListURL(value) {
+		e.check(name, fmt.Errorf("%q %w", value, errNotListURL))
+	}
+
+	return value
+}
+
+// listURLs reads a setting that is a list of the URLs lists are fetched
+// from. It is empty by default.
+func (e *environment) listURLs(name string) []string {
+	urls, err := parseListURLs(e.value(name, ""))
+	e.check(name, err)
+
+	return urls
+}
+
+// refresh reads the setting that is how long after a list was last
+// fetched or tried it is fetched again: a duration of an hour or more,
+// since the Spamhaus lists may be fetched no more often, which cannot be
+// off.
+func (e *environment) refresh(name, defaultValue string) time.Duration {
+	value := e.value(name, defaultValue)
+
+	duration, err := parseDuration(value)
+	if err != nil || duration < time.Hour {
+		e.check(name, fmt.Errorf("%q %w", value, errNotAnHourOrMore))
+	}
+
+	return duration
+}
+
+// action reads a setting that is what is done with a client a list names:
+// deny, log, or limit:<percent>, which it returns as limit and the
+// percentage.
+func (e *environment) action(name, defaultValue string) (string, int64) {
+	value := e.value(name, defaultValue)
+	if value == "deny" || value == "log" {
+		return value, 0
+	}
+
+	percentText, isLimit := strings.CutPrefix(value, "limit:")
+
+	percent, err := ParsePercent(percentText)
+	if !isLimit || err != nil {
+		e.check(name, fmt.Errorf("%q %w", value, errNotAction))
+	}
+
+	return "limit", percent
+}
+
 // lookupSource reads the setting that is where clients are looked up:
 // geojs, file, or off.
 func (e *environment) lookupSource(name, defaultValue string) string {
@@ -698,8 +780,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) {
 // checkCountriesAndLookups refuses a country on both country lists, and,
 // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
 // up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, the biased thresholds,
-// of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it
-// lowers a limit, and the anomaly thresholds per AS number.
+// SWWAF_ASN_LIMIT_PERCENT_URL among them, of which
+// SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a
+// limit, and the anomaly thresholds per AS number.
 func (e *environment) checkCountriesAndLookups(cfg *Config) {
 	for _, country := range cfg.ExclusivelyAllowedCountries {
 		if slices.Contains(cfg.DeniedCountries, country) {
@@ -724,6 +807,7 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
 		{"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0},
 		{"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0},
 		{"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100},
+		{"SWWAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
 		{"SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
 		{"SWWAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
 		{"SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -736,6 +820,15 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
 	}
 }

+// checkASNLimitPercentURL refuses SWWAF_ASN_LIMIT_PERCENT_URL naming a
+// blocklist too: the file at a URL is fetched as one list or the other.
+func (e *environment) checkASNLimitPercentURL(cfg *Config) {
+	if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
+		e.check("SWWAF_ASN_LIMIT_PERCENT_URL",
+			fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
+	}
+}
+
 // headerNames reads a setting that is a list of header names, and
 // returns them in lower case.
 func (e *environment) headerNames(name, defaultValue string) []string {
@@ -1187,7 +1280,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
 	netblocks := make([]netip.Prefix, 0, len(items))

 	for _, item := range items {
-		netblock, err := parseNetblock(item)
+		netblock, err := ParseNetblock(item)
 		if err != nil {
 			return nil, err
 		}
@@ -1198,9 +1291,10 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
 	return netblocks, nil
 }

-// parseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
-// address is a netblock of that address alone, a /32 or a /128.
-func parseNetblock(value string) (netip.Prefix, error) {
+// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
+// address is a netblock of that address alone, a /32 or a /128. A
+// blocklist's lines are read with it too.
+func ParseNetblock(value string) (netip.Prefix, error) {
 	if strings.Contains(value, "/") {
 		netblock, err := netip.ParsePrefix(value)
 		if err != nil {
@@ -1237,7 +1331,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
 			return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
 		}

-		netblock, err := parseNetblock(strings.TrimSpace(netblockText))
+		netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
 		if err != nil {
 			return nil, err
 		}
@@ -1337,10 +1431,11 @@ func parseCountry(value string) (string, error) {
 	return country, nil
 }

-// parseASN reads an AS number such as AS64496, in either case, and
+// ParseASN reads an AS number such as AS64496, in either case, and
 // returns it as the lookup gives it: AS and the number, in capitals and
-// without leading zeros.
-func parseASN(value string) (string, error) {
+// without leading zeros. The file SWWAF_ASN_LIMIT_PERCENT_URL names is
+// read with it too.
+func ParseASN(value string) (string, error) {
 	digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")

 	number, err := strconv.ParseUint(digits, 10, 32)
@@ -1376,7 +1471,7 @@ func parsePercents(
 			return nil, err
 		}

-		percent, err := parsePercent(percentText)
+		percent, err := ParsePercent(percentText)
 		if err != nil {
 			return nil, err
 		}
@@ -1391,8 +1486,9 @@ func parsePercents(
 	return percents, nil
 }

-// parsePercent reads a percentage, a whole number from 0 to 100.
-func parsePercent(value string) (int64, error) {
+// ParsePercent reads a percentage, a whole number from 0 to 100. The file
+// SWWAF_ASN_LIMIT_PERCENT_URL names is read with it too.
+func ParsePercent(value string) (int64, error) {
 	percent, err := strconv.ParseInt(value, 10, 64)
 	if err != nil || percent < 0 || percent > 100 {
 		return 0, fmt.Errorf("%q %w", value, errNotPercent)
@@ -1548,16 +1644,7 @@ func parseWebhookURL(value string) (*url.URL, string, error) {
 	}

 	webhook, err := url.Parse(value)
-	if err != nil {
-		return nil, "", errNotWebhookURL
-	}
-
-	port, err := strconv.ParseUint(webhook.Port(), 10, 16)
-
-	valid := (webhook.Scheme == "http" || webhook.Scheme == "https") &&
-		webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) &&
-		webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == ""
-	if !valid {
+	if err != nil || !isHTTPURL(webhook) {
 		return nil, "", errNotWebhookURL
 	}

@@ -1569,6 +1656,47 @@ func parseWebhookURL(value string) (*url.URL, string, error) {
 	return webhook, logged, nil
 }

+// isHTTPURL reports whether u is http or https, with a host, and an
+// optional port from 1 to 65535, path and query, without a user or a
+// fragment.
+func isHTTPURL(u *url.URL) bool {
+	port, err := strconv.ParseUint(u.Port(), 10, 16)
+
+	return (u.Scheme == "http" || u.Scheme == "https") && u.Hostname() != "" &&
+		(u.Port() == "" || (err == nil && port != 0)) &&
+		u.User == nil && u.Opaque == "" && u.Fragment == ""
+}
+
+// isListURL reports whether value is a URL a list can be fetched from, as
+// isHTTPURL says.
+func isListURL(value string) bool {
+	u, err := url.Parse(value)
+
+	return err == nil && isHTTPURL(u)
+}
+
+// parseListURLs reads a comma-separated list of the URLs lists are fetched
+// from. A URL listed twice is an error: it would be fetched twice as
+// often.
+func parseListURLs(value string) ([]string, error) {
+	urls, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, listURL := range urls {
+		if !isListURL(listURL) {
+			return nil, fmt.Errorf("%q %w", listURL, errNotListURL)
+		}
+
+		if slices.Contains(urls[:i], listURL) {
+			return nil, fmt.Errorf("%q %w", listURL, errListedTwice)
+		}
+	}
+
+	return urls, nil
+}
+
 // parseWebhookHeaders reads a comma-separated list of headers, each its
 // name, :, and its value, and returns them, and how the log shows them,
 // with each value as ********. An error names the item by its place in