From de9c71a7966c811729e7040bfb6c6e104292fd6f Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 9 Nov 2026 00:39:59 +0000 Subject: [PATCH] Request tags, with refusing and limits per tag (closes #175) Each request gets sorted tags from the lookup, new address lists, the reputation sources and the rule matches, logged as `tags`. New settings: `SWW_TAG_LISTS`, `SWW_TAG_DENY`, `SWW_TAG_LIMIT_PERCENT`. They replace `SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT` and `SWW_DENIED_COUNTRIES`. New heuristics `tag_lists` and `tag_deny`; the latter runs before the rate limits and again after the Core Rule Set. Model: opus-5-5 --- README.md | 63 ++-- SPEC.md | 158 ++++++--- docs/request-log.md | 7 +- docs/settings.md | 69 +++- internal/config/config.go | 343 +++++++++++++++++-- internal/config/config_test.go | 304 +++++++++++++--- internal/discriminator/abuseipdb.go | 17 +- internal/discriminator/answertime_test.go | 4 +- internal/discriminator/bans_test.go | 8 +- internal/discriminator/biased.go | 62 ++-- internal/discriminator/biased_test.go | 72 ++-- internal/discriminator/coreruleset.go | 8 +- internal/discriminator/coreruleset_test.go | 35 ++ internal/discriminator/countries.go | 27 +- internal/discriminator/countries_test.go | 16 +- internal/discriminator/crowdsec.go | 8 +- internal/discriminator/crowdsec_test.go | 1 + internal/discriminator/discriminator_test.go | 4 +- internal/discriminator/dnsbl.go | 9 +- internal/discriminator/heuristics.go | 5 + internal/discriminator/history_test.go | 2 +- internal/discriminator/lookup.go | 33 +- internal/discriminator/lookup_test.go | 6 +- internal/discriminator/metrics_test.go | 18 +- internal/discriminator/observe_test.go | 4 +- internal/discriminator/proxy.go | 12 +- internal/discriminator/proxy_test.go | 4 +- internal/discriminator/ratelimits_test.go | 8 +- internal/discriminator/refusallimit_test.go | 2 +- internal/discriminator/reputation_test.go | 62 +++- internal/discriminator/request.go | 9 + internal/discriminator/rulefiles.go | 6 +- internal/discriminator/rulefiles_test.go | 14 + internal/discriminator/staticlists_test.go | 21 +- internal/discriminator/tagdeny.go | 30 ++ internal/discriminator/tagdeny_test.go | 121 +++++++ internal/discriminator/taglists.go | 24 ++ internal/discriminator/taglists_test.go | 100 ++++++ internal/metrics/busiest.go | 6 +- internal/reputation/reputation.go | 55 ++- internal/requestlog/requestlog.go | 7 +- internal/sww/sww_test.go | 18 +- 42 files changed, 1420 insertions(+), 362 deletions(-) create mode 100644 internal/discriminator/tagdeny.go create mode 100644 internal/discriminator/tagdeny_test.go create mode 100644 internal/discriminator/taglists.go create mode 100644 internal/discriminator/taglists_test.go diff --git a/README.md b/README.md index e38335b..0cc5ef6 100644 --- a/README.md +++ b/README.md @@ -257,14 +257,14 @@ Every setting the code reads, with its default: | `SWW_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. | | `SWW_LOOKUP_TIMEOUT` | `800ms` | How long a request waits for its client's first GeoJS answer when a setting needs it, and how long a request to GeoJS may take. | | `SWW_ADD_LOOKUP_HEADERS` | `false` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. | -| `SWW_DENIED_COUNTRIES` | empty | Countries whose clients are refused, such as `cn,ru`. | | `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. | -| `SWW_ASN_LIMIT_PERCENT` | empty | AS numbers whose clients get a percentage of every request and byte limit, such as `AS14061:50,AS16276:25`. | -| `SWW_COUNTRY_LIMIT_PERCENT` | empty | The same by country, such as `cn:25`. | -| `SWW_ASN_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `SWW_ASN_LIMIT_PERCENT`. | -| `SWW_COUNTRY_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `SWW_COUNTRY_LIMIT_PERCENT`. | +| `SWW_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. | +| `SWW_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. | +| `SWW_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. | +| `SWW_ASN_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. | +| `SWW_COUNTRY_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. | | `SWW_UNKNOWN_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. | -| `SWW_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS:` lines, fetched like a blocklist; where it and `SWW_ASN_LIMIT_PERCENT` both give a percentage, the lower applies. | +| `SWW_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS:` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_TAG_LIMIT_PERCENT` (the lowest wins). | | `SWW_RULES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. | | `SWW_RULES_ENABLED` | `true` | `false` reads no rule file. | | `SWW_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. | @@ -414,8 +414,7 @@ services: SWW_RATE_LIMIT_PER_HOUR: "600" SWW_RATE_LIMIT_PER_DAY: "3000" # Hosting networks and one country get a share of those limits. - SWW_ASN_LIMIT_PERCENT: AS14061:50,AS16276:25 - SWW_COUNTRY_LIMIT_PERCENT: cn:25 + SWW_TAG_LIMIT_PERCENT: asn=14061:50,asn=16276:25,country=CN:25 # Paths only scanners ask for; one request bans the client. SWW_TRAP_PATHS: /wp-login.php,/xmlrpc.php # Answer a refused client with 429 rather than 403. @@ -507,31 +506,34 @@ of the request reaches the app. `discriminator` takes the checks: -3. The static lists, `SWW_ALLOW_NETS` and `SWW_DENY_NETS`. -4. The bans. -5. The lookup of the client's AS number and country. -6. The country lists. -7. The reputation sources. -8. The client's limit percentage, from its AS number, its country or a - reputation hit. -9. The rate limits. -10. The trap paths. -11. The rule files. -12. The header limits, and then the Core Rule Set. -13. A path under `/_sww/` is answered `404`, so that the app never sees that +3. The tag lists. +4. The static lists, `SWW_ALLOW_NETS` and `SWW_DENY_NETS`. +5. The bans. +6. The lookup of the client's AS number and country. +7. The country list. +8. The reputation sources. +9. The tag refusals of `SWW_TAG_DENY`. +10. The client's limit percentage, from its tags, its AS number, its country or + a reputation hit. +11. The rate limits. +12. The trap paths. +13. The rule files. +14. The header limits, and then the Core Rule Set. +15. The tag refusals again, so that `exploitsignature` can refuse too. +16. A path under `/_sww/` is answered `404`, so that the app never sees that prefix. `rproxy` takes the rest, with `discriminator` counting at the end: -14. The request is passed to the app, streaming. -15. After the answer, `rproxy` sends `discriminator` a report with the status +17. The request is passed to the app, streaming. +18. After the answer, `rproxy` sends `discriminator` a report with the status and the bytes each way. For an allowed request, `discriminator` counts the bytes toward the byte limits and adds the request to the client's history. A refused visitor is answered with `SWW_BAN_RESPONSE`. A `block` rule or a Core Rule Set match answers `403` and bans nobody, since the Core Rule Set has false positives; such refusals count toward `SWW_ERROR_BURST_THRESHOLD` instead. A -request over the header limits of step 12 is answered `431`, and bans nobody +request over the header limits of step 14 is answered `431`, and bans nobody either. `rproxy` itself answers `405` to a method it does not pass on, and the status for a size or time limit, such as `413` for a body too large. @@ -550,6 +552,21 @@ status for a size or time limit, such as `413` for a body too large. bans. An admin adds, keeps or lifts a ban by editing `bans.json` or through the ban endpoints (see [`docs/admin-endpoints.md`](docs/admin-endpoints.md)). +### Request tags + +Each request gets tags from what `discriminator` knows of its client, written in +the `tags` field of its log line: + +- `asn=` and `country=`, from the lookup; +- one tag per list of `SWW_TAG_LISTS` that holds the client, such as `torexit` + or `vpn=mullvad`; +- `crowdsec`, `dnsbl=` and `abuseipdb`, when that reputation source lists + the client; +- `exploitsignature`, when a rule file or the Core Rule Set matched the request. + +`SWW_TAG_DENY` refuses requests by tag, and `SWW_TAG_LIMIT_PERCENT` lowers a +client's limits by tag. See "Request tags" in [`SPEC.md`](SPEC.md). + ### Attack detection - The Core Rule Set reads the method, the URL with its query string, and the diff --git a/SPEC.md b/SPEC.md index 5337b41..3de44b6 100644 --- a/SPEC.md +++ b/SPEC.md @@ -119,9 +119,9 @@ to them. ## What happens to a request -`rproxy` runs steps 1, 2 and 14, and sends the report of step 15. -`discriminator` runs steps 3 to 13, and does the counting of step 15. The first -step that produces a final answer ends the checks. Step 15 runs for each request +`rproxy` runs steps 1, 2 and 17, and sends the report of step 18. +`discriminator` runs steps 3 to 16, and does the counting of step 18. The first +step that produces a final answer ends the checks. Step 18 runs for each request `discriminator` answers with `allow`, `deny` or `close`. `rproxy`'s size and time limits apply to the whole exchange. Each step names the section that holds its rules. A step that names a heuristic in parentheses runs that heuristic. The @@ -143,46 +143,54 @@ request log's `heuristic` field gives that name. `refusal_limit` and - An IPv6 client is its group: the netblock of `SWW_IPV6_GROUP_PREFIX` around its address, a /64 by default, since one abuser usually controls a whole /64. A client is therefore one IPv4 address or one IPv6 group. -3. The static lists. A client in `SWW_ALLOW_NETS` skips steps 4 to 12, the byte +3. The tag lists (`tag_lists`; see "Request tags"). Each list that holds the + client adds its tag. A client in `SWW_ALLOW_NETS` gets no tag. +4. The static lists. A client in `SWW_ALLOW_NETS` skips steps 5 to 15, the byte limits and the error burst. A client in `SWW_DENY_NETS` is refused (`deny_nets`); a client in both lists is let through. -4. Bans. A client under a ban is refused (`active_ban`; see "Bans"). -5. The lookup, unless `SWW_LOOKUP_SOURCE` is `off` (see "Lookup of AS number and +5. Bans. A client under a ban is refused (`active_ban`; see "Bans"). +6. The lookup, unless `SWW_LOOKUP_SOURCE` is `off` (see "Lookup of AS number and country"). The request goes on at once. It waits for the first GeoJS answer, up to `SWW_LOOKUP_TIMEOUT`, only when one of these needs it: - - a country list; + - the country list; - a biased threshold; + - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=` + tag; - `SWW_ADD_LOOKUP_HEADERS`. A request that ends before the answer comes is logged without it. -6. The country lists (`countries`; see "Country lists"). -7. The reputation sources (see "Reputation"), in this order: +7. The country list (`countries`; see "Country list"). +8. The reputation sources (see "Reputation"), in this order: - A client on a fetched blocklist gets `SWW_BLOCKLIST_ACTION` (`blocklists`). - A client on the CrowdSec decision list is banned (`crowdsec`). - A client listed by a DNS blocklist gets `SWW_REPUTATION_ACTION` (`dnsbl`). - A client scored at least `SWW_ABUSEIPDB_MIN_SCORE` by AbuseIPDB gets `SWW_REPUTATION_ACTION` (`abuseipdb`). -8. The client's limit percentage (see "Biased thresholds"). -9. Rate limiting (`rate_limits`; see "Request rate limits, per client"). A - request over a limit times the percentage is refused and bans the client. -10. The trap paths (`trap_paths`; see "Attack detection"). A request for one is +9. The tag refusals (`tag_deny`; see "Request tags"). A request carrying a tag + in `SWW_TAG_DENY` is refused. +10. The client's limit percentage (see "Biased thresholds"). +11. Rate limiting (`rate_limits`; see "Request rate limits, per client"). A + request over a limit times the percentage is refused and bans the client. +12. The trap paths (`trap_paths`; see "Attack detection"). A request for one is refused and bans the client. -11. The rule files (`rule_files`; see "Rule files"). Matching stops at the first +13. The rule files (`rule_files`; see "Rule files"). Matching stops at the first rule that refuses. -12. The header limits (`header_limits`; see "Answering in time"), and then the +14. The header limits (`header_limits`; see "Answering in time"), and then the Core Rule Set (`core_rule_set`; see "Attack detection"). A Core Rule Set match is refused with `403` while `SWW_WAF_MODE` is `block`, and only logged and alerted while it is `detect`. -13. A path under `/_sww/` is answered `404`, so that the app never sees that +15. The tag refusals again (`tag_deny`), so that `exploitsignature` can refuse + too. +16. A path under `/_sww/` is answered `404`, so that the app never sees that prefix. -14. `rproxy` passes the request to `SWRP_UPSTREAM_URL`, streaming (see "Passing +17. `rproxy` passes the request to `SWRP_UPSTREAM_URL`, streaming (see "Passing a request to the app"). With `SWW_ADD_LOOKUP_HEADERS`, `X-Client-ASN` and `X-Client-Country` are added for the app's own logs; a header whose value is unknown is left out. Whatever that setting says, any such header the client sent is removed. -15. After the visitor's answer has ended, `rproxy` sends the report (see "The +18. After the visitor's answer has ended, `rproxy` sends the report (see "The report"). `discriminator` then counts: - For a request it allowed, the response bytes, the request bytes or both are added to the byte counters, as `SWW_BYTES_COUNT` says. For a @@ -285,7 +293,8 @@ no offence: - `SWW_DENY_NETS`; - a blocklist or reputation hit whose action is `deny`; -- the country lists. +- the country list; +- `SWW_TAG_DENY`. Every ban carries notes with what an admin needs to decide whether to lift it, drawn from what `discriminator` already knows. No lookup is made for the notes @@ -392,7 +401,7 @@ Conventions: ### Static lists -- `SWW_ALLOW_NETS` skips the checks step 3 names, for monitoring or the owner's +- `SWW_ALLOW_NETS` skips the checks step 4 names, for monitoring or the owner's own networks. `SWW_RATE_LIMIT_EXEMPT_NETS` bypasses the request and byte limits only; the error burst, attack detection and bans still apply. - Many people can share one address, as in an office, on a mobile carrier or @@ -442,7 +451,7 @@ the same way. By default there is no minute limit, so a client can download in a burst at full speed. The hour limit defaults to 50 GiB and the day limit to 100 GiB. They allow several downloads of a few GiB each. A response's bytes are counted when -it ends (see step 15). +it ends (see step 18). ### Lookup of AS number and country @@ -471,12 +480,12 @@ service that keeps its visitors' addresses on its own host. dropped from that table. The answer reaches the client's history and ban notes when it comes. - `SWW_LOOKUP_TIMEOUT` is how long a visitor's request waits for its client's - first answer, when a setting needs it (see step 5). It also bounds each + first answer, when a setting needs it (see step 6). It also bounds each request to GeoJS. A client whose answer has not come in time counts as unknown until it comes, and its later requests do not wait. Its address is asked about again in the background. - A client the lookup cannot place has an unknown country and AS number, which - the country lists and the biased thresholds treat as described there. Nothing + the country list and the biased thresholds treat as described there. Nothing else treats it differently. - GeoJS publishes no rate limit, but its terms forbid "an excessive amount of API requests", judged by GeoJS alone, and let it block a caller. While GeoJS @@ -496,12 +505,12 @@ service that keeps its visitors' addresses on its own host. `file_error` alert and ignored. The database already loaded stays in use. A client whose address the file does not hold counts as unknown. -### Country lists +### Country list -Both are empty by default: the defaults judge a client by what it does, not by -where it comes from. +`SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` is empty by default: the defaults judge a +client by what it does, not by where it comes from. A `country=` tag in +`SWW_TAG_DENY` refuses one country (see "Request tags"). -- `SWW_DENIED_COUNTRIES` refuses every request from a listed country. - `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` lets through only clients placed in a listed country. Every request from any other country is refused. So is every request from a client the lookup cannot place, such as internal monitoring or @@ -510,45 +519,84 @@ where it comes from. client in whenever GeoJS stops answering. The cost: while GeoJS fails, this list refuses every new client. No other failure of GeoJS, a reputation source or an alert destination comes this close to taking the service down. -- Both may be set. `SWW_DENIED_COUNTRIES` then adds nothing, since the exclusive - list already refuses every other country. A code on both lists stops the start +- A code in this list whose `country=` tag is in `SWW_TAG_DENY` stops the start with a message naming it. - A refused request is answered with `SWW_BAN_RESPONSE`, as soon as the client's address has been looked up. It is logged with the action `country_denied`, and counted in the metrics. +### Request tags + +Each request gets tags from what `discriminator` knows of its client and what +the checks found. The operator refuses or slows clients by tag. Tags appear in +the log line as `tags`, sorted, with empty lists left out (see +`docs/request-log.md`). + +Each tag and where it comes from: + +- `asn=` and `country=`, such as `asn=64496` and `country=DE`, + from the lookup. A client the lookup cannot place has neither. A request + refused by `deny_nets` or `active_ban` has neither, because those checks come + before the lookup. +- One tag per list of `SWW_TAG_LISTS` that holds the client (`tag_lists`), the + tag being the list's name, such as `torexit` or `vpn=mullvad`. These lists are + fetched and kept like a blocklist. `discriminator` ships none of its own. +- `crowdsec`, `dnsbl=` and `abuseipdb`, when that reputation source lists + the client, whatever the source's action. A zone with a key is named with the + key masked, as the `reputation` field gives it. +- `exploitsignature`, when a rule file or the Core Rule Set matched the request. + For the Core Rule Set a match is a score at or over + `SWW_WAF_ANOMALY_THRESHOLD`. + +A client in `SWW_ALLOW_NETS` skips the checks and gets no tag. Every tag is in +lower case, except a country code, which is in capitals. `SWW_TAG_DENY` and +`SWW_TAG_LIMIT_PERCENT` accept tags in either case. + +Two settings act on tags: + +- `SWW_TAG_DENY` lists tags whose requests are refused with the action `denied` + (`tag_deny`). The check runs twice: just before the rate limits, and again + after the Core Rule Set, so that `exploitsignature` can refuse too. +- `SWW_TAG_LIMIT_PERCENT` lists tags with a percentage from 1 to 99. Their + clients get that percentage of every request and byte limit. The lowest + percentage among a client's tags applies. `exploitsignature` cannot be listed, + since a request gets it only after the rate limits have counted it. + +The tag settings are empty by default, for the same reason as the country list. +The lookup waits for GeoJS when either names an `asn=` or `country=` tag. A +setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`. + ### Biased thresholds -The biased thresholds lower the limits of clients in listed AS numbers and -countries. The lists are empty by default, for the same reason as the country -lists. The request log and the metrics show which AS numbers and countries a -service's abuse comes from. +The biased thresholds lower the limits of clients by AS number, country and tag. +The lists are empty by default, for the same reason as the country list. The +request log and the metrics show which AS numbers and countries a service's +abuse comes from. -- Each list gives AS numbers or countries a percentage, such as `AS14061:50`. - Their clients get that percentage of every request and byte limit - (`SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT`). A lowered limit is - rounded down. `0` is a zero allowance: the client's first request breaks a - limit and bans it. +- `SWW_TAG_LIMIT_PERCENT` gives a tag a percentage, such as `asn=14061:50` (see + "Request tags"). A lowered limit is rounded down. +- `SWW_UNKNOWN_LIMIT_PERCENT` at `0` is a zero allowance: the client's first + request breaks a limit and bans it. - `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` set the byte limits' percentage alone, when it should differ from the request limits'. For an AS - number they list, they replace its percentage from `SWW_ASN_LIMIT_PERCENT` and - `SWW_ASN_LIMIT_PERCENT_URL`; for a country, the one from - `SWW_COUNTRY_LIMIT_PERCENT`. + number they list, they replace its `asn=` tag's percentage and its entry in + `SWW_ASN_LIMIT_PERCENT_URL`; for a country, its `country=` tag's percentage. - `SWW_ASN_LIMIT_PERCENT_URL` names a text file of `AS:percent` lines, so one abuse-source list can be shared by every `discriminator` in the fleet. It is - fetched and kept like a blocklist (see "Reputation"). Where it and - `SWW_ASN_LIMIT_PERCENT` both give a percentage, the lower applies. + fetched and kept like a blocklist (see "Reputation"). Each entry counts like + an `asn=` tag in `SWW_TAG_LIMIT_PERCENT`. Where both give a percentage, the + lower applies. - A client's limit percentage is the lowest of those that apply to it: its AS - number's, its country's, or a reputation hit's `limit:`. A client - without a country gets `SWW_UNKNOWN_LIMIT_PERCENT` in place of a country's. - The percentage is 100 if none applies. Clients given a lower percentage are - banned after fewer requests, first for a while and then, for repeated abuse, - permanently. The log line of each counted request, and the notes of each ban, - say which setting lowered the limit. + number's, its country's, its tags', or a reputation hit's `limit:`. A + client without a country gets `SWW_UNKNOWN_LIMIT_PERCENT` in place of a + country's. The percentage is 100 if none applies. Clients given a lower + percentage are banned after fewer requests, first for a while and then, for + repeated abuse, permanently. The log line of each counted request, and the + notes of each ban, say which setting lowered the limit. - No budget is shared by a whole country or AS number. One abuser could use it up and lock out everyone else there: a denial of service nobody chose, and the one this tool exists to prevent. Refusing a whole country is left to the - operator's choice, through the country lists. + operator's choice, through `SWW_TAG_DENY` or the country list. ### Attack detection @@ -903,7 +951,7 @@ left unset sends no alert. An alert needs a destination only the operator can supply, and what counts as unusual depends on each service's normal traffic, which the metrics show. They reveal a widely distributed scraper, thousands of addresses at low rates each, which the per-client limits do not see. A low -`SWW_ASN_LIMIT_PERCENT` for its AS number is the response. +percentage for its `asn=` tag in `SWW_TAG_LIMIT_PERCENT` is the response. - Requests and bytes are counted per minute and per hour, for each of: - a client; @@ -1181,7 +1229,7 @@ the metrics. ### The answer -`discriminator` runs steps 3 to 13 of "What happens to a request", in that +`discriminator` runs steps 3 to 16 of "What happens to a request", in that order. Every answer but the last below carries `X-Sww-Disposition`. - `X-Sww-Disposition: allow`, status `200`, no body. `rproxy` passes the request @@ -1220,13 +1268,13 @@ seconds, and `SWRP_DEFAULT_DISPOSITION` then decides. So at the defaults, and whatever its outside sources do. GeoJS is the only outside source a question can wait on. A question waits only -when step 5 says so, and for at most `SWW_LOOKUP_TIMEOUT`, by default 800 +when step 6 says so, and for at most `SWW_LOOKUP_TIMEOUT`, by default 800 milliseconds. A client without an answer by then counts as unknown. No question waits on these, which run in the background: - the DNS blocklists and AbuseIPDB; -- fetching the blocklists, the CrowdSec decision list and the file - `SWW_ASN_LIMIT_PERCENT_URL` names. +- fetching the blocklists, the tag lists, the CrowdSec decision list and the + file `SWW_ASN_LIMIT_PERCENT_URL` names. In `enforce` mode, a question does not wait for GeoJS when it is a request: diff --git a/docs/request-log.md b/docs/request-log.md index 65a1fc3..ee10225 100644 --- a/docs/request-log.md +++ b/docs/request-log.md @@ -96,8 +96,8 @@ Every one of these lines has: - `action`: what was done: - `forward`: answered `allow`, so that `rproxy` passes it to the app; - - `denied`: refused for `SWW_DENY_NETS`, or for a reputation source with the - action `deny`; + - `denied`: refused for `SWW_DENY_NETS`, for a reputation source with the + action `deny`, or for a tag in `SWW_TAG_DENY`; - `banned`: refused under a ban, or for a `ban` rule, a trap path or a CrowdSec decision, each of which bans the client; - `country_denied`, `rate_limited`, `rule_blocked` and `waf_blocked`: @@ -115,6 +115,7 @@ Every one of these lines has: - `deny_nets`: `netblock`, the entry of `SWW_DENY_NETS` the client is in; - `active_ban`: `netblock`, the ban's netblock, and `ban_expires`; - `countries`: `country`, and `setting`, the country list that refused it; + - `tag_deny`: `tag`, the tag of `SWW_TAG_DENY` the request carries; - `blocklists`: `lists`, the blocklist URLs that list the client; - `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the decision ends; @@ -168,6 +169,8 @@ Every one of these lines has: - the CrowdSec decision list's URL; - DNS blocklist zones; - `abuseipdb`. +- `tags`: the request's tags, sorted, left out when it has none. Each tag and + where it comes from is under "Request tags" in [`SPEC.md`](../SPEC.md). - `offence`: `limit`, for a request that broke a limit. - `ban_expires`: when the client's ban ends, for a request that made one or was refused under one: a time, or `permanent`. diff --git a/docs/settings.md b/docs/settings.md index fb621eb..391327f 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -90,11 +90,11 @@ between the two (see "The image's command" below). ## Static lists - `SWW_ALLOW_NETS` (default empty): netblocks you trust, such as your monitoring - (see step 3 in [`SPEC.md`](../SPEC.md)). + (see step 4 in [`SPEC.md`](../SPEC.md)). - `SWW_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are exempt from rate limiting and the byte limits. - `SWW_DENY_NETS` (default empty): netblocks whose clients are refused (see step - 3 in [`SPEC.md`](../SPEC.md)). + 4 in [`SPEC.md`](../SPEC.md)). ## Request rate limits, per client @@ -130,9 +130,11 @@ between the two (see "The image's command" below). - `SWW_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off` stops the start while a setting that needs a lookup is set: - - the two country lists; + - `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES`; - `SWW_ADD_LOOKUP_HEADERS`; - - the four biased threshold lists and `SWW_ASN_LIMIT_PERCENT_URL`; + - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=` + tag; + - the two byte percentage lists and `SWW_ASN_LIMIT_PERCENT_URL`; - `SWW_UNKNOWN_LIMIT_PERCENT` below `100`; - an anomaly threshold per AS number. - `SWW_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its @@ -144,28 +146,61 @@ between the two (see "The image's command" below). the client's AS number in `X-Client-ASN` and its country in `X-Client-Country`. -## Country lists +## Country list -- `SWW_DENIED_COUNTRIES` (default empty): countries whose clients are refused, - such as `cn,ru,kp`. - `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only - countries whose clients get through, such as `us,de`. A country on both lists - stops the start. + countries whose clients get through, such as `us,de`. A country whose + `country=` tag is in `SWW_TAG_DENY` stops the start. + +## Request tags + +Every request gets tags from what `discriminator` knows of its client. See +"Request tags" in [`SPEC.md`](../SPEC.md). + +- `SWW_TAG_LISTS` (default empty): address lists that each add a tag, as + `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. A name is the + tag, and may hold `=`, such as `vpn=mullvad`. Each list is an `http` or + `https` URL fetched like a blocklist. A name or URL listed twice is refused. + So is a URL that is also in `SWW_BLOCKLIST_URLS`, `SWW_ASN_LIMIT_PERCENT_URL`, + or the CrowdSec decision list. +- `SWW_TAG_DENY` (default empty): tags whose requests are refused, such as + `torexit,country=CN`. A tag listed twice, an `asn=` tag without a number, or a + `country=` tag without a country, is refused. +- `SWW_TAG_LIMIT_PERCENT` (default empty): tags that lower a client's limits, as + tag and percentage pairs, such as `vpn:50,asn=14061:25`. A percentage is from + `1` to `99`, of every rate limit and byte limit. The lowest percentage among a + client's tags applies. A tag listed twice is refused. So is + `exploitsignature`: a request gets that tag after the rate limits have counted + it. + +`SWW_TAG_DENY` and `SWW_TAG_LIMIT_PERCENT` accept tags in either case. A tag +that no request can get stops the start. A request can get only these: + +- a name in `SWW_TAG_LISTS`; +- an `asn=` or `country=` tag; +- a `dnsbl=` tag of a zone in `SWW_DNSBL_ZONES`, in lower case, with a key + masked as the `reputation` field shows it; +- `abuseipdb`, while `SWW_ABUSEIPDB_KEY` is set; +- `exploitsignature`, in `SWW_TAG_DENY` only, while `SWW_RULES_ENABLED` is + `true`, or while neither `SWW_WAF_MODE` nor `SWW_WAF_ANOMALY_THRESHOLD` is + `off`. + +`crowdsec` stops the start in both settings, because a CrowdSec decision refuses +the request first. ## Biased thresholds -- `SWW_ASN_LIMIT_PERCENT` (default empty): a list of AS numbers, each with `:` - and a percentage, such as `AS14061:50,AS16276:50,AS45102:25`. An AS number - listed twice is refused. -- `SWW_COUNTRY_LIMIT_PERCENT` (default empty): the same by country, such as - `cn:25,ru:50`. -- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` (default empty): the - same, for the byte limits alone. +- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` (default empty): a + percentage of the byte limits alone, for the AS numbers or countries listed, + such as `AS14061:50` or `cn:25`. It takes the place of the percentage + `SWW_TAG_LIMIT_PERCENT` gives their `asn=` or `country=` tag. An AS number or + country listed twice is refused. - `SWW_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client the lookup cannot place. - `SWW_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a text file with one AS number and percentage on each line, such as - `AS64496:50`. A URL that is also in `SWW_BLOCKLIST_URLS` is refused. + `AS64496:50`. Each entry counts like an `asn=` tag in `SWW_TAG_LIMIT_PERCENT`. + A URL that is also in `SWW_BLOCKLIST_URLS` is refused. ## Attack detection diff --git a/internal/config/config.go b/internal/config/config.go index d19a26c..8f77579 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -93,29 +93,30 @@ type Config struct { LookupDBPath string LookupTimeout time.Duration AddLookupHeaders bool - // DeniedCountries are the countries whose clients are refused - // (SWW_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not - // empty, are the only countries whose clients are let through - // (SWW_EXCLUSIVELY_ALLOWED_COUNTRIES). Both hold two-letter codes in - // capitals, as GeoJS gives them. - DeniedCountries []string + // ExclusivelyAllowedCountries, when not empty, are the only countries + // whose clients are let through (SWW_EXCLUSIVELY_ALLOWED_COUNTRIES), + // two-letter codes in capitals, as GeoJS gives them. ExclusivelyAllowedCountries []string - // The biased thresholds. ASNLimitPercent and CountryLimitPercent give - // the clients of the AS numbers and the countries they list that - // percentage of every rate limit and byte limit - // (SWW_ASN_LIMIT_PERCENT and SWW_COUNTRY_LIMIT_PERCENT). - // ASNBytesPercent and CountryBytesPercent give those they list a - // percentage of the byte limits in place of that one + // TagLists are the address lists that each add a tag to the requests of + // the clients they hold (SWW_TAG_LISTS). TagDeny are the tags whose + // requests are refused (SWW_TAG_DENY). TagLimitPercent gives the clients + // with a tag that percentage of every rate limit and byte limit, from 1 + // to 99 (SWW_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in + // lower case, a country code in capitals. + TagLists []TagList + TagDeny []string + TagLimitPercent map[string]int64 + // The biased thresholds. ASNBytesPercent and CountryBytesPercent give + // those they list a percentage of the byte limits in place of the one + // TagLimitPercent gives their asn= or country= tag // (SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT). Each holds // percentages from 0 to 100, by AS number, written as AS64496, or by // country, a two-letter code in capitals, as the lookup gives them. // UnknownLimitPercent is the percentage of every limit a client without // a country gets (SWW_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is // where a file of AS:percent lines is fetched from, whose percentages - // count as those of ASNLimitPercent do (SWW_ASN_LIMIT_PERCENT_URL), "" - // while it is unset. - ASNLimitPercent map[string]int64 - CountryLimitPercent map[string]int64 + // count like asn= tags in TagLimitPercent (SWW_ASN_LIMIT_PERCENT_URL), + // "" while it is unset. ASNBytesPercent map[string]int64 CountryBytesPercent map[string]int64 UnknownLimitPercent int64 @@ -164,8 +165,8 @@ type Config struct { // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWW_BAN_RESPONSE). It answers a banned client, a request that - // breaks a rate limit or matches a ban rule, SWW_DENY_NETS and the - // country lists. + // breaks a rate limit or matches a ban rule, SWW_DENY_NETS, + // SWW_EXCLUSIVELY_ALLOWED_COUNTRIES and SWW_TAG_DENY. BanResponse int // LimitBanDuration is the ban for a first broken rate limit // (SWW_LIMIT_BAN_DURATION). A limit broken again within @@ -367,7 +368,7 @@ var ( errHeaderTakenOut = errors.New( "is taken out of every request by Go's HTTP server, so it can never " + "be logged") - errOnBothLists = errors.New("is in SWW_DENIED_COUNTRIES too") + errOnBothLists = errors.New("is in SWW_TAG_DENY too") errNotLookupSource = errors.New("is not geojs, file or off") errNeedsLookups = errors.New("it needs each client looked up") errNeedsDBPath = errors.New("it names the file to look clients up in") @@ -465,6 +466,25 @@ var ( errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL") errAnotherList = errors.New( "is in SWW_BLOCKLIST_URLS or is SWW_ASN_LIMIT_PERCENT_URL too") + errFetchedOtherwise = errors.New("is in SWW_BLOCKLIST_URLS, is " + + "SWW_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too") + errNotTagList = errors.New( + "is not a name, = and an http or https URL, such as " + + "torexit=https://example.com/tor.txt") + errNotTagPercentItem = errors.New( + "is not a tag, : and a percentage, such as vpn:50 or asn=14061:25") + errNotTagPercent = errors.New("is not a percentage from 1 to 99") + errNotASNTag = errors.New("is not asn= and a number, such as asn=14061") + errExploitTag = errors.New( + "cannot lower limits: the rate limits count a request before its " + + "rules are matched") + errCrowdSecTag = errors.New( + "cannot be used: a CrowdSec decision refuses the request first") + errNoRequestGetsTag = errors.New( + "is a tag no request can get: it is not a name in SWW_TAG_LISTS, an " + + "asn= or country= tag, a dnsbl= tag of a zone in SWW_DNSBL_ZONES, " + + "abuseipdb with SWW_ABUSEIPDB_KEY set, or exploitsignature with " + + "rule files or the Core Rule Set on") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -499,11 +519,11 @@ func FromEnvironment( LookupDBPath: env.value("SWW_LOOKUP_DB_PATH", ""), LookupTimeout: env.durationNotOff("SWW_LOOKUP_TIMEOUT", "800ms"), AddLookupHeaders: env.boolean("SWW_ADD_LOOKUP_HEADERS", "false"), - DeniedCountries: env.countries("SWW_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), - ASNLimitPercent: env.percents("SWW_ASN_LIMIT_PERCENT", ParseASN), - CountryLimitPercent: env.percents("SWW_COUNTRY_LIMIT_PERCENT", parseCountry), + TagLists: env.tagLists("SWW_TAG_LISTS"), + TagDeny: env.tags("SWW_TAG_DENY"), + TagLimitPercent: env.tagPercents("SWW_TAG_LIMIT_PERCENT"), ASNBytesPercent: env.percents("SWW_ASN_BYTES_PERCENT", ParseASN), CountryBytesPercent: env.percents("SWW_COUNTRY_BYTES_PERCENT", parseCountry), UnknownLimitPercent: env.percent("SWW_UNKNOWN_LIMIT_PERCENT", "100"), @@ -583,6 +603,8 @@ func FromEnvironment( env.checkCountriesAndLookups(cfg) env.checkASNLimitPercentURL(cfg) env.checkCrowdSec(cfg) + env.checkTagLists(cfg) + env.checkTags(cfg) if env.err != nil { return nil, env.err @@ -860,6 +882,31 @@ func (e *environment) percents( return percents } +// tagLists reads SWW_TAG_LISTS, which is empty by default. +func (e *environment) tagLists(name string) []TagList { + lists, err := parseTagLists(e.value(name, "")) + e.check(name, err) + + return lists +} + +// tags reads a setting that is a list of tags. It is empty by default. +func (e *environment) tags(name string) []string { + tags, err := parseTags(e.value(name, "")) + e.check(name, err) + + return tags +} + +// tagPercents reads a setting that is a list of tags, each with a +// percentage. It is empty by default. +func (e *environment) tagPercents(name string) map[string]int64 { + percents, err := parseTagPercents(e.value(name, "")) + e.check(name, err) + + return percents +} + // percent reads a setting that is a percentage, from 0 to 100. func (e *environment) percent(name, defaultValue string) int64 { percent, err := ParsePercent(e.value(name, defaultValue)) @@ -997,15 +1044,17 @@ func (e *environment) checkLookupDBPath(cfg *Config) { } } -// checkCountriesAndLookups refuses a country on both country lists, and, +// checkCountriesAndLookups refuses a country in +// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES whose tag is in SWW_TAG_DENY, and, // while SWW_LOOKUP_SOURCE is off, each setting that needs clients looked -// up: the country lists, SWW_ADD_LOOKUP_HEADERS, the biased thresholds, +// up: the country list, SWW_ADD_LOOKUP_HEADERS, the tag settings that name +// an asn= or country= tag, the biased thresholds, // SWW_ASN_LIMIT_PERCENT_URL among them, of which // SWW_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a // limit, and the anomaly thresholds per AS number. func (e *environment) checkCountriesAndLookups(cfg *Config) { for _, country := range cfg.ExclusivelyAllowedCountries { - if slices.Contains(cfg.DeniedCountries, country) { + if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) { e.check("SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", fmt.Errorf("%q %w", country, errOnBothLists)) } @@ -1019,11 +1068,10 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) { name string set bool }{ - {"SWW_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0}, {"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0}, {"SWW_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders}, - {"SWW_ASN_LIMIT_PERCENT", len(cfg.ASNLimitPercent) > 0}, - {"SWW_COUNTRY_LIMIT_PERCENT", len(cfg.CountryLimitPercent) > 0}, + {"SWW_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, isLookupTag)}, + {"SWW_TAG_LIMIT_PERCENT", tagLimitNeedsLookups(cfg)}, {"SWW_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0}, {"SWW_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0}, {"SWW_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100}, @@ -1069,6 +1117,90 @@ func (e *environment) checkCrowdSec(cfg *Config) { } } +// checkTagLists refuses a list of SWW_TAG_LISTS whose URL is fetched as +// another list too: the file at a URL is read as one kind of list. +func (e *environment) checkTagLists(cfg *Config) { + for _, list := range cfg.TagLists { + if slices.Contains(cfg.BlocklistURLs, list.URL) || + list.URL == cfg.ASNLimitPercentURL || list.URL == cfg.CrowdSecDecisionsURL { + e.check("SWW_TAG_LISTS", fmt.Errorf("%q %w", list.URL, errFetchedOtherwise)) + } + } +} + +// checkTags refuses a tag of SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT that would +// never apply: crowdsec, whose request is refused before tags are acted on, and +// any tag these settings cannot give a request. +func (e *environment) checkTags(cfg *Config) { + for _, tag := range cfg.TagDeny { + e.check("SWW_TAG_DENY", checkTag(cfg, tag)) + } + + for tag := range cfg.TagLimitPercent { + e.check("SWW_TAG_LIMIT_PERCENT", checkTag(cfg, tag)) + } +} + +// checkTag returns an error when tag would never apply, as checkTags says. +func checkTag(cfg *Config, tag string) error { + if tag == CrowdSecTag { + return fmt.Errorf("%q %w", tag, errCrowdSecTag) + } + + if !requestCanGetTag(cfg, tag) { + return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag) + } + + return nil +} + +// requestCanGetTag reports whether anything cfg sets up can give a request tag. +func requestCanGetTag(cfg *Config, tag string) bool { + isListName := func(list TagList) bool { return list.Name == tag } + if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) { + return true + } + + if tag == ExploitSignatureTag { + coreRuleSetOn := cfg.WAFMode != WAFModeOff && cfg.WAFAnomalyThreshold != 0 + + return cfg.RulesEnabled || coreRuleSetOn + } + + if tag == AbuseIPDBTag { + return cfg.AbuseIPDBKey != "" + } + + zone, isZoneTag := strings.CutPrefix(tag, DNSBLTagPrefix) + + return isZoneTag && slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool { + return strings.ToLower(MaskZoneKey(each)) == zone + }) +} + +// TagsNeedLookup reports whether SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT +// names an asn= or country= tag, which only the lookup can give a request. +func (c *Config) TagsNeedLookup() bool { + return slices.ContainsFunc(c.TagDeny, isLookupTag) || tagLimitNeedsLookups(c) +} + +// tagLimitNeedsLookups reports whether SWW_TAG_LIMIT_PERCENT names an asn= +// or country= tag. +func tagLimitNeedsLookups(cfg *Config) bool { + for tag := range cfg.TagLimitPercent { + if isLookupTag(tag) { + return true + } + } + + return false +} + +// isLookupTag reports whether tag is an asn= or country= tag. +func isLookupTag(tag string) bool { + return strings.HasPrefix(tag, ASNTagPrefix) || strings.HasPrefix(tag, CountryTagPrefix) +} + // headerNames reads a setting that is a list of header names, and // returns them in lower case. func (e *environment) headerNames(name, defaultValue string) []string { @@ -1785,6 +1917,161 @@ func ParseASN(value string) (string, error) { return "AS" + strconv.FormatUint(number, 10), nil } +// The tags a request can get that the settings can name, as discriminator +// writes them. The prefixes are followed by a value. +const ( + ASNTagPrefix = "asn=" + CountryTagPrefix = "country=" + DNSBLTagPrefix = "dnsbl=" + AbuseIPDBTag = "abuseipdb" + CrowdSecTag = "crowdsec" + // ExploitSignatureTag is the tag of a request a rule file or the Core + // Rule Set matched. + ExploitSignatureTag = "exploitsignature" +) + +// TagList is an address list of SWW_TAG_LISTS: the tag it adds to the +// requests of the clients it holds, and the URL it is fetched from. +type TagList struct { + Name string + URL string +} + +// parseTagLists reads a comma-separated list of name=URL pairs, such as +// torexit=https://example.com/tor.txt, and returns each list with its name +// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the +// = before http. A name or a URL listed twice is an error. +func parseTagLists(value string) ([]TagList, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + lists := make([]TagList, 0, len(items)) + + for _, item := range items { + end := strings.Index(item, "=http") + if end <= 0 || !isListURL(item[end+1:]) { + return nil, fmt.Errorf("%q %w", item, errNotTagList) + } + + list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]} + + if slices.ContainsFunc(lists, func(earlier TagList) bool { + return earlier.Name == list.Name || earlier.URL == list.URL + }) { + return nil, fmt.Errorf("%q %w", item, errListedTwice) + } + + lists = append(lists, list) + } + + return lists, nil +} + +// parseTags reads a comma-separated list of tags, which parseTag reads. A +// tag listed twice is an error, which shows the tag as MaskZoneKey does. +func parseTags(value string) ([]string, error) { + items, err := parseList(value) + if err != nil { + // parseList's error, for an empty item, shows the whole value, keys + // included. + return nil, errEmptyItem + } + + tags := make([]string, 0, len(items)) + + for _, item := range items { + tag, err := parseTag(item) + if err != nil { + return nil, err + } + + if slices.Contains(tags, tag) { + return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errListedTwice) + } + + tags = append(tags, tag) + } + + return tags, nil +} + +// parseTag reads a tag in either case, with spaces around it, and returns it +// as a request carries it: in lower case, but for a country code, which is in +// capitals, and an asn= tag's number without leading zeros. An asn= or +// country= tag must name a number or a country, so that a mistyped one stops +// the start instead of never matching. +func parseTag(value string) (string, error) { + tag := strings.ToLower(strings.TrimSpace(value)) + + if digits, isASN := strings.CutPrefix(tag, ASNTagPrefix); isASN { + number, err := strconv.ParseUint(digits, 10, 32) + if err != nil { + return "", fmt.Errorf("%q %w", value, errNotASNTag) + } + + return ASNTagPrefix + strconv.FormatUint(number, 10), nil + } + + if code, isCountry := strings.CutPrefix(tag, CountryTagPrefix); isCountry { + country, err := parseCountry(code) + if err != nil { + return "", err + } + + return CountryTagPrefix + country, nil + } + + return tag, nil +} + +// parseTagPercents reads a comma-separated list of tags, each with : and a +// percentage from 1 to 99, such as vpn:50 or asn=14061:25. A tag listed +// twice is an error, and so is ExploitSignatureTag, which a request gets +// only after the rate limits have counted it. +func parseTagPercents(value string) (map[string]int64, error) { + items, err := parseList(value) + if err != nil { + // parseList's error, for an empty item, shows the whole value, keys + // included. + return nil, errEmptyItem + } + + percents := make(map[string]int64, len(items)) + + for _, item := range items { + colon := strings.LastIndex(item, ":") + if colon <= 0 { + return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errNotTagPercentItem) + } + + tag, err := parseTag(item[:colon]) + if err != nil { + return nil, err + } + + percent, err := strconv.ParseInt(item[colon+1:], 10, 64) + if err != nil || percent < 1 || percent > 99 { + return nil, fmt.Errorf("%q %w", MaskZoneKey(item[colon+1:]), errNotTagPercent) + } + + if tag == ExploitSignatureTag { + return nil, fmt.Errorf("%q %w", tag, errExploitTag) + } + + if _, listed := percents[tag]; listed { + // tag, since item[:colon] may end in a space, which MaskZoneKey + // would not mask. + return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice) + } + + percents[tag] = percent + } + + return percents, nil +} + // parsePercents reads a comma-separated list of items, each an AS number // or a country, which parseCode reads, then : and a percentage, such as // AS64496:50 or cn:25, and returns each one's percentage. An empty value diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f8cfd56..7944890 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -42,10 +42,10 @@ const ( lookupDBPath = "SWW_LOOKUP_DB_PATH" lookupTimeout = "SWW_LOOKUP_TIMEOUT" addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS" - deniedCountries = "SWW_DENIED_COUNTRIES" allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES" - asnLimitPercent = "SWW_ASN_LIMIT_PERCENT" - countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT" + tagLists = "SWW_TAG_LISTS" + tagDeny = "SWW_TAG_DENY" + tagLimitPercent = "SWW_TAG_LIMIT_PERCENT" asnBytesPercent = "SWW_ASN_BYTES_PERCENT" countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT" unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT" @@ -208,6 +208,9 @@ const off = "off" // enabled is true, as a setting's value. const enabled = "true" +// disabled is false, as a setting's value. +const disabled = "false" + // defaultLookupSource is the default of SWW_LOOKUP_SOURCE, and // fileSource the source that is the lookup database. const ( @@ -274,9 +277,13 @@ func TestDefaults(t *testing.T) { wantNetblocks(t, cfg.AllowNets) wantNetblocks(t, cfg.RateLimitExemptNets) wantNetblocks(t, cfg.DenyNets) - wantCountries(t, deniedCountries, cfg.DeniedCountries) wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries) + if len(cfg.TagLists) != 0 || len(cfg.TagDeny) != 0 || len(cfg.TagLimitPercent) != 0 { + t.Errorf("tag settings %v, %v and %v by default, want all empty", + cfg.TagLists, cfg.TagDeny, cfg.TagLimitPercent) + } + hostname, err := os.Hostname() if err != nil || hostname == "" || cfg.InstanceName != hostname { t.Errorf("%s is %q, want the host's name %q (%v)", instanceName, @@ -327,7 +334,6 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", - deniedCountries: "cn, RU,kp,Xk", allowedCountries: "de", banResponse: "429", limitBanDuration: "15m", @@ -342,7 +348,7 @@ func TestValuesAsSet(t *testing.T) { metricsToken: token, metricsTopN: "10", rulesDir: "/srv/waf-rules", - rulesEnabled: "false", + rulesEnabled: disabled, }) wantSettings(t, cfg, config.Config{ @@ -370,7 +376,6 @@ func TestValuesAsSet(t *testing.T) { wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32") wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32") wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24") - wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") } @@ -1097,15 +1102,15 @@ func TestInstanceNameNotUTF8StopsTheStart(t *testing.T) { } } -func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { +func TestCodeInAllowedCountriesAndTagDenyStopsTheStart(t *testing.T) { t.Parallel() _, err := config.FromEnvironment(environment{ - deniedCountries: "cn,ru", + tagDeny: "country=cn,country=ru", allowedCountries: "de,RU", }.lookupEnv) if err == nil { - t.Fatal("ru on both country lists was accepted") + t.Fatal("ru in SWW_EXCLUSIVELY_ALLOWED_COUNTRIES and SWW_TAG_DENY was accepted") } if !strings.HasPrefix(err.Error(), allowedCountries+": ") || @@ -1176,12 +1181,11 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { t.Parallel() for name, value := range map[string]string{ - deniedCountries: "kp", allowedCountries: "de", addLookupHeaders: enabled, + tagDeny: "country=KP", + tagLimitPercent: "asn=64496:50", asnLimitPercentURL: asnURL, - asnLimitPercent: "AS64496:50", - countryLimitPercent: "cn:25", asnBytesPercent: "AS64496:50", countryBytesPercent: "cn:25", unknownLimitPercent: "99", @@ -1204,12 +1208,13 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { } // Set empty, the lists need nothing looked up, and nor does - // SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly - // threshold per AS number that is off, any other anomaly threshold, or - // a blocklist. + // SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, a tag the + // lookup does not give, an anomaly threshold per AS number that is off, + // any other anomaly threshold, or a blocklist. env := environment{ - lookupSource: off, deniedCountries: "", allowedCountries: "", - asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "", + lookupSource: off, allowedCountries: "", asnBytesPercent: "", + tagLists: "torexit=https://example.com/tor.txt,vpn=https://example.com/vpn.txt", + tagDeny: torExitTag, tagLimitPercent: "vpn:50", countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "", blocklistURLs: dropURL, } @@ -1355,34 +1360,194 @@ func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) { } } +// torExitTag, zenTag and exploitTag are tags these tests name. +const ( + torExitTag = "torexit" + zenTag = "dnsbl=zen.example" + exploitTag = "exploitsignature" +) + +func TestTagSettingsAsSet(t *testing.T) { + t.Parallel() + + const torURL = "https://example.com/tor.txt" + + // Names and tags in either case, an AS number with leading zeros, and a + // name holding a =. + cfg := fromEnvironment(t, environment{ + tagLists: "TorExit=" + torURL + + ", vpn=mullvad=https://example.com/mullvad.txt", + dnsblZones: "Zen.example", + tagDeny: "TorExit, Country=kp, asn=014061", + tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99", + }) + + wantLists := []config.TagList{ + {Name: torExitTag, URL: torURL}, + {Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"}, + } + if !slices.Equal(cfg.TagLists, wantLists) { + t.Errorf("%s gave %+v, want %+v", tagLists, cfg.TagLists, wantLists) + } + + wantDeny := []string{torExitTag, "country=KP", "asn=14061"} + if !slices.Equal(cfg.TagDeny, wantDeny) { + t.Errorf("%s gave %v, want %v", tagDeny, cfg.TagDeny, wantDeny) + } + + wantPercents := map[string]int64{ + "vpn=mullvad": 50, "asn=14061": 25, "country=CN": 1, zenTag: 99, + } + if !maps.Equal(cfg.TagLimitPercent, wantPercents) { + t.Errorf("%s gave %v, want %v", tagLimitPercent, cfg.TagLimitPercent, wantPercents) + } +} + +func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) { + t.Parallel() + + const ( + list = "torexit=https://example.com/tor.txt" + key = "abc123" + zone = key + ".xbl.dq.spamhaus.net" + masked = "********.xbl.dq.spamhaus.net" + ) + + for _, tc := range []struct { + name string + env environment + want string // "" is accepted + }{ + {"a list's name", environment{tagLists: list, tagDeny: torExitTag}, ""}, + { + "a mistyped list's name", environment{tagLists: list, tagDeny: "torexti"}, + `"torexti"`, + }, + {"a masked zone", environment{dnsblZones: zone, tagDeny: "dnsbl=" + masked}, ""}, + { + "a zone with its key", environment{dnsblZones: zone, tagDeny: "dnsbl=" + zone}, + masked, + }, + {"another zone", environment{dnsblZones: zone, tagDeny: zenTag}, zenTag}, + { + "abuseipdb with a key", + environment{abuseIPDBKey: "k", tagLimitPercent: "abuseipdb:50"}, "", + }, + {"crowdsec, though a list has its name", environment{ + tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec", + }, `"crowdsec" cannot be used`}, + {exploitTag, environment{tagDeny: exploitTag}, ""}, + { + exploitTag + " with rule files off", + environment{tagDeny: exploitTag, rulesEnabled: disabled}, "", + }, + { + exploitTag + " with the Core Rule Set off", + environment{tagDeny: exploitTag, wafMode: off}, "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(tc.env.lookupEnv) + + switch { + case tc.want == "" && err != nil: + t.Errorf("error %v, want none", err) + case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)): + t.Errorf("error %v, want one holding %s", err, tc.want) + case err != nil && strings.Contains(err.Error(), key): + t.Errorf("error %v shows the zone's key", err) + } + }) + } +} + +func TestErrorOfATagSettingLeavesOutAZoneKey(t *testing.T) { + t.Parallel() + + const ( + tag = "dnsbl=" + spamhaus + emptyItem = "has an empty item in its list" + twice = `"` + spamhausMasked + `" is listed twice` + ) + + for _, tc := range []struct{ name, value, want string }{ + {tagDeny, tag + "," + tag, twice}, + {tagDeny, tag + ",,", emptyItem}, + { + tagLimitPercent, tag, + `"` + spamhausMasked + `" is not a tag, : and a percentage, ` + + "such as vpn:50 or asn=14061:25", + }, + { + tagLimitPercent, "50:" + tag, + `"` + spamhausMasked + `" is not a percentage from 1 to 99`, + }, + {tagLimitPercent, tag + ":50," + tag + " :25", twice}, + {tagLimitPercent, tag + ":50,,", emptyItem}, + } { + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("%s=%s gave the error %v, want %s", tc.name, tc.value, err, want) + } + } +} + +func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) { + t.Parallel() + + const lapi = "http://172.17.0.1:8080" + + for _, tc := range []struct { + name, listURL string + env environment + }{ + {"a blocklist", dropURL, environment{blocklistURLs: dropURL}}, + {"the file of AS:percent lines", asnURL, environment{asnLimitPercentURL: asnURL}}, + { + "the CrowdSec decision list", lapi + "/v1/decisions", + environment{"SWW_CROWDSEC_LAPI_URL": lapi, "SWW_CROWDSEC_LAPI_KEY": "key"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + tc.env[tagLists] = "dup=" + tc.listURL + + want := tagLists + `: "` + tc.listURL + `" is in SWW_BLOCKLIST_URLS, is ` + + "SWW_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too" + + _, err := config.FromEnvironment(tc.env.lookupEnv) + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + func TestBiasedThresholdsAsSet(t *testing.T) { t.Parallel() cfg := fromEnvironment(t, environment{}) - if len(cfg.ASNLimitPercent) != 0 || len(cfg.CountryLimitPercent) != 0 || - len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 || + if len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 || cfg.UnknownLimitPercent != 100 { - t.Errorf("biased thresholds %v, %v, %v, %v and %d by default, "+ - "want four empty lists and 100", cfg.ASNLimitPercent, cfg.CountryLimitPercent, + t.Errorf("biased thresholds %v, %v and %d by default, "+ + "want two empty lists and 100", cfg.ASNBytesPercent, cfg.CountryBytesPercent, cfg.UnknownLimitPercent) } // AS numbers and countries in either case, an AS number with leading // zeros, 0 and 100. cfg = fromEnvironment(t, environment{ - asnLimitPercent: "AS14061:50, as16276:0,AS045102:100", - countryLimitPercent: "cn:25,RU:50", asnBytesPercent: "as16276:75", countryBytesPercent: "ru:10", unknownLimitPercent: "0", }) for name, tc := range map[string]struct{ got, want map[string]int64 }{ - asnLimitPercent: { - cfg.ASNLimitPercent, - map[string]int64{"AS14061": 50, "AS16276": 0, "AS45102": 100}, - }, - countryLimitPercent: {cfg.CountryLimitPercent, map[string]int64{"CN": 25, "RU": 50}}, asnBytesPercent: {cfg.ASNBytesPercent, map[string]int64{"AS16276": 75}}, countryBytesPercent: {cfg.CountryBytesPercent, map[string]int64{"RU": 10}}, } { @@ -1406,16 +1571,16 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) { ) for _, tc := range []struct{ name, value, want string }{ - {asnLimitPercent, "14061:50", `"14061"` + notASN}, - {asnLimitPercent, "AS4294967296:50", `"AS4294967296"` + notASN}, - {asnLimitPercent, "AS14061", `"AS14061"` + notItem}, - {asnLimitPercent, "AS14061:101", `"101"` + notPercent}, - {asnLimitPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`}, + {asnBytesPercent, "14061:50", `"14061"` + notASN}, + {asnBytesPercent, "AS4294967296:50", `"AS4294967296"` + notASN}, + {asnBytesPercent, "AS14061", `"AS14061"` + notItem}, + {asnBytesPercent, "AS14061:101", `"101"` + notPercent}, + {asnBytesPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`}, { - countryLimitPercent, "nk:25", + countryBytesPercent, "nk:25", `"nk" is not a two-letter country code such as de or kp`, }, - {countryLimitPercent, "cn:25,CN:50", `"CN" is listed twice`}, + {countryBytesPercent, "cn:25,CN:50", `"CN" is listed twice`}, {asnBytesPercent, "AS14061:-1", `"-1"` + notPercent}, {countryBytesPercent, "cn:50%", `"50%"` + notPercent}, {unknownLimitPercent, "101", `"101"` + notPercent}, @@ -2003,23 +2168,37 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""}, {lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"}, {addLookupHeaders, "yes"}, - {deniedCountries, "nk"}, - {deniedCountries, "kp,,ir"}, - {deniedCountries, "prk"}, - {deniedCountries, "408"}, - {deniedCountries, "k"}, - {deniedCountries, "eu"}, - {deniedCountries, "un"}, - {deniedCountries, "su"}, + {allowedCountries, "nk"}, + {allowedCountries, "kp,,ir"}, + {allowedCountries, "prk"}, + {allowedCountries, "408"}, + {allowedCountries, "k"}, + {allowedCountries, "eu"}, + {allowedCountries, "un"}, + {allowedCountries, "su"}, {allowedCountries, "ac"}, {allowedCountries, "uk"}, {allowedCountries, "zz"}, {allowedCountries, "de,germany"}, - {asnLimitPercent, "AS14061:50,,AS16276:50"}, {asnLimitPercent, "ASX:50"}, - {asnLimitPercent, "AS14061:"}, {asnLimitPercent, "AS14061 :50"}, - {asnLimitPercent, "AS14061:1.5"}, {asnLimitPercent, "AS-1:50"}, - {countryLimitPercent, "cn"}, {countryLimitPercent, "cn:"}, - {countryLimitPercent, "cn:25:50"}, {countryLimitPercent, "china:25"}, + {asnBytesPercent, "AS14061:50,,AS16276:50"}, {asnBytesPercent, "ASX:50"}, + {asnBytesPercent, "AS14061:"}, {asnBytesPercent, "AS14061 :50"}, + {asnBytesPercent, "AS14061:1.5"}, {asnBytesPercent, "AS-1:50"}, + {countryBytesPercent, "cn"}, {countryBytesPercent, "cn:"}, + {countryBytesPercent, "cn:25:50"}, {countryBytesPercent, "china:25"}, + {tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"}, + {tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"}, {tagDeny, "torexti"}, + {tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"}, + {tagLimitPercent, "torexti:50"}, {tagLimitPercent, "dnsbl=zen.example:50"}, + {tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"}, + {tagLimitPercent, "vpn :50"}, {tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, + {tagLimitPercent, "vpn:"}, {tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"}, + {tagLimitPercent, "vpn:1.5"}, {tagLimitPercent, "vpn:50,VPN:25"}, + {tagLimitPercent, "exploitsignature:50"}, {tagLimitPercent, "asn=abc:50"}, + {tagLists, torExitTag}, {tagLists, "=https://example.com/a.txt"}, + {tagLists, "a,b=https://example.com/a.txt"}, + {tagLists, "a=ftp://example.com/a.txt"}, + {tagLists, "a=https://example.com/a.txt,A=https://example.com/b.txt"}, + {tagLists, "a=https://example.com/a.txt,b=https://example.com/a.txt"}, {asnBytesPercent, "AS14061:101"}, {countryBytesPercent, "su:50"}, {unknownLimitPercent, ""}, {unknownLimitPercent, "-1"}, {unknownLimitPercent, "50%"}, @@ -2031,6 +2210,25 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {logLevel, "INFO"}, {logLevel, "warning"}, {logLevel, "trace"}, {logLevel, ""}, {rulesEnabled, "yes"}, {rulesEnabled, "True"}, }) + + wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t) +} + +// wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff checks that with rule +// files off, and either setting turning off the Core Rule Set, the tag +// exploitsignature in SWW_TAG_DENY stops the start. +func wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t *testing.T) { + t.Helper() + + for _, name := range []string{wafMode, wafAnomalyThreshold} { + _, err := config.FromEnvironment(environment{ + tagDeny: exploitTag, rulesEnabled: disabled, name: off, + }.lookupEnv) + if err == nil || !strings.HasPrefix(err.Error(), tagDeny+": ") { + t.Errorf("%s=off with rule files off gave the error %v, want one naming %s", + name, err, tagDeny) + } + } } func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) { @@ -2285,11 +2483,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { lookupSource: defaultLookupSource, lookupDBPath: "", lookupTimeout: "800ms", - addLookupHeaders: "false", - deniedCountries: "", + addLookupHeaders: disabled, allowedCountries: "", - asnLimitPercent: "", - countryLimitPercent: "", + tagLists: "", + tagDeny: "", + tagLimitPercent: "", asnBytesPercent: "", countryBytesPercent: "", unknownLimitPercent: "100", diff --git a/internal/discriminator/abuseipdb.go b/internal/discriminator/abuseipdb.go index 460929e..9f87a39 100644 --- a/internal/discriminator/abuseipdb.go +++ b/internal/discriminator/abuseipdb.go @@ -8,18 +8,20 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) -// abuseIPDB notes AbuseIPDB, as noteHit does, with the score, when its score -// of the client is a hit, and refuses the request when SWW_REPUTATION_ACTION -// is deny, as dnsbl does for a zone. While SWW_ABUSEIPDB_KEY is unset it does -// nothing. A client without a score is checked in the background, by the -// request's address, if its history counts an offence, and the request does -// not wait for the answer. The score is then used for each address of the -// client. Its figures are the score and SWW_ABUSEIPDB_MIN_SCORE. +// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with +// the score, when its score of the client is a hit, and refuses the request +// when SWW_REPUTATION_ACTION is deny, as dnsbl does for a zone. While +// SWW_ABUSEIPDB_KEY is unset it does nothing. A client without a score is +// checked in the background, by the request's address, if its history counts +// an offence, and the request does not wait for the answer. The score is then +// used for each address of the client. Its figures are the score and +// SWW_ABUSEIPDB_MIN_SCORE. type abuseIPDB struct{} func (abuseIPDB) name() string { return "abuseipdb" } @@ -39,6 +41,7 @@ func (abuseIPDB) check(ctx context.Context, rq *request, _ time.Time) *verdict { } rq.abuseIPDBHit = true + rq.addTag(config.AbuseIPDBTag) rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score}, "scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE") diff --git a/internal/discriminator/answertime_test.go b/internal/discriminator/answertime_test.go index 2b826e6..b7e4c02 100644 --- a/internal/discriminator/answertime_test.go +++ b/internal/discriminator/answertime_test.go @@ -335,7 +335,7 @@ func newDiscriminatorInBubble( ) (*discriminator.Server, *output) { t.Helper() - settings := map[string]string{deniedCountries: "KP"} + settings := map[string]string{tagDeny: "country=KP"} maps.Copy(settings, env) server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now, settings) @@ -542,7 +542,7 @@ func TestDiscriminatorAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs( return discriminator.New(params) }, held, time.Now, map[string]string{ - deniedCountries: "KP", wafMode: block, + tagDeny: "country=KP", wafMode: block, dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(), abuseIPDBKey: accountKey, blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt", diff --git a/internal/discriminator/bans_test.go b/internal/discriminator/bans_test.go index 776483d..9875a14 100644 --- a/internal/discriminator/bans_test.go +++ b/internal/discriminator/bans_test.go @@ -211,7 +211,7 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) { lookupTimeout: "1h", rateLimitPerMinute: "1", banScopeV4Prefix: "24", - deniedCountries: "kp", + tagDeny: denyKP, }) // fromDE's ban covers otherClient, which is refused unasked about. @@ -266,7 +266,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) { lookupTimeout: "1h", rateLimitPerMinute: "1", denyNets: denied, - deniedCountries: "kp", + tagDeny: denyKP, } if tc.setting != "" { @@ -276,7 +276,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) { s, _, _ := startWithClock(t, geojsURL, env) s.get(denied, tc.status, requestlog.ActionDenied) - s.get(fromKP, tc.status, requestlog.ActionCountryDenied) + s.get(fromKP, tc.status, requestlog.ActionDenied) s.get(fromDE, http.StatusOK, requestlog.ActionForward) s.get(fromDE, tc.status, requestlog.ActionRateLimited) s.get(fromDE, tc.status, requestlog.ActionBanned) @@ -291,7 +291,7 @@ func TestBanNotes(t *testing.T) { s, clk, server := startWithClock(t, geojsURL, map[string]string{ lookupTimeout: "1h", rateLimitPerMinute: "1", - deniedCountries: "kp", + tagDeny: denyKP, }) start := clk.Now() diff --git a/internal/discriminator/biased.go b/internal/discriminator/biased.go index c7a0654..ef97adc 100644 --- a/internal/discriminator/biased.go +++ b/internal/discriminator/biased.go @@ -17,30 +17,31 @@ type percentage struct { } // biasedThresholdsSet reports whether a biased threshold can lower a -// client's limits: one of its lists is not empty, -// SWW_UNKNOWN_LIMIT_PERCENT is below 100, or SWW_ASN_LIMIT_PERCENT_URL -// is set. The client's lookup is then needed before its request goes on. +// client's limits by its lookup: SWW_ASN_BYTES_PERCENT or +// SWW_COUNTRY_BYTES_PERCENT is not empty, SWW_UNKNOWN_LIMIT_PERCENT is +// below 100, or SWW_ASN_LIMIT_PERCENT_URL is set. The client's lookup is +// then needed before its request goes on. func biasedThresholdsSet(cfg *config.Config) bool { - return len(cfg.ASNLimitPercent) > 0 || len(cfg.CountryLimitPercent) > 0 || - len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 || + return len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 || cfg.UnknownLimitPercent < whole || cfg.ASNLimitPercentURL != "" } // limitPercentages returns the client's limit percentages, for the rate -// limits and for the byte limits, by its AS number and country as looked -// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB -// that list it. Each is the lowest of those the settings give it, the -// first of them in the order below when several are lowest: the -// percentage SWW_ASN_LIMIT_PERCENT gives its AS number, the one the file -// SWW_ASN_LIMIT_PERCENT_URL names gives it, the one -// SWW_COUNTRY_LIMIT_PERCENT gives its country, for a client without a -// country, unless its request skipped the wait for GeoJS, -// SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, the -// percentage of SWW_BLOCKLIST_ACTION while it is limit, and for a client -// a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, the -// percentage of SWW_REPUTATION_ACTION while it is limit. For the byte -// limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT take the -// place of the first three for an AS number or a country they list. +// limits and for the byte limits, by its tags, its AS number and country +// as looked up, each "" when unknown, and the blocklists, DNSBL zones and +// AbuseIPDB that list it. Each is the lowest of those the settings give +// it, the first of them in the order below when several are lowest: the +// percentage SWW_TAG_LIMIT_PERCENT gives its asn= tag, the one the file +// SWW_ASN_LIMIT_PERCENT_URL names gives its AS number, the one +// SWW_TAG_LIMIT_PERCENT gives its country= tag, the lowest it gives its +// other tags, for a client without a country, unless its request skipped +// the wait for GeoJS, SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist +// lists, the percentage of SWW_BLOCKLIST_ACTION while it is limit, and +// for a client a DNSBL zone's verdict lists, or whose AbuseIPDB score is +// a hit, the percentage of SWW_REPUTATION_ACTION while it is limit. For +// the byte limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT +// take the place of the first three for an AS number or a country they +// list. func (rq *request) limitPercentages() (percentage, percentage) { cfg := rq.h.config asn, country := rq.line.ASN, rq.line.Country @@ -65,10 +66,18 @@ func (rq *request) limitPercentages() (percentage, percentage) { reputationListed = percentage{cfg.ReputationLimitPercent, "SWW_REPUTATION_ACTION"} } - asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWW_ASN_LIMIT_PERCENT"), - fetched) - countryRequests := given(cfg.CountryLimitPercent, country, - "SWW_COUNTRY_LIMIT_PERCENT") + const tagSetting = "SWW_TAG_LIMIT_PERCENT" + + asnRequests := lowest(given(cfg.TagLimitPercent, asnTag(asn), tagSetting), fetched) + countryRequests := given(cfg.TagLimitPercent, countryTag(country), tagSetting) + + otherTags := percentage{percent: whole} + + for _, tag := range rq.line.Tags { + if tag != asnTag(asn) && tag != countryTag(country) { + otherTags = lowest(otherTags, given(cfg.TagLimitPercent, tag, tagSetting)) + } + } asnBytes, countryBytes := asnRequests, countryRequests if _, listed := cfg.ASNBytesPercent[asn]; listed { @@ -79,12 +88,13 @@ func (rq *request) limitPercentages() (percentage, percentage) { countryBytes = given(cfg.CountryBytesPercent, country, "SWW_COUNTRY_BYTES_PERCENT") } - return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed), - lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed) + return lowest(asnRequests, countryRequests, otherTags, unknown, blocklisted, + reputationListed), + lowest(asnBytes, countryBytes, otherTags, unknown, blocklisted, reputationListed) } // given returns the percentage percents, the setting named setting, gives -// code, an AS number or a country, or whole when it does not list code. +// code, an AS number, a country or a tag, or whole when it does not list code. func given(percents map[string]int64, code, setting string) percentage { percent, listed := percents[code] if !listed { diff --git a/internal/discriminator/biased_test.go b/internal/discriminator/biased_test.go index e9cb48d..81e4bac 100644 --- a/internal/discriminator/biased_test.go +++ b/internal/discriminator/biased_test.go @@ -22,8 +22,7 @@ import ( // The biased thresholds. const ( - asnLimitPercent = "SWW_ASN_LIMIT_PERCENT" - countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT" + tagLimitPercent = "SWW_TAG_LIMIT_PERCENT" asnBytesPercent = "SWW_ASN_BYTES_PERCENT" countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT" unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT" @@ -35,6 +34,11 @@ const ( asnDEHalf = asnDE + ":50" asnDEQuarter = asnDE + ":25" countryDEHalf = "de:50" + // The same, as SWW_TAG_LIMIT_PERCENT names fromDE's tags. + asnTagDEHalf = "asn=64496:50" + asnTagDEQuarter = "asn=64496:25" + countryTagDEHalf = "country=DE:50" + countryTagDEQuarter = "country=DE:25" // noCountry is in an AS of its own, AS64500, and in no country. noCountry = "192.0.2.80" // fourAMinute is the rate limit these tests set: half of it is 2 @@ -54,8 +58,8 @@ func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) { for _, tc := range []struct { setting, value, from string }{ - {asnLimitPercent, asnDEHalf, fromDE}, - {countryLimitPercent, countryDEHalf, fromDE}, + {tagLimitPercent, asnTagDEHalf, fromDE}, + {tagLimitPercent, countryTagDEHalf, fromDE}, {unknownLimitPercent, "50", unplaced}, } { t.Run(tc.setting, func(t *testing.T) { @@ -96,8 +100,8 @@ func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) { for _, tc := range []struct { setting, value, from string }{ - {asnLimitPercent, asnDEHalf, fromDE}, - {countryLimitPercent, "DE:50", fromDE}, + {tagLimitPercent, asnTagDEHalf, fromDE}, + {tagLimitPercent, "Country=de:50", fromDE}, {unknownLimitPercent, "50", unplaced}, {asnBytesPercent, "as64496:50", fromDE}, {countryBytesPercent, countryDEHalf, fromDE}, @@ -151,13 +155,13 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T) }, { "raising the byte limits back", - map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"}, - "50 from " + asnLimitPercent, none, "", + map[string]string{tagLimitPercent: asnTagDEHalf, asnBytesPercent: asnDE + ":100"}, + "50 from " + tagLimitPercent, none, "", }, { "raising the byte limits back, by country", - map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"}, - "50 from " + countryLimitPercent, none, "", + map[string]string{tagLimitPercent: countryTagDEHalf, countryBytesPercent: "de:100"}, + "50 from " + tagLimitPercent, none, "", }, } { t.Run(tc.name, func(t *testing.T) { @@ -184,19 +188,19 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T) func TestZeroPercentIsAZeroAllowance(t *testing.T) { t.Parallel() - s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"}) + s, _, _ := startWithLookups(t, map[string]string{unknownLimitPercent: "0"}) // The first request breaks the limit, and bans the client; the log line // gives the 0. - line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + line := s.get(unplaced, http.StatusForbidden, requestlog.ActionRateLimited) if line.fields["limit_percent"] != float64(0) || - line.fields["limit_percent_setting"] != asnLimitPercent { + line.fields["limit_percent_setting"] != unknownLimitPercent { t.Errorf("log line has limit_percent %v from %v, want 0 from %s", line.fields["limit_percent"], line.fields["limit_percent_setting"], - asnLimitPercent) + unknownLimitPercent) } - s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + s.get(unplaced, http.StatusForbidden, requestlog.ActionBanned) } func TestLowestPercentageApplies(t *testing.T) { @@ -211,29 +215,29 @@ func TestLowestPercentageApplies(t *testing.T) { }{ { "the country's", - map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"}, - fromDE, "25 from " + countryLimitPercent, + map[string]string{tagLimitPercent: asnTagDEHalf + "," + countryTagDEQuarter}, + fromDE, "25 from " + tagLimitPercent, }, { "the AS number's", - map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf}, - fromDE, "25 from " + asnLimitPercent, + map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEHalf}, + fromDE, "25 from " + tagLimitPercent, }, { "the AS number's, the first of two alike", - map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"}, - fromDE, "25 from " + asnLimitPercent, + map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEQuarter}, + fromDE, "25 from " + tagLimitPercent, }, { "that for a client without a country", - map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"}, + map[string]string{tagLimitPercent: "asn=64500:50", unknownLimitPercent: "25"}, noCountry, "25 from " + unknownLimitPercent, }, { // SWW_UNKNOWN_LIMIT_PERCENT is left at its default, 100. "the AS number's, for a client without a country", - map[string]string{asnLimitPercent: "AS64500:25"}, - noCountry, "25 from " + asnLimitPercent, + map[string]string{tagLimitPercent: "asn=64500:25"}, + noCountry, "25 from " + tagLimitPercent, }, } { t.Run(tc.name, func(t *testing.T) { @@ -304,8 +308,12 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) { setting, value string waits bool }{ - {asnLimitPercent, asnDEHalf, true}, - {countryLimitPercent, countryDEHalf, true}, + {tagLimitPercent, asnTagDEHalf, true}, + {tagLimitPercent, countryTagDEHalf, true}, + {tagDeny, denyKP, true}, + // A tag the lookup does not give needs no answer. + {tagLimitPercent, "vpn:50", false}, + {tagDeny, "vpn", false}, {asnBytesPercent, asnDEHalf, true}, {countryBytesPercent, countryDEHalf, true}, {unknownLimitPercent, "99", true}, @@ -320,7 +328,9 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) { // says, with a GeoJS that never answers. synctest.Test(t, func(t *testing.T) { server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now, - map[string]string{lookupTimeout: timeout.String(), tc.setting: tc.value}) + map[string]string{ + lookupTimeout: timeout.String(), tagLists: "vpn=" + vpnList, tc.setting: tc.value, + }) began := time.Now() serveFromDE(t, server, http.MethodPost, strings.NewReader("ab")) @@ -363,15 +373,15 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T { // A quarter of 12 requests a minute is 3: the fourth breaks it. "a rate limit", - map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter}, + map[string]string{rateLimitPerMinute: "12", tagLimitPercent: asnTagDEQuarter}, 3, http.StatusForbidden, requestlog.ActionRateLimited, - "requests per minute over the limit of 3", "25 from " + asnLimitPercent, + "requests per minute over the limit of 3", "25 from " + tagLimitPercent, }, { // The byte limits' percentage, not the rate limits'. "a byte limit", map[string]string{ - bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter, + bytesLimitPerMinute: twoUploads, tagLimitPercent: asnTagDEQuarter, asnBytesPercent: asnDEHalf, }, 0, http.StatusOK, requestlog.ActionForward, @@ -487,7 +497,7 @@ func wantPercent(t *testing.T, what string, percent *int64, setting, want string } // percentText gives a limit percentage and the setting that gave it as -// text, such as "50 from SWW_ASN_LIMIT_PERCENT", or none when both are +// text, such as "50 from SWW_TAG_LIMIT_PERCENT", or none when both are // left out. func percentText(percent *int64, setting string) string { switch { diff --git a/internal/discriminator/coreruleset.go b/internal/discriminator/coreruleset.go index 01a36f0..4ed57e4 100644 --- a/internal/discriminator/coreruleset.go +++ b/internal/discriminator/coreruleset.go @@ -17,9 +17,10 @@ import ( // SWW_WAF_MODE is off or SWW_WAF_EXEMPT_PATHS exempts its path, as // pathExempt decides, and notes the rules it matched and its score in the // log line, and the rules in the metrics. A score at or over -// SWW_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert, -// and in block mode refuses the request with 403, which is an offence its -// client's history counts. Its figures are the score and the threshold. +// SWW_WAF_ANOMALY_THRESHOLD is a match: it adds the tag exploitsignature, +// raises the waf_block alert, and in block mode refuses the request with 403, +// which is an offence its client's history counts. Its figures are the score +// and the threshold. type coreRuleSet struct{} func (coreRuleSet) name() string { return "core_rule_set" } @@ -50,6 +51,7 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict { return nil } + rq.addTag(config.ExploitSignatureTag) rq.alertWAFBlock(result) if cfg.WAFMode == config.WAFModeDetect { diff --git a/internal/discriminator/coreruleset_test.go b/internal/discriminator/coreruleset_test.go index 95ea986..054b457 100644 --- a/internal/discriminator/coreruleset_test.go +++ b/internal/discriminator/coreruleset_test.go @@ -61,6 +61,41 @@ func wantWAF(t *testing.T, line logLine, score *int, ruleIDs ...int) { } } +func TestCoreRuleSetMatchAddsTheExploitSignatureTag(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mode string + status int + action string + }{ + {block, block, http.StatusForbidden, requestlog.ActionWAFBlocked}, + {detect, detect, http.StatusOK, requestlog.ActionForward}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode}) + + wantTags(t, s.request(client, sqlInjection, tc.status, tc.action), + "exploitsignature") + wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward)) + }) + } +} + +func TestDetectedAttackIsRefusedByTagDenyLast(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + wafMode: detect, tagDeny: "exploitsignature", + }) + + line := s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionDenied) + wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`) +} + func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *testing.T) { t.Parallel() diff --git a/internal/discriminator/countries.go b/internal/discriminator/countries.go index b387be4..826f737 100644 --- a/internal/discriminator/countries.go +++ b/internal/discriminator/countries.go @@ -1,7 +1,6 @@ package discriminator -// The heuristic countries. It reads SWW_DENIED_COUNTRIES and -// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. +// The heuristic countries. It reads SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. import ( "context" @@ -11,32 +10,26 @@ import ( "sneak.berlin/go/smallwebwaf/internal/requestlog" ) -// countries refuses a request by the client's country as it was looked up. A -// client without a country, or whose country cannot be found, is refused only -// by SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. Its figures are the country and the -// setting that refused it. +// countries refuses a request when SWW_EXCLUSIVELY_ALLOWED_COUNTRIES is set and +// does not hold the client's country as it was looked up. A client without a +// country, or whose country cannot be found, is refused. Its figures are the +// country and the setting that refused it. type countries struct{} func (countries) name() string { return "countries" } func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict { - denied := rq.h.config.DeniedCountries allowed := rq.h.config.ExclusivelyAllowedCountries country := rq.line.Country - var setting string - - switch { - case slices.Contains(denied, country): - setting = "SWW_DENIED_COUNTRIES" - case len(allowed) > 0 && !slices.Contains(allowed, country): - setting = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES" - default: + if len(allowed) == 0 || slices.Contains(allowed, country) { return nil } return &verdict{ - action: requestlog.ActionCountryDenied, - figures: map[string]any{"country": country, "setting": setting}, + action: requestlog.ActionCountryDenied, + figures: map[string]any{ + "country": country, "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", + }, } } diff --git a/internal/discriminator/countries_test.go b/internal/discriminator/countries_test.go index 4b075e5..89311a2 100644 --- a/internal/discriminator/countries_test.go +++ b/internal/discriminator/countries_test.go @@ -33,13 +33,13 @@ func TestCountryLists(t *testing.T) { env map[string]string refused []string }{ - {"denied", map[string]string{deniedCountries: "kp"}, []string{fromKP}}, { "exclusively allowed", map[string]string{allowedCountries: "DE"}, []string{fromKP, unplaced}, }, { - "both", map[string]string{deniedCountries: "kp", allowedCountries: "de,fr"}, + "with a country in SWW_TAG_DENY", + map[string]string{tagDeny: denyKP, allowedCountries: "de,fr"}, []string{fromKP, unplaced}, }, } { @@ -95,12 +95,12 @@ func TestCountryRefusalNamesItsHeuristicCountryAndSetting(t *testing.T) { geojsURL, _ := startGeoJS(t) s, _, _ := startWithClock(t, geojsURL, map[string]string{ - lookupTimeout: "1h", deniedCountries: "kp", allowedCountries: "de", + lookupTimeout: "1h", allowedCountries: "de", }) line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied) wantHeuristic(t, line, "countries", - `{"country": "KP", "setting": "SWW_DENIED_COUNTRIES"}`) + `{"country": "KP", "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"}`) line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied) wantHeuristic(t, line, "countries", @@ -117,9 +117,9 @@ func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) { }) geojsURL, _ := startGeoJS(t) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ - trustedProxies: trustLocalhost, - lookupTimeout: "1h", - deniedCountries: "kp", + trustedProxies: trustLocalhost, + lookupTimeout: "1h", + allowedCountries: "de", }) req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body")) @@ -202,7 +202,7 @@ func TestPrivateAddressIsNeverLookedUp(t *testing.T) { env map[string]string }{ {"no setting needs the lookup", nil}, - {"a country list is set", map[string]string{deniedCountries: "kp"}}, + {"a tag setting needs it", map[string]string{tagDeny: denyKP}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/internal/discriminator/crowdsec.go b/internal/discriminator/crowdsec.go index 4ed27cf..12ec1f1 100644 --- a/internal/discriminator/crowdsec.go +++ b/internal/discriminator/crowdsec.go @@ -8,14 +8,15 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) // crowdSec refuses a request when a decision of the CrowdSec decision list on -// the client is in force at now. It then notes the list, as noteHit does, and -// bans the client until that decision ends. Its figures are the decision's -// scenario and when it ends. +// the client is in force at now. It then adds the tag crowdsec, notes the +// list, as noteHit does, and bans the client until that decision ends. Its +// figures are the decision's scenario and when it ends. type crowdSec struct{} func (crowdSec) name() string { return "crowdsec" } @@ -26,6 +27,7 @@ func (crowdSec) check(_ context.Context, rq *request, now time.Time) *verdict { return nil } + rq.addTag(config.CrowdSecTag) rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL}, "listed by the CrowdSec decision list") rq.banForCrowdSec(now, decision) diff --git a/internal/discriminator/crowdsec_test.go b/internal/discriminator/crowdsec_test.go index aad46ca..9fd3e9e 100644 --- a/internal/discriminator/crowdsec_test.go +++ b/internal/discriminator/crowdsec_test.go @@ -108,6 +108,7 @@ func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) { line := s.get(client, http.StatusForbidden, requestlog.ActionBanned) wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+ `"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`) + wantTags(t, line, "crowdsec") } func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) { diff --git a/internal/discriminator/discriminator_test.go b/internal/discriminator/discriminator_test.go index 7800ea3..b746940 100644 --- a/internal/discriminator/discriminator_test.go +++ b/internal/discriminator/discriminator_test.go @@ -414,7 +414,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { geojsURL, asked, release := startHeldGeoJS(t) s := startDiscriminator(t, geojsURL, map[string]string{ - lookupTimeout: "1h", deniedCountries: "kp", bytesLimitPerMinute: byteLimit, + lookupTimeout: "1h", tagDeny: denyKP, bytesLimitPerMinute: byteLimit, }) first := s.question(siteA, client, "/") @@ -765,7 +765,7 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) { geojsURL, _ := startGeoJS(t) s := startDiscriminator(t, geojsURL, map[string]string{ - addLookupHeaders: tc.setting, lookupTimeout: "1h", deniedCountries: "kp", + addLookupHeaders: tc.setting, lookupTimeout: "1h", tagDeny: denyKP, }) got := s.ask(fromDE, "/") diff --git a/internal/discriminator/dnsbl.go b/internal/discriminator/dnsbl.go index a9b7627..8c31da7 100644 --- a/internal/discriminator/dnsbl.go +++ b/internal/discriminator/dnsbl.go @@ -5,13 +5,16 @@ package discriminator import ( "context" + "strings" "time" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) // dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed -// does, and refuses the request when SWW_REPUTATION_ACTION is deny. Being +// does, adds the tag dnsbl= for each, the zone's key masked, and refuses +// the request when SWW_REPUTATION_ACTION is deny. Being // limit, it lowers the client's limits instead (see limitPercentages), and // being log, it does nothing more. A zone without a verdict on the client is // asked about it in the background, and the request does not wait for the @@ -25,6 +28,10 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict { rq.dnsblListed = len(listedBy) > 0 rq.noteListed(listedBy, "listed by a DNSBL zone") + for _, zone := range listedBy { + rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) + } + if !rq.dnsblListed || rq.h.config.ReputationAction != deny { return nil } diff --git a/internal/discriminator/heuristics.go b/internal/discriminator/heuristics.go index 28bffd9..beef660 100644 --- a/internal/discriminator/heuristics.go +++ b/internal/discriminator/heuristics.go @@ -43,12 +43,15 @@ const ( // country are looked up, so that a client they refuse is not looked up. func beforeLookup() []heuristic { return []heuristic{ + tagLists{}, denyNets{}, activeBan{}, } } // afterLookup are the heuristics that run once the lookup has been made. +// tag_deny runs twice: before the rate limits, on the tags known by then, and +// last, on the tag rule_files and core_rule_set can add. func afterLookup() []heuristic { return []heuristic{ countries{}, @@ -56,11 +59,13 @@ func afterLookup() []heuristic { crowdSec{}, dnsbl{}, abuseIPDB{}, + tagDeny{}, rateLimits{}, trapPaths{}, ruleFiles{}, headerLimits{}, coreRuleSet{}, + tagDeny{}, } } diff --git a/internal/discriminator/history_test.go b/internal/discriminator/history_test.go index 710e326..43471e3 100644 --- a/internal/discriminator/history_test.go +++ b/internal/discriminator/history_test.go @@ -20,7 +20,7 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { s, clk, server := startWithClock(t, geojsURL, map[string]string{ lookupTimeout: "1h", rateLimitPerMinute: "2", - deniedCountries: "kp", + tagDeny: denyKP, }) start := clk.Now() diff --git a/internal/discriminator/lookup.go b/internal/discriminator/lookup.go index b6cf86b..73558b2 100644 --- a/internal/discriminator/lookup.go +++ b/internal/discriminator/lookup.go @@ -4,8 +4,10 @@ import ( "context" "net/http" "net/netip" + "strings" "time" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" ) @@ -19,7 +21,8 @@ const ( ) // lookUp looks up the client's AS number and country, in the lookup -// database or through GeoJS, and notes them for the log line, unless +// database or through GeoJS, and notes them for the log line and as the +// request's asn= and country= tags, unless // SWW_LOOKUP_SOURCE is off or the client is on a private, loopback or // link-local address, which no lookup can place. The lookup database // answers at once. With GeoJS, while a setting needs the answer, such as a @@ -47,6 +50,34 @@ func (rq *request) lookUp(ctx context.Context, now time.Time) { rq.line.ASN = rq.lookupAnswer.ASN rq.line.ASName = rq.lookupAnswer.ASName rq.line.Country = rq.lookupAnswer.Country + + if tag := asnTag(rq.line.ASN); tag != "" { + rq.addTag(tag) + } + + if tag := countryTag(rq.line.Country); tag != "" { + rq.addTag(tag) + } +} + +// asnTag returns the tag of asn, an AS number as the lookup gives it, such as +// asn=64496 for AS64496, or "" for no AS number. +func asnTag(asn string) string { + if asn == "" { + return "" + } + + return config.ASNTagPrefix + strings.TrimPrefix(asn, "AS") +} + +// countryTag returns the tag of country, a code as the lookup gives it, such +// as country=DE for DE, or "" for no country. +func countryTag(country string) string { + if country == "" { + return "" + } + + return config.CountryTagPrefix + country } // addLookup adds answer, an answer about a client from the lookup diff --git a/internal/discriminator/lookup_test.go b/internal/discriminator/lookup_test.go index a5e70ba..92075a5 100644 --- a/internal/discriminator/lookup_test.go +++ b/internal/discriminator/lookup_test.go @@ -94,9 +94,9 @@ func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) { clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 58, 0, time.UTC)} addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now, map[string]string{ - trustedProxies: trustLocalhost, - lookupTimeout: "1h", - deniedCountries: "kp", + trustedProxies: trustLocalhost, + lookupTimeout: "1h", + tagDeny: denyKP, }) requestFromDE := func() *http.Request { diff --git a/internal/discriminator/metrics_test.go b/internal/discriminator/metrics_test.go index 19a09b9..72af353 100644 --- a/internal/discriminator/metrics_test.go +++ b/internal/discriminator/metrics_test.go @@ -195,10 +195,10 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { _, _ = io.WriteString(w, "hello") }) env := map[string]string{ - trustedProxies: trustLocalhost, - metricsToken: token, - metricsTopN: "2", - deniedCountries: "kp", + trustedProxies: trustLocalhost, + metricsToken: token, + metricsTopN: "2", + allowedCountries: "de,fr", } geojsURL, _ := startGeoJS(t) addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env) @@ -322,13 +322,13 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { app := startApp(t, func(http.ResponseWriter, *http.Request) {}) addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now, map[string]string{ - trustedProxies: trustLocalhost, - metricsToken: token, - deniedCountries: "kp", + trustedProxies: trustLocalhost, + metricsToken: token, + tagDeny: denyKP, }) - // GeoJS fails, so the client counts as coming from an unknown country, - // which SWW_DENIED_COUNTRIES does not refuse. + // GeoJS fails, so the client has no country= tag, and SWW_TAG_DENY does + // not refuse it. req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) req.Header.Set(forwardedFor, fromDE) wantStatus(t, do(t, req), http.StatusOK) diff --git a/internal/discriminator/observe_test.go b/internal/discriminator/observe_test.go index 150c6f9..5ca9fcc 100644 --- a/internal/discriminator/observe_test.go +++ b/internal/discriminator/observe_test.go @@ -50,7 +50,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) { lookupTimeout: "1h", rateLimitPerMinute: "1", denyNets: denied, - deniedCountries: "kp", + tagDeny: denyKP, } if tc.setting != "" { @@ -71,7 +71,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) { for _, sent := range []struct{ from, refusal string }{ {denied, requestlog.ActionDenied}, {banned, requestlog.ActionBanned}, - {fromKP, requestlog.ActionCountryDenied}, + {fromKP, requestlog.ActionDenied}, {fromDE, requestlog.ActionRateLimited}, } { if !tc.observe { diff --git a/internal/discriminator/proxy.go b/internal/discriminator/proxy.go index b4da7db..7b15d8b 100644 --- a/internal/discriminator/proxy.go +++ b/internal/discriminator/proxy.go @@ -155,11 +155,12 @@ func newHandler(params Params) *handler { h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, Timeout: params.Config.LookupTimeout, - // The country lists, the headers and the biased thresholds act on - // the answer before the request goes on. - Wait: len(params.Config.DeniedCountries) > 0 || - len(params.Config.ExclusivelyAllowedCountries) > 0 || - params.Config.AddLookupHeaders || biasedThresholdsSet(params.Config), + // The country list, the headers, the tag settings that name an asn= + // or country= tag and the biased thresholds act on the answer before + // the request goes on. + Wait: len(params.Config.ExclusivelyAllowedCountries) > 0 || + params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() || + biasedThresholdsSet(params.Config), Answered: h.addLookup, Now: params.Now, ProcessLog: params.ProcessLog, @@ -183,6 +184,7 @@ func newReputation( lists := reputation.New(reputation.Params{ BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, ASNLimitPercentURL: cfg.ASNLimitPercentURL, + TagLists: cfg.TagLists, CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey, Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts, }) diff --git a/internal/discriminator/proxy_test.go b/internal/discriminator/proxy_test.go index 2a2ea0a..e98f866 100644 --- a/internal/discriminator/proxy_test.go +++ b/internal/discriminator/proxy_test.go @@ -69,7 +69,9 @@ const ( lookupDBPath = "SWW_LOOKUP_DB_PATH" lookupTimeout = "SWW_LOOKUP_TIMEOUT" addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS" - deniedCountries = "SWW_DENIED_COUNTRIES" + tagDeny = "SWW_TAG_DENY" + // denyKP is the value of SWW_TAG_DENY that refuses fromKP's country. + denyKP = "country=KP" allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWW_BAN_RESPONSE" limitBanDuration = "SWW_LIMIT_BAN_DURATION" diff --git a/internal/discriminator/ratelimits_test.go b/internal/discriminator/ratelimits_test.go index 3e75ac0..926f22f 100644 --- a/internal/discriminator/ratelimits_test.go +++ b/internal/discriminator/ratelimits_test.go @@ -136,7 +136,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { rateLimitPerMinute: "1", rateLimitExemptPaths: "/assets/,/favicon.ico", denyNets: denied, - deniedCountries: "kp", + tagDeny: denyKP, }) // The answers are kept before the requests, so that none waits for @@ -162,12 +162,12 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { // /assets/, and breaks the limit. s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited) - // A ban, SWW_DENY_NETS and the country lists still refuse a path - // under a prefix. + // A ban, SWW_DENY_NETS and SWW_TAG_DENY still refuse a path under a + // prefix. s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned) s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied) s.request(fromKP, "/assets/app.js", - http.StatusForbidden, requestlog.ActionCountryDenied) + http.StatusForbidden, requestlog.ActionDenied) } func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) { diff --git a/internal/discriminator/refusallimit_test.go b/internal/discriminator/refusallimit_test.go index da14eee..09a34b9 100644 --- a/internal/discriminator/refusallimit_test.go +++ b/internal/discriminator/refusallimit_test.go @@ -191,7 +191,7 @@ func TestErrorBurstIsNotLoweredForAClientWithLowerLimits(t *testing.T) { lookupTimeout: "1h", errorBurstThreshold: "2", rulesDir: writeRules(t, testRules), - countryLimitPercent: countryDEHalf, + tagLimitPercent: countryTagDEHalf, }) // Half of the threshold would be one, which the second refusal is over. diff --git a/internal/discriminator/reputation_test.go b/internal/discriminator/reputation_test.go index a90d6be..c992117 100644 --- a/internal/discriminator/reputation_test.go +++ b/internal/discriminator/reputation_test.go @@ -168,11 +168,11 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(t *testing.T) } } -func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) { +func TestBlocklistsComeAfterAllowedCountriesAndSkipAllowNets(t *testing.T) { t.Parallel() s, server, queue := startWithLookups(t, map[string]string{ - blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE, + blocklistURLs: dropURL, allowedCountries: "de", allowNets: fromDE, }) loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}}) @@ -211,18 +211,18 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) // percentText gives them, and limitHit its limit_hit. want, limitHit string }{ - {limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, - {limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes}, + {limitHalf, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes}, + {limitQuarter, asnTagDEHalf, "25 from " + blocklistAction, minuteBytes}, // The AS number's, the first of two alike. - {limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, - {actionLog, asnDE + ":100", none, ""}, + {limitQuarter, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes}, + {actionLog, "asn=64511:25", none, ""}, } { t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) { t.Parallel() s, server, _ := startWithLookups(t, map[string]string{ bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL, - blocklistAction: tc.action, asnLimitPercent: tc.asnPercent, + blocklistAction: tc.action, tagLimitPercent: tc.asnPercent, }) loadLists(t, server, map[string][]string{dropURL: {fromDE}}) @@ -246,7 +246,7 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) const ( fromURL = "25 from " + asnLimitPercentURL - fromSetting = "25 from " + asnLimitPercent + fromSetting = "25 from " + tagLimitPercent ) for _, tc := range []struct { @@ -260,16 +260,16 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) {"the file's alone", nil, asnDEQuarter, fromURL, fromURL}, { "the file's, lower than the setting's", - map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL, + map[string]string{tagLimitPercent: asnTagDEHalf}, asnDEQuarter, fromURL, fromURL, }, { "the setting's, lower than the file's", - map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf, + map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEHalf, fromSetting, fromSetting, }, { "the setting's, the first of two alike", - map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter, + map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEQuarter, fromSetting, fromSetting, }, {"none, for an AS number the file does not list", nil, asnKP + ":25", none, none}, @@ -482,6 +482,46 @@ func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) { } } +func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + load func(*discriminator.Server) + want []string + }{ + { + "dnsbl", + map[string]string{ + dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionLog, + }, + func(server *discriminator.Server) { + loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}}) + }, + []string{"asn=64496", "country=DE", "dnsbl=" + dnsblZone}, + }, + { + abuseipdb, + map[string]string{abuseIPDBKey: accountKey, reputationAction: actionLog}, + func(server *discriminator.Server) { + loadScores(server, map[string]int64{fromDE: 80}) + }, + []string{abuseipdb, "asn=64496", "country=DE"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, tc.env) + tc.load(server) + + wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), + tc.want...) + }) + } +} + func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) { t.Parallel() diff --git a/internal/discriminator/request.go b/internal/discriminator/request.go index 0d55c46..dd69bb9 100644 --- a/internal/discriminator/request.go +++ b/internal/discriminator/request.go @@ -138,6 +138,15 @@ func (h *handler) startRequest( return rq } +// addTag adds tag to the request's tags, which stay sorted and hold each tag +// once. +func (rq *request) addTag(tag string) { + at, found := slices.BinarySearch(rq.line.Tags, tag) + if !found { + rq.line.Tags = slices.Insert(rq.line.Tags, at, tag) + } +} + // requestHeaders returns the headers of r that names lists, by name in // lower case, each with its values joined by ", ". Authorization, Cookie // and Set-Cookie are never among them, whatever names says. diff --git a/internal/discriminator/rulefiles.go b/internal/discriminator/rulefiles.go index 1d56916..27e9c17 100644 --- a/internal/discriminator/rulefiles.go +++ b/internal/discriminator/rulefiles.go @@ -8,12 +8,14 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) // ruleFiles checks the request against the rules of the rule files at now, -// notes the ids of those it matches in the log line, and refuses the request +// notes the ids of those it matches in the log line, adds the tag +// exploitsignature when it matches any, and refuses the request // when the last rule it matches refuses it: with ActionRuleBlocked for a block // rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock // for a clear sign of attack, or in observe mode raises the alert for the ban @@ -35,6 +37,8 @@ func (ruleFiles) check(_ context.Context, rq *request, now time.Time) *verdict { return nil } + rq.addTag(config.ExploitSignatureTag) + // Only the last rule matched can refuse the request. last := matched[len(matched)-1] figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action} diff --git a/internal/discriminator/rulefiles_test.go b/internal/discriminator/rulefiles_test.go index 4c9c81a..30a4e9f 100644 --- a/internal/discriminator/rulefiles_test.go +++ b/internal/discriminator/rulefiles_test.go @@ -93,6 +93,20 @@ func TestEachRuleAction(t *testing.T) { s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned) } +func TestEachRuleActionAddsTheExploitSignatureTag(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, "noted path log ^/noted$\n"+testRules), + }) + + wantTags(t, s.request(client, "/noted", http.StatusOK, requestlog.ActionForward), + "exploitsignature") + wantTags(t, s.request(client, "/blocked", http.StatusForbidden, + requestlog.ActionRuleBlocked), "exploitsignature") + wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward)) +} + func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) { t.Parallel() diff --git a/internal/discriminator/staticlists_test.go b/internal/discriminator/staticlists_test.go index 9712972..9ba8ad9 100644 --- a/internal/discriminator/staticlists_test.go +++ b/internal/discriminator/staticlists_test.go @@ -32,13 +32,12 @@ func TestAllowNetsSkipEveryCheck(t *testing.T) { trustedProxies: trustLocalhost, allowNets: "198.51.100.0/24", denyNets: fromKP, - deniedCountries: "kp", + tagDeny: denyKP, rateLimitPerMinute: "1", }) - // Neither SWW_DENY_NETS, the country lists nor the limit of one - // request a minute refuses the client, and its country is not looked - // up. + // Neither SWW_DENY_NETS, SWW_TAG_DENY nor the limit of one request a + // minute refuses the client, and its country is not looked up. wantAnswers(t, addr, out, []sentRequest{ {fromKP, http.StatusOK, requestlog.ActionForward}, {fromKP, http.StatusOK, requestlog.ActionForward}, @@ -83,9 +82,9 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) { }) geojsURL, asked := startGeoJS(t) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ - trustedProxies: trustLocalhost, - denyNets: "203.0.113.0/24", - deniedCountries: "kp", + trustedProxies: trustLocalhost, + denyNets: "203.0.113.0/24", + tagDeny: denyKP, }) req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body")) @@ -142,19 +141,19 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) { trustedProxies: trustLocalhost, lookupTimeout: "1h", rateLimitExemptNets: listedAddr + "," + fromKP, - deniedCountries: "kp", + tagDeny: denyKP, rateLimitPerMinute: "1", }) // listedAddr's requests are neither refused nor counted, so the first - // request from unlistedAddr is within the limit of one a minute. The - // country lists still refuse an exempt client. + // request from unlistedAddr is within the limit of one a minute. + // SWW_TAG_DENY still refuses an exempt client. wantAnswers(t, addr, out, []sentRequest{ {listedAddr, http.StatusOK, requestlog.ActionForward}, {listedAddr, http.StatusOK, requestlog.ActionForward}, {unlistedAddr, http.StatusOK, requestlog.ActionForward}, {unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited}, - {fromKP, http.StatusForbidden, requestlog.ActionCountryDenied}, + {fromKP, http.StatusForbidden, requestlog.ActionDenied}, }) } diff --git a/internal/discriminator/tagdeny.go b/internal/discriminator/tagdeny.go new file mode 100644 index 0000000..0437bdf --- /dev/null +++ b/internal/discriminator/tagdeny.go @@ -0,0 +1,30 @@ +package discriminator + +// The heuristic tag_deny. It reads SWW_TAG_DENY. + +import ( + "context" + "slices" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// tagDeny refuses a request that carries a tag in SWW_TAG_DENY. Its figure is +// the first such tag, in sorted order. +type tagDeny struct{} + +func (tagDeny) name() string { return "tag_deny" } + +func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict { + for _, tag := range rq.line.Tags { + if slices.Contains(rq.h.config.TagDeny, tag) { + return &verdict{ + action: requestlog.ActionDenied, + figures: map[string]any{"tag": tag}, + } + } + } + + return nil +} diff --git a/internal/discriminator/tagdeny_test.go b/internal/discriminator/tagdeny_test.go new file mode 100644 index 0000000..9817303 --- /dev/null +++ b/internal/discriminator/tagdeny_test.go @@ -0,0 +1,121 @@ +package discriminator_test + +import ( + "net/http" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestTagDenyRefusesALookupTagAndNamesTheTag(t *testing.T) { + t.Parallel() + + // The setting takes a country code in either case; the request's tag is + // in capitals. + s, _, _ := startWithLookups(t, map[string]string{tagDeny: "Country=kp"}) + + line := s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied) + wantHeuristic(t, line, "tag_deny", `{"tag": "country=KP"}`) + wantTags(t, line, "asn=64511", "country=KP") + + s.get(fromDE, http.StatusOK, requestlog.ActionForward) +} + +func TestTagDenyRefusesAnASNTag(t *testing.T) { + t.Parallel() + + s, _, _ := startWithLookups(t, map[string]string{tagDeny: "asn=64511"}) + + wantHeuristic(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied), + "tag_deny", `{"tag": "asn=64511"}`) + s.get(fromDE, http.StatusOK, requestlog.ActionForward) +} + +func TestTagDenyRefusesAListTagBeforeTheRateLimitsCount(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + tagLists: "torexit=" + torList, tagDeny: "torexit", rateLimitPerMinute: "1", + }) + loadLists(t, server, map[string][]string{torList: {fromDE}}) + + // Each is refused for its tag: none is counted to break the limit of one. + for range 3 { + wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied), + "tag_deny", `{"tag": "torexit"}`) + } +} + +func TestTagDenyRefusesAMixedCaseZoneByItsLowerCaseTag(t *testing.T) { + t.Parallel() + + const mixedCaseZone = "DNSBL.example" + + // The zone's tag is in lower case, however SWW_DNSBL_ZONES writes it. + s, server, _ := startWithLookups(t, map[string]string{ + dnsblZones: mixedCaseZone, dnsblResolver: noResolver, + reputationAction: actionLog, tagDeny: "dnsbl=" + dnsblZone, + }) + loadVerdicts(server, map[string][]string{fromDE: {mixedCaseZone}}) + + wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied), + "tag_deny", `{"tag": "dnsbl=`+dnsblZone+`"}`) +} + +func TestTagDenyRefusesAnExploitSignatureOfARuleThatOnlyLogs(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, "noted path log ^/\n"), tagDeny: "exploitsignature", + }) + + line := s.get(client, http.StatusForbidden, requestlog.ActionDenied) + wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`) + wantTags(t, line, "exploitsignature") +} + +func TestTagLimitPercentLowersALimitTheLowestWinning(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, + tagLists: "vpn=" + vpnList + ",tor=" + torList, + tagLimitPercent: "vpn:50,tor:25", + }) + loadLists(t, server, map[string][]string{vpnList: {fromDE}, torList: {fromDE}}) + + // A quarter of 4 requests a minute: the second breaks the limit. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + + line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + "25 from "+tagLimitPercent) +} + +func TestTagLimitPercentLowersAByteLimitByAListTag(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + bytesLimitPerMinute: twoUploads, + tagLists: "vpn=" + vpnList, + tagLimitPercent: "vpn:50", + }) + loadLists(t, server, map[string][]string{vpnList: {fromDE}}) + + // The upload's 100 bytes are over half of 199, 99. + line := s.uploadFrom(fromDE) + if line.LimitHit != minuteBytes { + t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes) + } + + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + "50 from "+tagLimitPercent) + + // fromKP, which the list does not hold, has the whole limit. + line = s.uploadFrom(fromKP) + if line.LimitHit != "" { + t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit) + } + + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none) +} diff --git a/internal/discriminator/taglists.go b/internal/discriminator/taglists.go new file mode 100644 index 0000000..dc3aba8 --- /dev/null +++ b/internal/discriminator/taglists.go @@ -0,0 +1,24 @@ +package discriminator + +// The heuristic tag_lists. It reads the address lists of SWW_TAG_LISTS through +// the reputation lists. + +import ( + "context" + "time" +) + +// tagLists adds to the request one tag for each list of SWW_TAG_LISTS that +// holds the client, the tag being the list's name. It refuses nothing, and runs +// first so that every refusal's log line carries these tags. +type tagLists struct{} + +func (tagLists) name() string { return "tag_lists" } + +func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict { + for _, name := range rq.h.lists.TagsOf(rq.client) { + rq.addTag(name) + } + + return nil +} diff --git a/internal/discriminator/taglists_test.go b/internal/discriminator/taglists_test.go new file mode 100644 index 0000000..cffc92d --- /dev/null +++ b/internal/discriminator/taglists_test.go @@ -0,0 +1,100 @@ +package discriminator_test + +import ( + "net/http" + "slices" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// tagLists is SWW_TAG_LISTS, and vpnList and torList lists it names. +const ( + tagLists = "SWW_TAG_LISTS" + vpnList = "https://lists.example/vpn.txt" + torList = "https://lists.example/torexit.txt" +) + +// wantTags checks the log line's tags. +func wantTags(t *testing.T, line logLine, want ...string) { + t.Helper() + + got, _ := line.fields["tags"].([]any) + + tags := make([]string, 0, len(got)) + for _, tag := range got { + text, _ := tag.(string) + tags = append(tags, text) + } + + if !slices.Equal(tags, want) { + t.Errorf("log line has tags %v, want %v", tags, want) + } +} + +func TestLookupAndTagListsGiveTheRequestItsTagsSorted(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + tagLists: "vpn=mullvad=" + vpnList + ",TorExit=" + torList, + }) + loadLists(t, server, map[string][]string{ + vpnList: {fromDE}, torList: {fromDE + "/32", fromKP}, + }) + + wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), + "asn=64496", "country=DE", "torexit", "vpn=mullvad") + wantTags(t, s.get(fromKP, http.StatusOK, requestlog.ActionForward), + "asn=64511", "country=KP", "torexit") +} + +func TestRequestWithoutATagLeavesTheFieldOut(t *testing.T) { + t.Parallel() + + // With no lookup, a client is not placed and no setting gives it a tag. + s, _, _ := startWithClock(t, "", map[string]string{lookupSource: "off"}) + + line := s.get(fromDE, http.StatusOK, requestlog.ActionForward) + if _, present := line.fields["tags"]; present { + t.Errorf("log line has tags %v, want none", line.fields["tags"]) + } +} + +func TestTagListsTagEvenARequestTheCountryListRefuses(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + tagLists: "vpn=" + vpnList, allowedCountries: "de", + }) + loadLists(t, server, map[string][]string{vpnList: {fromKP}}) + + wantTags(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied), + "asn=64511", "country=KP", "vpn") +} + +func TestTagListsTagARequestRefusedBeforeTheLookup(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + tagLists: "vpn=" + vpnList, denyNets: fromKP, rateLimitPerMinute: "1", + }) + loadLists(t, server, map[string][]string{vpnList: {fromDE, fromKP}}) + + line := s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied) + if line.Heuristic != "deny_nets" { + t.Errorf("log line has heuristic %q, want deny_nets", line.Heuristic) + } + + wantTags(t, line, "vpn") + + // The second request breaks the limit of one a minute, which bans the client. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + + line = s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + if line.Heuristic != "active_ban" { + t.Errorf("log line has heuristic %q, want active_ban", line.Heuristic) + } + + wantTags(t, line, "vpn") +} diff --git a/internal/metrics/busiest.go b/internal/metrics/busiest.go index 82148cf..0cf9215 100644 --- a/internal/metrics/busiest.go +++ b/internal/metrics/busiest.go @@ -25,7 +25,7 @@ type busiest struct { requests *prometheus.CounterVec requestBytes *prometheus.CounterVec responseBytes *prometheus.CounterVec - // refused are, by country, the requests the country lists refused; nil + // refused are, by country, the requests the country list refused; nil // by AS number. refused *prometheus.CounterVec @@ -42,7 +42,7 @@ type busiest struct { func newCountries(topN int) *busiest { countries := newBusiest(topN, "country", "the client's country") countries.refused = counterVec("sww_country_list_refusals_total", - "Requests the country lists refused, by the client's country.", + "Requests SWW_EXCLUSIVELY_ALLOWED_COUNTRIES refused, by the client's country.", []string{"country"}) return countries @@ -88,7 +88,7 @@ func (b *busiest) Collect(ch chan<- prometheus.Metric) { } // vecs returns the metrics: by AS number, those of requests and bytes; by -// country, the refusals by the country lists as well. +// country, the refusals by the country list as well. func (b *busiest) vecs() []*prometheus.CounterVec { vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes} if b.refused != nil { diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index a7a8421..64db688 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -1,7 +1,8 @@ // Package reputation fetches the lists the settings name by URL: the // blocklists of SWW_BLOCKLIST_URLS, the file of AS:percent lines -// SWW_ASN_LIMIT_PERCENT_URL names, and the decision list of the CrowdSec -// engine SWW_CROWDSEC_LAPI_URL names. It keeps the last good copy of +// SWW_ASN_LIMIT_PERCENT_URL names, the address lists of SWW_TAG_LISTS, and +// the decision list of the CrowdSec engine SWW_CROWDSEC_LAPI_URL names. It +// keeps the last good copy of // each, whole, comment lines included, which is used while a fetch fails, // and when each was last tried. It also asks the DNSBL zones of // SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks @@ -71,6 +72,8 @@ type Params struct { // (SWW_ASN_LIMIT_PERCENT_URL), "" while it is unset. BlocklistURLs []string ASNLimitPercentURL string + // TagLists are the address lists that add a tag (SWW_TAG_LISTS). + TagLists []config.TagList // CrowdSecDecisionsURL is the CrowdSec decision list, "" while // SWW_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey // (SWW_CROWDSEC_LAPI_KEY). @@ -112,11 +115,11 @@ type list struct { failures int } -// entries are what the lines of a copy say: for a blocklist, the netblocks -// it names, with the lengths among them, for the file of AS:percent lines, -// the percentage it gives each AS number, and for the CrowdSec decision -// list, the decision on each netblock that ends last, with the lengths -// among them. +// entries are what the lines of a copy say: for a blocklist or a tag list, the +// netblocks it names, with the lengths among them, for the file of AS:percent +// lines, the percentage it gives each AS number, and for the CrowdSec decision +// list, the decision on each netblock that ends last, with the lengths among +// them. type entries struct { netblocks map[netip.Prefix]bool lengths []int @@ -152,14 +155,18 @@ func New(params Params) *Lists { } // URLs returns the URL of every list: the blocklists' in the order -// SWW_BLOCKLIST_URLS names them, then SWW_ASN_LIMIT_PERCENT_URL, then -// the CrowdSec decision list's. +// SWW_BLOCKLIST_URLS names them, then SWW_ASN_LIMIT_PERCENT_URL, then the +// tag lists', then the CrowdSec decision list's. func (l *Lists) URLs() []string { urls := slices.Clone(l.params.BlocklistURLs) if l.params.ASNLimitPercentURL != "" { urls = append(urls, l.params.ASNLimitPercentURL) } + for _, list := range l.params.TagLists { + urls = append(urls, list.URL) + } + if l.params.CrowdSecDecisionsURL != "" { urls = append(urls, l.params.CrowdSecDecisionsURL) } @@ -184,6 +191,22 @@ func (l *Lists) ListedBy(addr netip.Addr) []string { return listedBy } +// TagsOf returns the names of the tag lists whose copy holds addr. +func (l *Lists) TagsOf(addr netip.Addr) []string { + l.mu.Lock() + defer l.mu.Unlock() + + var names []string + + for _, list := range l.params.TagLists { + if l.lists[list.URL].entries.contain(addr) { + names = append(names, list.Name) + } + } + + return names +} + // ASNLimitPercent returns the percentage the copy of the file of // AS:percent lines gives asn, and whether it lists asn. func (l *Lists) ASNLimitPercent(asn string) (int64, bool) { @@ -481,10 +504,10 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { } // parse reads the lines of the list at listURL, fetched at fetched: those -// of a blocklist, of the file of AS:percent lines, or of the CrowdSec -// decision list. In the first two, anything after a ; or a # on a line is -// left out, and so is a line left blank. Any other line that does not read -// is an error naming it by its number. +// of a blocklist or a tag list, of the file of AS:percent lines, or of the +// CrowdSec decision list. In all but the decision list, anything after a ; or +// a # on a line is left out, and so is a line left blank. Any other line that +// does not read is an error naming it by its number. func (l *Lists) parse( listURL string, lines []string, fetched time.Time, ) (entries, error) { @@ -498,8 +521,8 @@ func (l *Lists) parse( } } -// parseNetblocks reads a blocklist's lines, each an address or a netblock -// as the settings take them. +// parseNetblocks reads the lines of a blocklist or a tag list, each an address +// or a netblock as the settings take them. func parseNetblocks(lines []string) (entries, error) { found := entries{netblocks: map[netip.Prefix]bool{}} @@ -551,7 +574,7 @@ func parseNetblock(text string) (netip.Prefix, bool) { } // parsePercents reads the lines of the file of AS:percent lines, each an -// AS number, : and a percentage, as SWW_ASN_LIMIT_PERCENT takes them. An +// AS number, : and a percentage, such as AS64496:50. An // AS number listed more than once gets the lowest of its percentages. func parsePercents(lines []string) (entries, error) { found := entries{percents: map[string]int64{}} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 5ad8208..65bea8f 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -44,10 +44,11 @@ const ( // ActionDenied is a request refused because its client is in // SWW_DENY_NETS, in a blocklist while SWW_BLOCKLIST_ACTION is deny, // or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while - // SWW_REPUTATION_ACTION is deny. + // SWW_REPUTATION_ACTION is deny, or that carries a tag in SWW_TAG_DENY. ActionDenied = "denied" - // ActionCountryDenied is a request refused for its client's country. + // ActionCountryDenied is a request refused because its client's country + // is not in SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. ActionCountryDenied = "country_denied" // ActionAdmin is a request discriminator answered at one of its own // endpoints, or a question about a path under /_sww/. @@ -179,6 +180,8 @@ type Line struct { // whose verdict lists it, their keys masked, then abuseipdb when its // score is a hit. Reputation []string `json:"reputation,omitempty"` + // Tags are the request's tags, sorted. + Tags []string `json:"tags,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` // BanExpires is when the ban the request made, or was refused under, diff --git a/internal/sww/sww_test.go b/internal/sww/sww_test.go index 59059b5..a5b62c0 100644 --- a/internal/sww/sww_test.go +++ b/internal/sww/sww_test.go @@ -611,14 +611,14 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) { writeLookupDatabase(t, path, "DE") env := map[string]string{ - listenAddr: localhost + ":0", - stateDir: t.TempDir(), - rulesDir: t.TempDir(), - metricsToken: token, - instanceName: instance, - lookupSource: "file", - lookupDBPath: path, - "SWW_DENIED_COUNTRIES": "kp", + listenAddr: localhost + ":0", + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + metricsToken: token, + instanceName: instance, + lookupSource: "file", + lookupDBPath: path, + "SWW_TAG_DENY": "country=KP", // The requests sent until a replacement takes effect, and those // for the metrics, must not break a rate limit, whose ban would // refuse them too. @@ -1280,7 +1280,7 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) { "SWW_RATE_LIMIT_PER_HOUR": "1500", rateLimitPerDay: "6000", "SWW_RATE_LIMIT_EXEMPT_PATHS": "", - "SWW_DENIED_COUNTRIES": "", + "SWW_TAG_DENY": "", "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES": "", "SWW_BAN_RESPONSE": "403", "SWW_LIMIT_BAN_DURATION": "1h",