From e1ed444e68a83c0cf4dadc18e901923e95de7661 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 19 Nov 2026 07:59:24 +0000 Subject: [PATCH] waf: ban the clients on AbuseIPDB's blacklist (closes #411) While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS (60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30) with the new cause abuseipdb. The blacklist itself is not kept. These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address blacklist cannot push out the bans for limits and attacks. Each fetch forgets the abuseipdb bans that have ended. A netblock with an active ban is left alone, and so is one whose abuseipdb ban an admin lifted, until that ban would have ended. Observe mode makes no ban. Model: opus-5-5 --- README.md | 240 ++++++++++++----------- SPEC.md | 38 +++- docs/alerts.md | 3 +- docs/settings.md | 7 + docs/state-files.md | 14 +- internal/bans/abuseipdb_test.go | 165 ++++++++++++++++ internal/bans/bans.go | 175 ++++++++++++++--- internal/config/waf.go | 61 ++++++ internal/config/waf_test.go | 75 +++++++ internal/metrics/metrics.go | 2 +- internal/reputation/abuseipdb.go | 146 +++++++++++++- internal/reputation/blacklist_test.go | 249 ++++++++++++++++++++++++ internal/reputation/reputation.go | 3 +- internal/state/state.go | 7 +- internal/state/state_test.go | 5 +- internal/sww/sww.go | 9 +- internal/waf/abuseipdbblacklist.go | 39 ++++ internal/waf/abuseipdbblacklist_test.go | 69 +++++++ internal/waf/export_test.go | 13 +- internal/waf/proxy.go | 21 +- 20 files changed, 1162 insertions(+), 179 deletions(-) create mode 100644 internal/bans/abuseipdb_test.go create mode 100644 internal/reputation/blacklist_test.go create mode 100644 internal/waf/abuseipdbblacklist.go create mode 100644 internal/waf/abuseipdbblacklist_test.go diff --git a/README.md b/README.md index c947ac5..69c1cfd 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ Together, `rproxy` and `waf` protect against: - give the clients of chosen networks and countries a fraction of every limit, so that they are banned sooner; - refuse or limit clients on a blocklist, a DNS blocklist, or with a high - AbuseIPDB score. + AbuseIPDB score; + - ban the clients on AbuseIPDB's blacklist before they send a request. For the operator, it: @@ -290,123 +291,126 @@ account, a key, a token or an alert destination stays off until you give one. Every setting the code reads, with its default: -| Variable | Default | What it does | -| ------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. | -| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. | -| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. | -| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. | -| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. | -| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. | -| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. | -| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. | -| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. | -| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. | -| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. | -| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. | -| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. | -| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. | -| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. | -| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. | -| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps of its own; bans an admin made do not count. | -| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. | -| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. | -| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. | -| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. | -| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. | -| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. | -| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. | -| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | -| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | -| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | -| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. | -| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. | -| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. | -| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. | -| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. | -| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. | -| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. | -| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. | -| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. | -| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. | -| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. | -| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. | -| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. | -| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. | -| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. | -| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. | -| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. | -| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. | -| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. | -| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. | -| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS:` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). | -| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. | -| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. | -| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. | -| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. | -| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. | -| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. | -| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. | -| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. | -| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. | -| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. | -| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. | -| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. | -| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. | -| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:` of every limit, or `log`. | -| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. | -| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. | -| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. | -| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. | -| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. | -| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:` or `log`. | -| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. | -| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. | -| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. | -| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. | -| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. | -| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. | -| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. | -| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. | -| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. | -| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. | -| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. | -| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. | -| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. | -| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. | -| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. | -| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. | -| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. | -| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer `. | -| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. | -| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. | -| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. | -| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. | -| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. | -| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. | -| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Alert when one client makes more requests than this in a minute, a count. | -| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Alert when one client makes more requests than this in an hour, a count. | -| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in a minute, a size. | -| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in an hour, a size. | -| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Alert when the netblock around a client makes more requests than this in a minute, a count. | -| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Alert when the netblock around a client makes more requests than this in an hour, a count. | -| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in a minute, a size. | -| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in an hour, a size. | -| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. | -| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. | -| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Alert when one AS number makes more requests than this in a minute, a count. | -| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Alert when one AS number makes more requests than this in an hour, a count. | -| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in a minute, a size. | -| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in an hour, a size. | -| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Alert when the whole service gets more requests than this in a minute, a count. | -| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Alert when the whole service gets more requests than this in an hour, a count. | -| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in a minute, a size. | -| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in an hour, a size. | -| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. | -| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Alert when a watched netblock makes more requests than this in a minute, a count. | -| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Alert when a watched netblock makes more requests than this in an hour, a count. | -| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in a minute, a size. | -| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in an hour, a size. | +| Variable | Default | What it does | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. | +| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. | +| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. | +| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. | +| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. | +| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. | +| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. | +| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. | +| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. | +| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. | +| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. | +| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. | +| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. | +| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. | +| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. | +| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. | +| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps for broken limits and attacks; bans an admin made and bans for AbuseIPDB's blacklist do not count. | +| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. | +| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. | +| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. | +| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. | +| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. | +| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. | +| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. | +| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | +| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | +| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. | +| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. | +| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. | +| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. | +| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. | +| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. | +| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. | +| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. | +| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. | +| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. | +| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. | +| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. | +| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. | +| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. | +| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. | +| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. | +| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. | +| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. | +| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. | +| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. | +| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. | +| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS:` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). | +| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. | +| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. | +| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. | +| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. | +| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. | +| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. | +| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. | +| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. | +| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. | +| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. | +| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. | +| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. | +| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. | +| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:` of every limit, or `log`. | +| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. | +| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. | +| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account. Set, a client that has committed an offence is checked in the background, and the clients on AbuseIPDB's blacklist are banned. | +| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. | +| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. | +| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:` or `log`. | +| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. | +| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. | +| `SWW_WAF_ABUSEIPDB_THRESHOLD` | `90` | The least score, from 25 to 100, of the clients on AbuseIPDB's blacklist, which are banned. Do not set it below 50 to 75: lower scores list ordinary visitors. | +| `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` | `60` | How often AbuseIPDB's blacklist is fetched again, in minutes. | +| `SWW_WAF_ABUSEIPDB_BAN_DAYS` | `30` | How long a client on AbuseIPDB's blacklist is banned, in days. | +| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. | +| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. | +| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. | +| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. | +| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. | +| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. | +| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. | +| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. | +| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. | +| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. | +| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. | +| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. | +| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. | +| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. | +| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. | +| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer `. | +| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. | +| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. | +| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. | +| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. | +| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. | +| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. | +| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Alert when one client makes more requests than this in a minute, a count. | +| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Alert when one client makes more requests than this in an hour, a count. | +| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in a minute, a size. | +| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in an hour, a size. | +| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Alert when the netblock around a client makes more requests than this in a minute, a count. | +| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Alert when the netblock around a client makes more requests than this in an hour, a count. | +| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in a minute, a size. | +| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in an hour, a size. | +| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. | +| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. | +| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Alert when one AS number makes more requests than this in a minute, a count. | +| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Alert when one AS number makes more requests than this in an hour, a count. | +| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in a minute, a size. | +| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in an hour, a size. | +| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Alert when the whole service gets more requests than this in a minute, a count. | +| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Alert when the whole service gets more requests than this in an hour, a count. | +| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in a minute, a size. | +| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in an hour, a size. | +| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. | +| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Alert when a watched netblock makes more requests than this in a minute, a count. | +| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Alert when a watched netblock makes more requests than this in an hour, a count. | +| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in a minute, a size. | +| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in an hour, a size. | [`docs/settings.md`](docs/settings.md) gives the values each setting takes in full, and the limits that are fixed rather than settings. diff --git a/SPEC.md b/SPEC.md index ca67a0a..9d5d1e5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -92,7 +92,8 @@ never as a time. to count and report failures. - `internal/lookup`: looks clients up in the lookup database. - `internal/reputation`: fetches the lists, asks the DNS blocklists and - AbuseIPDB in the background, and keeps their answers. + AbuseIPDB in the background, and keeps their answers. It also fetches + AbuseIPDB's blacklist, which `internal/waf` bans. - `internal/routewatch`: asks a routewatch instance about clients in the background, and keeps its answers. - `internal/anomaly`: the anomaly counters and their alerts. @@ -299,6 +300,9 @@ to a ban, each by its own rule: Once it ends, only new requests can break a limit again. The client's history keeps its totals. +A client on AbuseIPDB's blacklist is banned before it sends a request, with the +cause `abuseipdb` (see "Reputation"). + A Core Rule Set match refuses only the request it matched, with `403` and without a ban, because the Core Rule Set has false positives. A `block` rule does the same. Those refusals count toward the refusal limit, so a client that @@ -325,11 +329,12 @@ does not count toward a longer ban; it refuses nothing. Deleting its entry from A permanent ban does not run out. It ends when an admin lifts it, or when `waf` drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans -of its own, it drops the earliest ban of one netblock: the netblock that has -gone longest without a request. A past, active or permanent ban is dropped -alike. A scanner whose ban was dropped that way is refused and banned again by -its next probe. A ban whose cause is `admin` is never dropped and does not count -toward `SWW_WAF_MAX_BANS`, so such bans can never fill the table. +for broken limits and clear signs of attack, it drops the earliest such ban of +one netblock: the netblock that has gone longest without a request. A past, +active or permanent ban is dropped alike. A scanner whose ban was dropped that +way is refused and banned again by its next probe. Bans whose cause is `admin` +or `abuseipdb` do not count toward `SWW_WAF_MAX_BANS` and are never dropped to +make room, so they can never push out the others. A request refused under a ban that runs out is logged with the action `temporarily_banned`. Under a permanent ban the action is `permanently_banned`. @@ -2036,6 +2041,25 @@ host, sharing one address, would often fetch too close together. - AbuseIPDB answers other than `200`; - the answer gives no score; - no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`. +- While `SWW_WAF_ABUSEIPDB_KEY` is set, AbuseIPDB's blacklist is fetched at the + start and every `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` minutes + (default `60`). It lists the clients AbuseIPDB scores at + `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`) or more. The blacklist itself is + not kept. + - Each client on it is banned for `SWW_WAF_ABUSEIPDB_BAN_DAYS` days (default + `30`), with the cause `abuseipdb`. The ban covers the client's netblock, + as any ban does. + - A netblock that already has an active ban is left as it is. So is one + whose `abuseipdb` ban an admin lifted, until that ban would have ended. + - Each fetch first forgets the `abuseipdb` bans that have ended, so that + `bans.json` holds only the clients listed within one ban's length. + - In `observe` mode the blacklist makes no ban. + - The threshold takes 25 to 100. It should not be set below 50 to 75, since + lower scores list ordinary visitors. AbuseIPDB serves free accounts the + blacklist at the threshold 100, whatever is asked for. + - AbuseIPDB limits the fetches of the blacklist a day: 5 for a free account. + A fetch over the limit fails, and is logged and alerted. The default + refresh fetches it 24 times a day. - `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is `limit:25` by default. Such verdicts are less certain than a blocklist, so a listed client gets a quarter of every request and byte limit rather than a @@ -2200,7 +2224,7 @@ lists the files, what each holds, and their sizes. these stop the process with a message naming the file and position: - a file that does not parse, or has an unknown `version`; - an entry without a field it needs; - - a ban whose cause is not one of the four; + - a ban whose cause is not `limit`, `attack`, `admin` or `abuseipdb`; - a ban whose `expires` is not after its `start`. Starting with empty state would silently forgive every repeat offender. diff --git a/docs/alerts.md b/docs/alerts.md index 6f14a37..0a11407 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -68,7 +68,8 @@ on one line: "earlier_bans": { "limit": 0, "attack": 0, - "admin": 0 + "admin": 0, + "abuseipdb": 0 } } }, diff --git a/docs/settings.md b/docs/settings.md index 0f07055..63e6472 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -338,6 +338,13 @@ tag that no request can get stops the start. A request can get only these: or AbuseIPDB's score is used. A duration. - `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a zone or a check with AbuseIPDB may take. A time in milliseconds. +- `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`): the least abuse confidence score + of the clients on AbuseIPDB's blacklist, a whole number from 25 to 100. It + should not be set below 50 to 75, since lower scores list ordinary visitors. +- `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` (default `60`): how often + the blacklist is fetched again, in minutes. A whole number above zero. +- `SWW_WAF_ABUSEIPDB_BAN_DAYS` (default `30`): how long the ban of a client on + the blacklist lasts, in days. A whole number above zero. ## Alerting diff --git a/docs/state-files.md b/docs/state-files.md index af6643d..8f8e261 100644 --- a/docs/state-files.md +++ b/docs/state-files.md @@ -21,11 +21,16 @@ top-level `version` of 1. ## `bans.json` -Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of -those `waf` made. Per entry: +Active and past bans: + +- every ban an admin made; +- up to `SWW_WAF_MAX_BANS` of those `waf` made for a broken limit or an attack; +- the bans for AbuseIPDB's blacklist that have not ended. + +Per entry: - `netblock`, `start` and `expires` (`null` for a permanent ban); -- `cause`: `attack`, `limit` or `admin`; +- `cause`: `attack`, `limit`, `admin` or `abuseipdb`; - `reason`: a short reason; - `lifted`: when an admin lifted it, if one did; - `notes`: the notes below. @@ -60,7 +65,8 @@ ban. The notes: - for a clear sign of attack, `rule_id` and `target`, or `trap_path`; - `reputation`: the sources the client was on, each with its `source` and, for AbuseIPDB, its `score`; -- `request`: the request that caused the ban, each text cut to 256 bytes: +- `request`: the request that caused the ban, each text cut to 256 bytes, and + empty for a ban for AbuseIPDB's blacklist: - `time`, `method` and `host`; - `path`, with its query string; - `status` and `user_agent`; diff --git a/internal/bans/abuseipdb_test.go b/internal/bans/abuseipdb_test.go new file mode 100644 index 0000000..71c670a --- /dev/null +++ b/internal/bans/abuseipdb_test.go @@ -0,0 +1,165 @@ +package bans_test + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultParams()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + expires := now.Add(30 * day) + + made := ledger.BanForAbuseIPDB( + now, expires, map[netip.Prefix]int64{netblock: 95}, + ) + if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 { + t.Fatalf("made %d bans, %d counted, want 1 and 1", + made, ledger.Made(bans.CauseAbuseIPDB)) + } + + ban, banned, madePermanent := ledger.Check(netblock.Addr(), now) + if !banned || madePermanent || !ban.Expires.Equal(expires) || + ban.Cause != bans.CauseAbuseIPDB || + ban.Reason != "on AbuseIPDB's blacklist with a score of 95" || + len(ban.Notes.Reputation) != 1 || + ban.Notes.Reputation[0].Source != "abuseipdb" || + *ban.Notes.Reputation[0].Score != 95 { + t.Errorf( + "ban %+v, banned %t, made permanent %t, want the blacklist's "+ + "ban until %s", + ban, + banned, + madePermanent, + expires, + ) + } + + if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned { + t.Error("the ban refuses a request at its end") + } +} + +func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultParams()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + scores := map[netip.Prefix]int64{netblock: 100} + + limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) + + if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 { + t.Errorf("made %d bans over a ban for a limit, want none", made) + } + + // Once that ban has ended the netblock is banned, and the blacklist + // fetched again makes no second ban while the first lasts. + later := limitBan.Expires + if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 { + t.Errorf("made %d bans after the ban for a limit, want 1", made) + } + + later = later.Add(time.Hour) + if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 { + t.Errorf("made %d bans while the blacklist's lasts, want none", made) + } + + wantBans(t, ledger, map[netip.Prefix]int{netblock: 2}) + + if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{ + Limit: 1, + }) { + t.Errorf("earlier bans %+v, want 1 for a limit", got) + } +} + +func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultParams()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + expires := now.Add(30 * day) + scores := map[netip.Prefix]int64{netblock: 100} + + ledger.BanForAbuseIPDB(now, expires, scores) + ledger.Lift(netblock.Addr(), now) + + // The admin lifted it: the blacklist fetched again until its end bans + // nothing. + later := expires.Add(-time.Hour) + if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 { + t.Errorf("made %d bans before the lifted ban's end, want none", made) + } + + // From its end on, the blacklist bans the netblock again, and the + // lifted ban, ended, is forgotten. + if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 { + t.Errorf("made %d bans at the lifted ban's end, want 1", made) + } + + wantBans(t, ledger, map[netip.Prefix]int{netblock: 1}) +} + +func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultParams()) + listed := netip.MustParsePrefix("203.0.113.9/32") + limited := netip.MustParsePrefix("203.0.113.10/32") + now := midnight() + expires := now.Add(30 * day) + + ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100}) + ledger.BanForLimit(limited, now, bans.Notes{}) + + // listed is off the blacklist when it is fetched after the ban's end. + if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 { + t.Errorf("made %d bans for an empty blacklist, want none", made) + } + + wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1}) +} + +func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) { + t.Parallel() + + params := defaultParams() + params.MaxBans = 1 + ledger := bans.New(params) + limited := netip.MustParsePrefix("203.0.113.1/32") + attacker := netip.MustParsePrefix("203.0.113.2/32") + now := midnight() + + scores := map[netip.Prefix]int64{} + for i := range 3 { + scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100 + } + + ledger.BanForLimit(limited, now, bans.Notes{}) + + if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 { + t.Errorf("made %d bans for 3 netblocks, want 3", made) + } + + wantBans(t, ledger, map[netip.Prefix]int{limited: 1}) + + // The ban for an attack takes the place of the one for a limit, and no + // blacklist ban goes. + ledger.BanForAttack(attacker, now, bans.Notes{}) + + want := map[netip.Prefix]int{limited: 0, attacker: 1} + for netblock := range scores { + want[netblock] = 1 + } + + wantBans(t, ledger, want) +} diff --git a/internal/bans/bans.go b/internal/bans/bans.go index b581382..2057c20 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -26,6 +26,9 @@ const ( // CauseAdmin is a ban an admin made, or one sww made that an // admin keeps. It is never dropped. CauseAdmin = "admin" + // CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's + // blacklist. It is forgotten once it has ended. + CauseAbuseIPDB = "abuseipdb" ) // repeatFactor is how many times as long as the netblock's last ban a ban @@ -50,10 +53,11 @@ type Params struct { // AttackBanDuration is how long a first ban for a clear sign of attack // lasts. AttackBanDuration time.Duration - // MaxBans is the most bans held whose cause is not CauseAdmin, at - // least one. Past it, the earliest such ban of the netblock that has - // gone longest without a request is dropped. Bans whose cause is - // CauseAdmin are held besides, and never dropped. + // MaxBans is the most bans held whose cause is CauseLimit or + // CauseAttack, at least one. Past it, the earliest such ban of the + // netblock that has gone longest without a request is dropped. Bans + // whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and + // never dropped this way. MaxBans int } @@ -63,10 +67,11 @@ type Ban struct { Start time.Time // Expires is when the ban ends, zero for a permanent ban. Expires time.Time - // Cause is CauseLimit, CauseAttack or CauseAdmin. + // Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB. Cause string - // Reason is a short text: for a ban sww made, the limit broken or - // the rule that matched; for an admin's, what the admin wrote. + // Reason is a short text: for a ban sww made, the limit broken, the + // rule that matched or AbuseIPDB's score; for an admin's, what the + // admin wrote. Reason string // Lifted is when an admin lifted the ban, zero while no admin has. A // lifted ban refuses nothing, and does not make the netblock's next @@ -127,7 +132,8 @@ type Notes struct { // log's reputation names them. It is left out when none did. Reputation []ReputationHit `json:"reputation,omitempty"` // Request is the request that broke the limit, or whose bytes broke - // it, or that was the clear sign of attack. + // it, or that was the clear sign of attack. A ban for AbuseIPDB's + // blacklist has none, and leaves it zero. Request Request `json:"request"` // Requests is how many requests the netblock has sent since it was // first seen, and Refused how many of them the ban has refused so @@ -150,9 +156,10 @@ type ReputationHit struct { // EarlierBans counts a netblock's bans before a ban, by cause. type EarlierBans struct { - Limit int `json:"limit"` - Attack int `json:"attack"` - Admin int `json:"admin"` + Limit int `json:"limit"` + Attack int `json:"attack"` + Admin int `json:"admin"` + AbuseIPDB int `json:"abuseipdb"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. @@ -180,8 +187,8 @@ type Ledger struct { // netblocks holds each banned netblock's bans, oldest first. Check and // Find make each netblock they find the most recently seen. netblocks *simplelru.LRU[netip.Prefix, *[]Ban] - // held is how many bans netblocks holds whose cause is not CauseAdmin, - // at most params.MaxBans. + // held is how many bans netblocks holds whose cause is CauseLimit or + // CauseAttack, at most params.MaxBans. held int // made is how many bans have been made since the start, by cause: by // the ledger, and by an admin, through BanForAdmin or in an edit of @@ -198,8 +205,9 @@ type Ledger struct { // New returns a Ledger with no ban yet. func New(params Params) *Ledger { // The ledger drops bans itself, and never those whose cause is - // CauseAdmin, however many there are, so the LRU has no limit of its - // own: it keeps the netblocks in the order they were last seen. + // CauseAdmin or CauseAbuseIPDB to make room, however many there are, so + // the LRU has no limit of its own: it keeps the netblocks in the order + // they were last seen. netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban]( math.MaxInt, nil, @@ -429,6 +437,75 @@ func (l *Ledger) BanForAdmin( return ban } +// BanForAbuseIPDB bans each netblock in scores at now until expires, and +// returns how many bans it made. scores are the netblocks of the clients +// on AbuseIPDB's blacklist, each with the highest score of its clients. +// A netblock is left as it is while a ban on it is active, or while a ban +// for the blacklist that an admin lifted has not reached its end: the +// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB +// and that has ended is forgotten first, so that the bans held for the +// blacklist are those of the clients listed within one ban's length. +func (l *Ledger) BanForAbuseIPDB( + now, expires time.Time, scores map[netip.Prefix]int64, +) int { + l.mu.Lock() + defer l.mu.Unlock() + + changed := l.forgetEndedAbuseIPDBBans(now) + made := 0 + + for netblock, score := range scores { + netblock = netblock.Masked() + + var notes Notes + + if held, found := l.netblocks.Peek(netblock); found { + if activeBan(*held, now) != nil || + liftedAbuseIPDBBan(*held, now) { + continue + } + + notes.EarlierBans = earlierBans(*held) + } + + // The source is named as the request log names AbuseIPDB. + notes.Reputation = []ReputationHit{ + {Source: CauseAbuseIPDB, Score: &score}, + } + + l.add(Ban{ + Netblock: netblock, + Start: now, + Expires: expires, + Cause: CauseAbuseIPDB, + Reason: fmt.Sprintf( + "on AbuseIPDB's blacklist with a score of %d", score, + ), + Notes: notes, + }) + + made++ + } + + l.made[CauseAbuseIPDB] += made + + if changed || made > 0 { + l.markChanged() + } + + return made +} + +// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose +// cause is CauseAbuseIPDB that an admin lifted, and whose end is not +// reached at now. +func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool { + return slices.ContainsFunc(bans, func(ban Ban) bool { + return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() && + (ban.Permanent() || now.Before(ban.Expires)) + }) +} + // Lift lifts, at now, every ban active then on a netblock client is in, // as an admin does, and returns those bans. A lifted ban is kept, refuses // nothing, and does not make the netblock's next ban longer. @@ -515,9 +592,9 @@ func (l *Ledger) AddLookup( } // Made returns how many bans for cause have been made since the start: -// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an -// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts -// them. The bans read from bans.json at the start are not among them. +// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for +// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as +// LoadEdit counts them. The bans read from bans.json at the start are not among them. func (l *Ledger) Made(cause string) int { l.mu.Lock() defer l.mu.Unlock() @@ -574,8 +651,8 @@ func (l *Ledger) Snapshot() []Ban { // seen. A ban without a cause is an admin's, and gets CauseAdmin. Each // netblock is masked to its length, so that 203.0.113.9/24 is // 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past -// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as -// when they are made. +// MaxBans the earliest bans that count toward it are dropped, as when they +// are made. func (l *Ledger) Load(bans []Ban) { l.mu.Lock() defer l.mu.Unlock() @@ -639,6 +716,35 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool { }) } +// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB +// and whose end is reached at now, lifted or not, and each netblock left +// without a ban. It reports whether it forgot one. +func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool { + forgot := false + + for _, netblock := range l.netblocks.Keys() { + bans, _ := l.netblocks.Peek(netblock) + + kept := slices.DeleteFunc(*bans, func(ban Ban) bool { + return ban.Cause == CauseAbuseIPDB && !ban.Permanent() && + !now.Before(ban.Expires) + }) + if len(kept) == len(*bans) { + continue + } + + forgot = true + + if len(kept) == 0 { + l.netblocks.Remove(netblock) + } else { + *bans = kept + } + } + + return forgot +} + // ban bans netblock at now for cause, CauseLimit or CauseAttack, with // reason and notes, as BanForLimit and BanForAttack describe, and returns // the ban, and whether it made it. Unless keep is true, the ban is not @@ -714,13 +820,16 @@ func earlierBans(held []Ban) EarlierBans { fromHere.Attack++ case CauseAdmin: fromHere.Admin++ + case CauseAbuseIPDB: + fromHere.AbuseIPDB++ } notes := ban.Notes.EarlierBans count := EarlierBans{ - Limit: notes.Limit + fromHere.Limit, - Attack: notes.Attack + fromHere.Attack, - Admin: notes.Admin + fromHere.Admin, + Limit: notes.Limit + fromHere.Limit, + Attack: notes.Attack + fromHere.Attack, + Admin: notes.Admin + fromHere.Admin, + AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB, } // Of counts that name as many bans, the earlier ban's is taken: it @@ -735,7 +844,7 @@ func earlierBans(held []Ban) EarlierBans { // total returns how many bans e counts, of all causes. func (e EarlierBans) total() int { - return e.Limit + e.Attack + e.Admin + return e.Limit + e.Attack + e.Admin + e.AbuseIPDB } // markChanged has Changed receive a value, unless one is waiting already. @@ -806,9 +915,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban { // add adds ban to its netblock's bans, after the last, and makes its // netblock the most recently seen. With MaxBans held, it drops one first, -// unless ban's cause is CauseAdmin, which does not count toward MaxBans. +// unless ban does not count toward MaxBans. func (l *Ledger) add(ban Ban) { - counted := ban.Cause != CauseAdmin + counted := countsTowardMaxBans(ban) if counted && l.held == l.params.MaxBans { l.dropOne() } @@ -889,7 +998,13 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time { return now.Add(l.params.AttackBanDuration) } -// dropOne drops the earliest ban whose cause is not CauseAdmin of the +// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause +// is CauseLimit or CauseAttack. +func countsTowardMaxBans(ban Ban) bool { + return ban.Cause == CauseLimit || ban.Cause == CauseAttack +} + +// dropOne drops the earliest ban that counts toward MaxBans of the // netblock that has gone longest without a request, of those that hold // such a ban, and the netblock with it if that was its only ban. It is // called with at least one such ban held. It looks at each netblock once @@ -898,11 +1013,9 @@ func (l *Ledger) dropOne() { for range l.netblocks.Len() { netblock, bans, _ := l.netblocks.GetOldest() - i := slices.IndexFunc(*bans, func(ban Ban) bool { - return ban.Cause != CauseAdmin - }) + i := slices.IndexFunc(*bans, countsTowardMaxBans) if i < 0 { - // Its bans are all an admin's, and never dropped. Get makes + // None of its bans is dropped to make room. Get makes // it the most recently seen, so that the next netblock is // looked at; when it was seen matters only for dropping a // ban, and a ban added to it makes it the most recently seen diff --git a/internal/config/waf.go b/internal/config/waf.go index 513d253..0691438 100644 --- a/internal/config/waf.go +++ b/internal/config/waf.go @@ -158,6 +158,14 @@ type WAF struct { DNSBLAbuseIPDBLimitPercent int64 IPReputationCacheTTL time.Duration DNSBLAbuseIPDBTimeout time.Duration + // While AbuseIPDBKey is set, AbuseIPDB's blacklist of the clients it + // scores AbuseIPDBThreshold or more (SWW_WAF_ABUSEIPDB_THRESHOLD), from + // 25 to 100, is fetched at the start and every AbuseIPDBBlacklistRefresh + // (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Each client on it + // is banned for AbuseIPDBBanDuration (SWW_WAF_ABUSEIPDB_BAN_DAYS). + AbuseIPDBThreshold int64 + AbuseIPDBBlacklistRefresh time.Duration + AbuseIPDBBanDuration time.Duration // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that @@ -330,6 +338,9 @@ const ( // netblock a provider is usually given: a shorter one would make one // client of the customers of several providers. minIPv6GroupPrefix = 32 + // minAbuseIPDBThreshold is the lowest score AbuseIPDB's blacklist can + // be cut at. + minAbuseIPDBThreshold = 25 // minTokenLength is the fewest characters a token may have. minTokenLength = 32 // maxParanoiaLevel is the Core Rule Set's highest paranoia level. @@ -446,6 +457,13 @@ var ( errNotPercent = errors.New( "is not a percentage, a whole number from 0 to 100", ) + errNotAbuseIPDBThreshold = errors.New( + "is not a whole number from 25 to 100, the scores AbuseIPDB's " + + "blacklist can be cut at", + ) + errNotWholeUnits = errors.New( + "is not a whole number above zero, such as 30, that fits a duration", + ) errListedTwice = errors.New("is listed twice") errNotListURL = errors.New( "is not an http or https URL without a user or a fragment, " + @@ -587,6 +605,20 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) { "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS", "2000", ), + AbuseIPDBThreshold: env.abuseIPDBThreshold( + "SWW_WAF_ABUSEIPDB_THRESHOLD", + "90", + ), + AbuseIPDBBlacklistRefresh: env.wholeUnits( + "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS", + "60", + time.Minute, + ), + AbuseIPDBBanDuration: env.wholeUnits( + "SWW_WAF_ABUSEIPDB_BAN_DAYS", + "30", + day, + ), BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff( "SWW_WAF_LIMIT_BAN_DURATION", @@ -957,6 +989,35 @@ func (e *environment) percent(name, defaultValue string) int64 { return percent } +// abuseIPDBThreshold reads the setting that is the least score of the +// clients on AbuseIPDB's blacklist: a whole number from 25 to 100, the +// range AbuseIPDB takes. +func (e *environment) abuseIPDBThreshold(name, defaultValue string) int64 { + value := e.value(name, defaultValue) + + threshold, err := strconv.ParseInt(value, 10, 64) + if err != nil || threshold < minAbuseIPDBThreshold || threshold > 100 { + e.check(name, fmt.Errorf("%q %w", value, errNotAbuseIPDBThreshold)) + } + + return threshold +} + +// wholeUnits reads a setting that is a whole number above zero of unit, +// such as minutes or days, and returns it as a duration. +func (e *environment) wholeUnits( + name, defaultValue string, unit time.Duration, +) time.Duration { + value := e.value(name, defaultValue) + + n, err := strconv.ParseInt(value, 10, 64) + if err != nil || n <= 0 || n > math.MaxInt64/int64(unit) { + e.check(name, fmt.Errorf("%q %w", value, errNotWholeUnits)) + } + + return time.Duration(n) * unit +} + // listURL reads a setting that is the URL a list is fetched from, "" while // it is unset or empty. func (e *environment) listURL(name string) string { diff --git a/internal/config/waf_test.go b/internal/config/waf_test.go index f3ea741..2f98d7f 100644 --- a/internal/config/waf_test.go +++ b/internal/config/waf_test.go @@ -61,6 +61,9 @@ const ( abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY" abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE" abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET" + abuseIPDBThreshold = "SWW_WAF_ABUSEIPDB_THRESHOLD" + abuseIPDBBlacklistRefresh = "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS" + abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS" dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION" ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL" dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS" @@ -2399,6 +2402,75 @@ func TestAbuseIPDBSettingsAsSet(t *testing.T) { } } +func TestAbuseIPDBBlacklistSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := wafFromEnvironment(t, environment{}) + if cfg.AbuseIPDBThreshold != 90 || + cfg.AbuseIPDBBlacklistRefresh != time.Hour || + cfg.AbuseIPDBBanDuration != 30*24*time.Hour { + t.Errorf( + "by default, the threshold %d, the refresh %s and the ban %s, "+ + "want 90, 1h and 720h", + cfg.AbuseIPDBThreshold, + cfg.AbuseIPDBBlacklistRefresh, + cfg.AbuseIPDBBanDuration, + ) + } + + cfg = wafFromEnvironment(t, environment{ + abuseIPDBThreshold: "25", abuseIPDBBlacklistRefresh: "1440", + abuseIPDBBanDays: "1", + }) + if cfg.AbuseIPDBThreshold != 25 || + cfg.AbuseIPDBBlacklistRefresh != 24*time.Hour || + cfg.AbuseIPDBBanDuration != 24*time.Hour { + t.Errorf( + "set, the threshold %d, the refresh %s and the ban %s, "+ + "want 25, 24h and 24h", + cfg.AbuseIPDBThreshold, + cfg.AbuseIPDBBlacklistRefresh, + cfg.AbuseIPDBBanDuration, + ) + } +} + +func TestInvalidAbuseIPDBBlacklistSettingStopsTheStartSayingWhatIsWrong( + t *testing.T, +) { + t.Parallel() + + const ( + notThreshold = " is not a whole number from 25 to 100, the scores " + + "AbuseIPDB's blacklist can be cut at" + notWhole = " is not a whole number above zero, such as 30, that fits " + + "a duration" + ) + + for _, tc := range []struct{ name, value, want string }{ + {abuseIPDBThreshold, "24", `"24"` + notThreshold}, + {abuseIPDBThreshold, "101", `"101"` + notThreshold}, + {abuseIPDBThreshold, off, `"off"` + notThreshold}, + {abuseIPDBBlacklistRefresh, "0", `"0"` + notWhole}, + {abuseIPDBBlacklistRefresh, "1h", `"1h"` + notWhole}, + {abuseIPDBBanDays, "-1", `"-1"` + notWhole}, + {abuseIPDBBanDays, "200000", `"200000"` + notWhole}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.WAFFromEnvironment( + environment{tc.name: tc.value}.viper(), + ) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong( t *testing.T, ) { @@ -2951,6 +3023,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { abuseIPDBKey: "", abuseIPDBMinScore: "75", abuseIPDBDailyBudget: "900", + abuseIPDBThreshold: "90", + abuseIPDBBlacklistRefresh: "60", + abuseIPDBBanDays: "30", dnsblAbuseIPDBAction: "limit:25", ipReputationCacheTTL: defaultIPReputationCacheTTL, dnsblAbuseIPDBTimeout: "2000", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2f787e6..6e66332 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -185,7 +185,7 @@ func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { for _, cause := range []string{ - bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, + bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB, } { m.registry.MustRegister( prometheus.NewCounterFunc(prometheus.CounterOpts{ diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go index f6e6ff0..429b3a5 100644 --- a/internal/reputation/abuseipdb.go +++ b/internal/reputation/abuseipdb.go @@ -11,6 +11,7 @@ import ( "net/netip" "net/url" "slices" + "strconv" "sync" "time" @@ -23,6 +24,9 @@ const ( // AbuseIPDBURL is where clients are checked: the check endpoint of // AbuseIPDB's API. AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check" + // AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from: + // the blacklist endpoint of AbuseIPDB's API. + AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist" // AbuseIPDBSource is how the request log, the alerts and the metrics // name AbuseIPDB. AbuseIPDBSource = "abuseipdb" @@ -32,6 +36,8 @@ const ( var ( errNoScore = errors.New("the answer gives no abuseConfidenceScore") + errNoBlacklist = errors.New("the answer gives no data") + errNotAddress = errors.New("is not an address") errBudgetUsedUp = errors.New( "checks spent; none is made until the day ends at 00:00 UTC") ) @@ -53,6 +59,13 @@ type ChecksSpent struct { Spent int } +// Listed is a client on AbuseIPDB's blacklist: its address and its abuse +// confidence score. +type Listed struct { + Addr netip.Addr + Score int64 +} + // AbuseIPDBParams are what NewAbuseIPDB needs. type AbuseIPDBParams struct { // URL is where clients are checked, normally AbuseIPDBURL, with Key, @@ -69,10 +82,19 @@ type AbuseIPDBParams struct { // take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS). CacheTTL time.Duration Timeout time.Duration + // BlacklistURL is where the blacklist is fetched from, normally + // AbuseIPDBBlacklistURL, with the clients scored Threshold or more + // (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh + // (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted + // receives the clients on each blacklist fetched. + BlacklistURL string + Threshold int64 + BlacklistRefresh time.Duration + Blacklisted func([]Listed) // Now tells the time, normally time.Now in UTC. Now func() time.Time - // ProcessLog receives each check that fails, and why, and the day's - // budget used up. + // ProcessLog receives each check or fetch of the blacklist that fails, + // and why, and the day's budget used up. ProcessLog *slog.Logger // Alerts receive a source_failure alert for each. Alerts *alerts.Queue @@ -256,6 +278,40 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) { a.day, a.spent = checks.Day, checks.Spent } +// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every +// BlacklistRefresh, until ctx is done, and gives the clients on each copy +// fetched to Blacklisted. While Key is unset it does nothing. A fetch that +// fails is logged and raised as a source_failure alert, unless it was cut +// off as ctx ended. +func (a *AbuseIPDB) RunBlacklist(ctx context.Context) { + if a.params.Key == "" { + return + } + + for { + listed, err := a.fetchBlacklist(ctx) + + switch { + case ctx.Err() != nil: + return + case err != nil: + a.alert("fetching AbuseIPDB's blacklist failed", err) + default: + a.params.Blacklisted(listed) + } + + timer := time.NewTimer(a.params.BlacklistRefresh) + + select { + case <-ctx.Done(): + timer.Stop() + + return + case <-timer.C: + } + } +} + // check checks client with AbuseIPDB by addr, one of its addresses, keeps // the score as client's, and notes the check as no longer under way. A // check that fails gives no score: it is counted, logged and raised as a @@ -344,6 +400,92 @@ func (a *AbuseIPDB) ask( return *answer.Data.AbuseConfidenceScore, nil } +// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored +// Threshold or more, sending the key in the header Key. An answer other +// than 200, one longer than maxListBytes or without the list, an entry +// whose address does not read, and none within fetchTimeout, fail. +func (a *AbuseIPDB) fetchBlacklist(ctx context.Context) ([]Listed, error) { + ctx, cancel := context.WithTimeout(ctx, fetchTimeout) + defer cancel() + + query := url.Values{ + "confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)}, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + a.params.BlacklistURL+"?"+query.Encode(), http.NoBody) + if err != nil { + return nil, fmt.Errorf("make the request: %w", err) + } + + req.Header.Set("Key", a.params.Key) + req.Header.Set("Accept", "application/json") + + res, err := a.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err)) + } + + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w %s", errStatus, res.Status) + } + + body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1)) + if err != nil { + return nil, fmt.Errorf("read the blacklist: %w", err) + } + + if len(body) > maxListBytes { + return nil, errTooLong + } + + return parseBlacklist(body) +} + +// parseBlacklist reads body, AbuseIPDB's answer with its blacklist. +func parseBlacklist(body []byte) ([]Listed, error) { + var answer struct { + Data *[]struct { + IPAddress string `json:"ipAddress"` + AbuseConfidenceScore int64 `json:"abuseConfidenceScore"` + } `json:"data"` + } + + err := json.Unmarshal(body, &answer) + if err != nil { + return nil, fmt.Errorf("read the blacklist: %w", err) + } + + if answer.Data == nil { + return nil, errNoBlacklist + } + + listed := make([]Listed, 0, len(*answer.Data)) + + for i, entry := range *answer.Data { + addr, err := netip.ParseAddr(entry.IPAddress) + if err != nil || addr.Zone() != "" { + // The error leaves out what the entry holds, as it does a + // client's address. + return nil, fmt.Errorf( + "entry %d's ipAddress %w", + i+1, + errNotAddress, + ) + } + + listed = append(listed, Listed{ + Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore, + }) + } + + return listed, nil +} + // alert raises a source_failure alert from AbuseIPDB with reason and err, // and logs them. func (a *AbuseIPDB) alert(reason string, err error) { diff --git a/internal/reputation/blacklist_test.go b/internal/reputation/blacklist_test.go new file mode 100644 index 0000000..d79b371 --- /dev/null +++ b/internal/reputation/blacklist_test.go @@ -0,0 +1,249 @@ +package reputation_test + +import ( + "context" + "fmt" + "io" + "net/http" + "net/netip" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// The tests of the blacklist run in synctest bubbles, as those of the +// checks do, with AbuseIPDB a stand-in reached without the network. + +// blacklistURL is where the tests fetch the blacklist from. +const blacklistURL = "https://abuseipdb.example/api/v2/blacklist" + +func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` + + `{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` + + `{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` + + `{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`} + got := runBlacklist(t, abuseIPDB, key, newQueue()) + + want := []reputation.Listed{ + {Addr: netip.MustParseAddr("203.0.113.9"), Score: 100}, + {Addr: netip.MustParseAddr("198.51.100.7"), Score: 95}, + {Addr: netip.MustParseAddr("2001:db8::9"), Score: 90}, + } + + synctest.Wait() + got.want(t, want) + + time.Sleep(time.Hour - time.Nanosecond) + synctest.Wait() + got.want(t, want) + + time.Sleep(time.Nanosecond) + synctest.Wait() + got.want(t, want, want) + + // Each fetch asks for the clients scored 90 or more, with the key + // in the header Key alone. + abuseIPDB.mu.Lock() + defer abuseIPDB.mu.Unlock() + + for _, req := range abuseIPDB.asked { + if req.URL.String() != blacklistURL+"?confidenceMinimum=90" || + req.Header.Get("Key") != key { + t.Errorf( + "asked %s with the key %q, want %s?confidenceMinimum=90 "+ + "with %q", + req.URL, + req.Header.Get("Key"), + blacklistURL, + key, + ) + } + } + }) +} + +func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh( + t *testing.T, +) { + t.Parallel() + + for _, tc := range []struct{ name, body, error string }{ + { + "a refusal", "", + "the server answered 429 Too Many Requests", + }, + { + "an answer without the list", `{"errors":[]}`, + "the answer gives no data", + }, + { + "an entry that is not an address", + `{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`, + "entry 2's ipAddress is not an address", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &blacklistStandIn{body: tc.body} + if tc.body == "" { + abuseIPDB.status = http.StatusTooManyRequests + } + + queue := newQueue() + got := runBlacklist(t, abuseIPDB, key, queue) + + synctest.Wait() + + got.want(t) + + alerted := waiting(queue) + if len(alerted) != 1 || + alerted[0].Event != alerts.EventSourceFailure || + alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" || + alerted[0].Detail["source"] != reputation.AbuseIPDBSource || + alerted[0].Detail["error"] != tc.error { + t.Errorf( + "alerts %+v, want the failure %q", + alerted, + tc.error, + ) + } + + abuseIPDB.answer(`{"data":[]}`) + time.Sleep(time.Hour) + synctest.Wait() + got.want(t, []reputation.Listed{}) + }) + }) + } +} + +func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &blacklistStandIn{body: `{"data":[]}`} + got := runBlacklist(t, abuseIPDB, "", newQueue()) + + time.Sleep(time.Hour) + synctest.Wait() + got.want(t) + + if len(abuseIPDB.asked) != 0 { + t.Errorf("fetched the blacklist %d times, want none", + len(abuseIPDB.asked)) + } + }) +} + +// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of +// the blacklist with status, 200 when it is zero, and body, and keeps the +// requests. +type blacklistStandIn struct { + mu sync.Mutex + status int + body string + asked []*http.Request +} + +// RoundTrip has the stand-in answer req, in place of the network. +func (s *blacklistStandIn) RoundTrip( + req *http.Request, +) (*http.Response, error) { + s.mu.Lock() + s.asked = append(s.asked, req) + status, body := s.status, s.body + s.mu.Unlock() + + if status == 0 { + status = http.StatusOK + } + + return &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(body)), + Request: req, + }, nil +} + +// answer has the stand-in answer each fetch with 200 and body. +func (s *blacklistStandIn) answer(body string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.status, s.body = 0, body +} + +// blacklisted holds the clients on each blacklist given to Blacklisted. +type blacklisted struct { + mu sync.Mutex + fetched [][]reputation.Listed +} + +// want checks the clients on each blacklist given so far. +func (b *blacklisted) want(t *testing.T, want ...[]reputation.Listed) { + t.Helper() + + b.mu.Lock() + defer b.mu.Unlock() + + if !slices.EqualFunc(b.fetched, want, slices.Equal) { + t.Errorf("blacklists %v, want %v", b.fetched, want) + } +} + +// runBlacklist runs RunBlacklist, until the test ends, with the key +// accountKey, the threshold 90 and a refresh of an hour, fetching from +// abuseIPDB and raising alerts into queue, and returns what it gives +// Blacklisted. +func runBlacklist( + t *testing.T, abuseIPDB *blacklistStandIn, accountKey string, + queue *alerts.Queue, +) *blacklisted { + t.Helper() + + got := &blacklisted{} + p := abuseIPDBParams() + p.Key = accountKey + p.BlacklistURL = blacklistURL + p.Threshold = 90 + p.BlacklistRefresh = time.Hour + p.Alerts = queue + p.Blacklisted = func(listed []reputation.Listed) { + got.mu.Lock() + defer got.mu.Unlock() + + got.fetched = append(got.fetched, listed) + } + + checker := reputation.NewAbuseIPDB(p) + checker.SetTransport(abuseIPDB) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + + go func() { + checker.RunBlacklist(ctx) + close(done) + }() + + t.Cleanup(func() { + cancel() + <-done + }) + + return got +} diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index 2b22cf7..a8790e9 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -7,7 +7,8 @@ // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // clients with AbuseIPDB, and keeps their scores and the checks spent // today. The state package writes all of these to files and reads them -// from there, so that a restart keeps them too. +// from there, so that a restart keeps them too. Last, it fetches +// AbuseIPDB's blacklist, which it does not keep. package reputation import ( diff --git a/internal/state/state.go b/internal/state/state.go index 9a04cc9..50bc991 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -104,7 +104,7 @@ var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. errMissing = errors.New("has no") - errCause = errors.New("is not limit, attack or admin") + errCause = errors.New("is not limit, attack, admin or abuseipdb") errExpires = errors.New("is not after its start") errDestination = errors.New("is not webhook, slack or ntfy") errScope = errors.New( @@ -967,7 +967,7 @@ func (e BanEntry) ban() bans.Ban { // each expires is read again as written. An expires not after the start // is refused too, since the netblock's next ban for a broken limit, three // times as long as this one, would then refuse nothing; and so is a cause -// other than limit, attack or admin, most likely misspelt. +// other than limit, attack, admin or abuseipdb, most likely misspelt. func (f *bansFile) check(data []byte) error { var written struct { Bans []struct { @@ -991,7 +991,8 @@ func (f *bansFile) check(data []byte) error { case entry.Expires != nil && !entry.Expires.After(entry.Start): return fmt.Errorf("entry %d's expires %w", i+1, errExpires) case entry.Cause != "" && entry.Cause != bans.CauseLimit && - entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin: + entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin && + entry.Cause != bans.CauseAbuseIPDB: return fmt.Errorf( "entry %d's cause %q %w", i+1, diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 2ee13d9..79609c8 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -114,7 +114,8 @@ const permanentBansJSON = `{ "earlier_bans": { "limit": 3, "attack": 1, - "admin": 1 + "admin": 1, + "abuseipdb": 0 } } } @@ -1190,7 +1191,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { `"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+ `{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "atack"}]}`, - `: entry 4's cause "atack" is not limit, attack or admin`) + `: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`) } func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) { diff --git a/internal/sww/sww.go b/internal/sww/sww.go index 7befcde..80e4f26 100644 --- a/internal/sww/sww.go +++ b/internal/sww/sww.go @@ -278,6 +278,8 @@ func newServer( Now: now, Rules: ruleFiles, Alerts: alertQueue, + + AbuseIPDBBlacklistURL: reputation.AbuseIPDBBlacklistURL, }) server.Metrics.AddAlerts(alertQueue) @@ -377,7 +379,8 @@ func startSending( // serve serves requests on listener, writes the state and cache files as they // are due, takes in an admin's edits of them, reads the rule files again as // they change, and the lookup database when it is replaced, fetches the -// lists the settings name by URL as they are due, and sends the alerts, +// lists the settings name by URL as they are due, bans the clients on +// AbuseIPDB's blacklist as it is fetched, and sends the alerts, // until ctx is done. Then it gives the requests in progress // shutdownTimeout to finish, and writes every state and cache file, // alerts.json with the alerts still waiting. @@ -404,6 +407,9 @@ func serve( } }) listsFetched := inBackground(func() { server.Lists.Run(writing) }) + blacklistFetched := inBackground(func() { + server.AbuseIPDB.RunBlacklist(writing) + }) alertsSent := inBackground(func() { alertQueue.Run(writing) }) select { @@ -446,6 +452,7 @@ func serve( <-rulesWatched <-lookupFileWatched <-listsFetched + <-blacklistFetched <-alertsSent err = files.WriteAll() diff --git a/internal/waf/abuseipdbblacklist.go b/internal/waf/abuseipdbblacklist.go new file mode 100644 index 0000000..0993047 --- /dev/null +++ b/internal/waf/abuseipdbblacklist.go @@ -0,0 +1,39 @@ +package waf + +// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and +// SWW_WAF_MODE; the blacklist is fetched by the reputation package. + +import ( + "net/netip" + + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// banAbuseIPDBBlacklist bans the netblock of each client on AbuseIPDB's +// blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS, as the ledger's +// BanForAbuseIPDB describes, and logs how many clients were listed and how +// many bans were made. Clients in one netblock make one ban, with the +// highest of their scores. In observe mode it makes no ban. +func (h *handler) banAbuseIPDBBlacklist(listed []reputation.Listed) { + scores := make(map[netip.Prefix]int64, len(listed)) + + for _, client := range listed { + netblock := h.netblock(client.Addr) + scores[netblock] = max(scores[netblock], client.Score) + } + + made := 0 + + if !h.config.Observe { + now := h.now() + made = h.ledger.BanForAbuseIPDB( + now, now.Add(h.config.AbuseIPDBBanDuration), scores, + ) + } + + h.processLog.Info( + "fetched AbuseIPDB's blacklist", + "clients", len(listed), + "bans_made", made, + ) +} diff --git a/internal/waf/abuseipdbblacklist_test.go b/internal/waf/abuseipdbblacklist_test.go new file mode 100644 index 0000000..738a756 --- /dev/null +++ b/internal/waf/abuseipdbblacklist_test.go @@ -0,0 +1,69 @@ +package waf_test + +import ( + "net/http" + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/reputation" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's +// blacklist lasts, in days. +const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS" + +func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{ + abuseIPDBBanDays: "2", banScopeV4Prefix: "24", + }) + + // Two clients in one netblock make one ban, with the higher score. + server.BanAbuseIPDBBlacklist([]reputation.Listed{ + {Addr: netip.MustParseAddr(client), Score: 95}, + {Addr: netip.MustParseAddr(otherClient), Score: 100}, + {Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90}, + }) + + held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet)) + if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB || + *held[0].Notes.Reputation[0].Score != 100 { + t.Errorf("bans on %s %+v, want one for the blacklist, with 100", + clientsNet, held) + } + + // Every client of a banned netblock is refused, the IPv6 address's /64 + // included. + for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} { + s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned) + } + + s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward) + + clk.advance(2 * 24 * time.Hour) + s.get(client, http.StatusOK, requestlog.ActionForward) +} + +func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock( + t, + "", + map[string]string{mode: "observe"}, + ) + + server.BanAbuseIPDBBlacklist([]reputation.Listed{ + {Addr: netip.MustParseAddr(client), Score: 100}, + }) + + if active, _ := server.Ledger.Count(clk.Now()); active != 0 { + t.Errorf("%d bans active, want none", active) + } + + s.get(client, http.StatusOK, requestlog.ActionForward) +} diff --git a/internal/waf/export_test.go b/internal/waf/export_test.go index 60b6e9b..e8d641c 100644 --- a/internal/waf/export_test.go +++ b/internal/waf/export_test.go @@ -1,6 +1,10 @@ package waf -import "time" +import ( + "time" + + "sneak.berlin/go/smallwebwaf/internal/reputation" +) // BesideLookupTimeLimit is besideLookupTimeLimit. const BesideLookupTimeLimit = besideLookupTimeLimit @@ -11,3 +15,10 @@ func (s *Server) SetBesideLookupTimeLimit(limit time.Duration) { h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one h.besideLookupLimit = limit } + +// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with +// the clients on listed does. +func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.Listed) { + h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one + h.banAbuseIPDBBlacklist(listed) +} diff --git a/internal/waf/proxy.go b/internal/waf/proxy.go index 9179ae4..7a0c029 100644 --- a/internal/waf/proxy.go +++ b/internal/waf/proxy.go @@ -62,6 +62,10 @@ type Params struct { // AbuseIPDBURL is where clients are checked with AbuseIPDB while // SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL. AbuseIPDBURL string + // AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from + // while SWW_WAF_ABUSEIPDB_KEY is set, normally + // reputation.AbuseIPDBBlacklistURL. + AbuseIPDBBlacklistURL string // LookupFile is the lookup database they are looked up in while // SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise. LookupFile *lookup.File @@ -120,7 +124,6 @@ func newHandler(params Params) *handler { params.Config.MetricsBusiestCountriesAndASNs, params.Config.InstanceName, ) - lists, dnsbl, abuseIPDB := newReputation(params, m) awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) if err != nil { @@ -161,15 +164,16 @@ func newHandler(params Params) *handler { Alerts: params.Alerts, }), lookupFile: params.LookupFile, - lists: lists, - dnsbl: dnsbl, - abuseIPDB: abuseIPDB, rules: params.Rules, coreRuleSet: params.CoreRuleSet, alerts: params.Alerts, besideLookupLimit: besideLookupTimeLimit, } + h.lists, h.dnsbl, h.abuseIPDB = newReputation( + params, m, h.banAbuseIPDBBlacklist, + ) + if params.Config.RouteWatchURL != "" { h.routeWatch = routewatch.New(routewatch.Params{ URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog, @@ -199,9 +203,10 @@ func newHandler(params Params) *handler { // newReputation returns the lists fetched from URLs, the DNSBL zones' // verdicts and AbuseIPDB's scores, as the settings in params name them, // with none fetched, asked for or checked yet, and adds their metrics to -// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set. +// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set. blacklisted receives +// the clients on each AbuseIPDB blacklist fetched. func newReputation( - params Params, m *metrics.Metrics, + params Params, m *metrics.Metrics, blacklisted func([]reputation.Listed), ) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) { cfg := params.Config lists := reputation.New(reputation.Params{ @@ -222,7 +227,9 @@ func newReputation( abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{ URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore, DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL, - Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog, + Timeout: cfg.DNSBLAbuseIPDBTimeout, BlacklistURL: params.AbuseIPDBBlacklistURL, + Threshold: cfg.AbuseIPDBThreshold, BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh, + Blacklisted: blacklisted, Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts, })