Compare commits

..
Author SHA1 Message Date
sneak f6df989e67 coreruleset: time worst-case inspections in processor time
check / check (push) Failing after 2m9s
Not offered as a PR: processor time alone does not keep the test
from failing on a busy host. See the issue for the decision needed.

Model: opus-5-5
2026-11-16 07:13:40 +00:00
102 changed files with 2303 additions and 5777 deletions

No files matched your search

-14
View File
@@ -161,20 +161,6 @@ RUN git checkout --quiet --detach b4b2c785308b1ce785b6155c7fe5f16879080193 \
# ubuntu 26.04, 2026-09-27
FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7
# The image's own labels, in place of the ones it would inherit from
# Ubuntu's. script/docker and script/cibuild pass the build arguments;
# a build that passes none labels the image unknown rather than leave
# Ubuntu's values.
ARG VERSION=unknown
ARG CREATED=unknown
ARG REVISION=unknown
LABEL org.opencontainers.image.title="smallwebwaf" \
org.opencontainers.image.description="A web application firewall for self-hosted web apps: sww rproxy in front of an app, and sww waf, which decides." \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.created="${CREATED}" \
org.opencontainers.image.revision="${REVISION}" \
org.opencontainers.image.source="https://git.eeqj.de/sneak/smallwebwaf"
# runit's install creates its _runit-log user with minsysusers, which
# reads this file in place of runit's /usr/lib/sysusers.d/runit.conf.
# runit's line leaves out the shell, and minsysusers prints a Perl
+128 -282
View File
@@ -75,8 +75,7 @@ Together, `rproxy` and `waf` protect against:
- give the clients of chosen networks and countries a fraction of every
limit, so that they are banned sooner;
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high
AbuseIPDB score;
- ban the clients on AbuseIPDB's blacklist before they send a request.
AbuseIPDB score.
For the operator, it:
@@ -285,136 +284,129 @@ account, a key, a token or an alert destination stays off until you give one.
`SWW_WAF_BYTES_COUNTED` says. Headers are never counted. What the visitor sent
over a WebSocket counts as request bytes, and what the app sent as response
bytes.
- An anomaly is more requests or bytes than an `SWW_WAF_ANOMALY_` threshold
allows. Crossing a threshold sends one `anomaly` alert to the alert
destinations, and does nothing else: no client is banned, rate limited or
blocked.
- An invalid setting stops the process at start with one log line naming the
variable and what is wrong with the value. The settings in effect are logged
on one line at start, with tokens, keys and webhook paths shown as `********`.
Every setting the code reads, with its default:
| Variable | Default | What it does |
| --------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. |
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. |
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps for broken limits and attacks; bans an admin made and bans for AbuseIPDB's blacklist do not count. |
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. |
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. |
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. |
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. |
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account. Set, a client that has committed an offence is checked in the background, and the clients on AbuseIPDB's blacklist are banned. |
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
| `SWW_WAF_ABUSEIPDB_THRESHOLD` | `90` | The least AbuseIPDB score, from 25 to 100, at which a client on its blacklist is banned. Do not set it below 50 to 75: lower scores list ordinary visitors. |
| `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` | `60` | How often AbuseIPDB's blacklist is fetched again, in minutes. |
| `SWW_WAF_ABUSEIPDB_BAN_DAYS` | `30` | How long a client on AbuseIPDB's blacklist is banned, in days. |
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. |
| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. |
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. Those thresholds only send alerts. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
| Variable | Default | What it does |
| ------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. |
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. |
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps of its own; bans an admin made do not count. |
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, and how long a request to GeoJS may take, in milliseconds. |
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. |
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. |
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. |
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. |
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. |
| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. |
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Alert when one client makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Alert when one client makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in a minute, a size. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in an hour, a size. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Alert when the netblock around a client makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Alert when the netblock around a client makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in a minute, a size. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in an hour, a size. |
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. |
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Alert when one AS number makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Alert when one AS number makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in a minute, a size. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in an hour, a size. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Alert when the whole service gets more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Alert when the whole service gets more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in a minute, a size. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in an hour, a size. |
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Alert when a watched netblock makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Alert when a watched netblock makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in a minute, a size. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in an hour, a size. |
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in
full, and the limits that are fixed rather than settings.
@@ -424,8 +416,8 @@ full, and the limits that are fixed rather than settings.
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
`make docker` gives the image; on another host, use the name you pushed it
under, by digest. Docker Compose puts the services on a private network of its
own, where each reaches the others by service name. Each example but "On the
open internet" publishes `rproxy`'s port on the host, for trying it out.
own, where each reaches the others by service name. Each example publishes
`rproxy`'s port on the host, for trying it out.
### Smallest setup
@@ -457,142 +449,6 @@ volumes:
waf-state:
```
### On the open internet
One gitea at `git.example.com`, with traefik in front of `rproxy`. traefik gets
the certificate from Let's Encrypt and is the only service with published ports.
`waf` sets what most sites set. More `waf` settings follow, commented out, each
with what it does and when to set it.
```yaml
services:
traefik:
# traefik v3.7.19, 2026-11-13
image: traefik@sha256:a6c0194dda77bcb804824580f36fa7edc56e315b1acfe933366c8b52844a78a4
restart: unless-stopped
command:
- --providers.docker=true
- --providers.docker.exposedByDefault=false
- --entryPoints.web.address=:80
- --entryPoints.websecure.address=:443
# Redirect plain HTTP to HTTPS.
- --entryPoints.web.http.redirections.entryPoint.to=websecure
- --entryPoints.web.http.redirections.entryPoint.scheme=https
# A Let's Encrypt certificate for each site, by the TLS-ALPN
# challenge on port 443.
- --entryPoints.websecure.http.tls.certResolver=le
- --certificatesResolvers.le.acme.email=hostmaster@example.com
- --certificatesResolvers.le.acme.storage=/letsencrypt/acme.json
- --certificatesResolvers.le.acme.tlsChallenge=true
# Give a request 10 minutes to arrive, rather than 1, for the
# large git pushes the rproxy lines below accept.
# - --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s
ports:
- "80:80"
- "443:443"
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- acme:/letsencrypt
app:
# Your app: here a gitea, which listens on port 3000.
image: registry.example/app@sha256:0000000000000000000000000000000000000000000000000000000000000000
restart: unless-stopped
rproxy:
image: smallwebwaf
restart: unless-stopped
environment:
SWW_RPROXY_WAF_BACKEND: http://waf:8080
SWW_RPROXY_UPSTREAM_URL: http://app:3000
# Accept git pushes up to 1 GiB, rather than 100 MiB, sent in
# up to 10 minutes, rather than 1, with traefik's readTimeout
# line above.
# SWW_RPROXY_MAX_REQUEST_SIZE: 1G
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
labels:
- traefik.enable=true
- traefik.http.routers.git.rule=Host(`git.example.com`)
- traefik.http.routers.git.entryPoints=websecure
- traefik.http.services.git.loadBalancer.server.port=8080
waf:
image: smallwebwaf
restart: unless-stopped
environment:
SWW_COMMAND: waf
# The name in every log line, alert and metric. The default,
# the container's host name, is a random id under Compose.
SWW_WAF_INSTANCE_NAME: host1/waf
# The app each hostname serves. Only that app's fixes to the
# Core Rule Set apply there.
SWW_WAF_APPS: git.example.com:gitea
# Monitoring skips every check.
SWW_WAF_ALLOW_NETS: 192.0.2.10
# The office and the CI runner, which would exceed the rate
# limits, are never rate limited.
SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
# WordPress paths, which only scanners ask a gitea for. One
# request for one bans the client.
SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php
# Spamhaus's list of netblocks hijacked or leased by
# criminals. Their clients are refused.
SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt
# The token for the ban endpoints, to lift a ban without
# editing bans.json.
SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
# Alerts, such as one for each ban, to an ntfy topic.
SWW_WAF_ALERT_NTFY_URL: https://ntfy.example.com/smallwebwaf
# Refuse nothing and ban no one, for the first days. Each log
# line says what enforce mode would have done.
# SWW_WAF_MODE: observe
# The access token for an ntfy topic that is not public.
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
# The token for /_sww/metrics, for Prometheus. Unset, the
# metrics are off.
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
# 600 requests an hour rather than 1500, to ban slower
# scrapers. A fast human reader can reach it too.
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
# A quarter of every limit for clients in two hosting
# networks, where scrapers run rather than people.
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:25,asn=16276:25
# For a site only for people in a few countries. Clients
# elsewhere are refused, and so are any the lookup cannot place.
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: de,at,ch
# Close a refused request's connection rather than answer
# 403. traefik then answers 502, so a scraper cannot tell a
# ban from a broken site.
# SWW_WAF_BAN_RESPONSE: close
# A copy of every log line to a syslog server.
# SWW_WAF_LOG_REMOTE_URL: syslog+tls://logs.example.com:6514
volumes:
- waf-state:/var/lib/sww
- /srv/sww/tokens:/etc/sww/tokens:ro
volumes:
acme:
waf-state:
```
Change these before the first start:
- `git.example.com`, in the router rule and in `SWW_WAF_APPS`;
- the email address for Let's Encrypt;
- the ntfy topic's URL;
- the app's image and port;
- the netblocks, which are documentation addresses.
Make the admin token file first, as for any token file:
```sh
mkdir -p /srv/sww/tokens
openssl rand -hex 32 > /srv/sww/tokens/admin
chown 65532:65532 /srv/sww/tokens/admin
chmod 0400 /srv/sww/tokens/admin
```
### Custom per-address limits
For a site with no API:
@@ -743,11 +599,6 @@ request reaches the app.
16. A path under `/_sww/` is answered `404`, so that the app never sees that
prefix.
Steps 12 to 14 only read the request. While the lookup waits for GeoJS, they run
at the same time as steps 6 to 11. The first of the two sides to refuse the
request ends the checks, and `waf` answers at once. Steps 12 to 14 have 500 ms
together.
`rproxy` takes the rest, with `waf` counting at the end:
17. The request is passed to the app, streaming.
@@ -760,20 +611,17 @@ A refused visitor is answered with `SWW_WAF_BAN_RESPONSE`. A `block` rule or a
Core Rule Set match answers `403` and bans nobody, since the Core Rule Set has
false positives; such refusals count toward `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE`
instead. A request over the header limits of step 3 is answered `414` for a
request line over 8 KiB and `431` otherwise. It bans nobody either. Nor does a
request that steps 12 to 14 have not finished within 500 ms, which is answered
`503`. `rproxy` itself answers `405` to a method it does not pass on, and the
status for a size or time limit, such as `413` for a body too large.
request line over 8 KiB and `431` otherwise. It bans nobody either. `rproxy`
itself answers `405` to a method it does not pass on, and the status for a size
or time limit, such as `413` for a body too large.
### Clients and bans
- A client is one IPv4 address or one IPv6 netblock of
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
- Each limit is counted over a sliding window with a few integers per client.
The table of clients and the bans for limits and attacks each have a bound
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`). The bans for AbuseIPDB's
blacklist are bounded by the clients it listed over the last
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days. So memory is bounded too.
The table of clients and the table of bans each have a bound
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`), so memory is bounded too.
- A ban refuses every request from its netblock until it ends. Requests refused
under a ban count toward no limit. A permanent ban ends only when an admin
lifts it.
@@ -896,16 +744,14 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
- `cmd/sww`: the binary, which only calls `internal/sww`.
- `internal/sww`: the process, as the command it is given, and the health check.
It is the only package that imports `go.uber.org/fx`, which builds the parts
of `waf` and `rproxy`.
- `internal/config`: reads the settings, the one place they are read.
- `internal/rproxy`: the reverse proxy in front of one app.
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
endpoints.
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each.
`internal/coreruleset`, `internal/lookup`, `internal/reputation`,
`internal/routewatch`, `internal/anomaly` and `internal/alerts`: one check or
source each.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
`internal/metrics`: the state and cache files, the log lines, the syslog
sender and the metrics.
+44 -123
View File
@@ -69,9 +69,7 @@ never as a time.
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
With no command, or one it does not know, the binary prints the commands
and exits with status 2. `internal/sww` is the only package that imports
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work.
and exits with status 2.
- `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
@@ -89,13 +87,10 @@ never as a time.
change, and tells which rules a request matches.
- `internal/coreruleset`: the Core Rule Set, with the five changes and the
apps' fixes listed under "Attack detection", as Coraza directives.
- `internal/geojs`: looks clients up through GeoJS and keeps the answers. It
imports no other package of this repository: `waf` gives it what it needs
to count and report failures.
- `internal/lookup`: looks clients up in the lookup database.
- `internal/lookup`: looks clients up through GeoJS or in the lookup
database.
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
AbuseIPDB in the background, and keeps their answers. It also fetches
AbuseIPDB's blacklist, which `internal/waf` bans.
AbuseIPDB in the background, and keeps their answers.
- `internal/routewatch`: asks a routewatch instance about clients in the
background, and keeps its answers.
- `internal/anomaly`: the anomaly counters and their alerts.
@@ -176,8 +171,7 @@ field gives that name. `refusal_limit` and `byte_limits` give theirs in
`country=` tag.
Otherwise the request goes on at once. A request that ends before the answer
comes is logged without it. While the request waits, steps 12 to 14 run at
the same time as steps 6 to 11 (see "Answering in time").
comes is logged without it.
7. The country list (`countries`; see "Country list").
8. The reputation sources (see "Reputation"), in this order:
@@ -241,15 +235,13 @@ apply.
load.
- Memory is bounded. When the table of clients holds
`SWW_WAF_MAX_TRACKED_CLIENTS`, the least recently seen client is dropped
first, with its history. The bans for broken limits and clear signs of attack
are bounded by `SWW_WAF_MAX_BANS` (see "Bans"). Those for AbuseIPDB's
blacklist are bounded by the clients it listed over the last
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days (see "Reputation"). The GeoJS answers, the
reputation verdicts and scores, the DNS queries under way and the anomaly
counters have fixed bounds, listed under "Fixed limits" in `docs/settings.md`.
Past its bound, the GeoJS answer or anomaly counter used longest ago goes
first, and the verdict or score fetched longest ago. Past the bound on DNS
queries under way, a zone is asked at the client's next request.
first, with its history. The bans are bounded by `SWW_WAF_MAX_BANS` (see
"Bans"). The GeoJS answers, the reputation verdicts and scores, the DNS
queries under way and the anomaly counters have fixed bounds, listed under
"Fixed limits" in `docs/settings.md`. Past its bound, the GeoJS answer or
anomaly counter used longest ago goes first, and the verdict or score fetched
longest ago. Past the bound on DNS queries under way, a zone is asked at the
client's next request.
## Bans
@@ -304,9 +296,6 @@ to a ban, each by its own rule:
Once it ends, only new requests can break a limit again. The client's
history keeps its totals.
A client on AbuseIPDB's blacklist is banned before it sends a request, with the
cause `abuseipdb` (see "Reputation").
A Core Rule Set match refuses only the request it matched, with `403` and
without a ban, because the Core Rule Set has false positives. A `block` rule
does the same. Those refusals count toward the refusal limit, so a client that
@@ -333,12 +322,11 @@ does not count toward a longer ban; it refuses nothing. Deleting its entry from
A permanent ban does not run out. It ends when an admin lifts it, or when `waf`
drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans
for broken limits and clear signs of attack, it drops the earliest such ban of
one netblock: the netblock that has gone longest without a request. A past,
active or permanent ban is dropped alike. A scanner whose ban was dropped that
way is refused and banned again by its next probe. Bans whose cause is `admin`
or `abuseipdb` do not count toward `SWW_WAF_MAX_BANS` and are never dropped to
make room, so they can never push out the others.
of its own, it drops the earliest ban of one netblock: the netblock that has
gone longest without a request. A past, active or permanent ban is dropped
alike. A scanner whose ban was dropped that way is refused and banned again by
its next probe. A ban whose cause is `admin` is never dropped and does not count
toward `SWW_WAF_MAX_BANS`, so such bans can never fill the table.
A request refused under a ban that runs out is logged with the action
`temporarily_banned`. Under a permanent ban the action is `permanently_banned`.
@@ -519,25 +507,19 @@ service that keeps its visitors' addresses on its own host.
dropped from that table. The answer reaches the client's history and ban notes
when it comes.
- `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` is how long a visitor's request
waits for its client's first answer, when a setting needs it (see step 6). A
client whose answer has not come in time counts as unknown until it comes, and
its later requests do not wait.
- A request to GeoJS may take up to 10 seconds, however long visitors wait. It
goes on after the visitor's request ends, and its answer is kept. Say GeoJS
answers after 5 seconds. The visitor's request is decided after 800
milliseconds, without a country. A request from the same client 6 seconds
later gets the answer.
- Requests from one client that come together share one request to GeoJS.
waits for its client's first answer, when a setting needs it (see step 6). It
also bounds each request to GeoJS. A client whose answer has not come in time
counts as unknown until it comes, and its later requests do not wait. Its
address is asked about again in the background.
- A client the lookup cannot place has an unknown country and AS number, which
the country list and the biased thresholds treat as described there. Nothing
else treats it differently.
- GeoJS publishes no rate limit, but its terms forbid "an excessive amount of
API requests", judged by GeoJS alone, and let it block a caller. A request to
GeoJS fails when it takes over 10 seconds or GeoJS answers with an error.
While GeoJS is slow, down or refusing `waf`, new clients count as unknown.
After a failure GeoJS is left alone for a second, and twice as long after each
further failure, up to five minutes. Each failure raises a `source_failure`
alert (see "Alerting"). A service that cannot accept this uses the file.
API requests", judged by GeoJS alone, and let it block a caller. While GeoJS
is slow, down or refusing `waf`, new clients count as unknown. GeoJS is then
left alone for a second, and twice as long after each further failure, up to
five minutes. Each failure raises a `source_failure` alert (see "Alerting"). A
service that cannot accept this uses the file.
- The file is the free IPinfo Lite database (`ipinfo_lite.mmdb`), whose `asn`,
`as_name` and `country_code` fields are read. The operator downloads it with a
free IPinfo account and mounts it read-only; `waf` never fetches it and holds
@@ -2045,26 +2027,6 @@ host, sharing one address, would often fetch too close together.
- AbuseIPDB answers other than `200`;
- the answer gives no score;
- no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`.
- While `SWW_WAF_ABUSEIPDB_KEY` is set, AbuseIPDB's blacklist is fetched at the
start and every `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` minutes
(default `60`). It lists the clients AbuseIPDB scores at
`SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`) or more. The blacklist itself is
not kept.
- Each client on it is banned for `SWW_WAF_ABUSEIPDB_BAN_DAYS` days (default
`30`), with the cause `abuseipdb`. The ban covers the client's netblock,
as any ban does.
- A netblock that already has an active ban is left as it is. So is one
whose `abuseipdb` ban an admin lifted, until that ban would have ended.
- An `abuseipdb` ban that has ended is forgotten at the next of these:
- a fetch of the blacklist in `enforce` mode;
- a read of `bans.json`, at the start or after an admin's edit.
- In `observe` mode the blacklist makes no ban.
- The threshold takes 25 to 100. It should not be set below 50 to 75, since
lower scores list ordinary visitors. AbuseIPDB serves free accounts the
blacklist at the threshold 100, whatever is asked for.
- AbuseIPDB limits the fetches of the blacklist a day: 5 for a free account.
A fetch over the limit fails, and is logged and alerted. The default
refresh fetches it 24 times a day.
- `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is
`limit:25` by default. Such verdicts are less certain than a blocklist, so a
listed client gets a quarter of every request and byte limit rather than a
@@ -2079,10 +2041,9 @@ host, sharing one address, would often fetch too close together.
- Alerts go to a webhook as JSON, and to Slack and ntfy as formatted messages.
`docs/alerts.md` lists the events, the JSON object and the messages.
`SWW_WAF_ALERT_EVENTS` chooses which of the seven event types are sent; the
hourly `summary` is sent whatever it says. The bans an admin makes, and the
bans for AbuseIPDB's blacklist, raise no alert. In `observe` mode, a request
that would have made a ban still raises the ban's alert, with `mode` set to
`observe` in its `detail`.
hourly `summary` is sent whatever it says. The bans an admin makes raise no
alert. In `observe` mode, a request that would have made a ban still raises
the ban's alert, with `mode` set to `observe` in its `detail`.
- A repeat is an alert with the same event, and the same netblock, source, file
or anomaly scope, as one sent less than `SWW_WAF_ALERT_COOLDOWN` before. An
anomaly scope is one of the things counted under "Anomaly thresholds". A
@@ -2230,7 +2191,7 @@ lists the files, what each holds, and their sizes.
these stop the process with a message naming the file and position:
- a file that does not parse, or has an unknown `version`;
- an entry without a field it needs;
- a ban whose cause is not `limit`, `attack`, `admin` or `abuseipdb`;
- a ban whose cause is not one of the four;
- a ban whose `expires` is not after its `start`.
Starting with empty state would silently forgive every repeat offender.
@@ -2410,9 +2371,8 @@ runs no check, writes no request log line and counts nothing in the metrics.
### The answer
`waf` runs steps 3 to 16 of "What happens to a request", in that order, apart
from the steps that run at the same time (see "Answering in time"). Every answer
but the last below carries `X-Sww-Disposition`.
`waf` runs steps 3 to 16 of "What happens to a request", in that order. Every
answer but the last below carries `X-Sww-Disposition`.
- `X-Sww-Disposition: allow`, status `200`, no body. `rproxy` passes the request
to the app. While `SWW_WAF_ADD_LOOKUP_HEADERS` is `true`, the answer also
@@ -2424,8 +2384,6 @@ but the last below carries `X-Sww-Disposition`.
- `SWW_WAF_BAN_RESPONSE`, for a refusal;
- `403`, for a `block` rule or the Core Rule Set;
- `414` or `431`, for a request over the header limits;
- `503`, for a request that steps 12 to 14 have not finished within 500
milliseconds;
- `404`, for a path under `/_sww/` that passes the checks, so that the app
never sees that prefix.
- `X-Sww-Disposition: close`, status `200`, no body. This takes the place of a
@@ -2446,22 +2404,15 @@ The Core Rule Set inspects the body a question carries only while
`rproxy` stops waiting for `waf` after `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`, by
default 2 seconds, and `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` then
decides. Two things in a decision take time:
- the wait for GeoJS, at most `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS`, by
default 800 milliseconds;
- the trap paths, the rule files and the Core Rule Set, which have 500
milliseconds.
They run at the same time, so `waf` answers within the longer of the two, well
within `rproxy`'s wait.
decides. The longest a question waits on an outside source is
`SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS`, by default 800 milliseconds, well
within that time.
GeoJS is the only outside source a question can wait on. A question waits only
when step 6 says so, and for at most
`SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS`, by default 800 milliseconds. A
client without an answer by then counts as unknown. The setting bounds only the
question's wait: the request to GeoJS goes on, and its answer is kept for later
questions. No question waits on these, which run in the background:
client without an answer by then counts as unknown. No question waits on these,
which run in the background:
- the DNS blocklists and AbuseIPDB;
- fetching the blocklists, the tag lists and the file
@@ -2486,38 +2437,10 @@ never wait. These can still wait for GeoJS:
- a country list, which needs the answer;
- a rate limit a percentage has lowered, which needs the answer;
- a DNS blocklist or AbuseIPDB, while `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` is
`deny`.
The trap paths, the rule files and the Core Rule Set, steps 12 to 14, run during
that wait. Once a request has passed the bans, the lookup starts. While it waits
for GeoJS, steps 12 to 14 run at the same time as steps 6 to 11. A lookup that
needs no wait has its answer at once, and steps 7 to 11 then run before steps 12
to 14 start.
- The first of the two sides to refuse the request ends the checks. `waf`
answers at once, and does not wait for the other side. The lookup goes on, and
its answer is kept.
- What the other side finds leaves no trace. It counts nothing, bans nothing,
raises no alert, counts no metric and adds nothing to the log line.
- When both sides would refuse, the one that finishes first is answered and
logged.
- When neither refuses, step 15 runs.
Steps 12 to 14 only read the request. They have 500 milliseconds together, from
their start. The limit is fixed, with no setting. A request they have not
finished by then is refused:
- the answer is `deny`, with `503`;
- the log line's action is `checks_timed_out`, and its `heuristic` names the
step that had not finished;
- the refusal makes no ban, is no offence and does not count toward the refusal
limit;
- it holds in every `SWW_WAF_CORE_RULE_SET_MODE`;
- in `observe` mode the request is answered `allow` with `would_action`, like
every refusal.
The limit is measured on the clock. While `waf`'s host is so busy that an
ordinary request takes 500 milliseconds, that request is refused too.
`deny`;
- a trap path;
- a rule file;
- the Core Rule Set.
The header limits are fixed, with no setting. A line's length is in bytes and
does not count its line ending. `waf` refuses a request whose request line is
@@ -2567,11 +2490,9 @@ body can add up to more than the body. From 1 KiB of JSON, Coraza makes at most
A request can reach all of these at once. At paranoia level 4, with no rule
switched off, the Core Rule Set takes under a second on one core of a current
server to inspect it. `waf` refuses it if the inspection takes over 500
milliseconds. Coraza cannot stop an inspection part-way, so the inspection runs
to its end with nobody waiting for it. These limits keep one request from
holding a processor for more than about a second. Enough such requests at once
can still overload `waf` past `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`.
server to inspect it. So `waf` answers it within the default 2 seconds, even
after the default 800 milliseconds of waiting for GeoJS. Enough such requests at
once can still overload `waf` past `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`.
### The report
-4
View File
@@ -9,10 +9,6 @@
ARG SWW_IMAGE=smallwebwaf
FROM ${SWW_IMAGE}
# Its own title and description, in place of the smallwebwaf image's.
LABEL org.opencontainers.image.title="smallwebwaf-example-app" \
org.opencontainers.image.description="An example app built on the smallwebwaf image."
# Packages the app needs, from the nixpkgs in the image.
RUN nix-env -iA nixpkgs.busybox
+2 -3
View File
@@ -6,7 +6,7 @@ back or dropped.
## Events
- `ban`: a ban `waf` made for a broken limit or a clear sign of attack.
- `ban`: a ban `waf` made.
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
a request.
- `core_rule_set_match`: the Core Rule Set scored a request at or over the
@@ -68,8 +68,7 @@ on one line:
"earlier_bans": {
"limit": 0,
"attack": 0,
"admin": 0,
"abuseipdb": 0
"admin": 0
}
}
},
-7
View File
@@ -105,8 +105,6 @@ Every one of these lines has:
`blocked_by_core_rule_set`: refused by that check;
- `request_line_too_long`: a request line over the header limits;
- `request_header_too_large`: headers over the header limits;
- `checks_timed_out`: refused with `503` because the trap paths, the rule
files and the Core Rule Set had not finished within 500 ms;
- `own_endpoint`: answered at one of `waf`'s own endpoints, or `404` for a
path under `/_sww/`.
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
@@ -136,11 +134,6 @@ Every one of these lines has:
`SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD`;
- `refusal_limit`: `window`, `count` and `limit`, of the refusal limit;
- `byte_limits`: `window`, `count` and `limit`, the count in bytes.
For a request refused as `checks_timed_out`, `heuristic` is the one of
`trap_paths`, `rule_files` and `core_rule_set` that had not finished, and
its figure is `time_limit_ms`, 500.
- `after_answer_heuristic`: the heuristic that banned the client after `waf`
answered the request, in `observe` mode too: `refusal_limit` or `byte_limits`.
- `after_answer_heuristic_figures`: an object of the figures that heuristic
+5 -20
View File
@@ -42,8 +42,8 @@ chooses between the two (see "The image's command" below).
client's group, from 32 to 128.
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
whole number above zero.
- `SWW_WAF_MAX_BANS` (default `5000`): the most bans `waf` keeps for broken
limits and attacks, a whole number above zero.
- `SWW_WAF_MAX_BANS` (default `5000`): how many of its own bans `waf` keeps, a
whole number above zero.
## Tokens
@@ -159,9 +159,8 @@ cut off; the client is banned.
- `SWW_WAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in
its `.mmdb` form, for `SWW_WAF_LOOKUP_SOURCE=file`.
- `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` (default `800`): how long a
request waits for its client's first answer from GeoJS. A time in
milliseconds; never `off`. It does not cut off the request to GeoJS, whose
answer is kept when it comes later.
request waits for its client's first answer from GeoJS, and how long a request
to GeoJS may take. A time in milliseconds; never `off`.
- `SWW_WAF_ADD_LOOKUP_HEADERS` (default `true`): `true` has `rproxy` pass the
app the client's AS number in `X-Client-ASN` and its country in
`X-Client-Country`. The default is `false` while `SWW_WAF_LOOKUP_SOURCE` is
@@ -338,13 +337,6 @@ tag that no request can get stops the start. A request can get only these:
or AbuseIPDB's score is used. A duration.
- `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a
zone or a check with AbuseIPDB may take. A time in milliseconds.
- `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`): the least abuse confidence score
of the clients on AbuseIPDB's blacklist, a whole number from 25 to 100. It
should not be set below 50 to 75, since lower scores list ordinary visitors.
- `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` (default `60`): how often
the blacklist is fetched again, in minutes. A whole number above zero.
- `SWW_WAF_ABUSEIPDB_BAN_DAYS` (default `30`): how long the ban of a client on
the blacklist lasts, in days. A whole number above zero.
## Alerting
@@ -371,11 +363,6 @@ tag that no request can get stops the start. A request can get only these:
## Anomaly thresholds
An anomaly is more requests or bytes than one of these thresholds allows.
Crossing a threshold sends one `anomaly` alert to the destinations under
"Alerting", and does nothing else: no client is banned, rate limited or blocked.
`SWW_WAF_ALERT_COOLDOWN` holds back repeats of that alert.
Each scope has four thresholds: the most requests and the most body bytes
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED`
names. Requests are counts, bytes are sizes. Each is `off` by default. The four
@@ -447,9 +434,7 @@ A few limits are fixed rather than settings:
`SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` is the longest a question waits on
an outside source. Keep it well under `rproxy`'s
`SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`, as the default is. The trap paths, the rule
files and the Core Rule Set run during that wait and have 500 ms, so `waf`
answers within the longer of the two times. No question waits for
`SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`, as the default is. No question waits for
`SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`, since the DNS blocklists and AbuseIPDB are
asked in the background (see "Answering in time" under "`waf`" in
[`SPEC.md`](../SPEC.md)).
+5 -19
View File
@@ -21,22 +21,11 @@ top-level `version` of 1.
## `bans.json`
Active and past bans:
- every ban an admin made;
- up to `SWW_WAF_MAX_BANS` of those `waf` made for a broken limit or an attack;
- the bans for AbuseIPDB's blacklist.
A ban for AbuseIPDB's blacklist that has ended is forgotten at the next of
these:
- a fetch of the blacklist in `enforce` mode;
- a read of `bans.json`, at the start or after an admin's edit.
Per entry:
Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of
those `waf` made. Per entry:
- `netblock`, `start` and `expires` (`null` for a permanent ban);
- `cause`: `attack`, `limit`, `admin` or `abuseipdb`;
- `cause`: `attack`, `limit` or `admin`;
- `reason`: a short reason;
- `lifted`: when an admin lifted it, if one did;
- `notes`: the notes below.
@@ -71,8 +60,7 @@ ban. The notes:
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
- `reputation`: the sources the client was on, each with its `source` and, for
AbuseIPDB, its `score`;
- `request`: the request that caused the ban, each text cut to 256 bytes, and
empty for a ban for AbuseIPDB's blacklist:
- `request`: the request that caused the ban, each text cut to 256 bytes:
- `time`, `method` and `host`;
- `path`, with its query string;
- `status` and `user_agent`;
@@ -163,9 +151,7 @@ A cache file. routewatch's answers from the last hour, one per line:
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
more. A ban for AbuseIPDB's blacklist takes about 1 KiB. A free account's
blacklist lists 10,000 clients, so it adds at least 10 MiB, and 1 GiB of
writes a day.
more.
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
full.
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
+1 -5
View File
@@ -12,8 +12,6 @@ require (
github.com/oschwald/maxminddb-golang/v2 v2.7.0
github.com/prometheus/client_golang v1.24.1
github.com/spf13/viper v1.21.0
go.uber.org/fx v1.24.0
golang.org/x/net v0.60.0
golang.org/x/sys v0.48.0
)
@@ -46,11 +44,9 @@ require (
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.26.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
-8
View File
@@ -97,16 +97,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+42 -55
View File
@@ -34,76 +34,65 @@ import (
"time"
)
// Event is what an alert is for, such as EventBan.
type Event string
// The events an alert is for, as SWW_WAF_ALERT_EVENTS names them.
const (
// EventBan is a ban sww made.
EventBan Event = "ban"
EventBan = "ban"
// EventPermanentBan is a permanent ban sww made, or a ban it
// made permanent.
EventPermanentBan Event = "permanent_ban"
EventPermanentBan = "permanent_ban"
// EventAnomaly is a count of requests or bytes over an anomaly
// threshold.
EventAnomaly Event = "anomaly"
EventAnomaly = "anomaly"
// EventCoreRuleSetMatch is a request the Core Rule Set scored at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, refused in block mode, let through in
// detect mode.
EventCoreRuleSetMatch Event = "core_rule_set_match"
EventCoreRuleSetMatch = "core_rule_set_match"
// EventReputationHit is a request whose client a blocklist or a DNSBL
// zone lists, or whose AbuseIPDB score is a hit.
EventReputationHit Event = "reputation_hit"
EventReputationHit = "reputation_hit"
// EventSourceFailure is GeoJS failing or refusing sww, a fetch
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
// failing or refused, or the day's AbuseIPDB checks used up.
EventSourceFailure Event = "source_failure"
EventSourceFailure = "source_failure"
// EventFileError is one of:
// - a rule file or state file edited while sww runs that does not parse;
// - a cache file that does not parse, at start or after an edit;
// - a state file or cache file that cannot be written;
// - a replacement of the lookup database that cannot be read.
EventFileError Event = "file_error"
EventFileError = "file_error"
// EventSummary is the summary sent as an hour ends: of the alerts held
// back in it past SWW_WAF_ALERT_MAX_PER_HOUR, and of the repeats held
// back by the cooldowns dropped as it ends, which no alert let through
// has given. It is sent with SWW_WAF_ALERT_MAX_PER_HOUR off too, for
// those repeats. SWW_WAF_ALERT_EVENTS does not name it.
EventSummary Event = "summary"
EventSummary = "summary"
)
// Events returns every event SWW_WAF_ALERT_EVENTS can name, which is its
// default.
func Events() []Event {
return []Event{
func Events() []string {
return []string{
EventBan, EventPermanentBan, EventCoreRuleSetMatch, EventAnomaly,
EventReputationHit, EventSourceFailure, EventFileError,
}
}
// DestinationName is the name of a destination alerts are sent to:
// DestinationWebhook, DestinationSlack or DestinationNtfy.
type DestinationName string
// The destinations alerts are sent to, as the metrics and alerts.json
// name them.
const (
// DestinationWebhook is the webhook SWW_WAF_ALERT_WEBHOOK_URL names.
DestinationWebhook DestinationName = "webhook"
DestinationWebhook = "webhook"
// DestinationSlack is the Slack incoming webhook
// SWW_WAF_ALERT_SLACK_WEBHOOK_URL names.
DestinationSlack DestinationName = "slack"
DestinationSlack = "slack"
// DestinationNtfy is the ntfy topic SWW_WAF_ALERT_NTFY_URL names.
DestinationNtfy DestinationName = "ntfy"
DestinationNtfy = "ntfy"
)
// Destinations returns every destination alerts can be sent to.
func Destinations() []DestinationName {
return []DestinationName{
DestinationWebhook,
DestinationSlack,
DestinationNtfy,
}
func Destinations() []string {
return []string{DestinationWebhook, DestinationSlack, DestinationNtfy}
}
const (
@@ -146,7 +135,7 @@ type Params struct {
NtfyURL *url.URL
NtfyToken string
// Events are the events alerts are sent for (SWW_WAF_ALERT_EVENTS).
Events []Event
Events []string
// Cooldown is how long a repeat of an alert is held back
// (SWW_WAF_ALERT_COOLDOWN), 0 for no time. MaxPerHour is the most alerts
// sent in an hour (SWW_WAF_ALERT_MAX_PER_HOUR), 0 for no limit.
@@ -170,7 +159,7 @@ type Params struct {
type Alert struct {
Instance string `json:"instance"`
Time time.Time `json:"time"`
Event Event `json:"event"`
Event string `json:"event"`
Host string `json:"host,omitempty"`
Client netip.Addr `json:"client"`
Netblock netip.Prefix `json:"netblock"`
@@ -197,7 +186,7 @@ type Alert struct {
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Cooldown struct {
Event Event `json:"event"`
Event string `json:"event"`
Netblock netip.Prefix `json:"netblock"`
File string `json:"file,omitempty"`
Source string `json:"source,omitempty"`
@@ -214,18 +203,18 @@ type Cooldown struct {
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Hour struct {
Start time.Time `json:"start"`
Sent int `json:"sent"`
HeldBack map[Event]int `json:"held_back"`
Start time.Time `json:"start"`
Sent int `json:"sent"`
HeldBack map[string]int `json:"held_back"`
}
// State is what alerts.json holds: the cooldowns, the hour under way, and
// for each destination set, the alerts waiting to be sent to it, oldest
// first.
type State struct {
Cooldowns []Cooldown `json:"cooldowns"`
Hour Hour `json:"hour"`
Waiting map[DestinationName][]Alert `json:"waiting"`
Cooldowns []Cooldown `json:"cooldowns"`
Hour Hour `json:"hour"`
Waiting map[string][]Alert `json:"waiting"`
}
// Counts are, for a destination, how many alerts it took, how many
@@ -259,7 +248,7 @@ type destination struct {
// name is how the metrics and alerts.json name the destination, and
// setting the setting that is its URL, which the log names in place
// of the URL.
name DestinationName
name string
setting string
url *url.URL
// message returns the body an alert is posted with, and the headers
@@ -284,7 +273,7 @@ type destination struct {
// scope with the same AS number or name, as its detail names them. Each
// is empty for an alert without one.
type cooldownKey struct {
event Event
event string
netblock netip.Prefix
file string
source string
@@ -317,7 +306,7 @@ func New(params Params) *Queue {
q := &Queue{
params: params,
cooldowns: map[cooldownKey]*Cooldown{},
hour: Hour{HeldBack: map[Event]int{}},
hour: Hour{HeldBack: map[string]int{}},
}
if params.WebhookURL != nil {
@@ -388,7 +377,7 @@ func (q *Queue) Raise(alert Alert) {
// event, no alert for event on netblock was let through less than
// Cooldown before, and fewer than MaxPerHour alerts have been let through
// in the hour under way. Unlike Raise, it counts nothing.
func (q *Queue) WouldSend(event Event, netblock netip.Prefix) bool {
func (q *Queue) WouldSend(event string, netblock netip.Prefix) bool {
if len(q.destinations) == 0 || !slices.Contains(q.params.Events, event) {
return false
}
@@ -444,7 +433,7 @@ func (q *Queue) Run(ctx context.Context) {
// Counts returns the counts of the destination name, all 0 for one not
// set.
func (q *Queue) Counts(name DestinationName) Counts {
func (q *Queue) Counts(name string) Counts {
for _, d := range q.destinations {
if d.name == name {
return Counts{
@@ -460,8 +449,8 @@ func (q *Queue) Counts(name DestinationName) Counts {
// DestinationsSet returns the destinations set, in the order of
// Destinations.
func (q *Queue) DestinationsSet() []DestinationName {
names := make([]DestinationName, 0, len(q.destinations))
func (q *Queue) DestinationsSet() []string {
names := make([]string, 0, len(q.destinations))
for _, d := range q.destinations {
names = append(names, d.name)
}
@@ -485,7 +474,7 @@ func (q *Queue) Snapshot() State {
state := State{
Cooldowns: make([]Cooldown, 0, len(q.cooldowns)),
Hour: q.hour,
Waiting: map[DestinationName][]Alert{},
Waiting: map[string][]Alert{},
}
state.Hour.HeldBack = maps.Clone(q.hour.HeldBack)
@@ -536,7 +525,7 @@ func (q *Queue) Load(state State) {
q.hour.HeldBack = maps.Clone(state.Hour.HeldBack)
if q.hour.HeldBack == nil {
q.hour.HeldBack = map[Event]int{}
q.hour.HeldBack = map[string]int{}
}
for _, d := range q.destinations {
@@ -548,7 +537,7 @@ func (q *Queue) Load(state State) {
// its URL, that URL, target, and message, which makes the messages sent
// to it.
func (q *Queue) addDestination(
name DestinationName, setting string, target *url.URL,
name, setting string, target *url.URL,
message func(alert *Alert) ([]byte, http.Header, error),
) {
q.destinations = append(q.destinations, &destination{
@@ -660,7 +649,7 @@ func (q *Queue) endHour(now time.Time) {
})
}
q.hour = Hour{Start: start, HeldBack: map[Event]int{}}
q.hour = Hour{Start: start, HeldBack: map[string]int{}}
}
// queue adds alert to the alerts waiting for each destination.
@@ -915,21 +904,20 @@ func (q *Queue) ntfyMessage(alert *Alert) ([]byte, http.Header, error) {
// ntfyPriority returns the priority an alert for event is published to
// ntfy with: high for an event the admin needs to look at.
func ntfyPriority(event Event) string {
func ntfyPriority(event string) string {
switch event {
case EventPermanentBan, EventAnomaly, EventSourceFailure, EventFileError:
return "high"
case EventReputationHit:
return "low"
case EventBan, EventCoreRuleSetMatch, EventSummary:
default: // ban, core_rule_set_match and summary
return "default"
}
return "default"
}
// ntfyTag returns the tag an alert for event is published to ntfy with,
// which ntfy shows as an emoji.
func ntfyTag(event Event) string {
func ntfyTag(event string) string {
switch event {
case EventBan, EventPermanentBan:
return "no_entry"
@@ -941,16 +929,15 @@ func ntfyTag(event Event) string {
return "label"
case EventSourceFailure, EventFileError:
return "warning"
case EventSummary:
default: // summary
return "bar_chart"
}
return "bar_chart"
}
// title returns the title of alert in Slack and ntfy: the instance and
// the event.
func title(alert *Alert) string {
return alert.Instance + ": " + string(alert.Event)
return alert.Instance + ": " + alert.Event
}
// text returns the text of alert in Slack and ntfy: its reason, then a
+20 -31
View File
@@ -126,7 +126,7 @@ func TestOnlyTheChosenEventsAreSent(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
params := newParams()
params.Events = []alerts.Event{
params.Events = []string{
alerts.EventSourceFailure,
alerts.EventFileError,
}
@@ -156,7 +156,7 @@ func TestWouldSendOnlyForTheChosenEvents(t *testing.T) {
t.Parallel()
params := newParams()
params.Events = []alerts.Event{
params.Events = []string{
alerts.EventSourceFailure,
alerts.EventFileError,
}
@@ -186,8 +186,8 @@ func TestRaiseDoesNothingWithoutADestination(t *testing.T) {
// No alert waits, no cooldown has started, and the hour counts none.
want := alerts.State{
Cooldowns: []alerts.Cooldown{},
Hour: alerts.Hour{HeldBack: map[alerts.Event]int{}},
Waiting: map[alerts.DestinationName][]alerts.Alert{},
Hour: alerts.Hour{HeldBack: map[string]int{}},
Waiting: map[string][]alerts.Alert{},
}
if got := q.Snapshot(); !reflect.DeepEqual(got, want) {
t.Errorf("state %+v, want %+v", got, want)
@@ -235,7 +235,7 @@ func TestRepeatWithinTheCooldownIsHeldBackAndCountedInTheNext(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
webhook, q := start(t, newParams())
raise := func(event alerts.Event, n int) {
raise := func(event string, n int) {
q.Raise(alerts.Alert{Event: event, Netblock: netblock(n)})
}
@@ -1014,7 +1014,7 @@ func TestNtfyTokenIsSentToNtfyAlone(t *testing.T) {
)
synctest.Wait()
for destination, want := range map[alerts.DestinationName][]string{
for destination, want := range map[string][]string{
alerts.DestinationWebhook: nil,
alerts.DestinationSlack: nil,
alerts.DestinationNtfy: tc.want,
@@ -1063,7 +1063,7 @@ func TestDestinationThatDoesNotAnswerHoldsUpNeitherOther(t *testing.T) {
// With no time passed, the webhook and ntfy have taken every alert,
// while Slack has not answered the first.
for destination, want := range map[alerts.DestinationName]alerts.Counts{
for destination, want := range map[string]alerts.Counts{
alerts.DestinationWebhook: {Sent: 3},
alerts.DestinationSlack: {},
alerts.DestinationNtfy: {Sent: 3},
@@ -1126,7 +1126,7 @@ func TestDropsAreCountedForEachDestination(t *testing.T) {
// the webhook takes every one.
const all = alerts.QueueSize + 1
for destination, want := range map[alerts.DestinationName]alerts.Counts{
for destination, want := range map[string]alerts.Counts{
alerts.DestinationWebhook: {Sent: all},
alerts.DestinationSlack: {Dropped: 1},
alerts.DestinationNtfy: {SendFailures: all, Dropped: all},
@@ -1152,21 +1152,11 @@ func TestEachDestinationIsSentOnlyTheAlertsWaitingForIt(t *testing.T) {
// As read from alerts.json. The alert waiting for the webhook, which
// is not set, is dropped.
q.Load(
roundTrip(
t,
alerts.State{
Waiting: map[alerts.DestinationName][]alerts.Alert{
alerts.DestinationWebhook: {waiting("first")},
alerts.DestinationSlack: {
waiting("second"),
waiting("third"),
},
alerts.DestinationNtfy: {waiting("fourth")},
},
},
),
)
q.Load(roundTrip(t, alerts.State{Waiting: map[string][]alerts.Alert{
alerts.DestinationWebhook: {waiting("first")},
alerts.DestinationSlack: {waiting("second"), waiting("third")},
alerts.DestinationNtfy: {waiting("fourth")},
}}))
synctest.Wait()
wantBodies(t, standIns[alerts.DestinationSlack],
@@ -1175,7 +1165,7 @@ func TestEachDestinationIsSentOnlyTheAlertsWaitingForIt(t *testing.T) {
wantBodies(t, standIns[alerts.DestinationNtfy], "fourth")
// alerts.json then lists Slack and ntfy alone, with no alert waiting.
want := map[alerts.DestinationName][]alerts.Alert{
want := map[string][]alerts.Alert{
alerts.DestinationSlack: {}, alerts.DestinationNtfy: {},
}
if got := q.Snapshot().Waiting; !reflect.DeepEqual(got, want) {
@@ -1344,11 +1334,11 @@ func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) {
func startAll(
t *testing.T,
params alerts.Params,
) (map[alerts.DestinationName]*standIn, *alerts.Queue) {
) (map[string]*standIn, *alerts.Queue) {
t.Helper()
q := alerts.New(params)
standIns := map[alerts.DestinationName]*standIn{}
standIns := map[string]*standIn{}
for _, destination := range q.DestinationsSet() {
standIns[destination] = &standIn{answers: answering}
@@ -1414,14 +1404,14 @@ func wantAlert(t *testing.T, got, want map[string]any) {
// wantEvents checks the events of the alerts the webhook was sent, in
// order.
func wantEvents(t *testing.T, webhook *standIn, want ...alerts.Event) {
func wantEvents(t *testing.T, webhook *standIn, want ...string) {
t.Helper()
got := make([]alerts.Event, 0, len(webhook.received()))
got := make([]string, 0, len(webhook.received()))
for _, request := range webhook.received() {
event, _ := request.alert["event"].(string)
got = append(got, alerts.Event(event))
got = append(got, event)
}
if !slices.Equal(got, want) {
@@ -1457,8 +1447,7 @@ func wantCounts(
// wantDestinationCounts checks q's counts for destination.
func wantDestinationCounts(
t *testing.T, q *alerts.Queue, destination alerts.DestinationName,
want alerts.Counts,
t *testing.T, q *alerts.Queue, destination string, want alerts.Counts,
) {
t.Helper()
+1 -3
View File
@@ -7,9 +7,7 @@ const QueueSize = queueSize
// SetTransport has q's requests to the destination name go through
// transport instead of the network.
func (q *Queue) SetTransport(
name DestinationName, transport http.RoundTripper,
) {
func (q *Queue) SetTransport(name string, transport http.RoundTripper) {
for _, d := range q.destinations {
if d.name == name {
d.httpClient.Transport = transport
+8 -10
View File
@@ -295,10 +295,9 @@ func (c *Counters) netAround(client netip.Addr) netip.Prefix {
// overThreshold is a count over its threshold: what it counts, requests or
// bytes, its window, the count and the threshold.
type overThreshold struct {
kind ratelimit.LimitKind
window string
count float64
threshold int64
kind, window string
count float64
threshold int64
}
// add counts a request of bytes at now in each of c's counts whose
@@ -347,10 +346,9 @@ func (c *Counter) add(
// bucketCount is one of a counter's four counts: requests or bytes, in a
// window of length, and the buckets they are counted in.
type bucketCount struct {
kind ratelimit.LimitKind
window string
length time.Duration
buckets *ratelimit.Buckets
kind, window string
length time.Duration
buckets *ratelimit.Buckets
}
// counts returns c's counts: requests and bytes in the minute, then in
@@ -406,8 +404,8 @@ func (k key) counter() *Counter {
// counted, the count and the threshold.
func alertFor(r Request, k key, o overThreshold) alerts.Alert {
detail := map[string]any{
"scope": k.scope, "window": o.window, "kind": string(o.kind),
"count": o.count, "threshold": o.threshold,
"scope": k.scope, "window": o.window, "kind": o.kind, "count": o.count,
"threshold": o.threshold,
}
var counted string
-169
View File
@@ -1,169 +0,0 @@
package bans_test
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
)
func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
expires := now.Add(30 * day)
made := ledger.BanForAbuseIPDB(
now, expires, map[netip.Prefix]int64{netblock: 95},
)
if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 {
t.Fatalf("made %d bans, %d counted, want 1 and 1",
made, ledger.Made(bans.CauseAbuseIPDB))
}
ban, banned, madePermanent := ledger.Check(netblock.Addr(), now)
if !banned || madePermanent || !ban.Expires.Equal(expires) ||
ban.Cause != bans.CauseAbuseIPDB ||
ban.Reason != "on AbuseIPDB's blacklist with a score of 95" ||
len(ban.Notes.Reputation) != 1 ||
ban.Notes.Reputation[0].Source != "abuseipdb" ||
*ban.Notes.Reputation[0].Score != 95 {
t.Errorf(
"ban %+v, banned %t, made permanent %t, want the blacklist's "+
"ban until %s",
ban,
banned,
madePermanent,
expires,
)
}
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
t.Error("the ban refuses a request at its end")
}
}
func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
}
// Once that ban has ended the netblock is banned, and the blacklist
// fetched again makes no second ban while the first lasts.
later := limitBan.Expires
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 {
t.Errorf("made %d bans after the ban for a limit, want 1", made)
}
later = later.Add(time.Hour)
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans while the blacklist's lasts, want none", made)
}
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2})
if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{
Limit: 1,
}) {
t.Errorf("earlier bans %+v, want 1 for a limit", got)
}
}
func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
expires := now.Add(30 * day)
scores := map[netip.Prefix]int64{netblock: 100}
ledger.BanForAbuseIPDB(now, expires, scores)
ledger.Lift(netblock.Addr(), now)
// The admin lifted it: the blacklist fetched again until its end bans
// nothing.
later := expires.Add(-time.Hour)
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans before the lifted ban's end, want none", made)
}
// From its end on, the blacklist bans the netblock again, and the
// lifted ban, ended, is forgotten.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 {
t.Errorf("made %d bans at the lifted ban's end, want 1", made)
}
wantBans(t, ledger, map[netip.Prefix]int{netblock: 1})
}
func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
listed := netip.MustParsePrefix("203.0.113.9/32")
limited := netip.MustParsePrefix("203.0.113.10/32")
now := midnight()
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
t.Errorf("made %d bans for an empty blacklist, want none", made)
}
wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1})
}
func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
t.Parallel()
params := defaultParams()
params.MaxBans = 1
ledger := bans.New(params)
limited := netip.MustParsePrefix("203.0.113.1/32")
attacker := netip.MustParsePrefix("203.0.113.2/32")
now := midnight()
scores := map[netip.Prefix]int64{}
for i := range 3 {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
}
wantBans(t, ledger, map[netip.Prefix]int{limited: 1})
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
want[netblock] = 1
}
wantBans(t, ledger, want)
}
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
}
// And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
kept := first
kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams())
limit, _ := ledger.Ban(
limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
)
byteLimit, _ := ledger.Ban(
byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
)
attack, _ := ledger.Ban(
attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
bans.Notes{RuleID: "git-dir", Target: "path"},
)
for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses")
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
}
// The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf(
"the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
made, _ := ledger.Ban(
made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams())
// An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"),
+128 -234
View File
@@ -1,9 +1,8 @@
// Package bans is the ban ledger: the bans sww makes on the
// netblocks of clients that break a rate limit, a byte limit or the refusal
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
// describes. The bans are kept in memory, and written to bans.json and read
// from it by the state package.
// limit or show a clear sign of attack, and those an admin makes, with their
// notes, as the "Bans" section of SPEC.md describes. The bans are kept in
// memory, and written to bans.json and read from it by the state package.
package bans
import (
@@ -16,26 +15,17 @@ import (
"time"
"github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// Cause is why a ban was made: CauseLimit, CauseAttack, CauseAdmin or
// CauseAbuseIPDB.
type Cause string
// The causes of bans.
const (
// CauseLimit is a ban sww made for a broken limit.
CauseLimit Cause = "limit"
CauseLimit = "limit"
// CauseAttack is a ban sww made for a clear sign of attack.
CauseAttack Cause = "attack"
CauseAttack = "attack"
// CauseAdmin is a ban an admin made, or one sww made that an
// admin keeps. It is never dropped.
CauseAdmin Cause = "admin"
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
// blacklist. It is forgotten once it has ended.
CauseAbuseIPDB Cause = "abuseipdb"
CauseAdmin = "admin"
)
// repeatFactor is how many times as long as the netblock's last ban a ban
@@ -60,11 +50,10 @@ type Params struct {
// AttackBanDuration is how long a first ban for a clear sign of attack
// lasts.
AttackBanDuration time.Duration
// MaxBans is the most bans held whose cause is CauseLimit or
// CauseAttack, at least one. Past it, the earliest such ban of the
// netblock that has gone longest without a request is dropped. Bans
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and
// never dropped this way.
// MaxBans is the most bans held whose cause is not CauseAdmin, at
// least one. Past it, the earliest such ban of the netblock that has
// gone longest without a request is dropped. Bans whose cause is
// CauseAdmin are held besides, and never dropped.
MaxBans int
}
@@ -74,10 +63,10 @@ type Ban struct {
Start time.Time
// Expires is when the ban ends, zero for a permanent ban.
Expires time.Time
Cause Cause
// Reason is a short text: for a ban sww made, the limit broken, the
// rule that matched or AbuseIPDB's score; for an admin's, what the
// admin wrote.
// Cause is CauseLimit, CauseAttack or CauseAdmin.
Cause string
// Reason is a short text: for a ban sww made, the limit broken or
// the rule that matched; for an admin's, what the admin wrote.
Reason string
// Lifted is when an admin lifted the ban, zero while no admin has. A
// lifted ban refuses nothing, and does not make the netblock's next
@@ -116,10 +105,10 @@ type Notes struct {
// request that broke the limit included.
// These are what counted toward the ban, and the window is the time
// over which they came.
Kind ratelimit.LimitKind `json:"kind,omitempty"`
Limit int64 `json:"limit,omitempty"`
Window string `json:"window,omitempty"`
Count float64 `json:"count,omitempty"`
Kind string `json:"kind,omitempty"`
Limit int64 `json:"limit,omitempty"`
Window string `json:"window,omitempty"`
Count float64 `json:"count,omitempty"`
// LimitPercent and LimitPercentSetting are, for a ban for a limit a
// biased threshold lowered, the client's percentage of that kind of
// limit, of which Limit is the result, and the setting that gave it.
@@ -138,8 +127,7 @@ type Notes struct {
// log's reputation names them. It is left out when none did.
Reputation []ReputationHit `json:"reputation,omitempty"`
// Request is the request that broke the limit, or whose bytes broke
// it, or that was the clear sign of attack. A ban for AbuseIPDB's
// blacklist has none, and leaves it zero.
// it, or that was the clear sign of attack.
Request Request `json:"request"`
// Requests is how many requests the netblock has sent since it was
// first seen, and Refused how many of them the ban has refused so
@@ -162,10 +150,9 @@ type ReputationHit struct {
// EarlierBans counts a netblock's bans before a ban, by cause.
type EarlierBans struct {
Limit int `json:"limit"`
Attack int `json:"attack"`
Admin int `json:"admin"`
AbuseIPDB int `json:"abuseipdb"`
Limit int `json:"limit"`
Attack int `json:"attack"`
Admin int `json:"admin"`
}
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -193,13 +180,13 @@ type Ledger struct {
// netblocks holds each banned netblock's bans, oldest first. Check and
// Find make each netblock they find the most recently seen.
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
// held is how many bans netblocks holds whose cause is CauseLimit or
// CauseAttack, at most params.MaxBans.
// held is how many bans netblocks holds whose cause is not CauseAdmin,
// at most params.MaxBans.
held int
// made is how many bans have been made since the start, by cause: by
// the ledger, and by an admin, through BanForAdmin or in an edit of
// bans.json.
made map[Cause]int
made map[string]int
// v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6
// netblocks that have been banned. Check looks for a ban at each of
// them, so that a ban read from bans.json refuses every client in its
@@ -211,9 +198,8 @@ type Ledger struct {
// New returns a Ledger with no ban yet.
func New(params Params) *Ledger {
// The ledger drops bans itself, and never those whose cause is
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so
// the LRU has no limit of its own: it keeps the netblocks in the order
// they were last seen.
// CauseAdmin, however many there are, so the LRU has no limit of its
// own: it keeps the netblocks in the order they were last seen.
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
math.MaxInt,
nil,
@@ -226,7 +212,7 @@ func New(params Params) *Ledger {
params: params,
changed: make(chan struct{}, 1),
netblocks: netblocks,
made: map[Cause]int{},
made: map[string]int{},
}
}
@@ -305,52 +291,83 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest
}
// Grounds are why a request makes a ban: its cause, CauseLimit or
// CauseAttack, and its notes.
type Grounds struct {
Cause Cause
Notes Notes
// BanForLimit bans netblock at now for a broken limit, with notes, and
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
// made within LimitBanRepeatWindow after the netblock's ban that ended
// last, other than one for a clear sign of attack or a lifted one, lasts
// repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
}
// Ban bans netblock at now on grounds, and returns the ban, and true. If
// a ban on netblock is still active, as when two of its requests break a
// limit at once, that ban is returned with false, and no other is made.
// The ledger fills in the notes' Refused and EarlierBans itself.
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
// WouldBanForLimit returns what BanForLimit would, without making the ban:
// what observe mode would have done.
func (l *Ledger) WouldBanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, true)
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
}
// WouldBan returns what Ban would, without making the ban: what observe
// mode would have done.
func (l *Ledger) WouldBan(
netblock netip.Prefix, now time.Time, grounds Grounds,
// BanForAttack bans netblock at now for a clear sign of attack, with
// notes, and returns the ban, and whether it made it, as BanForLimit
// does. A first ban lasts AttackBanDuration; once the netblock has had
// one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, false)
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
}
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It
// works out nothing else of the ban.
// or CauseAttack, made at now would be permanent, as BanForLimit or
// BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause Cause,
netblock netip.Prefix, now time.Time, cause string,
) bool {
l.mu.Lock()
defer l.mu.Unlock()
@@ -385,9 +402,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
// on netblock is active, since the admin asked for this one. The ledger
// fills in the notes' EarlierBans, and counts the ban among those made.
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// ban even while another on netblock is active, since the admin asked
// for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string,
) Ban {
@@ -411,88 +429,6 @@ func (l *Ledger) BanForAdmin(
return ban
}
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
// returns how many bans it made. scores are the netblocks of the clients
// on AbuseIPDB's blacklist, each with the highest score of its clients.
// A netblock is left as it is while a ban on it is active, or while a ban
// for the blacklist that an admin lifted has not reached its end: the
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
// and that has ended is forgotten first, so that the bans held for the
// blacklist are those of the clients listed within one ban's length.
func (l *Ledger) BanForAbuseIPDB(
now, expires time.Time, scores map[netip.Prefix]int64,
) int {
l.mu.Lock()
defer l.mu.Unlock()
changed := l.forgetEndedAbuseIPDBBans(now)
made := 0
for netblock, score := range scores {
netblock = netblock.Masked()
var notes Notes
if held, found := l.netblocks.Peek(netblock); found {
if activeBan(*held, now) != nil ||
liftedAbuseIPDBBan(*held, now) {
continue
}
notes.EarlierBans = earlierBans(*held)
}
// The source is named as the request log names AbuseIPDB.
notes.Reputation = []ReputationHit{
{Source: string(CauseAbuseIPDB), Score: &score},
}
l.add(Ban{
Netblock: netblock,
Start: now,
Expires: expires,
Cause: CauseAbuseIPDB,
Reason: fmt.Sprintf(
"on AbuseIPDB's blacklist with a score of %d", score,
),
Notes: notes,
})
made++
}
l.made[CauseAbuseIPDB] += made
if changed || made > 0 {
l.markChanged()
}
return made
}
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
// and that have ended at now, as BanForAbuseIPDB does first. The state
// package calls it each time it reads bans.json, so that these bans are
// also forgotten while no fetch of the blacklist reaches the ledger.
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
l.mu.Lock()
defer l.mu.Unlock()
if l.forgetEndedAbuseIPDBBans(now) {
l.markChanged()
}
}
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
// reached at now.
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
return slices.ContainsFunc(bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
(ban.Permanent() || now.Before(ban.Expires))
})
}
// Lift lifts, at now, every ban active then on a netblock client is in,
// as an admin does, and returns those bans. A lifted ban is kept, refuses
// nothing, and does not make the netblock's next ban longer.
@@ -579,10 +515,10 @@ func (l *Ledger) AddLookup(
}
// Made returns how many bans for cause have been made since the start:
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as
// LoadEdit counts them. The bans read from bans.json at the start are not among them.
func (l *Ledger) Made(cause Cause) int {
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
// them. The bans read from bans.json at the start are not among them.
func (l *Ledger) Made(cause string) int {
l.mu.Lock()
defer l.mu.Unlock()
@@ -638,8 +574,8 @@ func (l *Ledger) Snapshot() []Ban {
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
// netblock is masked to its length, so that 203.0.113.9/24 is
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
// MaxBans the earliest bans that count toward it are dropped, as when they
// are made.
// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
// when they are made.
func (l *Ledger) Load(bans []Ban) {
l.mu.Lock()
defer l.mu.Unlock()
@@ -703,42 +639,15 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
})
}
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
// and whose end is reached at now, lifted or not, and each netblock left
// without a ban. It reports whether it forgot one.
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
forgot := false
for _, netblock := range l.netblocks.Keys() {
bans, _ := l.netblocks.Peek(netblock)
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
!now.Before(ban.Expires)
})
if len(kept) == len(*bans) {
continue
}
forgot = true
if len(kept) == 0 {
l.netblocks.Remove(netblock)
} else {
*bans = kept
}
}
return forgot
}
// ban bans netblock at now on grounds, as Ban describes, and returns the
// ban, and whether it made it. Unless keep is true, the ban is not made,
// only returned: it is the ban that would have been made.
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban(
netblock netip.Prefix,
now time.Time,
grounds Grounds,
cause, reason string,
notes Notes,
keep bool,
) (Ban, bool) {
l.mu.Lock()
@@ -747,8 +656,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active.
var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock)
if found {
active := activeBan(*bans, now)
@@ -764,21 +671,15 @@ func (l *Ledger) ban(
ban := Ban{
Netblock: netblock,
Start: now,
Cause: grounds.Cause,
Cause: cause,
Reason: reason,
Notes: notes,
}
switch grounds.Cause {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
if cause == CauseAttack {
ban.Expires = l.attackExpiry(held, now)
case CauseAdmin, CauseAbuseIPDB:
fallthrough
default:
panic("bans: a request's ban has the cause " + string(grounds.Cause))
} else {
ban.Expires = l.limitExpiry(held, now)
}
if !keep {
@@ -786,7 +687,7 @@ func (l *Ledger) ban(
}
l.add(ban)
l.made[grounds.Cause]++
l.made[cause]++
l.markChanged()
return ban, true
@@ -813,16 +714,13 @@ func earlierBans(held []Ban) EarlierBans {
fromHere.Attack++
case CauseAdmin:
fromHere.Admin++
case CauseAbuseIPDB:
fromHere.AbuseIPDB++
}
notes := ban.Notes.EarlierBans
count := EarlierBans{
Limit: notes.Limit + fromHere.Limit,
Attack: notes.Attack + fromHere.Attack,
Admin: notes.Admin + fromHere.Admin,
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
Limit: notes.Limit + fromHere.Limit,
Attack: notes.Attack + fromHere.Attack,
Admin: notes.Admin + fromHere.Admin,
}
// Of counts that name as many bans, the earlier ban's is taken: it
@@ -837,7 +735,7 @@ func earlierBans(held []Ban) EarlierBans {
// total returns how many bans e counts, of all causes.
func (e EarlierBans) total() int {
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB
return e.Limit + e.Attack + e.Admin
}
// markChanged has Changed receive a value, unless one is waiting already.
@@ -908,9 +806,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
// add adds ban to its netblock's bans, after the last, and makes its
// netblock the most recently seen. With MaxBans held, it drops one first,
// unless ban does not count toward MaxBans.
// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
func (l *Ledger) add(ban Ban) {
counted := countsTowardMaxBans(ban)
counted := ban.Cause != CauseAdmin
if counted && l.held == l.params.MaxBans {
l.dropOne()
}
@@ -991,13 +889,7 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
return now.Add(l.params.AttackBanDuration)
}
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause
// is CauseLimit or CauseAttack.
func countsTowardMaxBans(ban Ban) bool {
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
}
// dropOne drops the earliest ban that counts toward MaxBans of the
// dropOne drops the earliest ban whose cause is not CauseAdmin of the
// netblock that has gone longest without a request, of those that hold
// such a ban, and the netblock with it if that was its only ban. It is
// called with at least one such ban held. It looks at each netblock once
@@ -1006,9 +898,11 @@ func (l *Ledger) dropOne() {
for range l.netblocks.Len() {
netblock, bans, _ := l.netblocks.GetOldest()
i := slices.IndexFunc(*bans, countsTowardMaxBans)
i := slices.IndexFunc(*bans, func(ban Ban) bool {
return ban.Cause != CauseAdmin
})
if i < 0 {
// None of its bans is dropped to make room. Get makes
// Its bans are all an admin's, and never dropped. Get makes
// it the most recently seen, so that the next netblock is
// looked at; when it was seen matters only for dropping a
// ban, and a ban added to it makes it the most recently seen
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
}
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netblock,
first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params)
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight()
for i := range 14 {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Expires.After(ban.Start) {
t.Fatalf(
"ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
}
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
if !made {
t.Error("the first ban was not made")
}
again, made := ledger.Ban(
again, made := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
)
}
again, made = ledger.Ban(
again, made = ledger.BanForAttack(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) {
t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64")
now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.BanForLimit(b, now, bans.Notes{})
ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's.
ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes
// first.
ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit.
third, _ := ledger.Ban(
third, _ := ledger.BanForAttack(
netblock,
second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// While the first ban lasts, none would be made.
during, would := ledger.WouldBan(
during, would := ledger.WouldBanForAttack(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if would || !reflect.DeepEqual(during, first) {
t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
}
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
bans.Notes{RuleID: "git-dir"})
limit, wouldLimit := ledger.WouldBanForLimit(
netblock,
first.Expires,
limitGrounds,
limitNotes,
)
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false)
// The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit)
}
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1)
}
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
now = ban.Expires
}
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
}
// Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent")
}
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
}
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
bans.Notes{Request: request},
)
cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
})
ledger.Ban(
netblock,
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write.
ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false)
// Two bans before the value is read leave one.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
wantChanged(t, ledger, true)
wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
v6,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
ledger.BanForLimit(high, midnight(), bans.Notes{})
ledger.BanForLimit(low, midnight(), bans.Notes{})
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
// Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse")
}
again, _ := after.Ban(
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(),
},
})
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
}
// A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
}
// The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
want := []bans.Ban{first, second, kept}
+25 -4
View File
@@ -19,8 +19,6 @@ import (
"time"
"github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// off is the value that switches a timeout, a size limit or a rate limit
@@ -42,7 +40,9 @@ var (
errNotPositive = errors.New("must be more than zero, or off")
errNotMilliseconds = errors.New(
"is not a whole number of milliseconds above zero, such as 2000")
errEmptyItem = errors.New("has an empty item in its list")
errEmptyItem = errors.New("has an empty item in its list")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotListenAddr = errors.New(
"is not an address to listen on, such as :8080")
errNotCommand = errors.New("is not rproxy or waf")
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
netblocks := make([]netip.Prefix, 0, len(items))
for _, item := range items {
netblock, err := reputation.ParseNetblock(item)
netblock, err := ParseNetblock(item)
if err != nil {
return nil, err
}
@@ -264,6 +264,27 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
return netblocks, nil
}
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// parseListenAddr checks an address to listen on: an optional host and a
// port number.
func parseListenAddr(value string) (string, error) {
+162 -249
View File
@@ -20,26 +20,11 @@ import (
"unicode/utf8"
"github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The names of the settings that waf writes into log lines and ban notes,
// as the setting that refused a request or lowered a client's limits.
const (
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
)
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
@@ -90,7 +75,7 @@ type WAF struct {
BytesLimitPerMinute int64
BytesLimitPerHour int64
BytesLimitPerDay int64
BytesCounted BytesCounted
BytesCounted string
// LookupSource is where each client's AS number and country are looked
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
@@ -101,7 +86,7 @@ type WAF struct {
// AddLookupHeaders is true when the app is passed the client's AS number
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
// LookupSource is off.
LookupSource LookupSource
LookupSource string
LookupDBPath string
ASNAndCountryLookupTimeout time.Duration
AddLookupHeaders bool
@@ -115,7 +100,7 @@ type WAF struct {
// with a tag that percentage of every rate limit and byte limit, from 1
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
// lower case, a country code in capitals.
TagLists []reputation.TagList
TagLists []TagList
TagDeny []string
TagLimitPercent map[string]int64
// RouteWatchURL is where the routewatch instance is that adds the asn=
@@ -146,7 +131,7 @@ type WAF struct {
// BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string
BlocklistRefresh time.Duration
BlocklistAction ReputationAction
BlocklistAction string
BlocklistLimitPercent int64
// DNSBLZones are the DNSBL zones clients are asked about
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
@@ -169,18 +154,10 @@ type WAF struct {
AbuseIPDBKey string
AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int
DNSBLAbuseIPDBAction ReputationAction
DNSBLAbuseIPDBAction string
DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration
// While AbuseIPDBKey is set, AbuseIPDB's blacklist of the clients it
// scores AbuseIPDBThreshold or more (SWW_WAF_ABUSEIPDB_THRESHOLD), from
// 25 to 100, is fetched at the start and every AbuseIPDBBlacklistRefresh
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Each client on it
// is banned for AbuseIPDBBanDuration (SWW_WAF_ABUSEIPDB_BAN_DAYS).
AbuseIPDBThreshold int64
AbuseIPDBBlacklistRefresh time.Duration
AbuseIPDBBanDuration time.Duration
// BanResponse is the status a refused client is answered with, 403
// or 429, or 0 to close the connection without an answer
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -252,7 +229,7 @@ type WAF struct {
// reads none.
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
CoreRuleSetMode CoreRuleSetMode
CoreRuleSetMode string
CoreRuleSetParanoiaLevel int
CoreRuleSetScoreThreshold int
CoreRuleSetDisabledRules []int
@@ -305,7 +282,7 @@ type WAF struct {
AlertSlackWebhookURL *url.URL
AlertNtfyURL *url.URL
AlertNtfyToken string
AlertEvents []alerts.Event
AlertEvents []string
AlertCooldown time.Duration
AlertMaxPerHour int
// The anomaly thresholds, which only raise alerts: the most requests
@@ -331,66 +308,20 @@ type WAF struct {
settings []slog.Attr
}
// BytesCounted is which body bytes count toward the byte limits and the
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
type BytesCounted string
// The values of SWW_WAF_BYTES_COUNTED.
const (
// BytesCountedResponse counts the response's body bytes.
BytesCountedResponse BytesCounted = "response"
// BytesCountedRequest counts the request's body bytes.
BytesCountedRequest BytesCounted = "request"
// BytesCountedBoth counts the body bytes of both.
BytesCountedBoth BytesCounted = "both"
)
// LookupSource is where each client's AS number and country are looked up,
// a value of SWW_WAF_LOOKUP_SOURCE.
type LookupSource string
// The values of SWW_WAF_LOOKUP_SOURCE.
const (
// LookupSourceGeoJS asks GeoJS.
LookupSourceGeoJS LookupSource = "geojs"
// LookupSourceFile looks clients up in the lookup database, the file
// SWW_WAF_LOOKUP_DB_PATH names.
LookupSourceFile LookupSource = "file"
// LookupSourceOff looks no client up.
LookupSourceOff LookupSource = "off"
)
// ReputationAction is what is done with a client a reputation source
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
type ReputationAction string
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
const (
// ReputationActionDeny refuses the client's requests.
ReputationActionDeny ReputationAction = "deny"
// ReputationActionLimit gives the client a percentage of every limit,
// written limit:<percent> in the setting.
ReputationActionLimit ReputationAction = "limit"
// ReputationActionLog only notes the hit in the log line and raises an
// alert.
ReputationActionLog ReputationAction = "log"
)
// CoreRuleSetMode is what the Core Rule Set does, a value of
// SWW_WAF_CORE_RULE_SET_MODE.
type CoreRuleSetMode string
// The values of SWW_WAF_CORE_RULE_SET_MODE.
const (
// CoreRuleSetModeOff runs no request through the Core Rule Set.
CoreRuleSetModeOff CoreRuleSetMode = "off"
CoreRuleSetModeOff = off
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
CoreRuleSetModeDetect = "detect"
// CoreRuleSetModeBlock refuses a match with 403.
CoreRuleSetModeBlock CoreRuleSetMode = "block"
CoreRuleSetModeBlock = "block"
)
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
const fileSource = "file"
const (
day = 24 * time.Hour
ipv4Bits = 32
@@ -399,9 +330,6 @@ const (
// netblock a provider is usually given: a shorter one would make one
// client of the customers of several providers.
minIPv6GroupPrefix = 32
// minAbuseIPDBThreshold is the lowest score AbuseIPDB's blacklist can
// be cut at.
minAbuseIPDBThreshold = 25
// minTokenLength is the fewest characters a token may have.
minTokenLength = 32
// maxParanoiaLevel is the Core Rule Set's highest paranoia level.
@@ -512,14 +440,11 @@ var (
"is not a whole number above zero, such as 60, or off",
)
errNotUTF8 = errors.New("is not valid UTF-8")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercentItem = errors.New(
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
errNotAbuseIPDBThreshold = errors.New(
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
"blacklist can be cut at",
)
errNotWholeUnits = errors.New(
"is not a whole number above zero, such as 30, that fits a duration",
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
errListedTwice = errors.New("is listed twice")
errNotListURL = errors.New(
@@ -615,13 +540,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
),
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
BytesCounted: env.bytesCounted(
"SWW_WAF_BYTES_COUNTED",
BytesCountedBoth,
),
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
LookupSource: env.lookupSource(
"SWW_WAF_LOOKUP_SOURCE",
LookupSourceGeoJS,
"geojs",
),
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
ASNAndCountryLookupTimeout: env.milliseconds(
@@ -629,24 +551,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"800",
),
ExclusivelyAllowedCountries: env.countries(
ExclusivelyAllowedCountriesSetting, ""),
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents(
ASNBytesLimitPercentSetting,
reputation.ParseASN,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
ParseASN,
),
CountryBytesLimitPercent: env.percents(
CountryBytesLimitPercentSetting,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
parseCountry,
),
UnknownCountryLimitPercent: env.percent(
UnknownCountryLimitPercentSetting,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
"100",
),
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
@@ -665,20 +587,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
"2000",
),
AbuseIPDBThreshold: env.abuseIPDBThreshold(
"SWW_WAF_ABUSEIPDB_THRESHOLD",
"90",
),
AbuseIPDBBlacklistRefresh: env.wholeUnits(
"SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS",
"60",
time.Minute,
),
AbuseIPDBBanDuration: env.wholeUnits(
"SWW_WAF_ABUSEIPDB_BAN_DAYS",
"30",
day,
),
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
LimitBanDuration: env.durationNotOff(
"SWW_WAF_LIMIT_BAN_DURATION",
@@ -789,8 +697,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
),
AlertNtfyURL: env.webhookURL("SWW_WAF_ALERT_NTFY_URL"),
AlertNtfyToken: env.secret("SWW_WAF_ALERT_NTFY_TOKEN"),
AlertEvents: env.alertEvents("SWW_WAF_ALERT_EVENTS"),
AlertCooldown: env.duration("SWW_WAF_ALERT_COOLDOWN", "15m"),
AlertEvents: env.alertEvents("SWW_WAF_ALERT_EVENTS",
strings.Join(alerts.Events(), ",")),
AlertCooldown: env.duration("SWW_WAF_ALERT_COOLDOWN", "15m"),
AlertMaxPerHour: env.numberOrOff(
"SWW_WAF_ALERT_MAX_PER_HOUR",
"60",
@@ -817,11 +726,11 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil)
// The app gets the headers by default, unless no client is looked up.
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
strconv.FormatBool(cfg.LookupSource != off))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
BlocklistActionSetting, string(ReputationActionDeny))
"SWW_WAF_BLOCKLIST_ACTION", "deny")
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
DNSBLAbuseIPDBActionSetting, "limit:25")
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg)
@@ -917,12 +826,9 @@ func (e *environment) count(name, defaultValue string) int64 {
// bytesCounted reads the setting that is which body bytes count toward the
// byte limits and the byte anomaly thresholds: response, request or both.
func (e *environment) bytesCounted(
name string, defaultValue BytesCounted,
) BytesCounted {
value := BytesCounted(e.value(name, string(defaultValue)))
if value != BytesCountedResponse && value != BytesCountedRequest &&
value != BytesCountedBoth {
func (e *environment) bytesCounted(name, defaultValue string) string {
value := e.value(name, defaultValue)
if value != "response" && value != "request" && value != "both" {
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
}
@@ -967,10 +873,8 @@ func (e *environment) trapPaths(name string) []string {
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
// detect or block.
func (e *environment) coreRuleSetMode(
name string, defaultValue CoreRuleSetMode,
) CoreRuleSetMode {
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
mode := e.value(name, defaultValue)
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
mode != CoreRuleSetModeBlock {
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
@@ -1021,7 +925,7 @@ func (e *environment) percents(
}
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
func (e *environment) tagLists(name string) []reputation.TagList {
func (e *environment) tagLists(name string) []TagList {
lists, err := parseTagLists(e.value(name, ""))
e.check(name, err)
@@ -1047,41 +951,12 @@ func (e *environment) tagPercents(name string) map[string]int64 {
// percent reads a setting that is a percentage, from 0 to 100.
func (e *environment) percent(name, defaultValue string) int64 {
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
percent, err := ParsePercent(e.value(name, defaultValue))
e.check(name, err)
return percent
}
// abuseIPDBThreshold reads the setting that is the least score of the
// clients on AbuseIPDB's blacklist: a whole number from 25 to 100, the
// range AbuseIPDB takes.
func (e *environment) abuseIPDBThreshold(name, defaultValue string) int64 {
value := e.value(name, defaultValue)
threshold, err := strconv.ParseInt(value, 10, 64)
if err != nil || threshold < minAbuseIPDBThreshold || threshold > 100 {
e.check(name, fmt.Errorf("%q %w", value, errNotAbuseIPDBThreshold))
}
return threshold
}
// wholeUnits reads a setting that is a whole number above zero of unit,
// such as minutes or days, and returns it as a duration.
func (e *environment) wholeUnits(
name, defaultValue string, unit time.Duration,
) time.Duration {
value := e.value(name, defaultValue)
n, err := strconv.ParseInt(value, 10, 64)
if err != nil || n <= 0 || n > math.MaxInt64/int64(unit) {
e.check(name, fmt.Errorf("%q %w", value, errNotWholeUnits))
}
return time.Duration(n) * unit
}
// listURL reads a setting that is the URL a list is fetched from, "" while
// it is unset or empty.
func (e *environment) listURL(name string) string {
@@ -1120,29 +995,25 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
// action reads a setting that is what is done with a client a blocklist
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
// as limit and the percentage.
func (e *environment) action(
name, defaultValue string,
) (ReputationAction, int64) {
value := ReputationAction(e.value(name, defaultValue))
if value == ReputationActionDeny || value == ReputationActionLog {
func (e *environment) action(name, defaultValue string) (string, int64) {
value := e.value(name, defaultValue)
if value == "deny" || value == "log" {
return value, 0
}
percentText, isLimit := strings.CutPrefix(
string(value), string(ReputationActionLimit)+":",
)
percentText, isLimit := strings.CutPrefix(value, "limit:")
percent, err := reputation.ParsePercent(percentText)
percent, err := ParsePercent(percentText)
if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
}
return ReputationActionLimit, percent
return "limit", percent
}
// zones reads the setting that is the list of DNSBL zones. It is empty by
// default. The log shows each zone with its key masked, as
// reputation.MaskZoneKey masks it.
// default. The log shows each zone with its key masked, as MaskZoneKey
// masks it.
func (e *environment) zones(name string) []string {
value, _ := e.lookup(name)
zones, err := parseZones(value)
@@ -1150,7 +1021,7 @@ func (e *environment) zones(name string) []string {
logged := make([]string, len(zones))
for i, zone := range zones {
logged[i] = reputation.MaskZoneKey(zone)
logged[i] = MaskZoneKey(zone)
}
e.settings = append(
@@ -1188,12 +1059,9 @@ func (e *environment) routeWatchURL(name string) string {
// lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off.
func (e *environment) lookupSource(
name string, defaultValue LookupSource,
) LookupSource {
source := LookupSource(e.value(name, string(defaultValue)))
if source != LookupSourceGeoJS && source != LookupSourceFile &&
source != LookupSourceOff {
func (e *environment) lookupSource(name, defaultValue string) string {
source := e.value(name, defaultValue)
if source != "geojs" && source != fileSource && source != off {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
}
@@ -1205,12 +1073,12 @@ func (e *environment) lookupSource(
// one source at a time.
func (e *environment) checkLookupDBPath(cfg *WAF) {
switch {
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
errNeedsDBPath,
))
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
cfg.LookupSource,
@@ -1230,12 +1098,12 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
e.check(ExclusivelyAllowedCountriesSetting,
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
fmt.Errorf("%q %w", country, errOnBothLists))
}
}
if cfg.LookupSource != LookupSourceOff {
if cfg.LookupSource != off {
return
}
@@ -1249,15 +1117,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
name string
set bool
}{
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
{TagLimitPercentSetting, slices.ContainsFunc(
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -1277,7 +1145,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
// blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check(ASNLimitPercentURLSetting,
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
}
}
@@ -1304,18 +1172,14 @@ func (e *environment) checkTags(cfg *WAF) {
}
for tag := range cfg.TagLimitPercent {
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
}
}
// checkTag returns an error when tag would never apply, as checkTags says.
func checkTag(cfg *WAF, tag string) error {
if !requestCanGetTag(cfg, tag) {
return fmt.Errorf(
"%q %w",
reputation.MaskZoneKey(tag),
errNoRequestGetsTag,
)
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
}
return nil
@@ -1323,7 +1187,7 @@ func checkTag(cfg *WAF, tag string) error {
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
func requestCanGetTag(cfg *WAF, tag string) bool {
isListName := func(list reputation.TagList) bool { return list.Name == tag }
isListName := func(list TagList) bool { return list.Name == tag }
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
return true
}
@@ -1347,7 +1211,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
return isZoneTag &&
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
return strings.ToLower(MaskZoneKey(each)) == zone
})
}
@@ -1685,15 +1549,9 @@ func (e *environment) secret(name string) string {
return value
}
// alertEvents reads the setting that is the events alerts are sent for,
// every event by default.
func (e *environment) alertEvents(name string) []alerts.Event {
every := make([]string, 0, len(alerts.Events()))
for _, event := range alerts.Events() {
every = append(every, string(event))
}
events, err := parseAlertEvents(e.value(name, strings.Join(every, ",")))
// alertEvents reads the setting that is the events alerts are sent for.
func (e *environment) alertEvents(name, defaultValue string) []string {
events, err := parseAlertEvents(e.value(name, defaultValue))
e.check(name, err)
return events
@@ -1851,9 +1709,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
}
netblock, err := reputation.ParseNetblock(
strings.TrimSpace(netblockText),
)
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
if err != nil {
return nil, err
}
@@ -2059,6 +1915,21 @@ func parseCountry(value string) (string, error) {
return country, nil
}
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// The tags a request can get that the settings can name, as waf
// writes them. The prefixes are followed by a value.
const (
@@ -2072,17 +1943,24 @@ const (
ExploitSignatureTag = "exploitsignature"
)
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// parseTagLists reads a comma-separated list of name=URL pairs, such as
// torexit=https://example.com/tor.txt, and returns each list with its name
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
// = before http. A name or a URL listed twice is an error.
func parseTagLists(value string) ([]reputation.TagList, error) {
func parseTagLists(value string) ([]TagList, error) {
items, err := parseList(value)
if err != nil {
return nil, err
}
lists := make([]reputation.TagList, 0, len(items))
lists := make([]TagList, 0, len(items))
for _, item := range items {
end := strings.Index(item, "=http")
@@ -2090,12 +1968,9 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
return nil, fmt.Errorf("%q %w", item, errNotTagList)
}
list := reputation.TagList{
Name: strings.ToLower(item[:end]),
URL: item[end+1:],
}
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
if slices.ContainsFunc(lists, func(earlier TagList) bool {
return earlier.Name == list.Name || earlier.URL == list.URL
}) {
return nil, fmt.Errorf("%q %w", item, errListedTwice)
@@ -2108,8 +1983,7 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
}
// parseTags reads a comma-separated list of tags, which parseTag reads. A
// tag listed twice is an error, which shows the tag as
// reputation.MaskZoneKey does.
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
func parseTags(value string) ([]string, error) {
items, err := parseList(value)
if err != nil {
@@ -2129,7 +2003,7 @@ func parseTags(value string) ([]string, error) {
if slices.Contains(tags, tag) {
return nil, fmt.Errorf(
"%q %w",
reputation.MaskZoneKey(item),
MaskZoneKey(item),
errListedTwice,
)
}
@@ -2198,7 +2072,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if colon <= 0 {
return nil, fmt.Errorf(
"%q %w",
reputation.MaskZoneKey(item),
MaskZoneKey(item),
errNotTagPercentItem,
)
}
@@ -2212,7 +2086,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if err != nil || percent < 1 || percent > 99 {
return nil, fmt.Errorf(
"%q %w",
reputation.MaskZoneKey(item[colon+1:]),
MaskZoneKey(item[colon+1:]),
errNotTagPercent,
)
}
@@ -2224,11 +2098,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if _, listed := percents[tag]; listed {
// tag, since item[:colon] may end in a space, which MaskZoneKey
// would not mask.
return nil, fmt.Errorf(
"%q %w",
reputation.MaskZoneKey(tag),
errListedTwice,
)
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
}
percents[tag] = percent
@@ -2262,7 +2132,7 @@ func parsePercents(
return nil, err
}
percent, err := reputation.ParsePercent(percentText)
percent, err := ParsePercent(percentText)
if err != nil {
return nil, err
}
@@ -2277,6 +2147,38 @@ func parsePercents(
return percents, nil
}
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// headerNameChars are the characters RFC 9110 allows in a header name:
// letters, digits and these marks.
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
"0123456789!#$%&'*+-.^_`|~"
// IsHeaderName reports whether name can be a header name: one or more of
// the characters RFC 9110 allows in one.
func IsHeaderName(name string) bool {
if name == "" {
return false
}
for _, char := range name {
if !strings.ContainsRune(headerNameChars, char) {
return false
}
}
return true
}
// parseHeaderNames reads a comma-separated list of header names in either
// case, and returns them in lower case. Host and Transfer-Encoding are
// refused: Go's HTTP server takes them out of the request's headers.
@@ -2289,7 +2191,7 @@ func parseHeaderNames(value string) ([]string, error) {
headers := make([]string, 0, len(items))
for _, item := range items {
if !httpguts.ValidHeaderFieldName(item) {
if !IsHeaderName(item) {
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
}
@@ -2445,8 +2347,8 @@ const (
// other name for one that does not exist, so that the zone would list no
// client. A zone listed twice is an error, whatever the case of its
// letters, which DNS names ignore, and whatever its key, since
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
// zone as it does.
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
// MaskZoneKey does.
func parseZones(value string) ([]string, error) {
zones, err := parseList(value)
if err != nil {
@@ -2456,14 +2358,11 @@ func parseZones(value string) ([]string, error) {
}
for i, zone := range zones {
shown := reputation.MaskZoneKey(zone)
shown := MaskZoneKey(zone)
listedBefore := slices.ContainsFunc(
zones[:i],
func(earlier string) bool {
return strings.EqualFold(
reputation.MaskZoneKey(earlier),
shown,
)
return strings.EqualFold(MaskZoneKey(earlier), shown)
},
)
@@ -2480,6 +2379,24 @@ func parseZones(value string) ([]string, error) {
return zones, nil
}
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return masked + "." + rest
}
// isZone reports whether each label of zone is as parseZones takes it. It
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
func isZone(zone string) bool {
@@ -2535,7 +2452,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
name, headerValue, found := strings.Cut(item, ":")
name = strings.TrimSpace(name)
if !found || !httpguts.ValidHeaderFieldName(name) ||
if !found || !IsHeaderName(name) ||
strings.ContainsAny(headerValue, "\r\n\x00") {
return nil, "", fmt.Errorf(
"item %d %w",
@@ -2553,20 +2470,16 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
// parseAlertEvents reads a comma-separated list of the events alerts can
// be sent for.
func parseAlertEvents(value string) ([]alerts.Event, error) {
names, err := parseList(value)
func parseAlertEvents(value string) ([]string, error) {
events, err := parseList(value)
if err != nil {
return nil, err
}
events := make([]alerts.Event, 0, len(names))
for _, name := range names {
event := alerts.Event(name)
for _, event := range events {
if !slices.Contains(alerts.Events(), event) {
return nil, fmt.Errorf("%q %w", name, errNotAlertEvent)
return nil, fmt.Errorf("%q %w", event, errNotAlertEvent)
}
events = append(events, event)
}
return events, nil
+27 -126
View File
@@ -18,13 +18,10 @@ import (
"time"
"github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The settings, by name.
@@ -64,9 +61,6 @@ const (
abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY"
abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE"
abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET"
abuseIPDBThreshold = "SWW_WAF_ABUSEIPDB_THRESHOLD"
abuseIPDBBlacklistRefresh = "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS"
abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
@@ -189,16 +183,6 @@ const (
defaultAlertCooldown = "15m"
)
// joinEvents returns events as SWW_WAF_ALERT_EVENTS lists them.
func joinEvents(events []alerts.Event) string {
names := make([]string, 0, len(events))
for _, event := range events {
names = append(names, string(event))
}
return strings.Join(names, ",")
}
// defaultIPReputationCacheTTL is the default of SWW_WAF_IP_REPUTATION_CACHE_TTL.
const defaultIPReputationCacheTTL = "24h"
@@ -627,7 +611,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
},
{
environment{
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
coreRuleSetMode: config.CoreRuleSetModeDetect,
coreRuleSetParanoiaLevel: "4",
coreRuleSetScoreThreshold: "10",
coreRuleSetDisabledRules: "942100, 920350",
@@ -877,32 +861,6 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
}
}
// config and rules check header names with httpguts.ValidHeaderFieldName.
// It takes a name of one or more of the characters RFC 9110 allows in one,
// as the check it replaced did: letters, digits and these marks.
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
t.Parallel()
const marks = "!#$%&'*+-.^_`|~"
if httpguts.ValidHeaderFieldName("") {
t.Error("the empty name is taken")
}
for b := range 256 {
char := byte(b)
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
// for a byte above 0x7f.
name := string([]byte{char})
if got := httpguts.ValidHeaderFieldName(name); got != want {
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
}
}
}
func TestRemoteLogSettingsDefaults(t *testing.T) {
t.Parallel()
@@ -1121,7 +1079,7 @@ func TestAlertSettingsDefaults(t *testing.T) {
if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 ||
cfg.AlertSlackWebhookURL != nil || cfg.AlertNtfyURL != nil ||
cfg.AlertNtfyToken != "" ||
joinEvents(cfg.AlertEvents) != defaultAlertEvents ||
strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents ||
cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 {
t.Errorf(
"alert settings %v, %v, %v, %v, %q, %v, %s and %d, want no URLs, "+
@@ -1168,10 +1126,7 @@ func TestAlertSettingsAsSet(t *testing.T) {
!reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) ||
cfg.AlertSlackWebhookURL.String() != slack || cfg.AlertNtfyURL.String() != ntfy ||
cfg.AlertNtfyToken != token ||
!slices.Equal(
cfg.AlertEvents,
[]alerts.Event{"ban", "file_error"},
) ||
!slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) ||
cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 {
t.Errorf(
"alert settings %v, %v, %v, %v, %q, %v, %s and %d",
@@ -1736,7 +1691,7 @@ func TestTagSettingsAsSet(t *testing.T) {
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
})
wantLists := []reputation.TagList{
wantLists := []config.TagList{
{Name: torExitTag, URL: torURL},
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
}
@@ -2075,9 +2030,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
}
for _, tc := range []struct {
value string
action config.ReputationAction
percent int64
value, action string
percent int64
}{
{actionDeny, actionDeny, 0},
{actionLog, actionLog, 0},
@@ -2210,7 +2164,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
// want are the zones, resolver, action and percent config.WAF gives.
want []string
wantResolver string
wantAction config.ReputationAction
wantAction string
wantPercent int64
}{
{
@@ -2330,6 +2284,25 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
}
}
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := config.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
t.Parallel()
@@ -2426,75 +2399,6 @@ func TestAbuseIPDBSettingsAsSet(t *testing.T) {
}
}
func TestAbuseIPDBBlacklistSettingsAsSet(t *testing.T) {
t.Parallel()
cfg := wafFromEnvironment(t, environment{})
if cfg.AbuseIPDBThreshold != 90 ||
cfg.AbuseIPDBBlacklistRefresh != time.Hour ||
cfg.AbuseIPDBBanDuration != 30*24*time.Hour {
t.Errorf(
"by default, the threshold %d, the refresh %s and the ban %s, "+
"want 90, 1h and 720h",
cfg.AbuseIPDBThreshold,
cfg.AbuseIPDBBlacklistRefresh,
cfg.AbuseIPDBBanDuration,
)
}
cfg = wafFromEnvironment(t, environment{
abuseIPDBThreshold: "25", abuseIPDBBlacklistRefresh: "1440",
abuseIPDBBanDays: "1",
})
if cfg.AbuseIPDBThreshold != 25 ||
cfg.AbuseIPDBBlacklistRefresh != 24*time.Hour ||
cfg.AbuseIPDBBanDuration != 24*time.Hour {
t.Errorf(
"set, the threshold %d, the refresh %s and the ban %s, "+
"want 25, 24h and 24h",
cfg.AbuseIPDBThreshold,
cfg.AbuseIPDBBlacklistRefresh,
cfg.AbuseIPDBBanDuration,
)
}
}
func TestInvalidAbuseIPDBBlacklistSettingStopsTheStartSayingWhatIsWrong(
t *testing.T,
) {
t.Parallel()
const (
notThreshold = " is not a whole number from 25 to 100, the scores " +
"AbuseIPDB's blacklist can be cut at"
notWhole = " is not a whole number above zero, such as 30, that fits " +
"a duration"
)
for _, tc := range []struct{ name, value, want string }{
{abuseIPDBThreshold, "24", `"24"` + notThreshold},
{abuseIPDBThreshold, "101", `"101"` + notThreshold},
{abuseIPDBThreshold, off, `"off"` + notThreshold},
{abuseIPDBBlacklistRefresh, "0", `"0"` + notWhole},
{abuseIPDBBlacklistRefresh, "1h", `"1h"` + notWhole},
{abuseIPDBBanDays, "-1", `"-1"` + notWhole},
{abuseIPDBBanDays, "200000", `"200000"` + notWhole},
} {
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
t.Parallel()
_, err := config.WAFFromEnvironment(
environment{tc.name: tc.value}.viper(),
)
want := tc.name + ": " + tc.want
if err == nil || err.Error() != want {
t.Errorf("error %v, want %s", err, want)
}
})
}
}
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(
t *testing.T,
) {
@@ -3047,9 +2951,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
abuseIPDBKey: "",
abuseIPDBMinScore: "75",
abuseIPDBDailyBudget: "900",
abuseIPDBThreshold: "90",
abuseIPDBBlacklistRefresh: "60",
abuseIPDBBanDays: "30",
dnsblAbuseIPDBAction: "limit:25",
ipReputationCacheTTL: defaultIPReputationCacheTTL,
dnsblAbuseIPDBTimeout: "2000",
@@ -3073,7 +2974,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
logLevel: "info",
ruleFilesDir: "/etc/sww/rules.d",
ruleFilesEnabled: "true",
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
coreRuleSetMode: config.CoreRuleSetModeBlock,
coreRuleSetParanoiaLevel: "1",
coreRuleSetScoreThreshold: "5",
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
+23 -65
View File
@@ -5,7 +5,6 @@ import (
"net/http"
"net/http/httptest"
"net/netip"
"regexp"
"runtime"
"slices"
"strconv"
@@ -35,46 +34,12 @@ const (
mostJSONRead = 1 << 10
)
// waf refuses a request the Core Rule Set has not inspected within 500
// milliseconds, but Coraza cannot stop an inspection part-way, so it runs to
// its end with nobody waiting for it. The test checks that an inspection ends
// within about 1 second, so that a visitor cannot keep a processor busy for
// longer with one request.
//
// A busy host slows all work, so the test does not compare an inspection with
// 1 second. It compares it with referenceTime, timed on the same thread just
// before and after. inspectionTimeLimitMultiple is how many times
// referenceTime an inspection may take.
//
// On the host that runs our checks, referenceTime took at least 78
// milliseconds, so 13 times it is about 1 second. With that host fully busy,
// no try of the slowest request took more than 9.5 times referenceTime.
const inspectionTimeLimitMultiple = 13
// referencePattern is what referenceTime matches. Like the Core Rule Set's
// rules, it is a regular expression with alternatives and repeats.
var referencePattern = regexp.MustCompile(
`(?i)(union|select|insert|delete)\s+\w+\s+(from|into)\s+\w+[0-9]+;`,
)
// referenceTime returns the processor time the calling thread takes to do a
// fixed amount of work unrelated to the Core Rule Set, so that a change to the
// rules does not change it. referencePattern matches nowhere in the text, so
// each match reads all of it.
func referenceTime(t *testing.T) time.Duration {
t.Helper()
referenceText := text(4 << 10)
began := threadCPUTime(t)
for range 200 {
if referencePattern.MatchString(referenceText) {
t.Fatal("the reference pattern matched")
}
}
return threadCPUTime(t) - began
}
// inspectionTimeLimit is the most time the Core Rule Set may take on one
// request. It is the default of rproxy's SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, 2
// seconds, less the default of SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, 800
// milliseconds, which a question can also wait for GeoJS, and less 200
// milliseconds for waf's other checks.
const inspectionTimeLimit = time.Second
// text returns n bytes of letters, digits and spaces, which the Core Rule Set
// takes longer over than one letter repeated.
@@ -354,16 +319,13 @@ func threadCPUTime(t *testing.T) time.Duration {
return time.Duration(now.Nano())
}
// inspectionTime returns the processor time crs takes to inspect tc, and the
// mean of referenceTime just before and after. It fails t unless crs reads all
// of tc's body and Coraza parses it.
//
// All of it runs on this goroutine, kept on one thread, so the thread's
// processor time measures it. Time other work holds the processor does not
// count.
// inspectionTime returns the processor time crs takes to inspect tc, and
// fails t unless it reads all of tc's body and Coraza parses it. Inspect does
// its work on the calling goroutine, which stays on one thread meanwhile, so
// that thread's processor time is the inspection's.
func inspectionTime(
t *testing.T, crs *coreruleset.CoreRuleSet, tc visitorRequest,
) (time.Duration, time.Duration) {
) time.Duration {
t.Helper()
req := httptest.NewRequestWithContext(
@@ -376,19 +338,14 @@ func inspectionTime(
}
runtime.LockOSThread()
defer runtime.UnlockOSThread()
before := referenceTime(t)
began := threadCPUTime(t)
result, read, _ := crs.Inspect(
req, netip.MustParseAddr("192.0.2.1"), strings.NewReader(tc.body),
)
took := threadCPUTime(t) - began
reference := (before + referenceTime(t)) / 2
if string(read) != tc.body {
t.Errorf("read %d bytes of %d", len(read), len(tc.body))
}
@@ -398,13 +355,15 @@ func inspectionTime(
t.Errorf("Coraza could not parse the body")
}
return took, reference
return took
}
// TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime runs one at a time, so
// that the tests running in parallel do not slow it. Other work on the host
// can still slow one try more than the reference work around it, so a request
// fails only when three tries in a row take too long.
// TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime measures processor
// time, not time on the clock, so that the time other work on the host holds
// the processor does not count. It runs one at a time, so that the tests
// running in parallel do not slow it. Other work can still slow one try a
// little, by sharing the processor's caches, so a request fails only when
// three tries in a row take too long.
//
//nolint:paralleltest // one at a time, as above
func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
@@ -418,19 +377,18 @@ func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
for _, tc := range worstCaseRequests() {
t.Run(tc.name, func(t *testing.T) {
var took, reference time.Duration
var took time.Duration
for range tries {
took, reference = inspectionTime(t, crs, tc)
if took <= inspectionTimeLimitMultiple*reference {
took = inspectionTime(t, crs, tc)
if took <= inspectionTimeLimit {
return
}
}
t.Errorf(
"took %s on the last of %d tries, over %d times the %s "+
"of the reference work",
took, tries, inspectionTimeLimitMultiple, reference,
"took %s on the last of %d tries, over %s",
took, tries, inspectionTimeLimit,
)
})
}
@@ -1,4 +1,4 @@
package geojs
package lookup
import "net/http"
+2 -6
View File
@@ -1,6 +1,3 @@
// Package lookup looks up each client's AS number and country in the lookup
// database, the IPinfo Lite file SWW_WAF_LOOKUP_DB_PATH names. Its answers
// take the form of GeoJS's, so that waf handles both alike.
package lookup
import (
@@ -17,7 +14,6 @@ import (
"github.com/oschwald/maxminddb-golang/v2"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/geojs"
)
// quietTime is how long the lookup database must go without a change
@@ -81,7 +77,7 @@ func OpenFile(params FileParams) (*File, error) {
// each "" when the database does not give it, as for an address missing
// from it. The database is asked about the client's first address, as
// GeoJS is.
func (f *File) LookUp(client netip.Prefix) geojs.Answer {
func (f *File) LookUp(client netip.Prefix) Answer {
f.mu.Lock()
reader := f.reader
f.mu.Unlock()
@@ -95,7 +91,7 @@ func (f *File) LookUp(client netip.Prefix) geojs.Answer {
found = record{}
}
return geojs.Answer{
return Answer{
Client: client,
ASN: found.ASN,
ASName: found.ASName,
+1 -2
View File
@@ -16,7 +16,6 @@ import (
"github.com/maxmind/mmdbwriter/mmdbtype"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
)
@@ -69,7 +68,7 @@ func TestFilePlacesClientsAndCountsAnAddressMissingFromItAsUnknown(
prefix := netip.MustParsePrefix(client)
got := f.LookUp(prefix)
if got != (geojs.Answer{
if got != (Answer{
Client: prefix, ASN: want.ASN, ASName: want.ASName, Country: want.Country,
Answered: now,
}) {
@@ -1,9 +1,9 @@
// Package geojs looks up clients' AS numbers and countries through the GeoJS
// web service. Its answers are kept in memory, for at most 100,000 clients and
// for 7 days each. A caller waits for an answer as long as it says. A request to
// GeoJS goes on after its callers stop waiting, and its answer is kept. The
// package imports no other package of this repository.
package geojs
// Package lookup looks up each client's AS number and country, through
// the GeoJS web service or in the lookup database, the IPinfo Lite file
// SWW_WAF_LOOKUP_DB_PATH names. GeoJS's answers are kept in memory, for at
// most 100,000 clients and for 7 days each, and are written to
// lookups.json and read from it by the state package.
package lookup
import (
"context"
@@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/netip"
"slices"
@@ -20,6 +21,8 @@ import (
"time"
"github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/metrics"
)
// URL is GeoJS's endpoint for an address's place and network. Asked about
@@ -40,9 +43,6 @@ const (
maxWaiting = 10000
// maxPerRequest is how many addresses one request to GeoJS asks about.
maxPerRequest = 200
// requestTimeout is how long a request to GeoJS may take before it is
// abandoned, which is a failure. It is apart from how long callers wait.
requestTimeout = 10 * time.Second
// unknownASN is the AS number GeoJS gives when it knows none.
unknownASN = 64512
// After a failure GeoJS is not asked again for a second, and for
@@ -64,32 +64,39 @@ var (
type Params struct {
// URL is where GeoJS is asked, normally URL.
URL string
// Timeout is how long a request waits for its client's first answer,
// and how long a request to GeoJS may take before it is abandoned
// (SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS).
Timeout time.Duration
// Wait is true when a setting needs each request's answer before the
// request goes on. Otherwise no request waits for one.
Wait bool
// Answered, unless nil, is given each answer GeoJS gives, once it is
// kept.
Answered func(Answer)
// Now tells the time, normally time.Now in UTC.
Now func() time.Time
// Asked is called for each request to GeoJS.
Asked func()
// Failed is called for each failure of GeoJS, with what went wrong and
// how long GeoJS is left alone after it. The error never names the
// addresses asked about.
Failed func(err error, askingAgainIn time.Duration)
// WentWithoutAnswer is called each time a caller that waits for an
// answer goes on without one.
WentWithoutAnswer func()
// ProcessLog receives GeoJS's failures.
ProcessLog *slog.Logger
// Metrics count the requests to GeoJS, those that failed, and the
// clients that go without an answer.
Metrics *metrics.Metrics
// Alerts receive a source_failure alert each time GeoJS fails.
Alerts *alerts.Queue
}
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
url string
timeout time.Duration
wait bool
answered func(Answer)
now func() time.Time
processLog *slog.Logger
metrics *metrics.Metrics
alerts *alerts.Queue
// httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure.
httpClient *http.Client
@@ -99,23 +106,20 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about.
waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way, and endAsking
// ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// asking is true while a request to GeoJS is under way.
asking bool
// retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration
retryAt time.Time
}
// Answer is what GeoJS said about a client: its AS number, such as AS64496,
// and the AS's name, both "" when GeoJS knows no AS number for it; its
// country, "" when GeoJS cannot place it; when GeoJS said so; and when the
// answer was last used. The zero Answer is that of a client with no answer.
// Answer is what GeoJS or the lookup database said about a client: its AS
// number, such as AS64496, and the AS's name, both "" when the source knows
// no AS number for it; its country, "" when the source cannot place it;
// when the source said so; and, for GeoJS's answers, which lookups.json
// holds, when the answer was last used. The zero Answer is that of a
// client with no answer.
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Answer struct {
@@ -133,7 +137,7 @@ type wait struct {
// replaced each time GeoJS fails before then.
asked chan struct{}
// late is true once the client has gone without an answer, for a
// whole wait or because GeoJS failed: its callers no longer wait.
// whole timeout or because GeoJS failed: its requests no longer wait.
late bool
}
@@ -145,12 +149,14 @@ func New(params Params) *GeoJS {
}
return &GeoJS{
url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
url: params.URL,
timeout: params.Timeout,
wait: params.Wait,
answered: params.Answered,
now: params.Now,
processLog: params.ProcessLog,
metrics: params.Metrics,
alerts: params.Alerts,
httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
@@ -164,22 +170,21 @@ func New(params Params) *GeoJS {
// LookUp returns the answer GeoJS gave about client, with its country as
// a two-letter code in capitals, or the zero Answer when there is none
// yet. An answer is kept for 7 days. Without one, the client is asked
// about in the background, and LookUp waits up to wait for the answer,
// unless the client has gone without one before. The wait ends early when
// ctx ends. Neither ends the request to GeoJS.
// about in the background, and, while Wait is set, the request waits up
// to Timeout for the answer, unless the client has gone without one
// before. ctx is the context of the client's request, and ends the wait
// when it ends.
//
// GeoJS is asked about the client's first address, which is the client's
// own address for IPv4, and an address in the same place for an IPv6
// netblock.
func (g *GeoJS) LookUp(
ctx context.Context, client netip.Prefix, wait time.Duration,
) Answer {
answer, asked := g.answerOrWait(ctx, client, true)
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
answer, asked := g.answerOrWait(ctx, client, g.wait)
if asked == nil {
return answer
}
timer := time.NewTimer(wait)
timer := time.NewTimer(g.timeout)
defer timer.Stop()
select {
@@ -193,7 +198,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client)
if !found {
g.wentWithoutAnswer()
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
}
w, waiting := g.waiting[client]
@@ -204,8 +209,8 @@ func (g *GeoJS) LookUp(
return answer
}
// LookUpWithoutWaiting is LookUp for a caller that does not need the answer:
// it returns the zero Answer at once when none is kept.
// LookUpWithoutWaiting is LookUp for a request that does not need the answer,
// whatever Wait says: it returns the zero Answer at once when none is kept.
func (g *GeoJS) LookUpWithoutWaiting(
ctx context.Context,
client netip.Prefix,
@@ -215,23 +220,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer
}
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock()
@@ -240,7 +228,8 @@ func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
return g.kept(client)
}
// Snapshot returns every answer kept, sorted by client.
// Snapshot returns every answer kept, sorted by client, as lookups.json
// lists them.
func (g *GeoJS) Snapshot() []Answer {
g.mu.Lock()
@@ -258,9 +247,9 @@ func (g *GeoJS) Snapshot() []Answer {
return answers
}
// Load keeps answers, such as those of an earlier Snapshot, in place of the
// answers it keeps, in the order they were last used, so that the one used
// longest ago is dropped first. Answers GeoJS gave keepFor ago or more are
// Load keeps answers read from lookups.json, in place of the answers it
// keeps, in the order they were last used, so that the one used longest
// ago is dropped first. Answers GeoJS gave keepFor ago or more are
// dropped.
func (g *GeoJS) Load(answers []Answer) {
answers = slices.Clone(answers)
@@ -306,23 +295,22 @@ func (g *GeoJS) answerOrWait(
g.ask(ctx)
if !mustWait {
return Answer{}, nil // the answer is not needed before the caller goes on
return Answer{}, nil // the answer is not needed before the request goes on
}
if w == nil {
g.wentWithoutAnswer()
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
return Answer{}, nil // too many clients wait already
}
if !g.asking {
// GeoJS is left alone after a failure, or asked nothing after
// StopAsking, so no answer can come.
// GeoJS is left alone after a failure, so no answer can come.
w.late = true
}
if w.late {
g.wentWithoutAnswer()
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
return Answer{}, nil
}
@@ -346,28 +334,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
}
// ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way, it is left alone after a failure, or StopAsking has
// been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking.
// it is under way or it is left alone after a failure. The requests to
// GeoJS are for every client waiting, so they go on when the client's
// request whose ctx is given ends.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.now().Before(g.retryAt) {
if g.asking || g.now().Before(g.retryAt) {
return
}
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
g.asking, g.endAsking = true, end
g.asking = true
g.askGoroutines.Go(func() {
defer end()
g.askAboutWaiting(askCtx)
})
go g.askAboutWaiting(context.WithoutCancel(ctx))
}
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
// answer kept is given to Answered, outside the lock, since Answered takes
// locks of its own.
// time, until none is left or GeoJS fails. Each answer kept is given to
// Answered, outside the lock, since Answered takes locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for {
clients := g.nextClients()
@@ -376,10 +358,6 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err)
if g.answered != nil {
@@ -395,13 +373,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
// nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, or StopAsking has been called, it returns none and
// notes that no request to GeoJS is under way.
// none is waiting, it returns none and notes that no request to GeoJS is
// under way.
func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.waiting) == 0 || g.stopped {
if len(g.waiting) == 0 {
g.asking = false
return nil
@@ -463,6 +441,8 @@ func (g *GeoJS) keep(
}
if err != nil {
g.metrics.GeoJSFailures.Inc()
g.retryDelay = min(
max(retryDelayFactor*g.retryDelay, firstRetryDelay),
maxRetryDelay,
@@ -477,7 +457,16 @@ func (g *GeoJS) keep(
w.late = true
}
g.failed(err, g.retryDelay)
g.processLog.Warn("asking GeoJS failed",
"error", err.Error(), "asking_again_in", g.retryDelay.String())
g.alerts.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: "asking GeoJS failed",
Detail: map[string]any{
"source": "geojs", "error": err.Error(),
"asking_again_in": g.retryDelay.String(),
},
})
return kept, false
}
@@ -490,8 +479,7 @@ func (g *GeoJS) keep(
// request asks GeoJS about clients in one request, and returns the answer
// for each address GeoJS's answer names: its AS number and the AS's name,
// both "" for the AS number 64512, which GeoJS gives when it knows none,
// and its country, in capitals. The request is abandoned after
// requestTimeout.
// and its country, in capitals.
func (g *GeoJS) request(
ctx context.Context, clients []netip.Prefix,
) (map[netip.Addr]Answer, error) {
@@ -501,7 +489,7 @@ func (g *GeoJS) request(
addrs = append(addrs, client.Addr().String())
}
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
ctx, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
req, err := http.NewRequestWithContext(
@@ -516,7 +504,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.asked()
g.metrics.GeoJSRequests.Inc()
res, err := g.httpClient.Do(req)
if err != nil {
File diff suppressed because it is too large. Load diff
@@ -1,4 +1,4 @@
package geojs_test
package lookup_test
import (
"net/netip"
@@ -7,7 +7,7 @@ import (
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
)
func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
@@ -25,7 +25,7 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
clock.advance(time.Hour)
wantCountry(t, g, placed, germany)
want := []geojs.Answer{
want := []lookup.Answer{
{Client: notPlaced, Answered: asked, Used: asked},
{
Client: placed, ASN: asn, ASName: asName, Country: germany,
@@ -41,24 +41,24 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
t.Parallel()
service, clock, g := start()
geojs, clock, g := start()
now := clock.Now()
kept := geojs.Answer{
kept := lookup.Answer{
Client: netip.MustParsePrefix("203.0.113.9/32"),
Country: "FR",
Answered: now.Add(-week + time.Second),
Used: now.Add(-time.Hour),
}
stale := geojs.Answer{
stale := lookup.Answer{
Client: netip.MustParsePrefix("203.0.113.10/32"),
Country: "FR",
Answered: now.Add(-week),
Used: now.Add(-time.Hour),
}
g.Load([]geojs.Answer{kept, stale})
g.Load([]lookup.Answer{kept, stale})
if got := g.Snapshot(); !slices.Equal(got, []geojs.Answer{kept}) {
if got := g.Snapshot(); !slices.Equal(got, []lookup.Answer{kept}) {
t.Errorf(
"kept %+v, want only the answer GeoJS gave less than 7 days ago",
got,
@@ -66,7 +66,7 @@ func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
}
wantCountry(t, g, kept.Client, "FR")
wantRequests(t, service, 0)
wantRequests(t, geojs, 0)
}
func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
@@ -80,11 +80,11 @@ func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
// lookups.json lists the answers by client. Here each was last used a
// second before the one listed before it, so the last listed is the
// one used longest ago, and the one dropped.
answers := make([]geojs.Answer, maxAnswers+1)
answers := make([]lookup.Answer, maxAnswers+1)
addr := netip.MustParseAddr("10.0.0.0")
for i := range answers {
answers[i] = geojs.Answer{
answers[i] = lookup.Answer{
Client: netip.PrefixFrom(addr, addr.BitLen()),
Country: germany,
Answered: now,
+15 -16
View File
@@ -14,6 +14,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -183,14 +184,14 @@ func New(busiestCount int, instanceName string) *Metrics {
func (m *Metrics) AddBansAndClients(
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
) {
for _, cause := range []bans.Cause{
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
for _, cause := range []string{
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin,
} {
m.registry.MustRegister(
prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "sww_bans_made_total",
Help: "Bans made, by cause.",
ConstLabels: prometheus.Labels{"cause": string(cause)},
ConstLabels: prometheus.Labels{"cause": cause},
}, func() float64 {
return float64(ledger.Made(cause))
}),
@@ -273,7 +274,8 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
// AddLookupFile adds the metrics of the lookup database, read as the
// metrics are asked for: when the file in use was read, which lastRead
// returns, and the replacements of it that could not be read, which
// readFailures returns.
// readFailures returns. The lookup package's File, which has both, cannot
// be named here: that package counts GeoJS's requests in these metrics.
func (m *Metrics) AddLookupFile(
lastRead func() time.Time,
readFailures func() int,
@@ -300,7 +302,7 @@ const sourceLabel = "source"
// AddReputation adds the metrics of the lists fetched from URLs and of the
// DNSBL zones, by source, each list's URL or each zone, its key masked as
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
// and, read from lists and dnsbl as the metrics are asked for, for a list,
// the fetches that failed and when the copy in use was fetched, and for a
@@ -319,9 +321,7 @@ func (m *Metrics) AddReputation(
m.registry.MustRegister(m.reputationHits)
for _, zone := range dnsbl.Zones() {
source := prometheus.Labels{
sourceLabel: reputation.MaskZoneKey(zone),
}
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
m.addReputationQueries(
source,
@@ -391,7 +391,7 @@ func (m *Metrics) ReputationHit(source string) {
// dropped. With no destination set, it adds none.
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
for _, name := range queue.DestinationsSet() {
destination := prometheus.Labels{"destination": string(name)}
destination := prometheus.Labels{"destination": name}
m.registry.MustRegister(
prometheus.NewCounterFunc(prometheus.CounterOpts{
@@ -466,7 +466,7 @@ func (m *Metrics) RequestReported(
// LimitHit counts a request that broke a rate limit, a byte limit or the
// refusal limit, by the window and the kind of hit.
func (m *Metrics) LimitHit(hit ratelimit.Hit) {
m.rateLimitHits.WithLabelValues(hit.Window, string(hit.Kind)).Inc()
m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc()
}
// Offences counts the offences of r, a request that has ended, as its
@@ -487,8 +487,8 @@ func (m *Metrics) Offences(r ratelimit.Request) {
// RuleMatched counts a request that matched the rule id, whose action is
// action.
func (m *Metrics) RuleMatched(id string, action rules.Action) {
m.ruleMatches.WithLabelValues(id, string(action)).Inc()
func (m *Metrics) RuleMatched(id, action string) {
m.ruleMatches.WithLabelValues(id, action).Inc()
}
// CoreRuleSetMatched counts a request that matched the Core Rule Set's rule id,
@@ -534,11 +534,10 @@ func (m *Metrics) count(
line *requestlog.Line, duration, upstreamDuration time.Duration,
) {
class := statusClass(line.Status)
action := string(line.Action)
m.requests.WithLabelValues(class, action).Inc()
m.requestBytes.WithLabelValues(class, action).
m.requests.WithLabelValues(class, line.Action).Inc()
m.requestBytes.WithLabelValues(class, line.Action).
Add(float64(line.RequestBytes))
m.responseBytes.WithLabelValues(class, action).
m.responseBytes.WithLabelValues(class, line.Action).
Add(float64(line.ResponseBytes))
m.requestDuration.Observe(duration.Seconds())
+6 -8
View File
@@ -19,19 +19,15 @@ import (
const day = 24 * time.Hour
// LimitKind is what a limit counts: KindRequests, KindBytes or
// KindRefusals.
type LimitKind string
// The kinds of limits, as the metrics name them.
const (
// KindRequests is a rate limit, on a client's requests.
KindRequests LimitKind = "requests"
KindRequests = "requests"
// KindBytes is a byte limit, on a client's bytes.
KindBytes LimitKind = "bytes"
KindBytes = "bytes"
// KindRefusals is the refusal limit, on a client's requests sww
// refused after a rule file match or for a missing or wrong token.
KindRefusals LimitKind = "refusals"
KindRefusals = "refusals"
)
// Limits are the most requests a client may make in a minute, an hour and
@@ -192,7 +188,9 @@ func New(limits Limits, maxClients int) *Limiter {
// Hit is a request that takes a client over a rate limit or the refusal
// limit, or whose bytes take it over a byte limit.
type Hit struct {
Kind LimitKind
// Kind is KindRequests for a rate limit, KindBytes for a byte limit,
// KindRefusals for the refusal limit.
Kind string
// Window is "minute", "hour" or "day".
Window string
// Limit is the window's limit, as the client's percentage of it.
+1 -1
View File
@@ -628,7 +628,7 @@ func wantBytesCount(
client netip.Prefix,
now time.Time,
bytes int64,
want ratelimit.LimitKind,
want string,
) {
t.Helper()
+18 -194
View File
@@ -11,7 +11,6 @@ import (
"net/netip"
"net/url"
"slices"
"strconv"
"sync"
"time"
@@ -24,9 +23,6 @@ const (
// AbuseIPDBURL is where clients are checked: the check endpoint of
// AbuseIPDB's API.
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
// the blacklist endpoint of AbuseIPDB's API.
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
// AbuseIPDBSource is how the request log, the alerts and the metrics
// name AbuseIPDB.
AbuseIPDBSource = "abuseipdb"
@@ -36,8 +32,6 @@ const (
var (
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
errNoBlacklist = errors.New("the answer gives no data")
errNotAddress = errors.New("is not an address")
errBudgetUsedUp = errors.New(
"checks spent; none is made until the day ends at 00:00 UTC")
)
@@ -59,13 +53,6 @@ type ChecksSpent struct {
Spent int
}
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
// abuse confidence score.
type BlacklistEntry struct {
Addr netip.Addr
Score int64
}
// AbuseIPDBParams are what NewAbuseIPDB needs.
type AbuseIPDBParams struct {
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
@@ -82,19 +69,10 @@ type AbuseIPDBParams struct {
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
CacheTTL time.Duration
Timeout time.Duration
// BlacklistURL is where the blacklist is fetched from, normally
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
// receives the clients on each blacklist fetched.
BlacklistURL string
Threshold int64
BlacklistRefresh time.Duration
Blacklisted func([]BlacklistEntry)
// Now tells the time, normally time.Now in UTC.
Now func() time.Time
// ProcessLog receives each check or fetch of the blacklist that fails,
// and why, and the day's budget used up.
// ProcessLog receives each check that fails, and why, and the day's
// budget used up.
ProcessLog *slog.Logger
// Alerts receive a source_failure alert for each.
Alerts *alerts.Queue
@@ -110,12 +88,8 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way, each with the
// function that ends it. stopped is true once StopChecking has been
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// checking are the clients whose check is under way.
checking map[netip.Prefix]bool
// day is the day, in UTC, of the checks spent counts.
day civil.Date
spent int
@@ -138,7 +112,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params,
httpClient: &http.Client{},
scores: scores,
checking: map[netip.Prefix]context.CancelFunc{},
checking: map[netip.Prefix]bool{},
}
}
@@ -148,11 +122,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, the day's checks have
// used up DailyBudget, or StopChecking has been called; Hit never waits
// for a check. The check that uses the budget up is logged and raised as a
// source_failure alert. ctx is the context of the client's request, and a
// check goes on after the request ends, until StopChecking.
// way, a check failed less than failureDelay ago, or the day's checks
// have used up DailyBudget; Hit never waits for a check. The check that
// uses the budget up is logged and raised as a source_failure alert. ctx
// is the context of the client's request, and a check goes on after the
// request ends.
func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) {
@@ -171,21 +145,14 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0
}
_, checking := a.checking[client]
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
if check {
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
a.checking[client] = end
a.checking[client] = true
a.checks++
a.spent++
a.checkGoroutines.Go(func() {
defer end()
a.check(checkCtx, client, addr)
})
go a.check(context.WithoutCancel(ctx), client, addr)
}
usedUp := check && a.spent == a.params.DailyBudget
@@ -200,24 +167,6 @@ func (a *AbuseIPDB) Hit(
return 0, false
}
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int {
a.mu.Lock()
@@ -307,45 +256,10 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
a.day, a.spent = checks.Day, checks.Spent
}
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
// fails is logged and raised as a source_failure alert, unless it was cut
// off as ctx ended.
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
if a.params.Key == "" {
return
}
for {
listed, err := a.fetchBlacklist(ctx)
switch {
case ctx.Err() != nil:
return
case err != nil:
a.alert("fetching AbuseIPDB's blacklist failed", err)
default:
a.params.Blacklisted(listed)
}
timer := time.NewTimer(a.params.BlacklistRefresh)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay. A check
// ended as ctx ends, by StopChecking, is no failure.
// source_failure alert, and no client is checked for failureDelay.
func (a *AbuseIPDB) check(
ctx context.Context,
client netip.Prefix,
@@ -353,26 +267,24 @@ func (a *AbuseIPDB) check(
) {
score, err := a.ask(ctx, addr)
now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock()
delete(a.checking, client)
switch {
case err == nil:
if err == nil {
a.scores.Add(
client,
Score{Client: client, Score: score, Fetched: now},
)
case isFailure:
} else {
a.failures++
a.retryAt = now.Add(failureDelay)
}
a.mu.Unlock()
if isFailure {
if err != nil {
a.alert("checking a client with AbuseIPDB failed", err)
}
}
@@ -432,94 +344,6 @@ func (a *AbuseIPDB) ask(
return *answer.Data.AbuseConfidenceScore, nil
}
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
// Threshold or more, sending the key in the header Key. An answer other
// than 200, one longer than maxListBytes or without the list, an entry
// whose address does not read, and none within fetchTimeout, fail.
func (a *AbuseIPDB) fetchBlacklist(
ctx context.Context,
) ([]BlacklistEntry, error) {
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
defer cancel()
query := url.Values{
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
if err != nil {
return nil, fmt.Errorf("make the request: %w", err)
}
req.Header.Set("Key", a.params.Key)
req.Header.Set("Accept", "application/json")
res, err := a.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
}
defer func() {
_ = res.Body.Close()
}()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
}
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
if err != nil {
return nil, fmt.Errorf("read the blacklist: %w", err)
}
if len(body) > maxListBytes {
return nil, errTooLong
}
return parseBlacklist(body)
}
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
var answer struct {
Data *[]struct {
IPAddress string `json:"ipAddress"`
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
} `json:"data"`
}
err := json.Unmarshal(body, &answer)
if err != nil {
return nil, fmt.Errorf("read the blacklist: %w", err)
}
if answer.Data == nil {
return nil, errNoBlacklist
}
listed := make([]BlacklistEntry, 0, len(*answer.Data))
for i, entry := range *answer.Data {
addr, err := netip.ParseAddr(entry.IPAddress)
if err != nil || addr.Zone() != "" {
// The error leaves out what the entry holds, as it does a
// client's address.
return nil, fmt.Errorf(
"entry %d's ipAddress %w",
i+1,
errNotAddress,
)
}
listed = append(listed, BlacklistEntry{
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
})
}
return listed, nil
}
// alert raises a source_failure alert from AbuseIPDB with reason and err,
// and logs them.
func (a *AbuseIPDB) alert(reason string, err error) {
-37
View File
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
})
}
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel()
-252
View File
@@ -1,252 +0,0 @@
package reputation_test
import (
"context"
"fmt"
"io"
"net/http"
"net/netip"
"slices"
"strings"
"sync"
"testing"
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The tests of the blacklist run in synctest bubbles, as those of the
// checks do, with AbuseIPDB a stand-in reached without the network.
// blacklistURL is where the tests fetch the blacklist from.
const blacklistURL = "https://abuseipdb.example/api/v2/blacklist"
func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` +
`{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` +
`{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` +
`{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`}
got := runBlacklist(t, abuseIPDB, key, newQueue())
want := []reputation.BlacklistEntry{
{Addr: netip.MustParseAddr("203.0.113.9"), Score: 100},
{Addr: netip.MustParseAddr("198.51.100.7"), Score: 95},
{Addr: netip.MustParseAddr("2001:db8::9"), Score: 90},
}
synctest.Wait()
got.want(t, want)
time.Sleep(time.Hour - time.Nanosecond)
synctest.Wait()
got.want(t, want)
time.Sleep(time.Nanosecond)
synctest.Wait()
got.want(t, want, want)
// Each fetch asks for the clients scored 90 or more, with the key
// in the header Key alone.
abuseIPDB.mu.Lock()
defer abuseIPDB.mu.Unlock()
for _, req := range abuseIPDB.asked {
if req.URL.String() != blacklistURL+"?confidenceMinimum=90" ||
req.Header.Get("Key") != key {
t.Errorf(
"asked %s with the key %q, want %s?confidenceMinimum=90 "+
"with %q",
req.URL,
req.Header.Get("Key"),
blacklistURL,
key,
)
}
}
})
}
func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh(
t *testing.T,
) {
t.Parallel()
for _, tc := range []struct{ name, body, error string }{
{
"a refusal", "",
"the server answered 429 Too Many Requests",
},
{
"an answer without the list", `{"errors":[]}`,
"the answer gives no data",
},
{
"an entry that is not an address",
`{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`,
"entry 2's ipAddress is not an address",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: tc.body}
if tc.body == "" {
abuseIPDB.status = http.StatusTooManyRequests
}
queue := newQueue()
got := runBlacklist(t, abuseIPDB, key, queue)
synctest.Wait()
got.want(t)
alerted := waiting(queue)
if len(alerted) != 1 ||
alerted[0].Event != alerts.EventSourceFailure ||
alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" ||
alerted[0].Detail["source"] != reputation.AbuseIPDBSource ||
alerted[0].Detail["error"] != tc.error {
t.Errorf(
"alerts %+v, want the failure %q",
alerted,
tc.error,
)
}
abuseIPDB.answer(`{"data":[]}`)
time.Sleep(time.Hour)
synctest.Wait()
got.want(t, []reputation.BlacklistEntry{})
})
})
}
}
func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: `{"data":[]}`}
got := runBlacklist(t, abuseIPDB, "", newQueue())
time.Sleep(time.Hour)
synctest.Wait()
got.want(t)
if len(abuseIPDB.asked) != 0 {
t.Errorf("fetched the blacklist %d times, want none",
len(abuseIPDB.asked))
}
})
}
// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of
// the blacklist with status, 200 when it is zero, and body, and keeps the
// requests.
type blacklistStandIn struct {
mu sync.Mutex
status int
body string
asked []*http.Request
}
// RoundTrip has the stand-in answer req, in place of the network.
func (s *blacklistStandIn) RoundTrip(
req *http.Request,
) (*http.Response, error) {
s.mu.Lock()
s.asked = append(s.asked, req)
status, body := s.status, s.body
s.mu.Unlock()
if status == 0 {
status = http.StatusOK
}
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(body)),
Request: req,
}, nil
}
// answer has the stand-in answer each fetch with 200 and body.
func (s *blacklistStandIn) answer(body string) {
s.mu.Lock()
defer s.mu.Unlock()
s.status, s.body = 0, body
}
// blacklisted holds the clients on each blacklist given to Blacklisted.
type blacklisted struct {
mu sync.Mutex
fetched [][]reputation.BlacklistEntry
}
// want checks the clients on each blacklist given so far.
func (b *blacklisted) want(
t *testing.T,
want ...[]reputation.BlacklistEntry,
) {
t.Helper()
b.mu.Lock()
defer b.mu.Unlock()
if !slices.EqualFunc(b.fetched, want, slices.Equal) {
t.Errorf("blacklists %v, want %v", b.fetched, want)
}
}
// runBlacklist runs RunBlacklist, until the test ends, with the key
// accountKey, the threshold 90 and a refresh of an hour, fetching from
// abuseIPDB and raising alerts into queue, and returns what it gives
// Blacklisted.
func runBlacklist(
t *testing.T, abuseIPDB *blacklistStandIn, accountKey string,
queue *alerts.Queue,
) *blacklisted {
t.Helper()
got := &blacklisted{}
p := abuseIPDBParams()
p.Key = accountKey
p.BlacklistURL = blacklistURL
p.Threshold = 90
p.BlacklistRefresh = time.Hour
p.Alerts = queue
p.Blacklisted = func(listed []reputation.BlacklistEntry) {
got.mu.Lock()
defer got.mu.Unlock()
got.fetched = append(got.fetched, listed)
}
checker := reputation.NewAbuseIPDB(p)
checker.SetTransport(abuseIPDB)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan struct{})
go func() {
checker.RunBlacklist(ctx)
close(done)
}()
t.Cleanup(func() {
cancel()
<-done
})
return got
}
+17 -67
View File
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
)
const (
@@ -31,8 +32,6 @@ const (
// it fails, and no client is checked with AbuseIPDB after a check
// fails, so that a source refusing them is not asked on every request.
failureDelay = time.Minute
// maskedKey is what MaskZoneKey shows in place of a zone's key.
maskedKey = "********"
)
var (
@@ -83,12 +82,8 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way, each with the function that ends
// it. stopped is true once StopAsking has been called, and
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// asking are the queries under way.
asking map[query]bool
// queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be
// asked again.
@@ -125,31 +120,13 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params,
resolver: resolver,
verdicts: verdicts,
asking: map[query]context.CancelFunc{},
asking: map[query]bool{},
queries: map[string]int{},
failures: map[string]int{},
retryAt: map[string]time.Time{},
}
}
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return maskedKey + "." + rest
}
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
func (d *DNSBL) Zones() []string {
return slices.Clone(d.params.Zones)
@@ -157,14 +134,13 @@ func (d *DNSBL) Zones() []string {
// ListedBy returns the zones whose verdict on addr, a client's address,
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
// masked, as MaskZoneKey masks it, since they go to the request
// masked, as config.MaskZoneKey masks it, since they go to the request
// log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, maxQueries are under way, or
// StopAsking has been called; ListedBy never waits for a query. ctx is the
// context of the client's request, and a query goes on after the request
// ends, until StopAsking.
// zone is left alone after a failure, or maxQueries are under way;
// ListedBy never waits for a query. ctx is the context of the client's
// request, and a query goes on after the request ends.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock()
defer d.mu.Unlock()
@@ -177,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed {
listedBy = append(listedBy, MaskZoneKey(zone))
listedBy = append(listedBy, config.MaskZoneKey(zone))
}
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
len(d.asking) < maxQueries:
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
d.asking[q] = true
d.queries[zone]++
d.queryGoroutines.Go(func() {
defer end()
d.ask(queryCtx, q)
})
go d.ask(context.WithoutCancel(ctx), q)
}
}
return listedBy
}
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int {
d.mu.Lock()
@@ -286,32 +238,30 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
// failureDelay.
func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q)
now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock()
delete(d.asking, q)
switch {
case err == nil:
if err == nil {
d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
})
case isFailure:
} else {
d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay)
}
d.mu.Unlock()
if isFailure {
if err != nil {
const failed = "asking a DNSBL zone failed"
shown := MaskZoneKey(q.zone)
shown := config.MaskZoneKey(q.zone)
// Raised before it is logged, so that the alert is there once the
// log line is.
+5 -72
View File
@@ -31,12 +31,10 @@ import (
// this package: Go's resolver counts the queries under way in one
// sync.WaitGroup for the whole process, and the process fails when
// queries from two bubbles, or from a bubble and from outside one, are
// under way at once. The one test outside a bubble is parallel, so it
// starts only once every test that is not parallel has ended; no other
// parallel test of this package queries a resolver. TestMain has the
// resolver make its configuration, which it makes on its first query,
// outside every bubble, since the configuration holds a channel, which
// the bubble it was made in would keep to itself.
// under way at once. TestMain has the resolver make its configuration,
// which it makes on its first query, outside every bubble, since the
// configuration holds a channel, which the bubble it was made in would
// keep to itself.
const (
// zone and otherZone are the DNSBL zones the tests name.
@@ -210,45 +208,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
@@ -566,9 +525,8 @@ func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(
}
}
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestQueriesGoToTheResolverSWWDNSBLResolverNames(t *testing.T) {
t.Parallel()
resolver := &resolverStandIn{answers: map[string]answer{
listedName: {addrs: []string{listing}},
}}
@@ -860,28 +818,3 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
t.Errorf("asked about %v, want %v", got, want)
}
}
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
const (
dronebl = "dnsbl.dronebl.org"
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
spamhausMasked = "********.xbl.dq.spamhaus.net"
)
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := reputation.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
+10 -77
View File
@@ -7,10 +7,7 @@
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to files and reads them
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
// blacklist, which it does not keep. Last, it reads the netblocks, AS
// numbers and percentages that both the settings and the lists hold, and
// masks the keys of DNSBL zones.
// from there, so that a restart keeps them too.
package reputation
import (
@@ -22,12 +19,12 @@ import (
"net/http"
"net/netip"
"slices"
"strconv"
"strings"
"sync"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
)
const (
@@ -42,28 +39,15 @@ const (
)
var (
errStatus = errors.New("the server answered")
errTooLong = errors.New("the list is longer than 16 MiB")
errNotNetblockLine = errors.New(
errStatus = errors.New("the server answered")
errTooLong = errors.New("the list is longer than 16 MiB")
errNotNetblock = errors.New(
"is not an address or a netblock, such as 192.0.2.0/24",
)
errNotASNPercent = errors.New(
"is not an AS number, : and a percentage, such as AS64496:50")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
)
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// List is a list as lists.json holds it: the URL it is fetched from,
// when it was last tried, the fetch failed or not, and its last good copy:
// when that was fetched, and its lines, as fetched, comment lines
@@ -83,7 +67,7 @@ type Params struct {
BlocklistURLs []string
ASNLimitPercentURL string
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
TagLists []TagList
TagLists []config.TagList
// Refresh is how long after a list was last fetched or tried it is
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
Refresh time.Duration
@@ -480,11 +464,7 @@ func parseNetblocks(lines []string) (entries, error) {
netblock, ok := parseNetblock(text)
if !ok {
return entries{}, fmt.Errorf(
"line %d %w",
i+1,
errNotNetblockLine,
)
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
}
found.netblocks[netblock] = true
@@ -504,7 +484,7 @@ func parseNetblocks(lines []string) (entries, error) {
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
// stands for none, is refused.
func parseNetblock(text string) (netip.Prefix, bool) {
netblock, err := ParseNetblock(text)
netblock, err := config.ParseNetblock(text)
if err != nil {
return netip.Prefix{}, false
}
@@ -539,9 +519,9 @@ func parsePercents(lines []string) (entries, error) {
}
asnText, percentText, _ := strings.Cut(text, ":")
asn, asnErr := ParseASN(asnText)
asn, asnErr := config.ParseASN(asnText)
percent, percentErr := ParsePercent(percentText)
percent, percentErr := config.ParsePercent(percentText)
if asnErr != nil || percentErr != nil {
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
}
@@ -555,53 +535,6 @@ func parsePercents(lines []string) (entries, error) {
return found, nil
}
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// withoutComment returns line without anything after a ; or a #, and
// without the spaces around what is left.
func withoutComment(line string) string {
+23 -39
View File
@@ -15,91 +15,74 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// Action is what waf did with a request, as its request line names it,
// such as ActionForward.
type Action string
// The action a request line names: what waf did with the request.
const (
// ActionForward is a request waf allowed, which rproxy passes to
// the app.
ActionForward Action = "forward"
ActionForward = "forward"
// ActionRequestLineTooLong is a request whose request line is too long for
// the header limits.
ActionRequestLineTooLong Action = "request_line_too_long"
ActionRequestLineTooLong = "request_line_too_long"
// ActionRequestHeaderTooLarge is a request whose headers are over the
// header limits.
ActionRequestHeaderTooLarge Action = "request_header_too_large"
ActionRequestHeaderTooLarge = "request_header_too_large"
// ActionClientIsRateLimited is a request refused because it took its
// client over a rate limit, which bans the client.
ActionClientIsRateLimited Action = "client_is_rate_limited"
ActionClientIsRateLimited = "client_is_rate_limited"
// ActionTemporarilyBanned is a request refused because a ban that runs
// out covers its client, or because it matched a ban rule or asked for a
// trap path, and so made such a ban.
ActionTemporarilyBanned Action = "temporarily_banned"
ActionTemporarilyBanned = "temporarily_banned"
// ActionPermanentlyBanned is a request refused because a ban that never
// runs out covers its client, or because the request made such a ban or
// made its client's ban permanent.
ActionPermanentlyBanned Action = "permanently_banned"
ActionPermanentlyBanned = "permanently_banned"
// ActionBlockedByBlockRule is a request refused because it matched a
// block rule.
ActionBlockedByBlockRule Action = "blocked_by_block_rule"
ActionBlockedByBlockRule = "blocked_by_block_rule"
// ActionBlockedByCoreRuleSet is a request refused because the Core Rule
// Set scored it at or over SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
ActionBlockedByCoreRuleSet Action = "blocked_by_core_rule_set"
ActionBlockedByCoreRuleSet = "blocked_by_core_rule_set"
// ActionClientDenied is a request refused because its client is in
// SWW_WAF_DENY_NETS, in a blocklist while SWW_WAF_BLOCKLIST_ACTION is deny,
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny, or that carries a tag in
// SWW_WAF_TAG_DENY.
ActionClientDenied Action = "client_denied"
ActionClientDenied = "client_denied"
// ActionCountryDenied is a request refused because its client's country
// is not in SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
ActionCountryDenied Action = "country_denied"
// ActionChecksTimedOut is a request refused because the trap paths, the
// rule files and the Core Rule Set had not finished with it within 500
// milliseconds.
ActionChecksTimedOut Action = "checks_timed_out"
ActionCountryDenied = "country_denied"
// ActionOwnEndpoint is a request waf answered at one of its own
// endpoints, or a question about a path under /_sww/.
ActionOwnEndpoint Action = "own_endpoint"
ActionOwnEndpoint = "own_endpoint"
)
// Offence is what a request was held as, as its request line names it.
type Offence string
// OffenceLimit is the offence a request line names for a request that
// broke a rate limit or the refusal limit, or whose bytes broke a byte
// limit.
const OffenceLimit Offence = "limit"
const OffenceLimit = "limit"
// LimitHitMinuteRefusals is the limit_hit a request line names for a request
// that broke the refusal limit.
const LimitHitMinuteRefusals = "minute_refusals"
// WAFLineKind is which of waf's two lines about a question a request line
// is: WAFLineDecision or WAFLineReport.
type WAFLineKind string
// The waf_line a request line names, which says which of
// waf's two lines about a question it is.
const (
// WAFLineDecision is waf's line at its decision.
WAFLineDecision WAFLineKind = "decision"
WAFLineDecision = "decision"
// WAFLineReport is waf's line when rproxy's report
// comes, with the decision's fields and the report's together.
WAFLineReport WAFLineKind = "report"
WAFLineReport = "report"
)
// timeLayout is RFC 3339 with milliseconds.
@@ -112,8 +95,9 @@ const timeLayout = "2006-01-02T15:04:05.000Z07:00"
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Line struct {
Type string `json:"type"`
// WAFLine is set on waf's two lines about a question.
WAFLine WAFLineKind `json:"waf_line,omitempty"`
// WAFLine is WAFLineDecision or WAFLineReport
// on waf's two lines about a question.
WAFLine string `json:"waf_line,omitempty"`
// The standard web log fields. Scheme is how the client reached rproxy, or
// the trusted proxy in front of it.
@@ -159,15 +143,14 @@ type Line struct {
Aborted bool `json:"aborted,omitempty"`
// The decision.
Action Action `json:"action"`
Action string `json:"action"`
// WouldAction is, in observe mode, the action enforce mode would have
// taken with a request it would have refused: ActionClientDenied,
// ActionTemporarilyBanned, ActionPermanentlyBanned, ActionCountryDenied,
// ActionClientIsRateLimited, ActionBlockedByBlockRule,
// ActionBlockedByCoreRuleSet, or, in waf, ActionRequestLineTooLong or
// ActionRequestHeaderTooLarge for the header limits, or
// ActionChecksTimedOut.
WouldAction Action `json:"would_action,omitempty"`
// ActionRequestHeaderTooLarge for the header limits.
WouldAction string `json:"would_action,omitempty"`
// Heuristic is the name of the heuristic that refused the request, in
// observe mode too, and HeuristicFigures are the figures it acted on, by
// name.
@@ -213,8 +196,9 @@ type Line struct {
// RouteWatchTagsMissing, while SWW_WAF_ROUTEWATCH_URL is set, says why the
// request got no tags from routewatch: no_answer_yet, request_failed or
// no_route.
RouteWatchTagsMissing string `json:"routewatch_tags_missing,omitempty"`
Offence Offence `json:"offence,omitempty"`
RouteWatchTagsMissing string `json:"routewatch_tags_missing,omitempty"`
// Offence is the offence the request was held as, OffenceLimit.
Offence string `json:"offence,omitempty"`
// BanExpires is when the ban the request made, or was refused under,
// ends: a time, or "permanent".
BanExpires string `json:"ban_expires,omitempty"`
+12 -43
View File
@@ -99,12 +99,8 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way, each with the
// function that ends it. stopped is true once StopAsking has been
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// asking are the clients whose query is under way.
asking map[netip.Prefix]bool
// retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time
}
@@ -120,7 +116,7 @@ func New(params Params) *Client {
params: params,
httpClient: &http.Client{},
answers: answers,
asking: map[netip.Prefix]context.CancelFunc{},
asking: map[netip.Prefix]bool{},
}
}
@@ -128,9 +124,8 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
// is the context of the client's request, and a query goes on after the request
// ends, until StopAsking.
// failureKept ago, or maxQueries are under way. ctx is the context of the
// client's request, and a query goes on after the request ends.
func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) {
@@ -152,38 +147,15 @@ func (c *Client) Route(
return Route{}, RequestFailed
}
_, asking := c.asking[client]
if !asking && !c.stopped && len(c.asking) < maxQueries {
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
if !c.asking[client] && len(c.asking) < maxQueries {
c.asking[client] = true
c.queryGoroutines.Go(func() {
defer end()
c.ask(queryCtx, client, addr)
})
go c.ask(context.WithoutCancel(ctx), client, addr)
}
return Route{}, NoAnswerYet
}
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them.
func (c *Client) Snapshot() []Answer {
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
// failure.
// those under way is logged.
func (c *Client) ask(
ctx context.Context,
client netip.Prefix,
@@ -242,20 +213,18 @@ func (c *Client) ask(
) {
route, err := c.query(ctx, addr)
now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock()
delete(c.asking, client)
firstFailure := isFailure && !now.Before(c.retryAt)
firstFailure := err != nil && !now.Before(c.retryAt)
switch {
case err == nil:
if err == nil {
c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
})
case isFailure:
} else {
c.retryAt = now.Add(failureKept)
}
-48
View File
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
})
}
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel()
-6
View File
@@ -2,12 +2,6 @@ package rproxy
import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct {
b breaker
-53
View File
@@ -71,59 +71,6 @@ func TestServerHasTheFixedLimits(t *testing.T) {
}
}
// getOfSize is a GET whose request line and headers, the blank line that ends
// them included, are size bytes long.
func getOfSize(size int) string {
const start, end = "GET / HTTP/1.1\r\nHost: site.example\r\nX-Pad: ", "\r\n\r\n"
return start + strings.Repeat("x", size-len(start)-len(end)) + end
}
func TestARequestLineAndHeadersOf32KiBReachTheApp(t *testing.T) {
t.Parallel()
standIn := startWAF(t, allow)
app := startApp(t, greet)
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
conn := dial(t, addr)
send(t, conn, getOfSize(32<<10))
if status := readStatus(t, conn); status != http.StatusOK {
t.Errorf("status %d, want the app's 200", status)
}
}
func TestARequestLineAndHeadersOver32KiBAreRefusedAndTheConnectionClosed(
t *testing.T,
) {
t.Parallel()
standIn := startWAF(t, allow)
app := startApp(t, func(http.ResponseWriter, *http.Request) {
t.Error("the app was asked")
})
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
conn := dial(t, addr)
send(t, conn, getOfSize(32<<10+1))
// io.ReadAll returns without an error only once rproxy has closed the
// connection.
text, err := io.ReadAll(conn)
if err != nil || !strings.HasPrefix(string(text), "HTTP/1.1 431 ") {
t.Errorf(
"read %q (%v), want 431 and the connection closed",
text,
err,
)
}
if len(standIn.questions) != 0 {
t.Error("waf was asked")
}
}
func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) {
t.Parallel()
+2 -2
View File
@@ -67,7 +67,7 @@ type request struct {
// disposition is what is done with the request: waf's valid disposition,
// or SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION. It is "" when neither
// gave one.
disposition waf.Disposition
disposition string
// answered is true when waf gave disposition, which makes a report due,
// and wafNoAnswer when SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION did.
answered bool
@@ -454,7 +454,7 @@ func (rq *request) finish(ctx context.Context) {
requestEnd.Sub(rq.start),
),
DurationResponseMS: requestlog.Milliseconds(end.Sub(requestEnd)),
Disposition: string(rq.disposition),
Disposition: rq.disposition,
WAFBackendNoAnswer: rq.wafNoAnswer,
},
)
+3 -6
View File
@@ -94,7 +94,6 @@ func New(params Params) *Server {
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(),
breakerClock: time.Now,
}
return &Server{
@@ -134,10 +133,8 @@ type handler struct {
wafTransport http.RoundTripper
// reportURL is where the reports go.
reportURL string
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
// the time.
breaker breaker
breakerClock func() time.Time
// breaker is the circuit breaker for asks to waf.
breaker breaker
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
@@ -218,7 +215,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// This panic has Go's server close the connection without an answer,
// and log nothing; the deferred finish still runs.
panic(http.ErrAbortHandler)
case waf.DispositionDeny, "": // "" is an error, or no valid disposition
default: // a deny, an error, or an answer without a valid disposition
rq.answerWith(answer)
}
}
+14 -62
View File
@@ -72,9 +72,9 @@ const (
// What the stand-in for waf answers.
const (
dispositionAllow = string(waf.DispositionAllow)
dispositionDeny = string(waf.DispositionDeny)
dispositionClose = string(waf.DispositionClose)
dispositionAllow = waf.DispositionAllow
dispositionDeny = waf.DispositionDeny
dispositionClose = waf.DispositionClose
// deniedBody is the body of a deny.
deniedBody = "refused by waf"
)
@@ -324,8 +324,7 @@ func greet(w http.ResponseWriter, r *http.Request) {
// startRProxy starts rproxy with waf at backend and the app at appURL, and
// the settings in env on top of the defaults. It returns where rproxy listens
// and what it writes. SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS is a minute unless env
// sets it, so that a busy host cannot make rproxy drop a question or a report.
// and what it writes.
func startRProxy(
t *testing.T, backend, appURL string, env map[string]string,
) (string, *output) {
@@ -356,22 +355,8 @@ func startServerLogging(
) (string, *rproxy.Server) {
t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
}
maps.Copy(settings, env)
@@ -380,7 +365,7 @@ func newServer(
t.Fatalf("settings %v: %v", settings, err)
}
return rproxy.New(rproxy.Params{
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger(
@@ -389,11 +374,6 @@ func newServer(
slog.LevelInfo,
),
})
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
t.Context(),
@@ -414,7 +394,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
server.WaitForReports()
})
return listener.Addr().String()
return listener.Addr().String(), server
}
// answered is the visitor's answer.
@@ -1444,14 +1424,7 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
failure(w, r)
},
)
nextID := make(chan string, 1)
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
if answering.Load() {
nextID <- r.Header.Get(requestIDHeader)
}
greet(w, r)
})
app := startApp(t, greet)
addr, out := startRProxy(
t,
standIn.URL,
@@ -1461,36 +1434,19 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
},
)
// An ask can time out before it reaches the stand-in, so these
// may leave fewer than five questions, or leave some late.
for range 5 {
wantGreeting(t, get(t, addr, "/"))
}
// rproxy writes this line before it answers a request it kept
// from asking waf, so no wait is needed.
if out.has("the circuit breaker is open") {
t.Error("the circuit breaker opened before five asks failed")
standIn.question(t)
}
answering.Store(true)
wantGreeting(t, get(t, addr, "/"))
out.waitFor(t, "the circuit breaker is open")
// Had the next request asked waf, its question would have reached
// the stand-in before the request reached the app.
var id string
select {
case id = <-nextID:
default:
t.Fatal("the app did not get the next request")
}
for len(standIn.questions) > 0 {
asked := <-standIn.questions
if asked.header.Get(requestIDHeader) == id {
t.Error("waf was asked, want it not to be")
}
case <-standIn.questions:
t.Error("waf was asked, want it not to be")
case <-time.After(10 * pollInterval):
}
})
}
@@ -1550,13 +1506,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r)
})
app := startApp(t, greet)
out := &output{}
server := newServer(t, standIn.URL, app.URL, nil, out, out)
// The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
wafBackendTimeout: "60000",
})
for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+5 -6
View File
@@ -10,6 +10,7 @@ import (
"maps"
"net/http"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -40,7 +41,7 @@ type wafAnswer struct {
// notes nothing, and no report is due for it: waf holds no decision to report
// on.
func (rq *request) noteAnswer(answer *wafAnswer) {
disposition := waf.Disposition(answer.header.Get(waf.DispositionHeader))
disposition := answer.header.Get(waf.DispositionHeader)
switch disposition {
case waf.DispositionAllow, waf.DispositionDeny,
@@ -76,9 +77,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
// https://git.eeqj.de/sneak/smallwebwaf/issues/182.
rq.h.processLog.Warn("waf gave no answer", "error", err.Error())
rq.disposition = waf.Disposition(
rq.h.config.WAFBackendNoAnswerDisposition,
)
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
rq.wafNoAnswer = true
if rq.disposition == waf.DispositionAllow {
@@ -98,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context,
) (*wafAnswer, error) {
b := &rq.h.breaker
if !b.mayAsk(rq.h.breakerClock()) {
if !b.mayAsk(time.Now()) {
return nil, errBreakerOpen
}
@@ -108,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil:
b.noteAnswer()
case ctx.Err() == nil:
b.noteFailure(rq.h.breakerClock())
b.noteFailure(time.Now())
}
return answer, err
+9 -12
View File
@@ -21,24 +21,20 @@ import (
"time"
"github.com/fsnotify/fsnotify"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
)
// Action is what a rule does when it matches: ActionLog, ActionBlock or
// ActionBan.
type Action string
// The actions a rule takes when it matches.
const (
// ActionLog notes the match in the request log, and does nothing else.
ActionLog Action = "log"
ActionLog = "log"
// ActionBlock refuses the request with 403.
ActionBlock Action = "block"
ActionBlock = "block"
// ActionBan refuses the request and bans the client's netblock: the
// request is a clear sign of attack.
ActionBan Action = "ban"
ActionBan = "ban"
)
// extension ends the name of every rule file.
@@ -99,7 +95,8 @@ type Rule struct {
// Target is what the regex is matched against, such as path or
// header:Accept.
Target string
Action Action
// Action is ActionLog, ActionBlock or ActionBan.
Action string
regex *regexp.Regexp
}
@@ -369,7 +366,7 @@ func parse(line string) (Rule, bool, error) {
return Rule{}, false, errNotRule
}
rule := Rule{ID: fields[1], Target: fields[2], Action: Action(fields[3])}
rule := Rule{ID: fields[1], Target: fields[2], Action: fields[3]}
headerName, isHeader := strings.CutPrefix(rule.Target, headerTarget)
switch {
@@ -381,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
rule.Target,
errNotTarget,
)
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
case isHeader && !config.IsHeaderName(headerName):
return Rule{}, false, fmt.Errorf(
"the target %q %w",
rule.Target,
@@ -397,7 +394,7 @@ func parse(line string) (Rule, bool, error) {
rule.Target,
errHeaderTakenOut,
)
case !slices.Contains([]Action{ActionLog, ActionBlock, ActionBan}, rule.Action):
case !slices.Contains([]string{ActionLog, ActionBlock, ActionBan}, rule.Action):
return Rule{}, false, fmt.Errorf(
"the action %q %w",
rule.Action,
+13 -16
View File
@@ -46,7 +46,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -104,7 +104,7 @@ var (
errVersion = errors.New("unknown version")
// errMissing is for an entry without a field it needs.
errMissing = errors.New("has no")
errCause = errors.New("is not limit, attack, admin or abuseipdb")
errCause = errors.New("is not limit, attack or admin")
errExpires = errors.New("is not after its start")
errDestination = errors.New("is not webhook, slack or ntfy")
errScope = errors.New(
@@ -135,7 +135,7 @@ type Params struct {
// sww runs.
Ledger *bans.Ledger
Limiter *ratelimit.Limiter
GeoJS *geojs.GeoJS
GeoJS *lookup.GeoJS
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
@@ -181,7 +181,7 @@ type BanEntry struct {
Netblock netip.Prefix `json:"netblock"`
Start time.Time `json:"start"`
Expires *time.Time `json:"expires"`
Cause bans.Cause `json:"cause"`
Cause string `json:"cause"`
Reason string `json:"reason,omitempty"`
Lifted *time.Time `json:"lifted,omitempty"`
Notes bans.Notes `json:"notes"`
@@ -195,8 +195,8 @@ type clientsFile struct {
// lookupsFile is lookups.json, with each answer on a line of its own.
type lookupsFile struct {
Version int `json:"version"`
Lookups []geojs.Answer `json:"lookups"`
Version int `json:"version"`
Lookups []lookup.Answer `json:"lookups"`
}
// abuseIPDBChecksSpentFile is abuseipdb-checks-spent.json, indented for an
@@ -237,11 +237,11 @@ type routeWatchFile struct {
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type alertsFile struct {
Version int `json:"version"`
Cooldowns []alerts.Cooldown `json:"cooldowns"`
Hour alerts.Hour `json:"hour"`
Waiting map[alerts.DestinationName][]alerts.Alert `json:"waiting"`
AnomalyCounters []anomaly.Counter `json:"anomaly_counters"`
Version int `json:"version"`
Cooldowns []alerts.Cooldown `json:"cooldowns"`
Hour alerts.Hour `json:"hour"`
Waiting map[string][]alerts.Alert `json:"waiting"`
AnomalyCounters []anomaly.Counter `json:"anomaly_counters"`
}
// stateFile is the struct of a state file. Once the file is decoded, its
@@ -638,8 +638,6 @@ func (f *Files) takeInState(
f.params.Ledger.Load(held)
}
f.params.Ledger.ForgetEndedAbuseIPDBBans(f.params.Now())
entries = len(held)
case clientsJSON:
var file clientsFile
@@ -969,7 +967,7 @@ func (e BanEntry) ban() bans.Ban {
// each expires is read again as written. An expires not after the start
// is refused too, since the netblock's next ban for a broken limit, three
// times as long as this one, would then refuse nothing; and so is a cause
// other than limit, attack, admin or abuseipdb, most likely misspelt.
// other than limit, attack or admin, most likely misspelt.
func (f *bansFile) check(data []byte) error {
var written struct {
Bans []struct {
@@ -993,8 +991,7 @@ func (f *bansFile) check(data []byte) error {
case entry.Expires != nil && !entry.Expires.After(entry.Start):
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
entry.Cause != bans.CauseAbuseIPDB:
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
return fmt.Errorf(
"entry %d's cause %q %w",
i+1,
+30 -74
View File
@@ -26,7 +26,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -114,8 +114,7 @@ const permanentBansJSON = `{
"earlier_bans": {
"limit": 3,
"attack": 1,
"admin": 1,
"abuseipdb": 0
"admin": 1
}
}
}
@@ -1191,7 +1190,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
`"expires": null, "cause": "atack"}]}`,
`: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`)
`: entry 4's cause "atack" is not limit, attack or admin`)
}
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
@@ -1284,18 +1283,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it.
first, _ := params.Ledger.Ban(
first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban(
second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1357,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run)
// A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay)
synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and
// another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1482,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1555,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
err = files.WriteAll()
@@ -1701,7 +1700,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
t,
lookupsJSON,
params.GeoJS.Snapshot(),
[]geojs.Answer{
[]lookup.Answer{
{Client: client, Country: "FR", Answered: midnight()},
},
)
@@ -1758,8 +1757,8 @@ func TestEditOfRouteWatchAndAlertsFilesTakenIn(t *testing.T) {
Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"),
Sent: midnight(),
}},
Hour: alerts.Hour{HeldBack: map[alerts.Event]int{}},
Waiting: map[alerts.DestinationName][]alerts.Alert{
Hour: alerts.Hour{HeldBack: map[string]int{}},
Waiting: map[string][]alerts.Alert{
alerts.DestinationWebhook: {
{Event: alerts.EventFileError, Time: midnight()},
},
@@ -2046,8 +2045,8 @@ func TestBanWithoutACauseTakenInAsAnAdmins(t *testing.T) {
var written struct {
Bans []struct {
Cause bans.Cause `json:"cause"`
Reason string `json:"reason"`
Cause string `json:"cause"`
Reason string `json:"reason"`
} `json:"bans"`
}
@@ -2091,39 +2090,6 @@ func TestLiftedBanReadAtTheStart(t *testing.T) {
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
}
func TestEndedAbuseIPDBBanForgottenWhenBansJSONIsReadAtTheStart(
t *testing.T,
) {
t.Parallel()
ended := netip.MustParsePrefix("203.0.113.9/32")
lasting := netip.MustParsePrefix("203.0.113.10/32")
limited := netip.MustParsePrefix("203.0.113.11/32")
// At midnight, the start's time, the first ban has ended and the second
// lasts. The third has ended too, but is for a limit.
dir := t.TempDir()
edit(t, dir, bansJSON, `{"version": 1, "bans": [`+
`{"netblock": "203.0.113.9/32", "start": "2026-09-06T00:00:00Z", `+
`"expires": "2026-10-06T00:00:00Z", "cause": "abuseipdb"}, `+
`{"netblock": "203.0.113.10/32", "start": "2026-09-07T00:00:00Z", `+
`"expires": "2026-10-07T00:00:00Z", "cause": "abuseipdb"}, `+
`{"netblock": "203.0.113.11/32", "start": "2026-10-05T00:00:00Z", `+
`"expires": "2026-10-05T01:00:00Z", "cause": "limit"}]}`)
params := newParams(dir)
load(t, params)
for netblock, want := range map[netip.Prefix]int{
ended: 0, lasting: 1, limited: 1,
} {
if got := len(params.Ledger.Bans(netblock)); got != want {
t.Errorf("the ledger holds %d bans on %s, want %d",
got, netblock, want)
}
}
}
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
t.Parallel()
@@ -2135,11 +2101,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned {
@@ -2453,7 +2415,9 @@ func newParams(dir string) state.Params {
MaxBans: 5000,
}),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}),
GeoJS: lookup.New(lookup.Params{
Now: midnight, ProcessLog: discard, Metrics: m,
}),
Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{
blocklistURL,
@@ -2514,15 +2478,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
})
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2534,7 +2494,7 @@ func fill(params state.Params) {
})
params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE")
params.GeoJS.Load([]geojs.Answer{
params.GeoJS.Load([]lookup.Answer{
{
Client: client, ASN: asn, ASName: asName, Country: "DE",
Answered: now.Add(-time.Hour), Used: now,
@@ -2708,11 +2668,7 @@ func wantLiftedBanKept(
}
// Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban(
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf(
"the next ban lasts %s, want 1h",
-198
View File
@@ -1,198 +0,0 @@
package sww
import (
"context"
"errors"
"io"
"log/slog"
"net"
"net/http"
"time"
"go.uber.org/fx"
"go.uber.org/fx/fxevent"
)
// shutdownTimeout is how long requests in progress may take to finish
// once sww is told to stop, before their connections are closed.
// runit and docker wait a little longer before they kill the process.
const shutdownTimeout = 5 * time.Second
// requestLog is where a daemon's server writes the request log: stdout, and
// the remote log sender while there is one. fx tells parts apart by type, so
// it has a type of its own.
type requestLog struct {
io.Writer
}
// servingFailed gets why serving failed, for run, which then stops the
// daemon. fx tells parts apart by type, so it has a type of its own.
type servingFailed chan error
// utcNow is the time the state files, the alerts and the rate limits go by.
func utcNow() time.Time {
return time.Now().UTC()
}
// startError is why a daemon cannot start: the message its process log gives,
// and the error it logs with it.
type startError struct {
message string
err error
}
func (e startError) Error() string {
return e.message + ": " + e.err.Error()
}
func (e startError) Unwrap() error {
return e.err
}
// cannotStart logs why a daemon cannot start, with the message of the
// startError in err, and returns its exit status.
func cannotStart(processLog *slog.Logger, err error) int {
message, cause := "cannot start", err
failed, ok := errors.AsType[startError](err)
if ok {
message, cause = failed.message, failed.err
}
processLog.Error(message, "error", cause.Error())
return 1
}
// listen listens on address, which the setting names.
func listen(address, setting string) (net.Listener, error) {
listener, err := (&net.ListenConfig{}).Listen(
context.Background(), "tcp", address)
if err != nil {
return nil, startError{"cannot listen on " + setting, err}
}
return listener, nil
}
// background returns the hook that runs loop on a goroutine of its own from
// the start, and at the stop ends it and waits for it to return.
func background(loop func(context.Context)) fx.Hook {
running, stop := context.WithCancel(context.Background())
ended := make(chan struct{})
return fx.StartStopHook(
func() {
go func() {
loop(running)
close(ended)
}()
},
func() {
stop()
<-ended
},
)
}
// serve serves requests on listener from the start. At the stop it gives the
// requests in progress shutdownTimeout to finish, then closes their
// connections. When serving fails, it logs why and sends the error to
// failed.
func serve(
lc fx.Lifecycle, server *http.Server, listener net.Listener,
processLog *slog.Logger, failed servingFailed,
) {
served := make(chan error, 1)
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
go func() {
err := server.Serve(listener)
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
failed <- err
}
served <- err
}()
return nil
},
OnStop: func(ctx context.Context) error {
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(ctx, shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
},
})
}
// run starts app, which serves until ctx is done or serving fails, as
// failed tells. Then it stops app and takes the daemon's last step. It
// returns the process's exit status, 1 when serving failed, a part could
// not stop in order or the last step failed; each logs why.
func run(
ctx context.Context, app *fx.App, failed servingFailed,
processLog *slog.Logger, lastStep func() error,
) int {
// The hooks only start goroutines, and end them at the stop, so neither
// is cut short by a deadline or by ctx.
err := app.Start(context.WithoutCancel(ctx))
if err != nil {
return cannotStart(processLog, err)
}
status := 0
select {
case <-ctx.Done():
case <-failed:
status = 1
}
err = app.Stop(context.WithoutCancel(ctx))
if err != nil {
status = 1
}
err = lastStep()
if err != nil {
status = 1
}
if status == 0 {
processLog.Info("stopped")
}
return status
}
// fxEventLogger returns what fx.WithLogger takes to write fx's own events to
// processLog at debug, which the daemons' log levels normally hold back. Its
// errors are at debug too: the daemon logs each one itself.
func fxEventLogger(processLog *slog.Logger) func() fxevent.Logger {
return func() fxevent.Logger {
logger := &fxevent.SlogLogger{Logger: processLog}
logger.UseLogLevel(slog.LevelDebug)
logger.UseErrorLevel(slog.LevelDebug)
return logger
}
}
+62 -41
View File
@@ -2,13 +2,12 @@ package sww
import (
"context"
"errors"
"log/slog"
"net"
"net/http"
"os"
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
@@ -31,31 +30,26 @@ func RunRProxy(ctx context.Context, params Params) int {
cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil {
return cannotStart(processLog, startError{"invalid setting", err})
processLog.Error("invalid setting", "error", err.Error())
return 1
}
var (
server *rproxy.Server
listener net.Listener
)
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: params.Stdout,
ProcessLog: processLog,
})
failed := make(servingFailed, 1)
app := fx.New(
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
fx.Provide(
newRProxyServer,
func(server *rproxy.Server) *http.Server { return server.Server },
newRProxyListener,
),
fx.Invoke(serve),
fx.Populate(&server, &listener),
fx.WithLogger(fxEventLogger(processLog)),
)
err = app.Err()
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
if err != nil {
return cannotStart(processLog, err)
processLog.Error(
"cannot listen on SWW_RPROXY_LISTEN_ADDR",
"error",
err.Error(),
)
return 1
}
processLog.Info("starting",
@@ -63,25 +57,52 @@ func RunRProxy(ctx context.Context, params Params) int {
"address", listener.Addr().String(),
"settings", cfg)
return run(ctx, app, failed, processLog, func() error {
server.WaitForReports()
return nil
})
return serveRProxy(ctx, server, listener, processLog)
}
// newRProxyServer returns the server rproxy runs.
func newRProxyServer(
cfg *config.RProxy, out requestLog, processLog *slog.Logger,
) *rproxy.Server {
return rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
})
}
// serveRProxy serves requests on listener until ctx is done, and then stops as
// RunRProxy says.
func serveRProxy(
ctx context.Context, server *rproxy.Server, listener net.Listener,
processLog *slog.Logger,
) int {
served := make(chan error, 1)
// newRProxyListener listens on SWW_RPROXY_LISTEN_ADDR.
func newRProxyListener(cfg *config.RProxy) (net.Listener, error) {
return listen(cfg.ListenAddr, "SWW_RPROXY_LISTEN_ADDR")
go func() {
served <- server.Serve(listener)
}()
select {
case err := <-served:
processLog.Error("serving failed", "error", err.Error())
return 1
case <-ctx.Done():
}
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
return 1
}
server.WaitForReports()
processLog.Info("stopped")
return 0
}
+22 -25
View File
@@ -51,9 +51,7 @@ func runRProxy(ctx context.Context, env map[string]string, out *output) int {
// startRProxy runs rproxy with waf at backend and the app at appURL, and
// returns its URL and what it writes. It believes X-Forwarded-For from the
// tests. It waits a minute for waf's answer, so that a slow answer on a busy
// host is not taken for no answer. It stops as the test ends, and that it
// stopped in order is checked.
// tests. It stops as the test ends, and that it stopped in order is checked.
func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
t.Helper()
@@ -67,7 +65,6 @@ func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
"SWW_RPROXY_UPSTREAM_URL": appURL,
"SWW_RPROXY_TRUSTED_PROXIES": localhost + "/32",
wafBackend: backend,
wafBackendTimeout: "60000",
}, out)
}()
@@ -164,7 +161,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
// One request's three lines carry one id, rproxy's with waf's answer.
line := firstOut.line(t, "path", bannedProbe)
if line["disposition"] != string(waf.DispositionDeny) ||
if line["disposition"] != waf.DispositionDeny ||
line["client_ip"] != placed {
t.Errorf("rproxy's line is %v", line)
}
@@ -249,7 +246,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
line := out.line(t, "path", "/after-stop")
if line["disposition"] != string(waf.DispositionAllow) ||
if line["disposition"] != waf.DispositionAllow ||
line["waf_backend_no_answer"] != true {
t.Errorf("rproxy's line is %v", line)
}
@@ -552,9 +549,7 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
// settings in env, each as NAME=value. It returns where rproxy listens and a
// function that sends rproxy SIGTERM and fails the test unless it exits with
// status 0 within waitLimit. It waits a minute for waf's answers unless env
// sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, so that a busy host cannot make it
// drop a question.
// status 0 within waitLimit.
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
t.Helper()
@@ -570,8 +565,7 @@ func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
cmd.Env = append(os.Environ(),
commandForMain+"=rproxy",
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0",
wafBackendTimeout+"=60000")
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0")
cmd.Env = append(cmd.Env, env...)
exited := startProcess(t, cmd)
@@ -608,8 +602,7 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
if r.Header.Get(waf.ClientHeader) != "" {
protocols <- r.Proto
w.Header().
Set(waf.DispositionHeader, string(waf.DispositionDeny))
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny)
w.WriteHeader(http.StatusForbidden)
}
}))
@@ -692,8 +685,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" {
w.Header().
Set(waf.DispositionHeader, string(waf.DispositionAllow))
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
return
}
@@ -757,7 +749,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
}
}
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper()
@@ -770,6 +763,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close()
})
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request)
if err != nil {
t.Fatalf("send: %v", err)
@@ -781,7 +779,10 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
t.Parallel()
const cutOffAfter = 5 * time.Second
const (
cutOffAfter = 5 * time.Second
upstreamSetting = "SWW_RPROXY_UPSTREAM_URL"
)
asked := make(chan struct{})
release := make(chan struct{})
@@ -795,8 +796,7 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.Header().
Set(waf.DispositionHeader, string(waf.DispositionAllow))
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
}))
t.Cleanup(backend.Close)
@@ -804,9 +804,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
defer stop()
env := map[string]string{
rproxyListenAddr: localhost + ":0",
rproxyUpstreamURL: app.URL,
wafBackend: backend.URL,
rproxyListenAddr: localhost + ":0",
upstreamSetting: app.URL,
wafBackend: backend.URL,
}
out := &output{}
exited := make(chan int, 1)
@@ -832,9 +832,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err)
+393 -5
View File
@@ -4,14 +4,17 @@
// stop, and then stops in an orderly way, writing the state and cache files.
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
// front of one app that asks waf about each request. `sww healthcheck` is the
// container's health check. fx builds the parts of each daemon, starts their
// background work and stops it.
// container's health check.
package sww
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
@@ -19,9 +22,28 @@ import (
"github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// shutdownTimeout is how long requests in progress may take to finish
// once sww is told to stop, before their connections are closed.
// runit and docker wait a little longer before they kill the process.
const shutdownTimeout = 5 * time.Second
// remoteLogStopTimeout is how long, as waf stops, the log lines still
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
// carried them.
const remoteLogStopTimeout = 2 * time.Second
// usageStatus is the exit status with no command, or one sww does not
// know.
const usageStatus = 2
@@ -44,9 +66,6 @@ type Params struct {
Viper *viper.Viper
// Stdout receives the request log and the process's own messages.
Stdout io.Writer
// BesideLookupTimeLimit is the waf.Params.BesideLookupTimeLimit RunWAF
// gives waf. Main leaves it zero, waf's own limit; tests lengthen it.
BesideLookupTimeLimit time.Duration
}
// Main runs the command os.Args names, and returns the process's exit status.
@@ -87,3 +106,372 @@ func printUsage(w io.Writer) int {
return usageStatus
}
// RunWAF is `sww waf`: it reads the settings, the rule files,
// the lookup database and the state and cache files, then serves requests until
// ctx is done. It returns the process's exit status, 1 when waf cannot start.
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
return cannotStart(processLog, "invalid setting", err)
}
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
// is sent there too.
stdout := params.Stdout
var remote *remotelog.Sender
if cfg.LogRemoteURL != nil {
remote = newRemoteLogSender(cfg)
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
defer stopSending()
}
return runWAFWithSettings(
ctx,
params.Version,
cfg,
stdout,
remote,
processLog,
)
}
// runWAFWithSettings is the rest of RunWAF once the settings are read and
// the process log goes where they say. remote is nil unless
// SWW_WAF_LOG_REMOTE_URL is set.
func runWAFWithSettings(
ctx context.Context,
version string,
cfg *config.WAF,
stdout io.Writer,
remote *remotelog.Sender,
processLog *slog.Logger,
) int {
// The state files and the alerts give times in UTC.
now := func() time.Time { return time.Now().UTC() }
alertQueue := newAlertQueue(cfg, now, processLog)
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir,
Enabled: cfg.RuleFilesEnabled,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return cannotStart(processLog, "cannot use the rule files", err)
}
crs, err := waf.NewCoreRuleSet(cfg)
if err != nil {
return cannotStart(processLog, "cannot use the Core Rule Set", err)
}
server, err := newServer(
cfg,
stdout,
processLog,
now,
ruleFiles,
alertQueue,
crs,
)
if err != nil {
return cannotStart(processLog, "cannot use the lookup database", err)
}
if remote != nil {
server.Metrics.AddRemoteLog(remote)
}
files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
if err != nil {
return cannotStart(
processLog,
"cannot use the state and cache files",
err,
)
}
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
if err != nil {
return cannotStart(
processLog,
"cannot listen on SWW_WAF_LISTEN_ADDR",
err,
)
}
processLog.Info("starting",
"version", version,
"address", listener.Addr().String(),
"settings", cfg)
return serve(
ctx,
server,
listener,
files,
ruleFiles,
alertQueue,
processLog,
)
}
// cannotStart logs why waf cannot start, and returns its exit status.
func cannotStart(processLog *slog.Logger, message string, err error) int {
processLog.Error(message, "error", err.Error())
return 1
}
// newServer returns the server waf runs, with the metrics of the alerts,
// after reading the lookup database while SWW_WAF_LOOKUP_SOURCE is file. A lookup
// database that cannot be read is an error.
func newServer(
cfg *config.WAF, stdout io.Writer, processLog *slog.Logger,
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
crs *coreruleset.CoreRuleSet,
) (*waf.Server, error) {
var lookupFile *lookup.File
if cfg.LookupSource == "file" {
var err error
lookupFile, err = lookup.OpenFile(lookup.FileParams{
Path: cfg.LookupDBPath,
Now: now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, err
}
}
server := waf.New(waf.Params{
Config: cfg,
RequestLog: stdout,
ProcessLog: processLog,
GeoJSURL: lookup.URL,
AbuseIPDBURL: reputation.AbuseIPDBURL,
LookupFile: lookupFile,
CoreRuleSet: crs,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
})
server.Metrics.AddAlerts(alertQueue)
if lookupFile != nil {
server.Metrics.AddLookupFile(
lookupFile.LastRead,
lookupFile.ReadFailures,
)
}
return server, nil
}
// loadStateFiles reads the state and cache files into the parts of server and
// into alertQueue, as state.Load does.
func loadStateFiles(
cfg *config.WAF, server *waf.Server, alertQueue *alerts.Queue,
now func() time.Time, processLog *slog.Logger,
) (*state.Files, error) {
return state.Load(state.Params{
Dir: cfg.StateDir,
CacheDir: cfg.CacheDir,
BansWriteDelay: cfg.BansWriteDelay,
WriteInterval: cfg.StateWriteInterval,
Ledger: server.Ledger,
Limiter: server.Limiter,
GeoJS: server.GeoJS,
Lists: server.Lists,
DNSBL: server.DNSBL,
AbuseIPDB: server.AbuseIPDB,
RouteWatch: server.RouteWatch,
Alerts: alertQueue,
Anomalies: server.Anomalies,
Now: now,
ProcessLog: processLog,
Metrics: server.Metrics,
})
}
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
// ntfy, with the settings for them.
func newAlertQueue(
cfg *config.WAF, now func() time.Time, processLog *slog.Logger,
) *alerts.Queue {
return alerts.New(alerts.Params{
WebhookURL: cfg.AlertWebhookURL,
WebhookHeaders: cfg.AlertWebhookHeaders,
SlackURL: cfg.AlertSlackWebhookURL,
NtfyURL: cfg.AlertNtfyURL,
NtfyToken: cfg.AlertNtfyToken,
Events: cfg.AlertEvents,
Cooldown: cfg.AlertCooldown,
MaxPerHour: cfg.AlertMaxPerHour,
Instance: cfg.InstanceName,
Now: now,
ProcessLog: processLog,
})
}
// newRemoteLogSender returns a sender of the log lines to
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
return remotelog.New(remotelog.Params{
URL: cfg.LogRemoteURL,
RootCAs: cfg.LogRemoteTLSCAs,
BufferLines: cfg.LogRemoteBufferLines,
Facility: cfg.LogRemoteFacility,
AppName: cfg.LogRemoteAppName,
})
}
// startSending runs remote until the function it returns is called, which
// then waits at most remoteLogStopTimeout for the lines still waiting to
// be sent. Sending goes on after ctx is done, so that the lines written
// while sww stops are sent too.
func startSending(
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
) func() {
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
sent := make(chan struct{})
go func() {
remote.Run(sending, processLog)
close(sent)
}()
return func() {
stop()
select {
case <-sent:
case <-time.After(remoteLogStopTimeout):
}
}
}
// serve serves requests on listener, writes the state and cache files as they
// are due, takes in an admin's edits of them, reads the rule files again as
// they change, and the lookup database when it is replaced, fetches the
// lists the settings name by URL as they are due, and sends the alerts,
// until ctx is done. Then it gives the requests in progress
// shutdownTimeout to finish, and writes every state and cache file,
// alerts.json with the alerts still waiting.
func serve(
ctx context.Context, server *waf.Server, listener net.Listener,
files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
processLog *slog.Logger,
) int {
served := make(chan error, 1)
go func() {
served <- server.Serve(listener)
}()
writing, stopWriting := context.WithCancel(ctx)
defer stopWriting()
written := inBackground(func() { files.Run(writing) })
watched := inBackground(func() { files.Watch(writing) })
rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
lookupFileWatched := inBackground(func() {
if server.LookupFile != nil {
server.LookupFile.Watch(writing)
}
})
listsFetched := inBackground(func() { server.Lists.Run(writing) })
alertsSent := inBackground(func() { alertQueue.Run(writing) })
select {
case err := <-served:
processLog.Error("serving failed", "error", err.Error())
return 1
case <-ctx.Done():
}
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
return 1
}
// Run and Watch have ended, so nothing else reads or writes the
// files, and no alert is being sent, so that alerts.json keeps every
// alert not yet sent. Every request has ended too, but for one cut off
// because Shutdown timed out, which adds to its client's history only as
// it ends. That can be after this write, and then that request is missing
// from clients.json.
<-written
<-watched
<-rulesWatched
<-lookupFileWatched
<-listsFetched
<-alertsSent
err = files.WriteAll()
if err != nil {
processLog.Error(
"writing the state and cache files failed",
"error",
err.Error(),
)
return 1
}
processLog.Info("stopped")
return 0
}
// inBackground runs task on a goroutine of its own, and returns a channel
// that is closed once task has returned.
func inBackground(task func()) <-chan struct{} {
done := make(chan struct{})
go func() {
task()
close(done)
}()
return done
}
+1 -6
View File
@@ -173,9 +173,6 @@ func run(ctx context.Context, env map[string]string, out *output) int {
Version: testVersion,
Viper: v,
Stdout: out,
// The race detector and a busy host can slow the Core Rule Set's
// inspections past waf's own limit.
BesideLookupTimeLimit: time.Hour,
})
}
@@ -660,9 +657,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
status := run(t.Context(), map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
"SWW_WAF_CORE_RULE_SET_MODE": string(
config.CoreRuleSetModeOff,
),
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
}, out)
if status != 1 {
-554
View File
@@ -1,554 +0,0 @@
package sww
import (
"context"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"time"
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// remoteLogStopTimeout is how long, as waf stops, the log lines still
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
// carried them.
const remoteLogStopTimeout = 2 * time.Second
// RunWAF is `sww waf`: it reads the settings, the rule files,
// the lookup database and the state and cache files, then serves requests until
// ctx is done. Then it gives the requests in progress shutdownTimeout to
// finish, ends its background work and writes every state and cache file,
// alerts.json with the alerts still waiting. It returns the process's exit
// status, 1 when waf cannot start or stop in order.
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
return cannotStart(processLog, startError{"invalid setting", err})
}
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
// is sent there too. The sender runs before any part is built, so that it
// sends why waf cannot start.
stdout := params.Stdout
var remote *remotelog.Sender
if cfg.LogRemoteURL != nil {
remote = newRemoteLogSender(cfg)
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
defer stopSending()
}
var (
files *state.Files
listener net.Listener
)
failed := make(servingFailed, 1)
app := fx.New(
fx.Supply(
params,
cfg,
processLog,
requestLog{stdout},
remote,
failed,
),
fx.Provide(
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
newLimiter, newLedger, newAnomalyCounters,
newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
newMetrics, newGeoJS, newWAFServer,
func(server *waf.Server) *http.Server { return server.Server },
newWAFListener, newStateFiles,
),
// fx stops hooks in the reverse of the order they were added in. The
// state files are read before serving is set up, so that serving
// stops before their loops do.
fx.Populate(&files),
fx.Invoke(serve),
fx.Populate(&listener),
fx.WithLogger(fxEventLogger(processLog)),
)
err = app.Err()
if err != nil {
return cannotStart(processLog, err)
}
processLog.Info("starting",
"version", params.Version,
"address", listener.Addr().String(),
"settings", cfg)
// The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert, and of every ask of
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll()
if err != nil {
processLog.Error("writing the state and cache files failed",
"error", err.Error())
}
return err
})
}
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
// ntfy, with the settings for them. Until waf stops, it sends them.
func newAlertQueue(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *alerts.Queue {
alertQueue := alerts.New(alerts.Params{
WebhookURL: cfg.AlertWebhookURL,
WebhookHeaders: cfg.AlertWebhookHeaders,
SlackURL: cfg.AlertSlackWebhookURL,
NtfyURL: cfg.AlertNtfyURL,
NtfyToken: cfg.AlertNtfyToken,
Events: cfg.AlertEvents,
Cooldown: cfg.AlertCooldown,
MaxPerHour: cfg.AlertMaxPerHour,
Instance: cfg.InstanceName,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(background(alertQueue.Run))
return alertQueue
}
// newRuleFiles reads the rule files, and reads them again as they change
// until waf stops.
func newRuleFiles(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*rules.Files, error) {
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir,
Enabled: cfg.RuleFilesEnabled,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, startError{"cannot use the rule files", err}
}
lc.Append(background(ruleFiles.Watch))
return ruleFiles, nil
}
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
crs, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, startError{
"cannot use the Core Rule Set",
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
}
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return crs, nil
}
// newLookupFile reads the lookup database while SWW_WAF_LOOKUP_SOURCE is
// file, and reads it again when it is replaced until waf stops. It returns
// nil while the source is anything else.
func newLookupFile(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*lookup.File, error) {
if cfg.LookupSource != config.LookupSourceFile {
return nil, nil //nolint:nilnil // no lookup database, and no error
}
lookupFile, err := lookup.OpenFile(lookup.FileParams{
Path: cfg.LookupDBPath,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, startError{"cannot use the lookup database", err}
}
lc.Append(background(lookupFile.Watch))
return lookupFile, nil
}
// newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
}
// newLedger returns the ledger of the bans, with the settings for them.
func newLedger(cfg *config.WAF) *bans.Ledger {
return bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
AttackBanDuration: cfg.AttackBanDuration,
MaxBans: cfg.MaxBans,
})
}
// newAnomalyCounters returns the counters of the anomaly thresholds.
func newAnomalyCounters(
cfg *config.WAF, alertQueue *alerts.Queue,
) *anomaly.Counters {
return anomaly.New(anomaly.Params{
Client: cfg.AnomalyClient,
Net: cfg.AnomalyNet,
ASN: cfg.AnomalyASN,
Total: cfg.AnomalyTotal,
WatchedNet: cfg.AnomalyWatchedNet,
NetV4Prefix: cfg.AnomalyNetV4Prefix,
NetV6Prefix: cfg.AnomalyNetV6Prefix,
NamedNetblocks: cfg.AnomalyWatchedNets,
Alerts: alertQueue,
})
}
// newLists returns the lists the settings name by URL, none fetched yet.
// Until waf stops, it fetches them as they are due.
func newLists(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.Lists {
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(background(lists.Run))
return lists
}
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
// ends the queries under way, as reputation.DNSBL.StopAsking says.
func newDNSBL(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.DNSBL {
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
}
// newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
// under way, as reputation.AbuseIPDB.StopChecking says.
func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger,
) *reputation.AbuseIPDB {
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: reputation.AbuseIPDBURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
BlacklistURL: reputation.AbuseIPDBBlacklistURL,
Threshold: cfg.AbuseIPDBThreshold,
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
Blacklisted: func(listed []reputation.BlacklistEntry) {
waf.BanAbuseIPDBBlacklist(
cfg, ledger, processLog, utcNow(), listed,
)
},
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB
}
// newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
// way, as routewatch.Client.StopAsking says.
func newRouteWatch(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client {
if cfg.RouteWatchURL == "" {
return nil
}
routeWatch := routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
}
// newMetrics returns the metrics, with those read from the other parts as
// the metrics are asked for: AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set,
// the lookup database's while there is one, and the remote log sender's
// while there is one.
func newMetrics(
cfg *config.WAF, ledger *bans.Ledger, limiter *ratelimit.Limiter,
ruleFiles *rules.Files, lists *reputation.Lists,
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
alertQueue *alerts.Queue, lookupFile *lookup.File,
remote *remotelog.Sender,
) *metrics.Metrics {
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
m.AddBansAndClients(ledger, limiter, utcNow)
m.AddRules(ruleFiles)
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
m.AddAlerts(alertQueue)
if lookupFile != nil {
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
}
if remote != nil {
m.AddRemoteLog(remote)
}
return m
}
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
// under way, as geojs.GeoJS.StopAsking says.
func newGeoJS(
lc fx.Lifecycle,
cfg *config.WAF,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
) *geojs.GeoJS {
geoJS := geojs.New(geojs.Params{
URL: geojs.URL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
},
Now: utcNow,
Asked: m.GeoJSRequests.Inc,
Failed: func(err error, askingAgainIn time.Duration) {
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
}
// newWAFServer returns the server waf runs, with its parts and the time
// limit params gives.
func newWAFServer(
params Params, cfg *config.WAF, out requestLog, processLog *slog.Logger,
m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger,
anomalies *anomaly.Counters, lists *reputation.Lists,
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
routeWatch *routewatch.Client, geoJS *geojs.GeoJS,
lookupFile *lookup.File, crs *coreruleset.CoreRuleSet,
ruleFiles *rules.Files, alertQueue *alerts.Queue,
) *waf.Server {
return waf.New(waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
Metrics: m,
Limiter: limiter,
Ledger: ledger,
Anomalies: anomalies,
Lists: lists,
DNSBL: dnsbl,
AbuseIPDB: abuseIPDB,
RouteWatch: routeWatch,
GeoJS: geoJS,
LookupFile: lookupFile,
CoreRuleSet: crs,
Now: utcNow,
Rules: ruleFiles,
Alerts: alertQueue,
BesideLookupTimeLimit: params.BesideLookupTimeLimit,
})
}
// newWAFListener listens on SWW_WAF_LISTEN_ADDR.
func newWAFListener(cfg *config.WAF) (net.Listener, error) {
return listen(cfg.ListenAddr, "SWW_WAF_LISTEN_ADDR")
}
// newStateFiles reads the state and cache files into the parts and the
// alert queue, as state.Load does. Until waf stops, it writes them as they
// are due and takes in an admin's edits of them.
func newStateFiles(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
m *metrics.Metrics, ledger *bans.Ledger, limiter *ratelimit.Limiter,
geoJS *geojs.GeoJS, lists *reputation.Lists, dnsbl *reputation.DNSBL,
abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client,
alertQueue *alerts.Queue, anomalies *anomaly.Counters,
) (*state.Files, error) {
files, err := state.Load(state.Params{
Dir: cfg.StateDir,
CacheDir: cfg.CacheDir,
BansWriteDelay: cfg.BansWriteDelay,
WriteInterval: cfg.StateWriteInterval,
Ledger: ledger,
Limiter: limiter,
GeoJS: geoJS,
Lists: lists,
DNSBL: dnsbl,
AbuseIPDB: abuseIPDB,
RouteWatch: routeWatch,
Alerts: alertQueue,
Anomalies: anomalies,
Now: utcNow,
ProcessLog: processLog,
Metrics: m,
})
if err != nil {
return nil, startError{"cannot use the state and cache files", err}
}
lc.Append(background(files.Run))
lc.Append(background(files.Watch))
return files, nil
}
// newRemoteLogSender returns a sender of the log lines to
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
return remotelog.New(remotelog.Params{
URL: cfg.LogRemoteURL,
RootCAs: cfg.LogRemoteTLSCAs,
BufferLines: cfg.LogRemoteBufferLines,
Facility: cfg.LogRemoteFacility,
AppName: cfg.LogRemoteAppName,
})
}
// startSending runs remote until the function it returns is called, which
// then waits at most remoteLogStopTimeout for the lines still waiting to
// be sent. Sending goes on after ctx is done, so that the lines written
// while sww stops are sent too.
func startSending(
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
) func() {
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
sent := make(chan struct{})
go func() {
remote.Run(sending, processLog)
close(sent)
}()
return func() {
stop()
select {
case <-sent:
case <-time.After(remoteLogStopTimeout):
}
}
}
+3 -8
View File
@@ -90,8 +90,6 @@ func TestMainRunsWAF(t *testing.T) {
cmd := exec.CommandContext(t.Context(), binary)
cmd.Stdout = out
// The Core Rule Set is off. Main keeps waf's own limit for its checks,
// and on a busy host the Core Rule Set's inspection can pass it.
cmd.Env = append(os.Environ(),
"TZ="+zone,
commandForMain+"="+wafCommand,
@@ -99,8 +97,7 @@ func TestMainRunsWAF(t *testing.T) {
stateDir+"="+dir,
cacheDir+"="+dir,
ruleFilesDir+"="+t.TempDir(),
lookupSource+"=off",
"SWW_WAF_CORE_RULE_SET_MODE=off")
lookupSource+"=off")
exited := startProcess(t, cmd)
@@ -208,9 +205,7 @@ func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
// ask sends waf at addr rproxy's question about a request for path from
// placed, checks the answer's status and disposition, and closes it.
func ask(
t *testing.T, addr, path string, status int, disposition waf.Disposition,
) {
func ask(t *testing.T, addr, path string, status int, disposition string) {
t.Helper()
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
@@ -234,7 +229,7 @@ func ask(
_ = res.Body.Close()
if res.StatusCode != status ||
res.Header.Get(waf.DispositionHeader) != string(disposition) {
res.Header.Get(waf.DispositionHeader) != disposition {
t.Errorf(
"answered %d with disposition %q, want %d with %q",
res.StatusCode,
+2 -2
View File
@@ -35,7 +35,7 @@ func (abuseIPDB) check(
return nil
}
client := clientGroup(rq.h.config, rq.client)
client := rq.h.clientGroup(rq.client)
held, _ := rq.h.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{}
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
)
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
-46
View File
@@ -1,46 +0,0 @@
package waf
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
import (
"log/slog"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
// clients were listed and how many bans were made. Clients in one netblock
// make one ban, with the highest of their scores. In observe mode it makes
// no ban.
func BanAbuseIPDBBlacklist(
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
now time.Time, listed []reputation.BlacklistEntry,
) {
scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed {
netblock := banNetblock(cfg, client.Addr)
scores[netblock] = max(scores[netblock], client.Score)
}
made := 0
if !cfg.Observe {
made = ledger.BanForAbuseIPDB(
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
)
}
processLog.Info(
"fetched AbuseIPDB's blacklist",
"clients", len(listed),
"bans_made", made,
)
}
-100
View File
@@ -1,100 +0,0 @@
package waf_test
import (
"context"
"net/http"
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
// blacklist lasts, in days.
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBKey: accountKey,
})
ctx, cancel := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
server.AbuseIPDB.RunBlacklist(ctx)
close(stopped)
}()
t.Cleanup(func() {
cancel()
<-stopped
})
waitUntil(func() bool {
active, _ := server.Ledger.Count(clk.Now())
return active == 1
})
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
}
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
})
// Two clients in one netblock make one ban, with the higher score.
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 95},
{Addr: netip.MustParseAddr(otherClient), Score: 100},
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
})
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
*held[0].Notes.Reputation[0].Score != 100 {
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
clientsNet, held)
}
// Every client of a banned netblock is refused, the IPv6 address's /64
// included.
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
}
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
clk.advance(2 * 24 * time.Hour)
s.get(client, http.StatusOK, requestlog.ActionForward)
}
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(
t,
"",
map[string]string{mode: "observe"},
)
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 100},
})
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
t.Errorf("%d bans active, want none", active)
}
s.get(client, http.StatusOK, requestlog.ActionForward)
}
+5 -5
View File
@@ -139,7 +139,7 @@ func (rq *request) addBan() {
)
if err == nil {
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
netblock, err = rq.h.banNetblock(toAdd.Netblock)
}
if err == nil {
@@ -190,12 +190,12 @@ func (rq *request) readBanToAdd() (banToAdd, error) {
return toAdd, nil
}
// parseBanNetblock reads value, a netblock such as 203.0.113.0/24, or a
// banNetblock reads value, a netblock such as 203.0.113.0/24, or a
// client's address, which stands for the netblock a ban on that client
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
// refused, since a client's address is looked up as IPv4 and a ban on it
// would refuse nothing, and so is a value with a zone.
func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
func (h *handler) banNetblock(value string) (netip.Prefix, error) {
netblock, err := netip.ParsePrefix(value)
if err == nil {
if netblock.Addr().Is4In6() {
@@ -224,7 +224,7 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
}
return banNetblock(h.config, addr), nil
return h.netblock(addr), nil
}
// expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +295,7 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr))
if seen {
answer.Client = &client
}
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"",
map[string]string{metricsToken: token},
)
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"),
clk.Now(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
before := server.Ledger.Snapshot()
+9 -11
View File
@@ -19,8 +19,6 @@ const (
alertMaxPerHour = "SWW_WAF_ALERT_MAX_PER_HOUR"
// alertInstance is the instance every alert of these tests gives.
alertInstance = "host1/blog"
// alertWebhook is where the tests send alerts.
alertWebhook = "https://alerts.example/sww"
)
func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) {
@@ -130,9 +128,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make
// permanent.
group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
})
attackBan, _ := server.Ledger.BanForAttack(
group,
start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack.
@@ -270,7 +270,7 @@ func startAppWithAlerts(
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
settings := map[string]string{
trustedProxies: trustLocalhost,
alertWebhookURL: alertWebhook,
alertWebhookURL: "https://alerts.example/sww",
instanceName: alertInstance,
}
maps.Copy(settings, env)
@@ -292,7 +292,7 @@ func startAppWithAlerts(
// the time raised, for ban, with its netblock, cause, reason and notes,
// which ends at expires, as the log line gives it.
func banAlert(
event alerts.Event,
event string,
raised time.Time,
client string,
ban bans.Ban,
@@ -307,9 +307,7 @@ func banAlert(
Netblock: ban.Netblock,
Reason: ban.Reason,
Detail: map[string]any{
"cause": string(
ban.Cause,
), "ban_expires": expires, "notes": ban.Notes,
"cause": ban.Cause, "ban_expires": expires, "notes": ban.Notes,
},
}
}
@@ -317,7 +315,7 @@ func banAlert(
// attackAlert is banAlert for a ban for the probe rule of testRules, with
// the netblock and the notes of ban.
func attackAlert(
event alerts.Event,
event string,
raised time.Time,
client string,
ban bans.Ban,
+5 -5
View File
@@ -86,10 +86,10 @@ func TestEachScopeAndWindowOverItsThresholdAlertsOncePerCooldown(
value int64
count float64
}{
{requestsPerMinute, string(ratelimit.KindRequests), minute, 2, 3},
{requestsPerHour, string(ratelimit.KindRequests), hour, 2, 3},
{bytesPerMinute, string(ratelimit.KindBytes), minute, 250, 300},
{bytesPerHour, string(ratelimit.KindBytes), hour, 250, 300},
{requestsPerMinute, ratelimit.KindRequests, minute, 2, 3},
{requestsPerHour, ratelimit.KindRequests, hour, 2, 3},
{bytesPerMinute, ratelimit.KindBytes, minute, 250, 300},
{bytesPerHour, ratelimit.KindBytes, hour, 250, 300},
} {
setting := scope.prefix + threshold.end
value := strconv.FormatInt(threshold.value, 10)
@@ -186,7 +186,7 @@ func TestEveryRequestIsCountedWhateverIsDoneWithIt(t *testing.T) {
for _, sent := range []struct {
from, path string
status int
action requestlog.Action
action string
}{
{allowed, "/", http.StatusOK, requestlog.ActionForward},
{exempt, "/", http.StatusOK, requestlog.ActionForward},
+21 -21
View File
@@ -20,7 +20,7 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -134,7 +134,7 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
// cookies is how many cookies the request carries.
cookies int
status int
action requestlog.Action
action string
}{
{
"at the limit", lineLimit, 0, http.StatusOK,
@@ -591,7 +591,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
s := startWAF(t, geojsURL, map[string]string{
unknownCountryLimitPercent: "0", rateLimitPerMinute: "1",
})
s.server.GeoJS.Load([]geojs.Answer{{
s.server.GeoJS.Load([]lookup.Answer{{
Client: netip.MustParsePrefix(client + "/32"), ASN: "AS64512",
Answered: s.clock.Now(), Used: s.clock.Now(),
}})
@@ -686,10 +686,10 @@ func askDirectly(
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"),
now,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
}
@@ -730,7 +730,7 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
// cookie is the request's Cookie header.
cookie string
status int
action requestlog.Action
action string
}{
{
name: "SWW_WAF_DENY_NETS", env: map[string]string{denyNets: client},
@@ -770,8 +770,6 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
// TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays says,
// so that a request that does not wait takes no time at all.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out := newWAFInBubble(t, tc.env)
tc.prepare(t, server, time.Now())
@@ -813,7 +811,7 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
// prepare gets the server ready, at now, before the question.
prepare func(t *testing.T, server *waf.Server, now time.Time)
status int
action requestlog.Action
action string
}{
{"nothing refuses it", nil, leaveAsItIs, http.StatusOK, requestlog.ActionForward},
{
@@ -840,8 +838,6 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
t.Parallel()
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out := newWAFInBubble(t, tc.env)
tc.prepare(t, server, time.Now())
@@ -877,13 +873,18 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
held, _, _ := startHeldGeoJS(t)
resolver, releaseResolver := startHeldResolver(t)
server, out, _ := newWAFWith(t, waf.New,
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver,
abuseIPDBKey: accountKey,
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
})
server, out, _ := newWAFWith(t, func(
params waf.Params,
) *waf.Server {
params.AbuseIPDBURL = silentAbuseIPDBURL
return waf.New(params)
}, silentGeoJSURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver,
abuseIPDBKey: accountKey,
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
})
// The lists are fetched outside the synctest bubble below: a wait on the
// network inside it would keep its clock from moving on.
@@ -949,11 +950,10 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
t.Errorf("%d checks with AbuseIPDB, want 1", checked)
}
// The query is answered, and the check with AbuseIPDB and the request
// to GeoJS abandoned, before the bubble ends.
// The query is answered, and the check with AbuseIPDB abandoned,
// before the bubble ends.
releaseResolver()
time.Sleep(defaultDNSBLAbuseIPDBWait)
waitForGeoJSRequestsToBeAbandoned()
})
}
+67 -93
View File
@@ -10,14 +10,13 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// banResponse is a refusal answered with SWW_WAF_BAN_RESPONSE, and logged
// with action.
func (rq *request) banResponse(action requestlog.Action) *refusal {
func (rq *request) banResponse(action string) *refusal {
return &refusal{status: rq.h.config.BanResponse, action: action}
}
@@ -63,7 +62,7 @@ func (activeBan) check(
// banAction is the action of a request refused under a ban or for making one:
// permanent says the ban never runs out.
func banAction(permanent bool) requestlog.Action {
func banAction(permanent bool) string {
if permanent {
return requestlog.ActionPermanentlyBanned
}
@@ -86,14 +85,13 @@ func (rq *request) countedBytes() int64 {
response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted {
case config.BytesCountedResponse:
case "response":
return response
case config.BytesCountedRequest:
case "request":
return request
case config.BytesCountedBoth:
default: // both
return response + request
}
return response + request
}
// banForLimit bans the client's netblock at now for a broken limit, the
@@ -103,8 +101,9 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what
// happens instead.
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit(
now time.Time,
hit ratelimit.Hit,
@@ -115,18 +114,30 @@ func (rq *request) banForLimit(
rq.line.LimitHit = hit.Window + "_bytes" // as counts names the byte totals
case ratelimit.KindRefusals:
rq.line.LimitHit = requestlog.LimitHitMinuteRefusals
case ratelimit.KindRequests:
default:
rq.line.LimitHit = hit.Window
}
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := rq.h.netblock(rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
}
switch hit.Kind {
@@ -134,59 +145,49 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.rateLimitPercent.logged()
case ratelimit.KindBytes:
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
case ratelimit.KindRefusals: // the refusal limit is not lowered
}
rq.banClient(
now,
status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
)
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(rq.h.clientGroup(rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
}
// banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead.
func (rq *request) banForAttack(
now time.Time, notes bans.Notes,
) requestlog.Action {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
netblock := rq.h.netblock(rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) requestlog.Action {
netblock := banNetblock(rq.h.config, rq.client)
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
@@ -194,13 +195,7 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
rq.line.BanExpires = banExpires(ban)
if made {
@@ -210,32 +205,13 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
// netblock's requests, which can mean going through every client.
func (rq *request) wouldAlertBan(
netblock netip.Prefix, now time.Time, cause bans.Cause,
netblock netip.Prefix, now time.Time, cause string,
) bool {
event := alerts.EventBan
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
@@ -257,9 +233,7 @@ func (rq *request) alertBan(ban bans.Ban) {
}
detail := map[string]any{
"cause": string(ban.Cause),
"ban_expires": banExpires(ban),
"notes": ban.Notes,
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
}
if rq.h.config.Observe {
detail["mode"] = "observe"
@@ -299,16 +273,16 @@ func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// banNetblock is the netblock a ban on client covers: its IPv4 address,
// netblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// counts it in.
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
func (h *handler) netblock(client netip.Addr) netip.Prefix {
addr := client.Unmap()
if addr.Is4() {
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked()
}
return clientGroup(cfg, addr)
return h.clientGroup(addr)
}
// banExpires is when ban ends, as the log line gives it: a time, or
+9 -12
View File
@@ -504,9 +504,7 @@ type sender struct {
}
// get sends a GET request for / from the client at from.
func (s *sender) get(
from string, status int, action requestlog.Action,
) logLine {
func (s *sender) get(from string, status int, action string) logLine {
s.t.Helper()
return s.request(from, "/", status, action)
@@ -519,7 +517,7 @@ func (s *sender) get(
func (s *sender) request(
from, path string,
status int,
action requestlog.Action,
action string,
) logLine {
s.t.Helper()
@@ -529,7 +527,7 @@ func (s *sender) request(
// requestWithHeader is request with header, such as "Authorization:
// Bearer x", added to the request unless it is "".
func (s *sender) requestWithHeader(
from, path, header string, status int, action requestlog.Action,
from, path, header string, status int, action string,
) logLine {
s.t.Helper()
@@ -550,14 +548,8 @@ func (s *sender) requestWithHeader(
// body is sent as it is after the headers, header holding its
// Content-Length or Transfer-Encoding. header may hold several lines,
// separated by "\r\n". It returns the whole answer.
//
// It waits for the answer as long as it takes, since a busy host can slow
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
// whose answer never comes.
func (s *sender) requestWithBody(
method, from, path, header, body string,
status int,
action requestlog.Action,
method, from, path, header, body string, status int, action string,
) (logLine, answer) {
s.t.Helper()
@@ -574,6 +566,11 @@ func (s *sender) requestWithBody(
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
if err != nil {
s.t.Fatalf("set read deadline: %v", err)
}
var got answer
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
-172
View File
@@ -1,172 +0,0 @@
package waf
// The heuristics of besideLookup, and how they run beside the lookup.
import (
"context"
"sync/atomic"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
)
// besideLookupTimeLimit is how long the heuristics of besideLookup have for a
// request, from when they start. They only read the request, and no request
// should take them that long. A request they have not finished by then is
// refused with 503, as ActionChecksTimedOut.
const besideLookupTimeLimit = 500 * time.Millisecond
// besideLookupFindings is what the heuristics of besideLookup found of a
// request, before any of it is applied to the request.
type besideLookupFindings struct {
// trapPath is the request's path when it is in SWW_WAF_TRAP_PATHS, and ""
// otherwise.
trapPath string
// rules are the rules of the rule files the request matched.
rules []rules.Rule
// inspected is true when the Core Rule Set inspected the request.
// inspection, read and inspectionTime are then what it found, what it read
// of the body, and how long it took.
inspected bool
inspection coreruleset.Result
read []byte
inspectionTime time.Duration
// refuses is true when one of the heuristics refuses the request, or would
// in observe mode. Those after it did no work.
refuses bool
}
// runBesideLookup runs the heuristics of besideLookup while the lookup waits
// for GeoJS, and returns the first verdict, or nil when none gives one.
// lookedUp is the channel the lookup's result comes on, or nil once afterLookup
// has let the request through.
//
// The first of the two to refuse the request ends the checks: waf answers at
// once, and what the other found is dropped. besideLookup has
// besideLookupTimeLimit; past it, the request is refused as
// ActionChecksTimedOut. Once both have let the request through, the heuristics
// of afterBesideLookup run. ctx is the request's own context.
func (rq *request) runBesideLookup(
ctx context.Context, lookedUp <-chan lookupResult,
) *verdict {
var running atomic.Pointer[string]
running.Store(new(trapPaths{}.name()))
found := make(chan *besideLookupFindings, 1)
go func() { found <- rq.findBesideLookup(&running) }()
timeUp := time.NewTimer(rq.h.besideLookupLimit)
defer timeUp.Stop()
var findings *besideLookupFindings
for findings == nil || lookedUp != nil {
select {
case result := <-lookedUp:
lookedUp = nil
v := rq.runAfterLookup(ctx, result)
if v != nil {
return v
}
case findings = <-found:
found = nil
timeUp.Stop()
if findings.refuses {
return rq.applyFindings(ctx, findings)
}
case <-timeUp.C:
return rq.timedOut(*running.Load())
}
}
v := rq.applyFindings(ctx, findings)
if v != nil {
return v
}
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
}
// findBesideLookup does the work of the heuristics of besideLookup on the
// request, in their order, until one refuses it, and returns what they found.
// It changes nothing of the request or of what waf keeps, so that it can run
// while the lookup and afterLookup do. running holds the name of the heuristic
// at work, the first one's when findBesideLookup is called.
func (rq *request) findBesideLookup(
running *atomic.Pointer[string],
) *besideLookupFindings {
findings := &besideLookupFindings{}
findings.trapPath = rq.trapPath()
if findings.trapPath != "" {
findings.refuses = true
return findings
}
running.Store(new(ruleFiles{}.name()))
findings.rules = rq.h.rules.Match(rq.in)
if refusedByRule(findings.rules) {
findings.refuses = true
return findings
}
running.Store(new(coreRuleSet{}.name()))
cfg := rq.h.config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
return findings
}
start := time.Now()
// A question's body has been read into memory, so that reading it here
// cannot fail.
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
rq.in, rq.client, rq.in.Body,
)
findings.inspectionTime = time.Since(start)
findings.inspected = true
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
return findings
}
// applyFindings applies findings, what findBesideLookup found, to the
// request: it runs the heuristics of besideLookup, which read them, at the
// time it is called, and returns the first verdict, or nil when none gives
// one.
func (rq *request) applyFindings(
ctx context.Context, findings *besideLookupFindings,
) *verdict {
rq.findings = findings
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
}
// timedOut refuses the request for the heuristics of besideLookup not having
// finished with it within besideLookupTimeLimit. running is the name of the
// one at work, which the log line's heuristic gives. The refusal makes no ban
// and is no offence.
func (rq *request) timedOut(running string) *verdict {
v := &verdict{
action: requestlog.ActionChecksTimedOut,
figures: map[string]any{
"time_limit_ms": rq.h.besideLookupLimit.Milliseconds(),
},
}
rq.line.Heuristic = running
rq.line.HeuristicFigures = v.figures
return v
}
-426
View File
@@ -1,426 +0,0 @@
package waf_test
import (
"io"
"maps"
"net/http"
"net/http/httptest"
"net/netip"
"strconv"
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// checksTimeLimit is how long the trap paths, the rule files and the Core Rule
// Set have for a request, as SPEC.md gives it, and checksTimedOut the status of
// a request they have not finished by then.
const (
checksTimeLimit = 500 * time.Millisecond
checksTimedOut = http.StatusServiceUnavailable
)
// heldCoreRuleSet is a stand-in for the Core Rule Set: each inspection takes
// took, then finds result. asked counts the inspections.
type heldCoreRuleSet struct {
took time.Duration
result coreruleset.Result
asked atomic.Int64
}
// Inspect inspects a request as the stand-in does, and reads all of body.
func (c *heldCoreRuleSet) Inspect(
_ *http.Request, _ netip.Addr, body io.Reader,
) (coreruleset.Result, []byte, error) {
c.asked.Add(1)
time.Sleep(c.took)
read, err := io.ReadAll(body)
return c.result, read, err
}
// coreRuleSetMatch returns what the stand-in finds of an attack: rule 942100,
// with a score of 5, the default SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
func coreRuleSetMatch() coreruleset.Result {
return coreruleset.Result{RuleIDs: []int{942100}, Score: 5}
}
// newWAFHoldingTheCoreRuleSet makes a waf, to be asked in a synctest bubble,
// with crs in place of the Core Rule Set, in block mode. GeoJS never answers,
// a setting needs its answer, alerts go to a webhook, and everything else is
// at its default but for env. No request to it may use the network.
func newWAFHoldingTheCoreRuleSet(
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
settings := map[string]string{
tagDeny: denyKP, coreRuleSetMode: block,
alertWebhookURL: alertWebhook,
}
maps.Copy(settings, env)
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
params.CoreRuleSet = crs
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
return waf.New(params)
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
return server, out, queue
}
// lookupWait is SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS for wait.
func lookupWait(wait time.Duration) string {
return strconv.FormatInt(wait.Milliseconds(), 10)
}
// wantAnswer checks the status of an answer and how long it took.
func wantAnswer(
t *testing.T, status int, took time.Duration,
wantStatus int, wantTook time.Duration, out *output,
) {
t.Helper()
if status != wantStatus || took != wantTook {
t.Errorf(
"answered %d after %s, want %d after %s; output:\n%s",
status, took, wantStatus, wantTook, out.text(),
)
}
}
// wantNoCoreRuleSetTrace checks that what the Core Rule Set found of a request
// left no trace: line has none of its fields, and server has counted no match
// of it and raised no alert for one.
func wantNoCoreRuleSetTrace(
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
) {
t.Helper()
if line.DurationCoreRuleSetMS != nil || line.CoreRuleSetScore != nil ||
line.CoreRuleSetRuleIDs != nil {
t.Errorf(
"log line has Core Rule Set time %v, score %v and rule ids %v, want none",
line.DurationCoreRuleSetMS,
line.CoreRuleSetScore,
line.CoreRuleSetRuleIDs,
)
}
recorder := httptest.NewRecorder()
server.Metrics.ServeHTTP(
recorder,
httptest.NewRequestWithContext(
t.Context(),
http.MethodGet,
waf.MetricsPath,
http.NoBody,
),
)
if strings.Contains(
recorder.Body.String(),
"sww_core_rule_set_matches_total{",
) {
t.Errorf("a Core Rule Set match is counted")
}
for _, alert := range queue.Snapshot().Waiting[alerts.DestinationWebhook] {
if alert.Event == alerts.EventCoreRuleSetMatch {
t.Errorf("alert raised: %+v", alert)
}
}
}
func TestTheLookupAndTheCoreRuleSetTakeTheLongerOfTheirTimesNotTheirSum(
t *testing.T,
) {
t.Parallel()
inspection := checksTimeLimit - time.Millisecond
for _, tc := range []struct {
name string
// wait is how long the request waits for GeoJS.
wait time.Duration
}{
{"the lookup waits longer", defaultLookupWait},
{"the inspection takes longer", inspection / 2},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{took: inspection}
server, out, _ := newWAFHoldingTheCoreRuleSet(
t,
crs,
map[string]string{
asnAndCountryLookupTimeout: lookupWait(tc.wait),
},
)
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
status, took := askDirectly(
t,
server,
func(*http.Request) {},
)
wantAnswer(t, status, took, http.StatusOK,
max(tc.wait, inspection), out)
wantLine(t, out.requestLine(t), http.StatusOK,
requestlog.ActionForward)
})
})
}
}
// firstToRefuse is a case of
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered: a request
// under env, whose inspection by the Core Rule Set takes inspection, refused as
// action after took.
type firstToRefuse struct {
name string
env map[string]string
inspection time.Duration
took time.Duration
action requestlog.Action
}
// firstToRefuseCases are the cases of
// TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered.
func firstToRefuseCases() []firstToRefuse {
return []firstToRefuse{
{
name: "a trap path, before GeoJS answers",
env: map[string]string{trapPaths: "/", allowedCountries: "DE"},
took: 0, action: requestlog.ActionTemporarilyBanned,
},
{
name: "the Core Rule Set, before GeoJS answers",
env: map[string]string{allowedCountries: "DE"},
inspection: 100 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionBlockedByCoreRuleSet,
},
{
name: "the country list, while the Core Rule Set inspects",
env: map[string]string{
allowedCountries: "DE", asnAndCountryLookupTimeout: "100",
},
inspection: 400 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionCountryDenied,
},
{
name: "the rate limits, while the Core Rule Set inspects",
env: map[string]string{
rateLimitPerMinute: "2", unknownCountryLimitPercent: "50",
asnAndCountryLookupTimeout: "100",
},
inspection: 400 * time.Millisecond,
took: 100 * time.Millisecond, action: requestlog.ActionClientIsRateLimited,
},
}
}
func TestTheFirstOfTheLookupAndTheRequestChecksToRefuseIsAnswered(
t *testing.T,
) {
t.Parallel()
const forbidden = http.StatusForbidden
for _, tc := range firstToRefuseCases() {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{
took: tc.inspection,
result: coreRuleSetMatch(),
}
server, out, queue := newWAFHoldingTheCoreRuleSet(t, crs, tc.env)
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Under the rate limits of the last case, the client's next
// request takes it over a limit only at the percentage of an
// unknown country, so it waits for GeoJS.
countClient(t, server, time.Now())
status, took := askDirectly(
t,
server,
func(*http.Request) {},
)
wantAnswer(t, status, took, forbidden, tc.took, out)
line := out.requestLine(t)
wantLine(t, line, forbidden, tc.action)
if tc.action == requestlog.ActionBlockedByCoreRuleSet {
return
}
if tc.action != requestlog.ActionClientIsRateLimited &&
line.Counts != (ratelimit.Counts{}) {
t.Errorf(
"the rate limits counted the request: %+v",
line.Counts,
)
}
// What the Core Rule Set finds once the request is answered is
// dropped.
time.Sleep(tc.inspection)
wantNoCoreRuleSetTrace(t, server, queue, line)
})
})
}
}
func TestARequestRefusedBeforeTheLookupStartsNoInspection(t *testing.T) {
t.Parallel()
crs := &heldCoreRuleSet{took: time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs, nil)
synctest.Test(t, func(t *testing.T) {
banClient(t, server, time.Now())
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusForbidden, 0, out)
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
t.Errorf(
"%d inspections and %v requests to GeoJS, want none",
asked,
requests,
)
}
})
}
func TestAClientInTheAllowNetsIsAllowedAtOnceAndNothingElseStarts(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: time.Second, result: coreRuleSetMatch()}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{allowNets: client, allowedCountries: "DE"})
synctest.Test(t, func(t *testing.T) {
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusOK, 0, out)
requests := testutil.ToFloat64(server.Metrics.GeoJSRequests)
if asked := crs.asked.Load(); asked != 0 || requests != 0 {
t.Errorf(
"%d inspections and %v requests to GeoJS, want none",
asked,
requests,
)
}
})
}
func TestARequestTheCoreRuleSetHasNotFinishedWithIn500MillisecondsIsRefused(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{refusalsLimitPerMinute: "1"})
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Were it counted toward the refusal limit, the second refusal would
// ban the client, and the third request would be refused for the ban.
for i := range 3 {
status, took := askDirectly(t, server, func(req *http.Request) {
req.Header.Set(requestIDHeader, "id-"+strconv.Itoa(i))
})
wantAnswer(t, status, took, checksTimedOut, checksTimeLimit, out)
line := out.requestLines(t, i+1)[i]
wantLine(
t,
line,
checksTimedOut,
requestlog.ActionChecksTimedOut,
)
wantHeuristic(t, line, "core_rule_set", `{"time_limit_ms": 500}`)
if line.Offence != "" || line.BanExpires != "" {
t.Errorf(
"log line has offence %q and ban_expires %q, want neither",
line.Offence,
line.BanExpires,
)
}
}
if bans := server.Ledger.Snapshot(); len(bans) != 0 {
t.Errorf("bans %+v, want none", bans)
}
offences := historyOf(t, server, client).Offences
if offences != (ratelimit.Offences{}) {
t.Errorf("history counts the offences %+v, want none", offences)
}
time.Sleep(crs.took)
})
}
func TestObserveModeAllowsARequestTheCoreRuleSetHasNotFinishedWith(
t *testing.T,
) {
t.Parallel()
crs := &heldCoreRuleSet{took: checksTimeLimit + time.Second}
server, out, _ := newWAFHoldingTheCoreRuleSet(t, crs,
map[string]string{mode: observe})
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
status, took := askDirectly(t, server, func(*http.Request) {})
wantAnswer(t, status, took, http.StatusOK, checksTimeLimit, out)
line := out.requestLine(t)
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
if line.WouldAction != requestlog.ActionChecksTimedOut {
t.Errorf(
"log line has would_action %q, want %q",
line.WouldAction,
requestlog.ActionChecksTimedOut,
)
}
time.Sleep(crs.took)
})
}
+9 -10
View File
@@ -53,34 +53,33 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if country == "" && !rq.skippedLookupWait {
unknown = percentage{
cfg.UnknownCountryLimitPercent,
config.UnknownCountryLimitPercentSetting,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
}
}
fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
}
blocklisted := percentage{percent: whole}
if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit {
if rq.blocklisted && cfg.BlocklistAction == "limit" {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
config.BlocklistActionSetting,
"SWW_WAF_BLOCKLIST_ACTION",
}
}
dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
cfg.DNSBLAbuseIPDBAction == "limit" {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
config.DNSBLAbuseIPDBActionSetting,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
}
}
const tagSetting = config.TagLimitPercentSetting
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
@@ -108,7 +107,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
asnBytes = given(
cfg.ASNBytesLimitPercent,
asn,
config.ASNBytesLimitPercentSetting,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
)
}
@@ -116,7 +115,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
countryBytes = given(
cfg.CountryBytesLimitPercent,
country,
config.CountryBytesLimitPercentSetting,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
)
}
+2 -6
View File
@@ -73,7 +73,7 @@ func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) {
// Half of 4 requests a minute: the third breaks the limit.
for _, sent := range []struct {
status int
action requestlog.Action
action string
}{
{http.StatusOK, requestlog.ActionForward},
{http.StatusOK, requestlog.ActionForward},
@@ -339,8 +339,6 @@ func TestClientWithoutAnAnswerInTimeHasTheUnknownCountryLimitPercent(
// TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays says, with
// a GeoJS that never answers.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out, _ := newWAF(t, silentGeoJSURL, time.Now,
map[string]string{unknownCountryLimitPercent: "0"})
@@ -392,8 +390,6 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
// with a GeoJS that never answers. The headers, which need the
// answer too, are off.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out, _ := newWAF(t, silentGeoJSURL, time.Now,
map[string]string{
asnAndCountryLookupTimeout: strconv.FormatInt(
@@ -452,7 +448,7 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(
// limit, which is answered with status and logged with action.
before int
status int
action requestlog.Action
action string
// reason and want are the ban's reason, and its notes' limit
// percentage, as percentText gives it.
reason, want string
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -42,6 +41,5 @@ func (blocklists) check(
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
}
+1 -1
View File
@@ -30,7 +30,7 @@ func (byteLimits) check(
}
counts, hit, over := rq.h.limiter.CountBytes(
clientGroup(rq.h.config, rq.client),
rq.h.clientGroup(rq.client),
now,
rq.countedBytes(),
rq.bytesLimitPercent.percent,
+2 -4
View File
@@ -5,8 +5,6 @@ import (
"net/http"
"net/netip"
"slices"
"sneak.berlin/go/smallwebwaf/internal/config"
)
// peerAddress is the address of the request's TCP peer.
@@ -47,10 +45,10 @@ func scheme(r *http.Request, isQuestion bool) string {
// address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
func (h *handler) clientGroup(addr netip.Addr) netip.Prefix {
addr = addr.Unmap()
if addr.Is6() {
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
return netip.PrefixFrom(addr, h.config.IPv6GroupPrefix).Masked()
}
return netip.PrefixFrom(addr, addr.BitLen())
+18 -26
View File
@@ -13,11 +13,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// coreRuleSet takes what the Core Rule Set found of the request, which
// findBesideLookup inspected with it unless SWW_WAF_CORE_RULE_SET_MODE is off
// or SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS exempts its path, as pathExempt
// decides. It notes the rules the request matched and its score in the log
// line, and the rules in the metrics. A score at or over
// coreRuleSet inspects the request with the Core Rule Set, unless
// SWW_WAF_CORE_RULE_SET_MODE is off or SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
// exempts its path, as pathExempt decides, and notes the rules it matched and
// its score in the log line, and the rules in the metrics. A score at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD is a match: it adds the tag
// exploitsignature, raises the core_rule_set_match alert, and in block mode
// refuses the request with 403, which is an offence its client's history
@@ -31,23 +30,27 @@ func (coreRuleSet) check(
rq *request,
_ time.Time,
) *verdict {
findings := rq.findings
if !findings.inspected {
cfg := rq.h.config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
return nil
}
cfg := rq.h.config
result := findings.inspection
rq.requestBytes = int64(len(findings.read))
start := time.Now()
// A question's body has been read into memory, so that reading it here
// cannot fail.
result, read, _ := rq.h.coreRuleSet.Inspect(rq.in, rq.client, rq.in.Body)
rq.requestBytes = int64(len(read))
rq.line.DurationCoreRuleSetMS = new(
requestlog.Milliseconds(findings.inspectionTime),
requestlog.Milliseconds(time.Since(start)),
)
rq.line.CoreRuleSetRuleIDs = result.RuleIDs
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
}
threshold := cfg.CoreRuleSetScoreThreshold
@@ -58,7 +61,7 @@ func (coreRuleSet) check(
rq.addTag(config.ExploitSignatureTag)
rq.alertCoreRuleSetMatch(result)
if !rq.refusedByCoreRuleSet(result) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeDetect {
return nil
}
@@ -73,17 +76,6 @@ func (coreRuleSet) check(
}
}
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
// Rule Set found result of: whether the score is at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
cfg := rq.h.config
threshold := cfg.CoreRuleSetScoreThreshold
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
threshold != 0 && result.Score >= threshold
}
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
// which the Core Rule Set scored at result, at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
@@ -99,7 +91,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect)
detail["mode"] = config.CoreRuleSetModeDetect
case rq.h.config.Observe:
detail["mode"] = "observe"
}
@@ -108,7 +100,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
Event: alerts.EventCoreRuleSetMatch,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: rq.h.clientGroup(rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+5 -7
View File
@@ -84,7 +84,7 @@ func TestCoreRuleSetMatchAddsTheExploitSignatureTag(t *testing.T) {
name string
mode string
status int
action requestlog.Action
action string
}{
{block, block, http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet},
{detect, detect, http.StatusOK, requestlog.ActionForward},
@@ -158,9 +158,8 @@ func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(
t.Parallel()
for _, tc := range []struct {
mode string
action requestlog.Action
status int
mode, action string
status int
}{
{block, requestlog.ActionBlockedByCoreRuleSet, http.StatusForbidden},
{detect, requestlog.ActionForward, http.StatusOK},
@@ -653,9 +652,8 @@ func TestCoreRuleSetMatchRaisesTheCoreRuleSetMatchAlert(t *testing.T) {
// status and action are what the request is answered and logged
// with, and alertMode what the alert's detail gives as mode, if
// anything.
status int
action requestlog.Action
alertMode string
status int
action, alertMode string
}{
{
"block", map[string]string{coreRuleSetMode: block},
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"slices"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -34,8 +33,7 @@ func (countries) check(
return &verdict{
action: requestlog.ActionCountryDenied,
figures: map[string]any{
"country": country,
"setting": config.ExclusivelyAllowedCountriesSetting,
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
},
}
}
+1 -1
View File
@@ -316,7 +316,7 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) {
name string
allowNets string
status int
action requestlog.Action
action string
}{
{
"not in SWW_WAF_ALLOW_NETS", "",
+1 -2
View File
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
}
if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
-13
View File
@@ -1,13 +0,0 @@
package waf
import "sneak.berlin/go/smallwebwaf/internal/reputation"
// BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
// the clients on listed does.
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
}
+12 -23
View File
@@ -3,8 +3,6 @@ package waf
import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// To add a heuristic, write one file with a type that implements heuristic,
@@ -22,10 +20,10 @@ type heuristic interface {
}
// verdict is a heuristic refusing a request, or banning its client: the action
// the log line names, and the figures the heuristic acted on, by name, such as
// the count and the limit.
// the log line names, one of requestlog's Action constants, and the figures the
// heuristic acted on, by name, such as the count and the limit.
type verdict struct {
action requestlog.Action
action string
figures map[string]any
}
@@ -56,6 +54,8 @@ func beforeLookup() []heuristic {
}
// afterLookup are the heuristics that run once the lookup has been made.
// tag_deny runs twice: before the rate limits, on the tags known by then, and
// last, on the tag rule_files and core_rule_set can add.
func afterLookup() []heuristic {
return []heuristic{
countries{},
@@ -64,26 +64,9 @@ func afterLookup() []heuristic {
abuseIPDB{},
tagDeny{},
rateLimits{},
}
}
// besideLookup are the heuristics that only read the request. Their work runs
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
// of the request. Their check then applies what that work found, and runs only
// once waf takes it.
func besideLookup() []heuristic {
return []heuristic{
trapPaths{},
ruleFiles{},
coreRuleSet{},
}
}
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second time,
// on the tag rule_files and core_rule_set can add.
func afterBesideLookup() []heuristic {
return []heuristic{
tagDeny{},
}
}
@@ -106,7 +89,13 @@ func onReport() []heuristic {
}
}
// runHeuristics runs list, one of those checkClient runs, on the request at now,
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
}
// runHeuristics runs list, beforeLookup or afterLookup, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its
// figures. ctx is the request's own context.
+30 -107
View File
@@ -2,18 +2,13 @@ package waf
import (
"context"
"log/slog"
"net/http"
"net/netip"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/lookup"
)
// The headers in which an answer of allow gives rproxy the client's AS number
@@ -25,81 +20,33 @@ const (
countryHeader = "X-Client-Country"
)
// lookupResult is what the lookup gave for a request: whether its client was
// looked up, the answer, and whether the request skipped the wait for GeoJS.
type lookupResult struct {
lookedUp, skippedWait bool
answer geojs.Answer
}
// startLookUp starts looking up the client's AS number and country, in the
// lookup database or through GeoJS, unless SWW_WAF_LOOKUP_SOURCE is off or the
// client is on a private, loopback or link-local address, which no lookup can
// place. It returns the channel the result comes on. A result that needs no
// wait is on it when startLookUp returns: the lookup database's, an answer
// GeoJS gave earlier, or none. With GeoJS, while a setting needs the answer,
// such as a country list or a biased threshold, a new client's request waits
// for it up to SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, unless
// refusedWhateverTheAnswer reports true. The wait goes on to its end even once
// the request has been answered, so that it ends as it would have for a
// request that waited. startLookUp changes nothing of the request. ctx is the
// request's own context.
func (rq *request) startLookUp(
ctx context.Context, now time.Time,
) <-chan lookupResult {
result := make(chan lookupResult, 1)
if rq.h.config.LookupSource == config.LookupSourceOff ||
!canBePlaced(rq.client) {
result <- lookupResult{}
return result
}
client := clientGroup(rq.h.config, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile {
result <- lookupResult{
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
}
return result
}
answer := rq.h.geojs.LookUpWithoutWaiting(ctx, client)
refused := rq.refusedWhateverTheAnswer(now)
if !answer.Answered.IsZero() || refused || !rq.h.waitForGeoJS {
result <- lookupResult{
lookedUp: true,
skippedWait: refused && answer.Answered.IsZero(),
answer: answer,
}
return result
}
go func() {
result <- lookupResult{
lookedUp: true,
answer: rq.h.geojs.LookUp(context.WithoutCancel(ctx), client,
rq.h.config.ASNAndCountryLookupTimeout),
}
}()
return result
}
// noteLookup notes result for the log line and as the request's asn= and
// country= tags.
func (rq *request) noteLookup(result lookupResult) {
if !result.lookedUp {
// lookUp looks up the client's AS number and country, in the lookup
// database or through GeoJS, and notes them for the log line and as the
// request's asn= and country= tags, unless
// SWW_WAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
// link-local address, which no lookup can place. The lookup database
// answers at once. With GeoJS, while a setting needs the answer, such as a
// country list or a biased threshold, a new client's request waits for it,
// unless refusedWhateverTheAnswer reports true. ctx is the request's own
// context.
func (rq *request) lookUp(ctx context.Context, now time.Time) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
return
}
client := rq.h.clientGroup(rq.client)
switch {
case rq.h.config.LookupSource == "file":
rq.lookupAnswer = rq.h.lookupFile.LookUp(client)
case rq.refusedWhateverTheAnswer(now):
rq.lookupAnswer = rq.h.geojs.LookUpWithoutWaiting(ctx, client)
rq.skippedLookupWait = rq.lookupAnswer.Answered.IsZero()
default:
rq.lookupAnswer = rq.h.geojs.LookUp(ctx, client)
}
rq.lookedUp = true
rq.lookupAnswer = result.answer
rq.skippedLookupWait = result.skippedWait
rq.line.ASN = rq.lookupAnswer.ASN
rq.line.ASName = rq.lookupAnswer.ASName
rq.line.Country = rq.lookupAnswer.Country
@@ -133,40 +80,16 @@ func countryTag(country string) string {
return config.CountryTagPrefix + country
}
// AddLookup adds answer, an answer about a client from the lookup database
// or GeoJS, to the client's history in limiter, and to the notes of the bans
// in ledger on its netblock that have no AS number, AS name or country yet.
// cfg sets the netblock's size.
func AddLookup(
cfg *config.WAF, limiter *ratelimit.Limiter, ledger *bans.Ledger,
answer geojs.Answer,
) {
limiter.AddLookup(answer.Client, answer.Answered,
// addLookup adds answer, an answer about a client from the lookup
// database or GeoJS, to the client's history, and to the notes of the bans
// on its netblock that have no AS number, AS name or country yet.
func (h *handler) addLookup(answer lookup.Answer) {
h.limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
}
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
// source_failure alert for it on alertQueue and counts it in m.
// askingAgainIn is how long GeoJS is left alone after it.
func GeoJSFailed(
processLog *slog.Logger, alertQueue *alerts.Queue, m *metrics.Metrics,
err error, askingAgainIn time.Duration,
) {
processLog.Warn("asking GeoJS failed",
"error", err.Error(), "asking_again_in", askingAgainIn.String())
alertQueue.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: "asking GeoJS failed",
Detail: map[string]any{
"source": "geojs", "error": err.Error(),
"asking_again_in": askingAgainIn.String(),
},
})
m.GeoJSFailures.Inc()
}
// setLookupHeaders sets the headers that give the client's AS number and
// country, leaving out one that is unknown.
func setLookupHeaders(header http.Header, asn, country string) {
+5 -18
View File
@@ -12,7 +12,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -138,7 +138,7 @@ func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
// of lookups.json would keep it, while the first request still waits
// for GeoJS. The second request goes on at once, and is counted.
clk.advance(2500 * time.Millisecond)
server.GeoJS.Load([]geojs.Answer{{
server.GeoJS.Load([]lookup.Answer{{
Client: netip.MustParsePrefix(
fromDE + "/32",
), ASN: asnDE, ASName: asNameDE,
@@ -190,7 +190,7 @@ func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(
clk.Now,
map[string]string{
trustedProxies: trustLocalhost,
alertWebhookURL: alertWebhook,
alertWebhookURL: "https://alerts.example/sww",
rateLimitPerMinute: "1",
},
)
@@ -199,7 +199,7 @@ func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(
// The answer is kept before the requests, so GeoJS is not asked, and
// gives no answer of its own.
netblock := netip.MustParsePrefix(fromDE + "/32")
server.GeoJS.Load([]geojs.Answer{{
server.GeoJS.Load([]lookup.Answer{{
Client: netblock, ASN: asnDE, ASName: asNameDE, Country: "DE",
Answered: clk.Now(), Used: clk.Now(),
}})
@@ -249,7 +249,7 @@ func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
)
// Even an answer kept from before is not used.
server.GeoJS.Load([]geojs.Answer{{
server.GeoJS.Load([]lookup.Answer{{
Client: netip.MustParsePrefix(
fromDE + "/32",
), ASN: asnDE, ASName: asNameDE,
@@ -450,8 +450,6 @@ func TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays(
// network, which would keep that clock from moving on: the request is
// handed to waf's handler, and GeoJS is one that never answers.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Not the default. The exclusive list needs the answer, and the app
// is never reached.
const timeout = 3 * time.Second
@@ -497,17 +495,6 @@ func TestMain(m *testing.M) {
m.Run()
}
// geojsRequestTimeout is how long a request to GeoJS may take before it is
// abandoned.
const geojsRequestTimeout = 10 * time.Second
// waitForGeoJSRequestsToBeAbandoned sleeps until every request to the GeoJS at
// silentGeoJSURL made so far has been abandoned. A synctest bubble that asks
// it defers this: the bubble cannot end while a request waits on it.
func waitForGeoJSRequestsToBeAbandoned() {
time.Sleep(geojsRequestTimeout)
}
// silentGeoJS is the GeoJS at silentGeoJSURL.
type silentGeoJS struct{}
+14 -50
View File
@@ -1,7 +1,6 @@
package waf_test
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
@@ -11,9 +10,8 @@ import (
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -250,7 +248,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
// The answers are kept before the requests, so that none waits for
// GeoJS.
server.GeoJS.Load([]geojs.Answer{
server.GeoJS.Load([]lookup.Answer{
keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"),
keptAnswer(fromFR, "FR"),
})
@@ -331,13 +329,13 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(
// The answers are kept before the requests, so that GeoJS gives none
// of its own. Each client is in an AS of its own.
answer := func(addr, asn string) geojs.Answer {
return geojs.Answer{
answer := func(addr, asn string) lookup.Answer {
return lookup.Answer{
Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
Answered: clk.Now(), Used: clk.Now(),
}
}
server.GeoJS.Load([]geojs.Answer{
server.GeoJS.Load([]lookup.Answer{
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
})
@@ -359,23 +357,21 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(
`sww_asn_requests_total{asn="AS64502",instance="app"}`)
}
func TestGeoJSFailureIsCountedLoggedAndAlerted(t *testing.T) {
func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
t.Parallel()
failingGeoJS := httptest.NewServer(http.HandlerFunc(
geojs := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
t.Cleanup(failingGeoJS.Close)
t.Cleanup(geojs.Close)
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
addr, out, server, queue := startProxyWithAlerts(t, app.URL,
failingGeoJS.URL, time.Now,
addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
map[string]string{
trustedProxies: trustLocalhost,
metricsToken: token,
tagDeny: denyKP,
alertWebhookURL: "https://alerts.example/waf",
trustedProxies: trustLocalhost,
metricsToken: token,
tagDeny: denyKP,
})
// GeoJS fails, so the client has no country= tag, and SWW_WAF_TAG_DENY does
@@ -405,46 +401,14 @@ func TestGeoJSFailureIsCountedLoggedAndAlerted(t *testing.T) {
`sww_requests_without_geojs_answer_total{instance="app"}`,
1,
)
// The process log's line names the failure but not the address asked
// about.
logged := ""
for line := range strings.Lines(out.text()) {
if strings.Contains(line, "asking GeoJS failed") {
logged = line
}
}
if logged == "" || strings.Contains(logged, fromDE) {
t.Errorf(
"logged %q, want the failure without the address asked about",
logged,
)
}
// fmt prints a map sorted by its keys.
const want = "map[asking_again_in:1s " +
"error:GeoJS answered 503 Service Unavailable source:geojs]"
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventSourceFailure ||
waiting[0].Reason != "asking GeoJS failed" ||
fmt.Sprint(waiting[0].Detail) != want {
t.Errorf(
"alerts waiting %+v, want one source_failure alert with %v",
waiting,
want,
)
}
}
// keptAnswer returns GeoJS's answer that the client at addr is in
// country, given now.
func keptAnswer(addr, country string) geojs.Answer {
func keptAnswer(addr, country string) lookup.Answer {
now := time.Now()
return geojs.Answer{
return lookup.Answer{
Client: netip.MustParsePrefix(addr + "/32"), Country: country,
Answered: now, Used: now,
}
+5 -8
View File
@@ -22,12 +22,12 @@ func TestBanActionSaysWhetherTheBanRunsOut(t *testing.T) {
for _, tc := range []struct {
name string
cause bans.Cause
cause string
// endsIn is how long after the request the ban ends: negative for a
// ban that has ended, and zero for a permanent ban.
endsIn time.Duration
path string
want requestlog.Action
want string
}{
{
"temporary ban", bans.CauseAdmin, time.Hour, "/",
@@ -170,10 +170,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
// and its second breaks it.
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
for _, sent := range []struct {
from string
refusal requestlog.Action
}{
for _, sent := range []struct{ from, refusal string }{
{denied, requestlog.ActionClientDenied},
{banned, requestlog.ActionTemporarilyBanned},
{fromKP, requestlog.ActionClientDenied},
@@ -256,7 +253,7 @@ func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) {
// wantWouldAction checks the request log line's would_action, and that a
// line that should have none has no such field.
func wantWouldAction(t *testing.T, line logLine, want requestlog.Action) {
func wantWouldAction(t *testing.T, line logLine, want string) {
t.Helper()
got, present := line.fields["would_action"]
@@ -264,7 +261,7 @@ func wantWouldAction(t *testing.T, line logLine, want requestlog.Action) {
switch {
case want == "" && present:
t.Errorf("log line has would_action %v, want none", got)
case want != "" && got != string(want):
case want != "" && got != want:
t.Errorf("log line has would_action %v, want %s", got, want)
}
}
+156 -77
View File
@@ -1,15 +1,13 @@
// Package waf is the server the waf command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each. It also bans the clients on AbuseIPDB's
// blacklist each time the reputation package fetches it.
// writes a log line for each.
package waf
import (
"cmp"
"fmt"
"io"
"log/slog"
"net/http"
"net/netip"
"time"
lru "github.com/hashicorp/golang-lru/v2"
@@ -19,7 +17,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -57,34 +54,18 @@ type Params struct {
RequestLog io.Writer
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// Metrics count what waf does. Whoever builds them adds the metrics
// read from the other parts.
Metrics *metrics.Metrics
// Limiter is the table of clients the rate limits count, with each
// client's history.
Limiter *ratelimit.Limiter
// Ledger holds the bans.
Ledger *bans.Ledger
// Anomalies are the counters of the anomaly thresholds.
Anomalies *anomaly.Counters
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
// RouteWatch asks routewatch about clients, and is nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
RouteWatch *routewatch.Client
// GeoJS looks clients' AS numbers and countries up while
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
// and GeoJSFailed as its Failed.
GeoJS *geojs.GeoJS
// GeoJSURL is where clients' AS numbers and countries are looked up
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
GeoJSURL string
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
AbuseIPDBURL string
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet *coreruleset.CoreRuleSet
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
// normally time.Now in UTC, the time the state files give.
@@ -100,29 +81,19 @@ type Params struct {
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
// or the day's AbuseIPDB checks used up.
Alerts *alerts.Queue
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
// for a request. Zero means besideLookupTimeLimit. Tests that run the
// Core Rule Set lengthen it, since the race detector and a busy host can
// slow an inspection past besideLookupTimeLimit.
BesideLookupTimeLimit time.Duration
}
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
type CoreRuleSetInspector interface {
Inspect(
r *http.Request, client netip.Addr, body io.Reader,
) (coreruleset.Result, []byte, error)
}
// Server is the server waf runs, with the parts of its Params. The tests
// read them.
// Server is the server waf runs, with the parts whose state the state
// files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
// RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
type Server struct {
*http.Server
Ledger *bans.Ledger
Limiter *ratelimit.Limiter
GeoJS *geojs.GeoJS
GeoJS *lookup.GeoJS
Anomalies *anomaly.Counters
LookupFile *lookup.File
Lists *reputation.Lists
@@ -132,45 +103,159 @@ type Server struct {
Metrics *metrics.Metrics
}
// newHandler returns the handler with what every request shares, the parts
// in params among it.
// newHandler returns the handler with what every request shares, as the
// settings in params make it.
func newHandler(params Params) *handler {
m := metrics.New(
params.Config.MetricsBusiestCountriesAndASNs,
params.Config.InstanceName,
)
lists, dnsbl, abuseIPDB := newReputation(params, m)
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
return &handler{
h := &handler{
config: params.Config,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
metrics: params.Metrics,
metrics: m,
awaitingReport: awaitingReport,
limiter: params.Limiter,
ledger: params.Ledger,
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
limiter: ratelimit.New(ratelimit.Limits{
PerMinute: params.Config.RateLimitPerMinute,
PerHour: params.Config.RateLimitPerHour,
PerDay: params.Config.RateLimitPerDay,
BytesPerMinute: params.Config.BytesLimitPerMinute,
BytesPerHour: params.Config.BytesLimitPerHour,
BytesPerDay: params.Config.BytesLimitPerDay,
}, params.Config.MaxTrackedClients),
ledger: bans.New(bans.Params{
LimitBanDuration: params.Config.LimitBanDuration,
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
LimitBanPermanentAbove: params.Config.LimitBanPermanentAbove,
AttackBanDuration: params.Config.AttackBanDuration,
MaxBans: params.Config.MaxBans,
}),
anomalies: anomaly.New(anomaly.Params{
Client: params.Config.AnomalyClient,
Net: params.Config.AnomalyNet,
ASN: params.Config.AnomalyASN,
Total: params.Config.AnomalyTotal,
WatchedNet: params.Config.AnomalyWatchedNet,
NetV4Prefix: params.Config.AnomalyNetV4Prefix,
NetV6Prefix: params.Config.AnomalyNetV6Prefix,
NamedNetblocks: params.Config.AnomalyWatchedNets,
Alerts: params.Alerts,
}),
lookupFile: params.LookupFile,
lists: lists,
dnsbl: dnsbl,
abuseIPDB: abuseIPDB,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
}
if params.Config.RouteWatchURL != "" {
h.routeWatch = routewatch.New(routewatch.Params{
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
})
}
h.geojs = lookup.New(lookup.Params{
URL: params.GeoJSURL,
Timeout: params.Config.ASNAndCountryLookupTimeout,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders ||
params.Config.TagsNeedLookup() ||
Wait: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
Answered: h.addLookup,
Now: params.Now,
ProcessLog: params.ProcessLog,
Metrics: m,
Alerts: params.Alerts,
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
return h
}
// newReputation returns the lists fetched from URLs, the DNSBL zones'
// verdicts and AbuseIPDB's scores, as the settings in params name them,
// with none fetched, asked for or checked yet, and adds their metrics to
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
func newReputation(
params Params, m *metrics.Metrics,
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
cfg := params.Config
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
return lists, dnsbl, abuseIPDB
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, fmt.Errorf(
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
err,
)
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return coreRuleSet, nil
}
// handler is waf's handler. It holds what every request shares; what
@@ -183,7 +268,7 @@ type handler struct {
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
geojs *lookup.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
lists *reputation.Lists
@@ -191,14 +276,8 @@ type handler struct {
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet CoreRuleSetInspector
coreRuleSet *coreruleset.CoreRuleSet
alerts *alerts.Queue
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
// besideLookupLimit is how long the heuristics of besideLookup have for a
// request, as Params.BesideLookupTimeLimit says.
besideLookupLimit time.Duration
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
+24 -182
View File
@@ -16,18 +16,10 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf"
@@ -193,11 +185,11 @@ func (o *output) requestLine(t *testing.T) logLine {
// endedLines waits for count requests that went through rproxy, or to waf's
// own endpoints, to have ended, and returns the last log line of each, in the
// order waf took them. A question's last line is the one its report writes.
// It waits as long as it takes, as requestWithBody does.
func (o *output) endedLines(t *testing.T, count int) []logLine {
t.Helper()
for {
deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
found := o.lastLines(t)
ended := len(found) >= count
@@ -213,6 +205,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
time.Sleep(pollInterval)
}
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
return nil
}
// lastLines returns the last request log line written so far of each request,
@@ -428,8 +424,7 @@ func startRProxy(
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
// fetched from abuseIPDBBlacklistURL.
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
// Core Rule Set have their requests inspected by it.
func newWAF(
@@ -440,15 +435,13 @@ func newWAF(
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
return newWAFWith(t, waf.New, geojsURL, now, env)
}
// newWAFWith is newWAF for the server build makes, with clients checked with
// AbuseIPDB at abuseIPDBCheckURL.
// newWAFWith is newWAF for the server build makes.
func newWAFWith(
t *testing.T, build func(waf.Params) *waf.Server,
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
env map[string]string,
geojsURL string, now func() time.Time, env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
@@ -501,174 +494,25 @@ func newWAFWith(
}
}
params := waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in.
BesideLookupTimeLimit: time.Hour,
}
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
server := build(params)
return server, out, alertQueue
}
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
func addCoreRuleSet(t *testing.T, params *waf.Params) {
t.Helper()
cfg := params.Config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params.CoreRuleSet = coreRuleSet
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
func addCounters(params *waf.Params) {
cfg := params.Config
params.Metrics = metrics.New(
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
AttackBanDuration: cfg.AttackBanDuration,
MaxBans: cfg.MaxBans,
})
params.Anomalies = anomaly.New(anomaly.Params{
Client: cfg.AnomalyClient,
Net: cfg.AnomalyNet,
ASN: cfg.AnomalyASN,
Total: cfg.AnomalyTotal,
WatchedNet: cfg.AnomalyWatchedNet,
NetV4Prefix: cfg.AnomalyNetV4Prefix,
NetV6Prefix: cfg.AnomalyNetV6Prefix,
NamedNetblocks: cfg.AnomalyWatchedNets,
Alerts: params.Alerts,
server := build(waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
GeoJSURL: geojsURL,
AbuseIPDBURL: abuseIPDBURL,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
})
params.Metrics.AddBansAndClients(
params.Ledger,
params.Limiter,
params.Now,
)
params.Metrics.AddRules(params.Rules)
}
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
// setting is set, and GeoJS at geojsURL, as sww builds
// them from params' settings and the parts addCounters added. The metrics
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
func addOutsideSources(
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
) {
cfg := params.Config
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
processLog, alertQueue := params.ProcessLog, params.Alerts
params.Lists = reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.DNSBL = reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: abuseIPDBCheckURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
BlacklistURL: abuseIPDBBlacklistURL,
Threshold: cfg.AbuseIPDBThreshold,
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
Blacklisted: func(listed []reputation.BlacklistEntry) {
waf.BanAbuseIPDBBlacklist(
cfg, ledger, processLog, params.Now(), listed,
)
},
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if cfg.RouteWatchURL != "" {
params.RouteWatch = routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: params.Now,
ProcessLog: processLog,
})
}
params.GeoJS = geojs.New(geojs.Params{
URL: geojsURL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
},
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: func(err error, askingAgainIn time.Duration) {
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddReputation(params.Lists, params.DNSBL)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(params.AbuseIPDB)
}
return server, out, alertQueue
}
// newClient returns an HTTP client that sends requests as they are made,
@@ -790,9 +634,7 @@ func readResponse(t *testing.T, conn net.Conn) answer {
}
// wantLine checks the request log line's status and action.
func wantLine(
t *testing.T, line logLine, status int, action requestlog.Action,
) {
func wantLine(t *testing.T, line logLine, status int, action string) {
t.Helper()
if line.Status != status || line.Action != action {
+3 -6
View File
@@ -38,11 +38,8 @@ func (rateLimits) check(
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count(
clientGroup(rq.h.config, rq.client),
now,
rq.rateLimitPercent.percent,
)
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
rq.rateLimitPercent.percent)
rq.line.Counts = counts
if !over {
@@ -62,7 +59,7 @@ func (rateLimits) check(
// limit. It counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool {
return rq.countedByRateLimits() &&
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
}
// countedByRateLimits reports whether the rate limits count the request: its
+3 -3
View File
@@ -5,7 +5,7 @@ import (
"sync/atomic"
"testing"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -121,7 +121,7 @@ func TestIPv6GroupPrefixSetsTheClientTheLimitsCount(t *testing.T) {
// status and action are those of the request that breaks the
// limit: a rate limit refuses it, a byte limit passes it on.
status int
action requestlog.Action
action string
}{
{rateLimitPerMinute, "1", http.StatusForbidden, requestlog.ActionClientIsRateLimited},
{bytesLimitPerMinute, byteLimit, http.StatusOK, requestlog.ActionForward},
@@ -162,7 +162,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
// The answers are kept before the requests, so that none waits for
// GeoJS.
server.GeoJS.Load([]geojs.Answer{
server.GeoJS.Load([]lookup.Answer{
keptAnswer(client, "DE"), keptAnswer(fromKP, "KP"),
})
+2 -13
View File
@@ -36,11 +36,8 @@ func (refusalLimit) check(
return nil
}
hit, over := rq.h.limiter.CountRefusal(
clientGroup(rq.h.config, rq.client),
now,
cfg.RefusalsLimitPerMinute,
)
hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
cfg.RefusalsLimitPerMinute)
if !over {
return nil
}
@@ -55,14 +52,6 @@ func (refusalLimit) check(
case requestlog.ActionTemporarilyBanned,
requestlog.ActionPermanentlyBanned:
status = cfg.BanResponse
case requestlog.ActionForward,
requestlog.ActionRequestLineTooLong,
requestlog.ActionRequestHeaderTooLarge,
requestlog.ActionClientIsRateLimited,
requestlog.ActionClientDenied,
requestlog.ActionCountryDenied,
requestlog.ActionChecksTimedOut,
requestlog.ActionOwnEndpoint:
}
rq.banForLimit(now, hit, status)
+1 -1
View File
@@ -74,7 +74,7 @@ func TestRefusalLimitBreaksAtOneOverTheLimit(t *testing.T) {
refusals []refused
// nextAction is the action of the client's next request: the next
// request after a ban for a ban rule makes it permanent.
nextAction requestlog.Action
nextAction string
}{
{
"block rule",
+5 -1
View File
@@ -7,6 +7,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
)
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason.
@@ -33,7 +37,7 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
Event: alerts.EventReputationHit,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: rq.h.clientGroup(rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+21 -38
View File
@@ -58,22 +58,22 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(
// statuses and actions are those of a listed client's three
// requests, and percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []requestlog.Action
actions []string
percent string
}{
{
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
[]requestlog.Action{denied, denied, denied}, none,
[]string{denied, denied, denied}, none,
},
{
// Half of 4 requests a minute: the third breaks the limit.
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
[]requestlog.Action{forward, forward, requestlog.ActionClientIsRateLimited},
[]string{forward, forward, requestlog.ActionClientIsRateLimited},
"50 from " + blocklistAction,
},
{
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
[]requestlog.Action{forward, forward, forward}, none,
[]string{forward, forward, forward}, none,
},
} {
t.Run(tc.action, func(t *testing.T) {
@@ -442,22 +442,22 @@ func TestEachDNSBLAbuseIPDBActionForAClientADNSBLZoneLists(t *testing.T) {
// statuses and actions are those of a listed client's three
// requests, and percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []requestlog.Action
actions []string
percent string
}{
{
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
[]requestlog.Action{denied, denied, denied}, none,
[]string{denied, denied, denied}, none,
},
{
// Half of 4 requests a minute: the third breaks the limit.
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
[]requestlog.Action{forward, forward, requestlog.ActionClientIsRateLimited},
[]string{forward, forward, requestlog.ActionClientIsRateLimited},
"50 from " + dnsblAbuseIPDBAction,
},
{
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
[]requestlog.Action{forward, forward, forward}, none,
[]string{forward, forward, forward}, none,
},
} {
t.Run(tc.action, func(t *testing.T) {
@@ -822,39 +822,22 @@ const abuseipdb = reputation.AbuseIPDBSource
// through which AbuseIPDB is asked.
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
// from: at abuseIPDBStandIn too.
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
// score 100, at once and without the network. Its blacklist lists client
// alone, with the score 100, to a fetch with accountKey that asks for the
// clients scored 90 or more, the default threshold; it refuses any other
// fetch of it with 400.
// score 100, at once and without the network.
type abuseIPDBStandIn struct{}
// RoundTrip answers req with the score 100, or with the blacklist.
// RoundTrip answers req with the score 100.
func (abuseIPDBStandIn) RoundTrip(
req *http.Request,
) (*http.Response, error) {
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
if req.URL.Path == "/api/v2/blacklist" {
body = `{"data":[{"ipAddress":"` + client +
`","abuseConfidenceScore":100}]}`
if req.Header.Get("Key") != accountKey ||
req.URL.RawQuery != "confidenceMinimum=90" {
status, body = http.StatusBadRequest, `{"errors":[]}`
}
}
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
StatusCode: http.StatusOK,
Status: "200 OK",
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(body)),
Request: req,
Body: io.NopCloser(
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
),
Request: req,
}, nil
}
@@ -954,7 +937,7 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(
// history then counts.
path string
status int
action requestlog.Action
action string
want ratelimit.Offences
}{
{
@@ -1018,22 +1001,22 @@ func TestEachDNSBLAbuseIPDBActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(
// statuses and actions are those of fromDE's three requests, and
// percent their rate_limit_percent, as percentText gives it.
statuses []int
actions []requestlog.Action
actions []string
percent string
}{
{
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
[]requestlog.Action{denied, denied, denied}, none,
[]string{denied, denied, denied}, none,
},
{
// Half of 4 requests a minute: the third breaks the limit.
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
[]requestlog.Action{forward, forward, requestlog.ActionClientIsRateLimited},
[]string{forward, forward, requestlog.ActionClientIsRateLimited},
"50 from " + dnsblAbuseIPDBAction,
},
{
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
[]requestlog.Action{forward, forward, forward}, none,
[]string{forward, forward, forward}, none,
},
} {
t.Run(tc.action, func(t *testing.T) {
+20 -64
View File
@@ -11,7 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -21,7 +21,7 @@ import (
// to close its connection without an answer, and the action the log line names.
type refusal struct {
status int
action requestlog.Action
action string
}
// decision is what the checks note of a request: its log line, and what
@@ -42,7 +42,7 @@ type decision struct {
// looked up, whether or not an answer was there, and lookupAnswer is
// what the lookup gave then, the zero Answer while GeoJS had given none.
lookedUp bool
lookupAnswer geojs.Answer
lookupAnswer lookup.Answer
// skippedLookupWait is true for a request that did not wait for GeoJS and
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
// so that not waiting never lowers its client's limits.
@@ -81,9 +81,6 @@ type request struct {
checked time.Time
// refused is what the request was refused with, nil if it was not.
refused *refusal
// findings is what the heuristics of besideLookup found of the request,
// once waf has taken it, and nil before.
findings *besideLookupFindings
// requestBytes is how many bytes of the request's body have been read.
requestBytes int64
}
@@ -122,7 +119,7 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion),
PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup(h.config, client).String(),
ClientGroup: h.clientGroup(client).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders(
r,
@@ -180,9 +177,9 @@ func requestHeaders(r *http.Request, names []string) map[string]string {
// check is the one place where a request can be refused. It returns nil to let
// the request through. The heuristics of checkClient decide: a request they
// refuse is answered with SWW_WAF_BAN_RESPONSE, 403 for a block rule or the Core
// Rule Set, 503 for heuristics that ran out of time, or 414 or 431 for the
// header limits. In observe mode nothing is refused, and the log line names
// what enforce mode would have done. ctx is the request's own context.
// Rule Set, or 414 or 431 for the header limits. In observe mode nothing is refused,
// and the log line names what enforce mode would have done. ctx is the
// request's own context.
func (rq *request) check(ctx context.Context) *refusal {
v := rq.checkClient(ctx)
if v == nil {
@@ -199,11 +196,6 @@ func (rq *request) check(ctx context.Context) *refusal {
case action == requestlog.ActionBlockedByBlockRule ||
action == requestlog.ActionBlockedByCoreRuleSet:
return &refusal{status: http.StatusForbidden, action: action}
case action == requestlog.ActionChecksTimedOut:
return &refusal{
status: http.StatusServiceUnavailable,
action: action,
}
case action == requestlog.ActionRequestLineTooLong:
return &refusal{status: http.StatusRequestURITooLong, action: action}
case action == requestlog.ActionRequestHeaderTooLarge:
@@ -216,16 +208,11 @@ func (rq *request) check(ctx context.Context) *refusal {
}
}
// checkClient runs the heuristics on the request, and returns the first
// verdict, or nil when none gives one. A client in SWW_WAF_ALLOW_NETS skips them
// all, and is not looked up. ctx is the request's own context.
//
// The heuristics of beforeLookup run first. Then the client's AS number and
// country are looked up, and afterLookup runs once the lookup has its result.
// While the lookup waits for GeoJS, besideLookup runs too, as runBesideLookup
// says. A lookup that needs no wait is done, and afterLookup with it, before
// besideLookup starts, so that besideLookup does no work for a request
// afterLookup refuses.
// checkClient runs the heuristics of beforeLookup on the request, then looks
// up the client's AS number and country, then runs those of afterLookup. It
// returns the first verdict, or nil when none gives one. A client in
// SWW_WAF_ALLOW_NETS skips them all, and is not looked up. ctx is the request's
// own context.
func (rq *request) checkClient(ctx context.Context) *verdict {
if isInside(rq.client, rq.h.config.AllowNets) {
return nil
@@ -238,28 +225,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
return v
}
lookedUp := rq.startLookUp(ctx, now)
select {
case result := <-lookedUp:
v := rq.runAfterLookup(ctx, result)
if v != nil {
return v
}
return rq.runBesideLookup(ctx, nil)
default:
return rq.runBesideLookup(ctx, lookedUp)
}
}
// runAfterLookup notes result, the lookup's, and runs the heuristics of
// afterLookup. It returns the first verdict, or nil when none gives one. ctx is
// the request's own context.
func (rq *request) runAfterLookup(
ctx context.Context, result lookupResult,
) *verdict {
rq.noteLookup(result)
rq.lookUp(ctx, now)
// The lookup can wait for GeoJS, so the checks after it read the time
// again: a request counted at the time it arrived could be counted
@@ -345,16 +311,6 @@ func timing(start, end time.Time) *float64 {
return new(requestlog.Milliseconds(end.Sub(start)))
}
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
// It then counts the request for the anomaly thresholds and adds it to its
// client's history. list runs first, so that a refusal limit it finds broken
// is in that history. ctx is the request's own context.
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
rq.runBanningHeuristics(ctx, list)
rq.countAnomalies()
rq.addToHistory()
}
// addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and
@@ -375,7 +331,7 @@ func (rq *request) addToHistory() {
}
rq.h.limiter.AddToHistory(
clientGroup(rq.h.config, rq.client),
rq.h.clientGroup(rq.client),
rq.h.now(),
request,
)
@@ -383,7 +339,7 @@ func (rq *request) addToHistory() {
answer, found := rq.answerAtTheEnd()
if found {
AddLookup(rq.h.config, rq.h.limiter, rq.h.ledger, answer)
rq.h.addLookup(answer)
}
}
@@ -404,7 +360,7 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site,
Client: rq.client,
ClientGroup: clientGroup(rq.h.config, rq.client),
ClientGroup: rq.h.clientGroup(rq.client),
ASN: answer.ASN,
ASName: answer.ASName,
Country: answer.Country,
@@ -426,16 +382,16 @@ func anomalyThresholdsSet(cfg *config.WAF) bool {
// answer about it as the request ends, and whether there is one: the
// lookup database's, which was there at once, or the one GeoJS has given
// by then, which a request does not wait for unless a setting needs it.
func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
func (rq *request) answerAtTheEnd() (lookup.Answer, bool) {
if !rq.lookedUp {
return geojs.Answer{}, false
return lookup.Answer{}, false
}
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
return rq.lookupAnswer, true
}
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
}
// responseWriter is waf's answer to a request. It notes the status and
+1 -1
View File
@@ -32,7 +32,7 @@ func (routeWatchTags) check(
route, missing := rq.h.routeWatch.Route(
ctx,
clientGroup(rq.h.config, rq.client),
rq.h.clientGroup(rq.client),
rq.client,
)
if missing != "" {
+5 -18
View File
@@ -13,8 +13,8 @@ import (
"sneak.berlin/go/smallwebwaf/internal/rules"
)
// ruleFiles takes the rules of the rule files the request matched, which
// findBesideLookup found, notes their ids in the log line, adds the tag
// ruleFiles checks the request against the rules of the rule files at now,
// notes the ids of those it matches in the log line, adds the tag
// exploitsignature when it matches any, and refuses the request
// when the last rule it matches refuses it: with ActionBlockedByBlockRule for a
// block rule, and for a ban rule with the action for the ban it makes. A ban
@@ -31,7 +31,7 @@ func (ruleFiles) check(
rq *request,
now time.Time,
) *verdict {
matched := rq.findings.rules
matched := rq.h.rules.Match(rq.in)
for _, rule := range matched {
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
@@ -66,20 +66,7 @@ func (ruleFiles) check(
),
figures: figures,
}
case rules.ActionLog:
default:
return nil
}
return nil
}
// refusedByRule reports whether ruleFiles refuses a request that matched
// matched: whether the last of them is a block rule or a ban rule.
func refusedByRule(matched []rules.Rule) bool {
if len(matched) == 0 {
return false
}
action := matched[len(matched)-1].Action
return action == rules.ActionBlock || action == rules.ActionBan
}
+1 -1
View File
@@ -184,7 +184,7 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) {
type sentRequest struct {
client string
status int
action requestlog.Action
action string
}
// wantAnswers sends requests to sww at addr one after another and
+6 -17
View File
@@ -11,8 +11,9 @@ import (
"sneak.berlin/go/smallwebwaf/internal/rules"
)
// trapPaths refuses a request for a path in SWW_WAF_TRAP_PATHS, as trapPath
// finds it. Such a request is a clear sign of attack, as a ban rule's match is:
// trapPaths refuses a request for a path in SWW_WAF_TRAP_PATHS: its path as a
// path rule sees it, without the query, as sent or once percent-decoded, is one
// of them. Such a request is a clear sign of attack, as a ban rule's match is:
// it bans the client's netblock, or in observe mode raises the alert for the
// ban it would have made. The ban's notes give the path as sent, and so does
// its figure.
@@ -25,8 +26,9 @@ func (trapPaths) check(
rq *request,
now time.Time,
) *verdict {
path := rq.findings.trapPath
if path == "" {
path := rules.Path(rq.in)
if !slices.Contains(rq.h.config.TrapPaths, path) &&
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
return nil
}
@@ -37,16 +39,3 @@ func (trapPaths) check(
figures: map[string]any{"path": path},
}
}
// trapPath returns the request's path as sent, when its path as a path rule
// sees it, without the query, as sent or once percent-decoded, is in
// SWW_WAF_TRAP_PATHS, and "" otherwise.
func (rq *request) trapPath() string {
path := rules.Path(rq.in)
if !slices.Contains(rq.h.config.TrapPaths, path) &&
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
return ""
}
return path
}
+34 -31
View File
@@ -32,22 +32,18 @@ const (
DispositionHeader = "X-Sww-Disposition"
)
// Disposition is what waf tells rproxy to do with a request, in
// DispositionHeader.
type Disposition string
// The values of DispositionHeader.
const (
// DispositionAllow tells rproxy to pass the request to the app.
DispositionAllow Disposition = "allow"
DispositionAllow = "allow"
// DispositionDeny tells rproxy to give the visitor the answer's status,
// headers and body.
DispositionDeny Disposition = "deny"
DispositionDeny = "deny"
// DispositionClose tells rproxy to close the visitor's connection
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
// waf closing the question's connection instead would read as
// waf being down, and rproxy would then let the request through.
DispositionClose Disposition = "close"
DispositionClose = "close"
)
// ReportPath is where rproxy sends the report of each request waf
@@ -181,13 +177,11 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
peer.Unmap(),
true,
)
// These run even when a check panics. finish runs first, so that what is
// held has the finished log line. Go's server sends the answer only once
// this function has returned, so the report cannot come before it is held.
defer func() {
rq.finish()
rq.holdForReport()
}()
// The last deferred runs first, so that what is held has the finished log
// line. Go's server sends the answer only once this function has
// returned, so the report cannot come before it is held.
defer rq.holdForReport()
defer rq.finish()
rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision
@@ -196,28 +190,35 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), afterAnswer())
defer rq.runAfterAnswer(r.Context())
rq.allow()
return
}
// The last deferred runs first: runAfterAnswer before addToHistory, so that
// a broken refusal limit is in the client's history.
defer rq.addToHistory()
defer rq.countAnomalies()
defer rq.runAfterAnswer(r.Context())
if refused != nil {
rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, string(DispositionDeny))
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
return
}
rq.banAndCount(r.Context(), afterAnswer())
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
}
// readQuestion returns the visitor's address and the body a question carries,
@@ -324,7 +325,7 @@ func (rq *request) allow() {
rq.allowed = true
header := rq.out.Header()
header.Set(DispositionHeader, string(DispositionAllow))
header.Set(DispositionHeader, DispositionAllow)
if rq.h.config.AddLookupHeaders {
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
@@ -347,13 +348,13 @@ func (rq *request) deny(r refusal) {
if r.status == 0 {
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
// status stays 0: the visitor is sent nothing.
rq.out.Header().Set(DispositionHeader, string(DispositionClose))
rq.out.Header().Set(DispositionHeader, DispositionClose)
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
return
}
rq.out.Header().Set(DispositionHeader, string(DispositionDeny))
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(rq.out, http.StatusText(r.status), r.status)
}
@@ -490,7 +491,9 @@ func (h *handler) finishWith(
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, onReport())
rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.h.metrics.RequestReported(
&rq.line,
+3 -3
View File
@@ -205,13 +205,13 @@ func wantDisposition(
t *testing.T,
got answer,
status int,
disposition waf.Disposition,
disposition string,
) {
t.Helper()
wantStatus(t, got, status)
if answered := got.header.Get(waf.DispositionHeader); answered !=
string(disposition) {
disposition {
t.Errorf(
"answered with disposition %q, want %q",
answered,
@@ -1480,7 +1480,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
env map[string]string
path string
status int
action requestlog.Action
action string
}{
{
"deny", map[string]string{denyNets: client}, "/", http.StatusForbidden,
Loaded 100 of 102 files, more files were not shown because too many files have changed in this diff. Show more