Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
23eaa869dd | ||
|
|
8219664ad1 | ||
|
|
58b9985b6d | ||
|
|
677df6a235 | ||
|
|
8188602a09 | ||
|
|
ee550be738 | ||
|
|
493aac02a1 | ||
|
|
51245804d0 |
No files matched your search
@@ -288,7 +288,9 @@ account, a key, a token or an alert destination stays off until you give one.
|
|||||||
- An anomaly is more requests or bytes than an `SWW_WAF_ANOMALY_` threshold
|
- An anomaly is more requests or bytes than an `SWW_WAF_ANOMALY_` threshold
|
||||||
allows. Crossing a threshold sends one `anomaly` alert to the alert
|
allows. Crossing a threshold sends one `anomaly` alert to the alert
|
||||||
destinations, and does nothing else: no client is banned, rate limited or
|
destinations, and does nothing else: no client is banned, rate limited or
|
||||||
blocked.
|
blocked. The thresholds per netblock and per AS number are on by default, set
|
||||||
|
high enough to alert only on obvious abuse. They are a starting point: tune
|
||||||
|
them to your own site's traffic, which the metrics show.
|
||||||
- An invalid setting stops the process at start with one log line naming the
|
- An invalid setting stops the process at start with one log line naming the
|
||||||
variable and what is wrong with the value. The settings in effect are logged
|
variable and what is wrong with the value. The settings in effect are logged
|
||||||
on one line at start, with tokens, keys and webhook paths shown as `********`.
|
on one line at start, with tokens, keys and webhook paths shown as `********`.
|
||||||
@@ -396,16 +398,16 @@ Every setting the code reads, with its default:
|
|||||||
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `1200` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `15000` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `5G` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `50G` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `6000`, or `off` while the lookup source is `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `75000`, or `off` while the lookup source is `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `25G`, or `off` while the lookup source is `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `250G`, or `off` while the lookup source is `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
|
|||||||
@@ -2113,12 +2113,14 @@ host, sharing one address, would often fetch too close together.
|
|||||||
|
|
||||||
### Anomaly thresholds
|
### Anomaly thresholds
|
||||||
|
|
||||||
These only alert; nothing is blocked. None is set by default, and a threshold
|
These only alert; nothing is blocked. The thresholds per netblock and per AS
|
||||||
left unset sends no alert. An alert needs a destination only the operator can
|
number have defaults high enough to alert only on obvious abuse. The others are
|
||||||
supply, and what counts as unusual depends on each service's normal traffic,
|
off by default. What counts as unusual depends on each service's normal traffic,
|
||||||
which the metrics show. They reveal a widely distributed scraper, thousands of
|
which the metrics show, so the defaults are a starting point for the operator to
|
||||||
addresses at low rates each, which the per-client limits do not see. A low
|
tune. An alert goes nowhere until the operator sets a destination. The
|
||||||
percentage for its `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` is the response.
|
thresholds reveal a widely distributed scraper, thousands of addresses at low
|
||||||
|
rates each, which the per-client limits do not see. A low percentage for its
|
||||||
|
`asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` is the response.
|
||||||
|
|
||||||
- Requests and body bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted
|
- Requests and body bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted
|
||||||
per minute and per hour, for each of:
|
per minute and per hour, for each of:
|
||||||
|
|||||||
@@ -0,0 +1,214 @@
|
|||||||
|
services:
|
||||||
|
exampleapp:
|
||||||
|
# Your app, built from the Dockerfile in this directory, listening
|
||||||
|
# on port 8081. Publish no port of it: it is reached through rproxy.
|
||||||
|
build: .
|
||||||
|
|
||||||
|
rproxy:
|
||||||
|
image: git.eeqj.de/sneak/smallwebwaf:latest
|
||||||
|
environment:
|
||||||
|
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
||||||
|
SWW_RPROXY_UPSTREAM_URL: http://exampleapp:8081
|
||||||
|
|
||||||
|
# The netblocks whose X-Forwarded-For rproxy believes. By default
|
||||||
|
# the private ranges, which cover a TLS-terminating proxy on the
|
||||||
|
# Docker network. A proxy elsewhere needs its address added; a
|
||||||
|
# list you give replaces the default, so repeat the private
|
||||||
|
# ranges if they are still wanted. Set but empty, rproxy believes
|
||||||
|
# no forwarded header.
|
||||||
|
# SWW_RPROXY_TRUSTED_PROXIES: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,203.0.113.5,2001:db8:1::/64
|
||||||
|
|
||||||
|
# By default rproxy passes a request to the app while waf gives
|
||||||
|
# no answer. Uncomment this to answer 503 instead.
|
||||||
|
# SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION: deny
|
||||||
|
|
||||||
|
# The largest request body (default 100M) and response body
|
||||||
|
# (default 1G): a size with K, M or G, or `off`. Raise them for
|
||||||
|
# uploads and downloads larger than that, such as git pushes and
|
||||||
|
# clones or image layers.
|
||||||
|
# SWW_RPROXY_MAX_REQUEST_SIZE: 2G
|
||||||
|
# SWW_RPROXY_MAX_RESPONSE_SIZE: 4G
|
||||||
|
|
||||||
|
# How long a request may take to reach the app (default 60000,
|
||||||
|
# one minute) and how long its answer may take to reach the
|
||||||
|
# visitor (default 1800000, 30 minutes), in milliseconds or
|
||||||
|
# `off`. Raise the first for slow uploads, the second for long
|
||||||
|
# downloads.
|
||||||
|
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
|
||||||
|
# SWW_RPROXY_MAX_RESPONSE_TIME_MS: "7200000"
|
||||||
|
|
||||||
|
# traefik terminates TLS and forwards to rproxy:8080 over its own
|
||||||
|
# network, which the `networks` block at the end of this file names.
|
||||||
|
# No port is published.
|
||||||
|
networks:
|
||||||
|
- default
|
||||||
|
- traefik
|
||||||
|
labels:
|
||||||
|
traefik.enable: "true"
|
||||||
|
# rproxy is on two networks; traefik reaches it on this one.
|
||||||
|
traefik.docker.network: traefik
|
||||||
|
traefik.http.routers.exampleapp.rule: Host(`app.example.com`)
|
||||||
|
traefik.http.routers.exampleapp.entrypoints: websecure
|
||||||
|
traefik.http.routers.exampleapp.tls.certresolver: letsencrypt
|
||||||
|
traefik.http.services.exampleapp.loadbalancer.server.port: "8080"
|
||||||
|
|
||||||
|
waf:
|
||||||
|
image: git.eeqj.de/sneak/smallwebwaf:latest
|
||||||
|
environment:
|
||||||
|
SWW_COMMAND: waf
|
||||||
|
|
||||||
|
# `observe` refuses nothing and makes no ban; each log line says
|
||||||
|
# what `enforce` (the default) would have done. Start here to
|
||||||
|
# see what the defaults would refuse on your site.
|
||||||
|
# SWW_WAF_MODE: observe
|
||||||
|
|
||||||
|
# waf's own messages below `info` (the default), while finding
|
||||||
|
# out why something is refused.
|
||||||
|
# SWW_WAF_LOG_LEVEL: debug
|
||||||
|
|
||||||
|
# Netblocks whose clients skip every check, such as monitoring.
|
||||||
|
# SWW_WAF_ALLOW_NETS: 192.0.2.10,2001:db8:2::/48
|
||||||
|
|
||||||
|
# Netblocks whose clients are refused.
|
||||||
|
# SWW_WAF_DENY_NETS: 198.51.100.128/25,2001:db8:3::/48
|
||||||
|
|
||||||
|
# Netblocks exempt from rate limiting and the byte limits, such
|
||||||
|
# as the office and an automation host that calls an API. Their
|
||||||
|
# requests are still checked for attacks.
|
||||||
|
# SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
|
||||||
|
|
||||||
|
# Refuse every request from a country, by its two-letter code.
|
||||||
|
# This refuses North Korea, Cuba, Russia and Iran. A client the
|
||||||
|
# lookup cannot place gets through; see SWW_WAF_LOOKUP_SOURCE.
|
||||||
|
# SWW_WAF_TAG_DENY: country=KP,country=CU,country=RU,country=IR
|
||||||
|
|
||||||
|
# Or from a network, by its AS number: here two hosting networks
|
||||||
|
# that send only scrapers, M247 (AS9009) and WorldStream
|
||||||
|
# (AS49981). Countries and networks go in the one list together.
|
||||||
|
# SWW_WAF_TAG_DENY: asn=9009,asn=49981,country=KP,country=RU
|
||||||
|
|
||||||
|
# Or let only these countries through; a client the lookup
|
||||||
|
# cannot place is refused too.
|
||||||
|
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: us,de
|
||||||
|
|
||||||
|
# Address lists that each tag their clients' requests, as
|
||||||
|
# name=URL pairs; this one tags Tor exit nodes. A tag is refused
|
||||||
|
# by adding its name to SWW_WAF_TAG_DENY, such as
|
||||||
|
# `torexit,country=KP`, or given a share of the limits in
|
||||||
|
# SWW_WAF_TAG_LIMIT_PERCENT, such as `torexit:25`.
|
||||||
|
# SWW_WAF_TAG_LISTS: torexit=https://check.torproject.org/torbulkexitlist
|
||||||
|
|
||||||
|
# A percentage, from 1 to 99, of every limit for a tag: here
|
||||||
|
# half for DigitalOcean (AS14061), a quarter for OVH (AS16276)
|
||||||
|
# and a quarter for China. The lowest of a client's tags applies.
|
||||||
|
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:50,asn=16276:25,country=CN:25
|
||||||
|
|
||||||
|
# The most requests a client may make in a minute, an hour and a
|
||||||
|
# day (defaults 120, 1500 and 6000), a count or `off`; the next
|
||||||
|
# one bans it. Lower them for a small site, raise them for a
|
||||||
|
# busy API.
|
||||||
|
# SWW_WAF_RATE_LIMIT_PER_MINUTE: "60"
|
||||||
|
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
|
||||||
|
# SWW_WAF_RATE_LIMIT_PER_DAY: "3000"
|
||||||
|
|
||||||
|
# Path prefixes exempt from rate limiting and the byte limits,
|
||||||
|
# such as static assets that one page loads many of.
|
||||||
|
# SWW_WAF_RATE_LIMIT_EXEMPT_PATHS: /assets/,/static/
|
||||||
|
|
||||||
|
# Paths only scanners ask for; one request bans the client. Pick
|
||||||
|
# paths your app never serves.
|
||||||
|
# SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php,/wp-admin/
|
||||||
|
|
||||||
|
# How a refused client is answered: 403 (the default), 429 or
|
||||||
|
# `close`, which drops the connection.
|
||||||
|
# SWW_WAF_BAN_RESPONSE: "429"
|
||||||
|
|
||||||
|
# The ban for a first broken limit (default 1h) and for a first
|
||||||
|
# clear sign of attack (default 7d): a duration with m, h or d.
|
||||||
|
# A repeat within SWW_WAF_LIMIT_BAN_REPEAT_WINDOW (default 24h)
|
||||||
|
# gets a ban three times as long.
|
||||||
|
# SWW_WAF_LIMIT_BAN_DURATION: 30m
|
||||||
|
# SWW_WAF_ATTACK_BAN_DURATION: 30d
|
||||||
|
|
||||||
|
# The Core Rule Set, which finds attacks in requests. `block`
|
||||||
|
# (the default) refuses a match, `detect` only logs and alerts
|
||||||
|
# it, `off` runs no rule. Use `detect` while finding out what it
|
||||||
|
# would refuse of your app's ordinary requests.
|
||||||
|
# SWW_WAF_CORE_RULE_SET_MODE: detect
|
||||||
|
|
||||||
|
# Path prefixes the Core Rule Set does not inspect, for a part of
|
||||||
|
# the app that it keeps matching, such as an editor that posts
|
||||||
|
# HTML or code.
|
||||||
|
# SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS: /api/editor/,/admin/
|
||||||
|
|
||||||
|
# The apps each hostname serves, as hostname:app pairs separated
|
||||||
|
# by semicolons, so that each hostname gets its own apps' fixes
|
||||||
|
# to the attack detection and no other's. "Attack detection" in
|
||||||
|
# the README lists the apps.
|
||||||
|
# SWW_WAF_APPS: git.example.com:gitea;blog.example.com:wordpress
|
||||||
|
|
||||||
|
# Blocklists of addresses and netblocks, text files fetched once
|
||||||
|
# a day, whose clients are refused.
|
||||||
|
# SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt,https://www.spamhaus.org/drop/dropv6.txt
|
||||||
|
|
||||||
|
# With the key of an AbuseIPDB account, a client that has
|
||||||
|
# committed an offence is checked there in the background; one
|
||||||
|
# scored 75 or more (SWW_WAF_ABUSEIPDB_MIN_SCORE) gets what
|
||||||
|
# SWW_WAF_DNSBL_ABUSEIPDB_ACTION says: a quarter of every limit
|
||||||
|
# by default, or `deny`. At most 900 checks a day
|
||||||
|
# (SWW_WAF_ABUSEIPDB_DAILY_BUDGET), under the free plan's 1000.
|
||||||
|
# SWW_WAF_ABUSEIPDB_KEY: api-key-.....
|
||||||
|
# SWW_WAF_DNSBL_ABUSEIPDB_ACTION: deny
|
||||||
|
|
||||||
|
# By default waf looks each client's address up at GeoJS, a web
|
||||||
|
# service, for its country and AS number. Set `off`, it makes no
|
||||||
|
# external requests for lookups. Every setting at its default
|
||||||
|
# works without them; one that needs a lookup, such as a country
|
||||||
|
# or ASN setting above, then stops the start. Set `file`, it
|
||||||
|
# reads an IPinfo Lite database instead, mounted by its
|
||||||
|
# directory as under `volumes` below, so that a replaced file is
|
||||||
|
# seen.
|
||||||
|
# SWW_WAF_LOOKUP_SOURCE: "off"
|
||||||
|
# SWW_WAF_LOOKUP_SOURCE: file
|
||||||
|
# SWW_WAF_LOOKUP_DB_PATH: /var/lib/ipinfo/ipinfo_lite.mmdb
|
||||||
|
|
||||||
|
# Alerts about bans, attacks and failures, to an ntfy topic, a
|
||||||
|
# Slack incoming webhook or any webhook. The ntfy token is the
|
||||||
|
# access token ntfy issued for the topic.
|
||||||
|
# SWW_WAF_ALERT_NTFY_URL: https://ntfy.sh/my-alerts
|
||||||
|
# SWW_WAF_ALERT_NTFY_TOKEN: hunter2
|
||||||
|
# SWW_WAF_ALERT_SLACK_WEBHOOK_URL: https://hooks.slack.com/services/...
|
||||||
|
|
||||||
|
# Bearer tokens you make up, at least 32 characters each. The
|
||||||
|
# admin token switches on the ban and client endpoints, through
|
||||||
|
# which a ban is lifted, your own included, as
|
||||||
|
# docs/admin-endpoints.md describes. The metrics token switches
|
||||||
|
# on /_sww/metrics.
|
||||||
|
# SWW_WAF_ADMIN_TOKEN: hunter2
|
||||||
|
# SWW_WAF_METRICS_TOKEN: hunter2
|
||||||
|
|
||||||
|
# Any setting can be read from a file instead, by adding _FILE
|
||||||
|
# to its name: the file's contents are the value, less one
|
||||||
|
# newline at the end. That keeps a secret out of this file and
|
||||||
|
# out of `docker inspect`. Mount the files as under `volumes`
|
||||||
|
# below, and set each setting in one form only.
|
||||||
|
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
|
||||||
|
# SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
|
||||||
|
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
|
||||||
|
# SWW_WAF_ABUSEIPDB_KEY_FILE: /etc/sww/tokens/abuseipdb
|
||||||
|
volumes:
|
||||||
|
# waf's state files: its clients and bans, kept across restarts.
|
||||||
|
- waf-state:/var/lib/sww
|
||||||
|
# Token files, each owned by uid 65532 (the sww user) with mode
|
||||||
|
# 0400, for the _FILE settings above.
|
||||||
|
# - /srv/sww/tokens:/etc/sww/tokens:ro
|
||||||
|
# The IPinfo Lite database, for SWW_WAF_LOOKUP_SOURCE=file.
|
||||||
|
# - /srv/ipinfo:/var/lib/ipinfo:ro
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
waf-state:
|
||||||
|
|
||||||
|
# traefik's network, made outside this file.
|
||||||
|
networks:
|
||||||
|
traefik:
|
||||||
|
external: true
|
||||||
+21
-2
@@ -378,8 +378,9 @@ Crossing a threshold sends one `anomaly` alert to the destinations under
|
|||||||
|
|
||||||
Each scope has four thresholds: the most requests and the most body bytes
|
Each scope has four thresholds: the most requests and the most body bytes
|
||||||
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED`
|
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED`
|
||||||
names. Requests are counts, bytes are sizes. Each is `off` by default. The four
|
names. Requests are counts, bytes are sizes. Each is `off` by default, except
|
||||||
thresholds per client are:
|
those per netblock and per AS number listed below. The four thresholds per
|
||||||
|
client are:
|
||||||
|
|
||||||
- `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE`;
|
- `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE`;
|
||||||
- `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR`;
|
- `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR`;
|
||||||
@@ -403,6 +404,24 @@ as `REQUESTS_PER_MINUTE`:
|
|||||||
`office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
`office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
||||||
name, or with a bad netblock, is refused. So is a name used twice.
|
name, or with a bad netblock, is refused. So is a name used twice.
|
||||||
|
|
||||||
|
The thresholds per netblock and per AS number have defaults set high enough to
|
||||||
|
alert only on obvious abuse:
|
||||||
|
|
||||||
|
- per netblock:
|
||||||
|
- `REQUESTS_PER_MINUTE` `1200` and `REQUESTS_PER_HOUR` `15000`, ten clients
|
||||||
|
at the default rate limits;
|
||||||
|
- `BYTES_PER_HOUR` `50G`, one client at the default byte limit per hour;
|
||||||
|
- `BYTES_PER_MINUTE` `5G`, a tenth of that.
|
||||||
|
- per AS number, five times those: `REQUESTS_PER_MINUTE` `6000`,
|
||||||
|
`REQUESTS_PER_HOUR` `75000`, `BYTES_PER_HOUR` `250G` and `BYTES_PER_MINUTE`
|
||||||
|
`25G`.
|
||||||
|
|
||||||
|
While `SWW_WAF_LOOKUP_SOURCE` is `off`, no request has an AS number, so the
|
||||||
|
thresholds per AS number are `off` by default. Setting one then stops the start.
|
||||||
|
|
||||||
|
These defaults are a starting point. Tune them to your own site's traffic, which
|
||||||
|
the metrics show.
|
||||||
|
|
||||||
## Settings given as files
|
## Settings given as files
|
||||||
|
|
||||||
To give a token as a file, as "Tokens" under "Deployment" in
|
To give a token as a file, as "Tokens" under "Deployment" in
|
||||||
|
|||||||
+51
-13
@@ -313,7 +313,9 @@ type WAF struct {
|
|||||||
// per netblock around a client (SWW_WAF_ANOMALY_NET_*), per AS number
|
// per netblock around a client (SWW_WAF_ANOMALY_NET_*), per AS number
|
||||||
// (SWW_WAF_ANOMALY_ASN_*), for the whole service (SWW_WAF_ANOMALY_TOTAL_*)
|
// (SWW_WAF_ANOMALY_ASN_*), for the whole service (SWW_WAF_ANOMALY_TOTAL_*)
|
||||||
// and per watched netblock (SWW_WAF_ANOMALY_WATCHED_NET_*), each 0 while
|
// and per watched netblock (SWW_WAF_ANOMALY_WATCHED_NET_*), each 0 while
|
||||||
// it is off. AnomalyNetV4Prefix and AnomalyNetV6Prefix are the lengths
|
// it is off. Only the thresholds per netblock and per AS number are on by
|
||||||
|
// default, those per AS number only while clients are looked up.
|
||||||
|
// AnomalyNetV4Prefix and AnomalyNetV6Prefix are the lengths
|
||||||
// of the netblock around a client (SWW_WAF_ANOMALY_NET_V4_PREFIX and
|
// of the netblock around a client (SWW_WAF_ANOMALY_NET_V4_PREFIX and
|
||||||
// SWW_WAF_ANOMALY_NET_V6_PREFIX), and AnomalyWatchedNets the watched
|
// SWW_WAF_ANOMALY_NET_V6_PREFIX), and AnomalyWatchedNets the watched
|
||||||
// netblocks (SWW_WAF_ANOMALY_WATCHED_NETS).
|
// netblocks (SWW_WAF_ANOMALY_WATCHED_NETS).
|
||||||
@@ -555,11 +557,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"SWW_WAF_ALERT_MAX_PER_HOUR",
|
"SWW_WAF_ALERT_MAX_PER_HOUR",
|
||||||
"60",
|
"60",
|
||||||
),
|
),
|
||||||
AnomalyClient: env.thresholds("SWW_WAF_ANOMALY_CLIENT_"),
|
|
||||||
AnomalyNet: env.thresholds("SWW_WAF_ANOMALY_NET_"),
|
|
||||||
AnomalyASN: env.thresholds("SWW_WAF_ANOMALY_ASN_"),
|
|
||||||
AnomalyTotal: env.thresholds("SWW_WAF_ANOMALY_TOTAL_"),
|
|
||||||
AnomalyWatchedNet: env.thresholds("SWW_WAF_ANOMALY_WATCHED_NET_"),
|
|
||||||
AnomalyNetV4Prefix: env.v4Prefix(
|
AnomalyNetV4Prefix: env.v4Prefix(
|
||||||
"SWW_WAF_ANOMALY_NET_V4_PREFIX",
|
"SWW_WAF_ANOMALY_NET_V4_PREFIX",
|
||||||
"24",
|
"24",
|
||||||
@@ -578,6 +575,7 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
// The app gets the headers by default, unless no client is looked up.
|
// The app gets the headers by default, unless no client is looked up.
|
||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
|
env.anomalyThresholds(cfg)
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
BlocklistActionSetting, string(ReputationActionDeny))
|
BlocklistActionSetting, string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
@@ -2693,15 +2691,55 @@ func (e *environment) duration(name, defaultValue string) time.Duration {
|
|||||||
return duration
|
return duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// thresholdDefaults are the defaults of a scope's four anomaly thresholds,
|
||||||
|
// each a count, a size or off.
|
||||||
|
type thresholdDefaults struct {
|
||||||
|
requestsPerMinute, requestsPerHour, bytesPerMinute, bytesPerHour string
|
||||||
|
}
|
||||||
|
|
||||||
|
// anomalyThresholds reads the anomaly thresholds of every scope into cfg.
|
||||||
|
// Only those per netblock and per AS number are on by default, high
|
||||||
|
// enough to alert on obvious abuse alone. Those per netblock count the
|
||||||
|
// requests of ten clients at the default per-client rate limits, and the
|
||||||
|
// bytes of one client at its default byte limit per hour, a tenth of
|
||||||
|
// that in a minute. Those per AS number are five times those per
|
||||||
|
// netblock. They are off by default while no client is looked up, since
|
||||||
|
// no request then has an AS number.
|
||||||
|
func (e *environment) anomalyThresholds(cfg *WAF) {
|
||||||
|
allOff := thresholdDefaults{off, off, off, off}
|
||||||
|
asnDefaults := thresholdDefaults{
|
||||||
|
requestsPerMinute: "6000", requestsPerHour: "75000",
|
||||||
|
bytesPerMinute: "25G", bytesPerHour: "250G",
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.LookupSource == LookupSourceOff {
|
||||||
|
asnDefaults = allOff
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.AnomalyClient = e.thresholds("SWW_WAF_ANOMALY_CLIENT_", allOff)
|
||||||
|
cfg.AnomalyNet = e.thresholds("SWW_WAF_ANOMALY_NET_", thresholdDefaults{
|
||||||
|
requestsPerMinute: "1200", requestsPerHour: "15000",
|
||||||
|
bytesPerMinute: "5G", bytesPerHour: "50G",
|
||||||
|
})
|
||||||
|
cfg.AnomalyASN = e.thresholds("SWW_WAF_ANOMALY_ASN_", asnDefaults)
|
||||||
|
cfg.AnomalyTotal = e.thresholds("SWW_WAF_ANOMALY_TOTAL_", allOff)
|
||||||
|
cfg.AnomalyWatchedNet = e.thresholds("SWW_WAF_ANOMALY_WATCHED_NET_",
|
||||||
|
allOff)
|
||||||
|
}
|
||||||
|
|
||||||
// thresholds reads the four anomaly thresholds whose settings' names
|
// thresholds reads the four anomaly thresholds whose settings' names
|
||||||
// start with prefix: requests and bytes per minute and per hour. Each is
|
// start with prefix: requests and bytes per minute and per hour.
|
||||||
// off by default.
|
func (e *environment) thresholds(
|
||||||
func (e *environment) thresholds(prefix string) anomaly.Thresholds {
|
prefix string, defaults thresholdDefaults,
|
||||||
|
) anomaly.Thresholds {
|
||||||
return anomaly.Thresholds{
|
return anomaly.Thresholds{
|
||||||
RequestsPerMinute: e.count(prefix+"REQUESTS_PER_MINUTE", off),
|
RequestsPerMinute: e.count(prefix+"REQUESTS_PER_MINUTE",
|
||||||
RequestsPerHour: e.count(prefix+"REQUESTS_PER_HOUR", off),
|
defaults.requestsPerMinute),
|
||||||
BytesPerMinute: e.size(prefix+"BYTES_PER_MINUTE", off),
|
RequestsPerHour: e.count(prefix+"REQUESTS_PER_HOUR",
|
||||||
BytesPerHour: e.size(prefix+"BYTES_PER_HOUR", off),
|
defaults.requestsPerHour),
|
||||||
|
BytesPerMinute: e.size(prefix+"BYTES_PER_MINUTE",
|
||||||
|
defaults.bytesPerMinute),
|
||||||
|
BytesPerHour: e.size(prefix+"BYTES_PER_HOUR", defaults.bytesPerHour),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -178,6 +178,17 @@ func loggedAnomalyDefaults() map[string]string {
|
|||||||
logged[name] = off
|
logged[name] = off
|
||||||
}
|
}
|
||||||
|
|
||||||
|
maps.Copy(logged, map[string]string{
|
||||||
|
anomalyNet + requestsPerMinute: "1200",
|
||||||
|
anomalyNet + requestsPerHour: "15000",
|
||||||
|
anomalyNet + bytesPerMinute: "5G",
|
||||||
|
anomalyNet + bytesPerHour: "50G",
|
||||||
|
anomalyASN + requestsPerMinute: "6000",
|
||||||
|
anomalyASN + requestsPerHour: "75000",
|
||||||
|
anomalyASN + bytesPerMinute: "25G",
|
||||||
|
anomalyASN + bytesPerHour: "250G",
|
||||||
|
})
|
||||||
|
|
||||||
return logged
|
return logged
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1565,7 +1576,35 @@ func TestAnomalySettingsDefaults(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := wafFromEnvironment(t, environment{})
|
cfg := wafFromEnvironment(t, environment{})
|
||||||
wantAllOff(t, cfg)
|
|
||||||
|
const gibibyte = int64(1) << 30
|
||||||
|
|
||||||
|
want := map[string]anomaly.Thresholds{
|
||||||
|
anomalyClient: {},
|
||||||
|
anomalyNet: {
|
||||||
|
RequestsPerMinute: 1200, RequestsPerHour: 15000,
|
||||||
|
BytesPerMinute: 5 * gibibyte, BytesPerHour: 50 * gibibyte,
|
||||||
|
},
|
||||||
|
anomalyASN: {
|
||||||
|
RequestsPerMinute: 6000, RequestsPerHour: 75000,
|
||||||
|
BytesPerMinute: 25 * gibibyte, BytesPerHour: 250 * gibibyte,
|
||||||
|
},
|
||||||
|
anomalyTotal: {}, anomalyWatchedNet: {},
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := thresholdsByScope(cfg); !maps.Equal(got, want) {
|
||||||
|
t.Errorf("thresholds by scope\n%+v\nwant\n%+v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// With no client looked up, the thresholds per AS number are off.
|
||||||
|
want[anomalyASN] = anomaly.Thresholds{}
|
||||||
|
|
||||||
|
got := thresholdsByScope(
|
||||||
|
wafFromEnvironment(t, environment{lookupSource: off}))
|
||||||
|
if !maps.Equal(got, want) {
|
||||||
|
t.Errorf("with %s off, thresholds by scope\n%+v\nwant\n%+v",
|
||||||
|
lookupSource, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
if cfg.AnomalyNetV4Prefix != 24 || cfg.AnomalyNetV6Prefix != 48 ||
|
if cfg.AnomalyNetV4Prefix != 24 || cfg.AnomalyNetV6Prefix != 48 ||
|
||||||
len(cfg.AnomalyWatchedNets) != 0 {
|
len(cfg.AnomalyWatchedNets) != 0 {
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
// The heuristic active_ban. It reads SWW_WAF_MODE and SWW_WAF_BAN_RESPONSE; the
|
||||||
|
// bans are the ban ledger's.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// activeBan refuses a client a ban covers, with SWW_WAF_BAN_RESPONSE: a ban on
|
||||||
|
// a netblock the client is in, in force at now. It notes for the log line when
|
||||||
|
// that ban ends. A request that makes the ban permanent, or in observe mode
|
||||||
|
// would have, raises the alert for it. Its figures are the ban's netblock and
|
||||||
|
// when it ends.
|
||||||
|
type activeBan struct {
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
|
func (activeBan) name() string { return "active_ban" }
|
||||||
|
|
||||||
|
func (a activeBan) check(
|
||||||
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
now time.Time,
|
||||||
|
) *verdict {
|
||||||
|
ledger := a.banMaker.ledger
|
||||||
|
|
||||||
|
check := ledger.Check
|
||||||
|
if a.banMaker.observe {
|
||||||
|
check = ledger.Find // the ban refuses nothing, and stays as it is
|
||||||
|
}
|
||||||
|
|
||||||
|
ban, banned, madePermanent := check(rq.client, now)
|
||||||
|
|
||||||
|
var expires string
|
||||||
|
if banned {
|
||||||
|
expires = banExpires(ban)
|
||||||
|
rq.line.BanExpires = expires
|
||||||
|
}
|
||||||
|
|
||||||
|
if madePermanent {
|
||||||
|
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
||||||
|
a.banMaker.alertBan(rq, ban)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !banned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &verdict{
|
||||||
|
action: banAction(ban.Permanent()),
|
||||||
|
status: a.banMaker.banResponse,
|
||||||
|
figures: map[string]any{
|
||||||
|
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,6 +32,22 @@ const (
|
|||||||
bytesPerHour = "BYTES_PER_HOUR"
|
bytesPerHour = "BYTES_PER_HOUR"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// netAndASNThresholdsOff returns the anomaly thresholds per netblock and
|
||||||
|
// per AS number, the ones on by default, each set to off.
|
||||||
|
func netAndASNThresholdsOff() map[string]string {
|
||||||
|
settings := map[string]string{}
|
||||||
|
|
||||||
|
for _, scope := range []string{anomalyNet, anomalyASN} {
|
||||||
|
for _, end := range []string{
|
||||||
|
requestsPerMinute, requestsPerHour, bytesPerMinute, bytesPerHour,
|
||||||
|
} {
|
||||||
|
settings[scope+end] = off
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return settings
|
||||||
|
}
|
||||||
|
|
||||||
// The other anomaly settings.
|
// The other anomaly settings.
|
||||||
const (
|
const (
|
||||||
anomalyNetV4Prefix = "SWW_WAF_ANOMALY_NET_V4_PREFIX"
|
anomalyNetV4Prefix = "SWW_WAF_ANOMALY_NET_V4_PREFIX"
|
||||||
|
|||||||
+2
-53
@@ -1,11 +1,9 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
// The heuristic active_ban, and banMaker, which makes the bans of several
|
// banMaker, which makes the bans of several heuristics. The bans are the ban
|
||||||
// heuristics. active_ban reads SWW_WAF_MODE and SWW_WAF_BAN_RESPONSE; the bans
|
// ledger's.
|
||||||
// are the ban ledger's.
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -17,55 +15,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// activeBan refuses a client a ban covers, with SWW_WAF_BAN_RESPONSE: a ban on
|
|
||||||
// a netblock the client is in, in force at now. It notes for the log line when
|
|
||||||
// that ban ends. A request that makes the ban permanent, or in observe mode
|
|
||||||
// would have, raises the alert for it. Its figures are the ban's netblock and
|
|
||||||
// when it ends.
|
|
||||||
type activeBan struct {
|
|
||||||
banMaker banMaker
|
|
||||||
}
|
|
||||||
|
|
||||||
func (activeBan) name() string { return "active_ban" }
|
|
||||||
|
|
||||||
func (a activeBan) check(
|
|
||||||
_ context.Context,
|
|
||||||
rq *request,
|
|
||||||
now time.Time,
|
|
||||||
) *verdict {
|
|
||||||
ledger := a.banMaker.ledger
|
|
||||||
|
|
||||||
check := ledger.Check
|
|
||||||
if a.banMaker.observe {
|
|
||||||
check = ledger.Find // the ban refuses nothing, and stays as it is
|
|
||||||
}
|
|
||||||
|
|
||||||
ban, banned, madePermanent := check(rq.client, now)
|
|
||||||
|
|
||||||
var expires string
|
|
||||||
if banned {
|
|
||||||
expires = banExpires(ban)
|
|
||||||
rq.line.BanExpires = expires
|
|
||||||
}
|
|
||||||
|
|
||||||
if madePermanent {
|
|
||||||
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
|
||||||
a.banMaker.alertBan(rq, ban)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !banned {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &verdict{
|
|
||||||
action: banAction(ban.Permanent()),
|
|
||||||
status: a.banMaker.banResponse,
|
|
||||||
figures: map[string]any{
|
|
||||||
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// banAction is the action of a request refused under a ban or for making one:
|
// banAction is the action of a request refused under a ban or for making one:
|
||||||
// permanent says the ban never runs out.
|
// permanent says the ban never runs out.
|
||||||
func banAction(permanent bool) requestlog.Action {
|
func banAction(permanent bool) requestlog.Action {
|
||||||
|
|||||||
@@ -80,6 +80,26 @@ func (rq *request) startLookUp(
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
||||||
|
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
||||||
|
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
|
||||||
|
// says it will refuse it. In observe mode, which refuses nothing, none is.
|
||||||
|
// rproxy lets a request through when waf answers too late.
|
||||||
|
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
||||||
|
if rq.h.config.Observe {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, each := range rq.h.afterLookup {
|
||||||
|
knows, ok := each.(refusesWithoutLookup)
|
||||||
|
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// noteLookup notes result for the log line and as the request's asn= and
|
// noteLookup notes result for the log line and as the request's asn= and
|
||||||
// country= tags.
|
// country= tags.
|
||||||
func (rq *request) noteLookup(result lookupResult) {
|
func (rq *request) noteLookup(result lookupResult) {
|
||||||
|
|||||||
@@ -1,329 +0,0 @@
|
|||||||
// Package waf is the server the waf command runs. It answers
|
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
|
||||||
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
|
||||||
// blacklist each time the reputation package fetches it.
|
|
||||||
package waf
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cmp"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
lru "github.com/hashicorp/golang-lru/v2"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MetricsPath is where the metrics are, for a request that carries
|
|
||||||
// SWW_WAF_METRICS_TOKEN.
|
|
||||||
const MetricsPath = "/_sww/metrics"
|
|
||||||
|
|
||||||
// BansPath is where an admin lists and adds bans, and, followed by / and
|
|
||||||
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
|
||||||
const BansPath = "/_sww/bans"
|
|
||||||
|
|
||||||
// ClientsPath is where an admin asks what waf knows of a client, by
|
|
||||||
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
|
||||||
const ClientsPath = "/_sww/clients/"
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
Config *config.WAF
|
|
||||||
// RequestLog receives two JSON lines per question, one at its decision and
|
|
||||||
// one when its report comes, and one per request to waf's own
|
|
||||||
// endpoints.
|
|
||||||
RequestLog io.Writer
|
|
||||||
// ProcessLog receives the process's own messages.
|
|
||||||
ProcessLog *slog.Logger
|
|
||||||
// Metrics count what waf does. Whoever builds them adds the metrics
|
|
||||||
// read from the other parts.
|
|
||||||
Metrics *metrics.Metrics
|
|
||||||
// Limiter is the table of clients the rate limits count, with each
|
|
||||||
// client's history.
|
|
||||||
Limiter *ratelimit.Limiter
|
|
||||||
// Ledger holds the bans.
|
|
||||||
Ledger *bans.Ledger
|
|
||||||
// Anomalies are the counters of the anomaly thresholds.
|
|
||||||
Anomalies *anomaly.Counters
|
|
||||||
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
|
||||||
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
|
||||||
Lists *reputation.Lists
|
|
||||||
DNSBL *reputation.DNSBL
|
|
||||||
AbuseIPDB *reputation.AbuseIPDB
|
|
||||||
// RouteWatch asks routewatch about clients, and is nil while
|
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
|
||||||
RouteWatch *routewatch.Client
|
|
||||||
// GeoJS looks clients' AS numbers and countries up while
|
|
||||||
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
|
||||||
// and GeoJSFailed as its Failed.
|
|
||||||
GeoJS *geojs.GeoJS
|
|
||||||
// LookupFile is the lookup database they are looked up in while
|
|
||||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
|
||||||
LookupFile *lookup.File
|
|
||||||
// CoreRuleSet is what inspects requests while
|
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
|
||||||
CoreRuleSet CoreRuleSetInspector
|
|
||||||
// Now tells the time by which requests are counted for the rate
|
|
||||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
|
||||||
// normally time.Now in UTC, the time the state files give.
|
|
||||||
Now func() time.Time
|
|
||||||
// Rules are the rule files' rules, which each request is checked
|
|
||||||
// against.
|
|
||||||
Rules *rules.Files
|
|
||||||
// Alerts receive the alert for each ban waf makes or makes
|
|
||||||
// permanent, for each count over an anomaly threshold, for each request
|
|
||||||
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
|
||||||
// request the Core Rule Set scores at or over
|
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
|
||||||
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
|
||||||
// or the day's AbuseIPDB checks used up.
|
|
||||||
Alerts *alerts.Queue
|
|
||||||
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
|
|
||||||
// for a request. Zero means besideLookupTimeLimit. Tests that run the
|
|
||||||
// Core Rule Set lengthen it, since the race detector and a busy host can
|
|
||||||
// slow an inspection past besideLookupTimeLimit.
|
|
||||||
BesideLookupTimeLimit time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
|
||||||
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
|
||||||
type CoreRuleSetInspector interface {
|
|
||||||
Inspect(
|
|
||||||
r *http.Request, client netip.Addr, body io.Reader,
|
|
||||||
) (coreruleset.Result, []byte, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server is the server waf runs.
|
|
||||||
type Server struct {
|
|
||||||
httpServer *http.Server
|
|
||||||
h *handler
|
|
||||||
}
|
|
||||||
|
|
||||||
// newHandler returns the handler with what every request shares, the parts
|
|
||||||
// in params among it. It builds the heuristics, each with the settings and
|
|
||||||
// parts it reads, and the lists they run in.
|
|
||||||
//
|
|
||||||
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
|
||||||
func newHandler(params Params) *handler {
|
|
||||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
||||||
if err != nil {
|
|
||||||
panic(err) // New fails only for a size below one
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := params.Config
|
|
||||||
h := &handler{
|
|
||||||
config: cfg,
|
|
||||||
requestLog: params.RequestLog,
|
|
||||||
processLog: params.ProcessLog,
|
|
||||||
now: params.Now,
|
|
||||||
metrics: params.Metrics,
|
|
||||||
awaitingReport: awaitingReport,
|
|
||||||
limiter: params.Limiter,
|
|
||||||
ledger: params.Ledger,
|
|
||||||
geojs: params.GeoJS,
|
|
||||||
anomalies: params.Anomalies,
|
|
||||||
lookupFile: params.LookupFile,
|
|
||||||
// The country list, the headers, the tag settings that name an asn=
|
|
||||||
// or country= tag and the biased thresholds act on the answer before
|
|
||||||
// the request goes on.
|
|
||||||
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
|
||||||
cfg.AddLookupHeaders ||
|
|
||||||
cfg.TagsNeedLookup() ||
|
|
||||||
biasedThresholdsSet(cfg),
|
|
||||||
besideLookupLimit: cmp.Or(
|
|
||||||
params.BesideLookupTimeLimit,
|
|
||||||
besideLookupTimeLimit,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
sharedBanMaker := banMaker{
|
|
||||||
ledger: params.Ledger,
|
|
||||||
limiter: params.Limiter,
|
|
||||||
metrics: params.Metrics,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
observe: cfg.Observe,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
}
|
|
||||||
hitNoter := reputationHitNoter{
|
|
||||||
metrics: params.Metrics,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
|
||||||
h.ruleFiles = ruleFiles{
|
|
||||||
rules: params.Rules,
|
|
||||||
metrics: params.Metrics,
|
|
||||||
banMaker: sharedBanMaker,
|
|
||||||
}
|
|
||||||
h.coreRuleSet = coreRuleSet{
|
|
||||||
inspector: params.CoreRuleSet,
|
|
||||||
metrics: params.Metrics,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
mode: cfg.CoreRuleSetMode,
|
|
||||||
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
|
||||||
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
|
||||||
observe: cfg.Observe,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.beforeLookup = []heuristic{
|
|
||||||
headerLimits{},
|
|
||||||
tagLists{lists: params.Lists},
|
|
||||||
routeWatchTags{
|
|
||||||
routeWatch: params.RouteWatch,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
},
|
|
||||||
denyNets{netblocks: cfg.DenyNets, banResponse: cfg.BanResponse},
|
|
||||||
activeBan{banMaker: sharedBanMaker},
|
|
||||||
}
|
|
||||||
h.afterLookup = []heuristic{
|
|
||||||
countries{
|
|
||||||
allowed: cfg.ExclusivelyAllowedCountries,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
},
|
|
||||||
blocklists{
|
|
||||||
lists: params.Lists,
|
|
||||||
action: cfg.BlocklistAction,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
hitNoter: hitNoter,
|
|
||||||
},
|
|
||||||
dnsbl{
|
|
||||||
dnsbl: params.DNSBL,
|
|
||||||
action: cfg.DNSBLAbuseIPDBAction,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
hitNoter: hitNoter,
|
|
||||||
},
|
|
||||||
abuseIPDB{
|
|
||||||
abuseIPDB: params.AbuseIPDB,
|
|
||||||
limiter: params.Limiter,
|
|
||||||
keySet: cfg.AbuseIPDBKey != "",
|
|
||||||
minScore: cfg.AbuseIPDBMinScore,
|
|
||||||
action: cfg.DNSBLAbuseIPDBAction,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
hitNoter: hitNoter,
|
|
||||||
},
|
|
||||||
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
|
||||||
rateLimits{
|
|
||||||
limiter: params.Limiter,
|
|
||||||
exemptNets: cfg.RateLimitExemptNets,
|
|
||||||
exemptPaths: cfg.RateLimitExemptPaths,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
biasedThresholds: biasedThresholds{
|
|
||||||
lists: params.Lists,
|
|
||||||
tagLimitPercent: cfg.TagLimitPercent,
|
|
||||||
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
|
||||||
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
|
||||||
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
|
||||||
blocklistAction: cfg.BlocklistAction,
|
|
||||||
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
|
||||||
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
|
||||||
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
|
||||||
},
|
|
||||||
banMaker: sharedBanMaker,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
|
||||||
h.afterBesideLookup = []heuristic{
|
|
||||||
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
|
||||||
}
|
|
||||||
h.afterAnswer = []heuristic{
|
|
||||||
refusalLimit{
|
|
||||||
limiter: params.Limiter,
|
|
||||||
perMinute: cfg.RefusalsLimitPerMinute,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
banMaker: sharedBanMaker,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
h.onReport = []heuristic{
|
|
||||||
byteLimits{
|
|
||||||
limiter: params.Limiter,
|
|
||||||
bytesCounted: cfg.BytesCounted,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
banMaker: sharedBanMaker,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ownEndpoints = h.routes()
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
|
||||||
// belongs to one request is in a request.
|
|
||||||
type handler struct {
|
|
||||||
config *config.WAF
|
|
||||||
requestLog io.Writer
|
|
||||||
processLog *slog.Logger
|
|
||||||
now func() time.Time
|
|
||||||
metrics *metrics.Metrics
|
|
||||||
limiter *ratelimit.Limiter
|
|
||||||
ledger *bans.Ledger
|
|
||||||
geojs *geojs.GeoJS
|
|
||||||
anomalies *anomaly.Counters
|
|
||||||
lookupFile *lookup.File
|
|
||||||
// waitForGeoJS is true when a setting needs each request's answer from
|
|
||||||
// GeoJS before the request goes on.
|
|
||||||
waitForGeoJS bool
|
|
||||||
// besideLookupLimit is how long the heuristics of besideLookup have for a
|
|
||||||
// request, as Params.BesideLookupTimeLimit says.
|
|
||||||
besideLookupLimit time.Duration
|
|
||||||
// awaitingReport holds the decision on each answered question by its id
|
|
||||||
// until its report comes, and nil for a question still being handled.
|
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
|
||||||
// ownEndpoints answers every request that is not a question, as routes
|
|
||||||
// lists the endpoints.
|
|
||||||
ownEndpoints http.Handler
|
|
||||||
|
|
||||||
// The lists of heuristics, each in the order its heuristics run. A client
|
|
||||||
// in SWW_WAF_ALLOW_NETS skips them all.
|
|
||||||
|
|
||||||
// beforeLookup are the heuristics that run before the client's AS number
|
|
||||||
// and country are looked up, so that a client they refuse is not looked
|
|
||||||
// up. header_limits comes first: it only measures and counts, and a
|
|
||||||
// request it refuses causes no other work.
|
|
||||||
beforeLookup []heuristic
|
|
||||||
// afterLookup are the heuristics that run once the lookup has been made.
|
|
||||||
afterLookup []heuristic
|
|
||||||
// besideLookup are the heuristics that only read the request. Their work
|
|
||||||
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
|
||||||
// changes nothing of the request. Their check then applies what that work
|
|
||||||
// found, and runs only once waf takes it.
|
|
||||||
besideLookup []heuristic
|
|
||||||
// afterBesideLookup are the heuristics that run once afterLookup and
|
|
||||||
// besideLookup have both let the request through. tag_deny runs a second
|
|
||||||
// time, on the tag rule_files and core_rule_set can add.
|
|
||||||
afterBesideLookup []heuristic
|
|
||||||
// afterAnswer are the heuristics that run on a request once waf has
|
|
||||||
// answered it, before its decision line is written. They ban the client,
|
|
||||||
// and do not change what the request got: their verdict's action is not
|
|
||||||
// used.
|
|
||||||
afterAnswer []heuristic
|
|
||||||
// onReport are the heuristics that run on an allowed request when
|
|
||||||
// rproxy's report of it comes, before its report line is written. Like
|
|
||||||
// afterAnswer, they only ban.
|
|
||||||
onReport []heuristic
|
|
||||||
|
|
||||||
// The heuristics of besideLookup, whose work findBesideLookup does.
|
|
||||||
trapPaths trapPaths
|
|
||||||
ruleFiles ruleFiles
|
|
||||||
coreRuleSet coreRuleSet
|
|
||||||
}
|
|
||||||
@@ -456,7 +456,9 @@ func (w *wafAndParts) banAbuseIPDBBlacklist(
|
|||||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
|
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
|
||||||
// fetched from abuseIPDBBlacklistURL.
|
// fetched from abuseIPDBBlacklistURL.
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
||||||
// Core Rule Set have their requests inspected by it.
|
// Core Rule Set have their requests inspected by it. Likewise the anomaly thresholds
|
||||||
|
// per netblock and per AS number are off unless env sets them, so that only the tests
|
||||||
|
// of the anomaly thresholds count requests.
|
||||||
func newWAF(
|
func newWAF(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
geojsURL string,
|
geojsURL string,
|
||||||
@@ -486,6 +488,7 @@ func newWAFWith(
|
|||||||
settings[lookupSource] = off
|
settings[lookupSource] = off
|
||||||
}
|
}
|
||||||
|
|
||||||
|
maps.Copy(settings, netAndASNThresholdsOff())
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(configtest.Viper(settings))
|
cfg, err := config.WAFFromEnvironment(configtest.Viper(settings))
|
||||||
|
|||||||
@@ -0,0 +1,235 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
|
)
|
||||||
|
|
||||||
|
// answerQuestion runs the checks on the visitor's request a question
|
||||||
|
// carries, and answers rproxy what to do with it. A denied request's counting
|
||||||
|
// is finished here, and an allowed one's by its report.
|
||||||
|
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||||
|
question, ok := h.readQuestion(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rq := h.startRequest(
|
||||||
|
w,
|
||||||
|
question.Visitor,
|
||||||
|
question.Client,
|
||||||
|
question.Peer,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
// These run even when a check panics. finish runs first, so that what is
|
||||||
|
// held has the finished log line. Go's server sends the answer only once
|
||||||
|
// this function has returned, so the report cannot come before it is held.
|
||||||
|
defer func() {
|
||||||
|
rq.finish()
|
||||||
|
rq.holdForReport()
|
||||||
|
}()
|
||||||
|
|
||||||
|
rq.site = r.Host
|
||||||
|
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||||
|
refused := rq.check(r.Context())
|
||||||
|
rq.checked = time.Now()
|
||||||
|
underOwnPrefix := strings.HasPrefix(
|
||||||
|
r.URL.Path,
|
||||||
|
wafprotocol.OwnEndpointPrefix,
|
||||||
|
)
|
||||||
|
|
||||||
|
if refused == nil && !underOwnPrefix {
|
||||||
|
rq.allow()
|
||||||
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
|
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if refused != nil {
|
||||||
|
rq.deny(*refused)
|
||||||
|
} else {
|
||||||
|
// The app never sees this prefix. waf's own endpoints are at its
|
||||||
|
// own address, so this answers as a path with no endpoint does.
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||||
|
)
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusNotFound),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.banAndCount(r.Context(), rq.h.afterAnswer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readQuestion reads a question, and reports whether waf takes it. It answers
|
||||||
|
// a question it does not take with 400 and no disposition. rproxy passes that
|
||||||
|
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
|
||||||
|
// not apply. The id of a question waf takes is held from here on, without a
|
||||||
|
// decision yet, so that another question with that id is refused while this
|
||||||
|
// one is being handled.
|
||||||
|
//
|
||||||
|
// A question whose body stops arriving gets no answer, and is not looked at.
|
||||||
|
func (h *handler) readQuestion(
|
||||||
|
w http.ResponseWriter, r *http.Request,
|
||||||
|
) (wafprotocol.Question, bool) {
|
||||||
|
question, err := wafprotocol.ReadQuestion(w, r)
|
||||||
|
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
|
||||||
|
h.processLog.Warn(
|
||||||
|
"a question's body stopped arriving",
|
||||||
|
"error",
|
||||||
|
err.Error(),
|
||||||
|
)
|
||||||
|
// This panic has Go's server close the connection without an answer,
|
||||||
|
// and log nothing.
|
||||||
|
panic(http.ErrAbortHandler)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
h.refuseQuestion(w, err.Error())
|
||||||
|
|
||||||
|
return question, false
|
||||||
|
}
|
||||||
|
|
||||||
|
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
|
||||||
|
if held {
|
||||||
|
h.refuseQuestion(
|
||||||
|
w,
|
||||||
|
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
|
||||||
|
)
|
||||||
|
|
||||||
|
return question, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return question, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// refuseQuestion answers a question waf does not take, for reason.
|
||||||
|
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
||||||
|
h.processLog.Warn("a question is refused", "reason", reason)
|
||||||
|
http.Error(w, reason, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// allow answers a question with wafprotocol.DispositionAllow and, while
|
||||||
|
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
||||||
|
func (rq *request) allow() {
|
||||||
|
rq.allowed = true
|
||||||
|
|
||||||
|
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
||||||
|
if rq.h.config.AddLookupHeaders {
|
||||||
|
answer.ASN = rq.lookupAnswer.ASN
|
||||||
|
answer.Country = rq.lookupAnswer.Country
|
||||||
|
}
|
||||||
|
|
||||||
|
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
||||||
|
rq.out.WriteHeader(http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdForReport holds a copy of what the report's steps need, by the request's
|
||||||
|
// id. A report only ever reaches that copy, never the request being handled.
|
||||||
|
func (rq *request) holdForReport() {
|
||||||
|
held := rq.decision
|
||||||
|
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
|
||||||
|
}
|
||||||
|
|
||||||
|
// deny answers a question with what the visitor is to get for r.
|
||||||
|
func (rq *request) deny(r refusal) {
|
||||||
|
rq.refused = &r
|
||||||
|
|
||||||
|
if r.status == 0 {
|
||||||
|
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
||||||
|
// status stays 0: the visitor is sent nothing.
|
||||||
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
|
||||||
|
)
|
||||||
|
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||||
|
)
|
||||||
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
|
||||||
|
// the report names, if it holds that request's decision, and ignores the
|
||||||
|
// report otherwise.
|
||||||
|
func (h *handler) handleReport() http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
report, err := wafprotocol.ReadReport(w, r)
|
||||||
|
if err != nil {
|
||||||
|
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||||
|
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
held, found := h.awaitingReport.Peek(report.RequestID)
|
||||||
|
// Remove reports true to one report only, of several that name the
|
||||||
|
// id. A question still being handled is held as nil.
|
||||||
|
if found && held != nil &&
|
||||||
|
h.awaitingReport.Remove(report.RequestID) {
|
||||||
|
h.finishWith(r.Context(), held, report)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishWith finishes a request with what its report says, and writes the
|
||||||
|
// request's last log line, which holds its decision's fields and the report's.
|
||||||
|
// For an allowed request it first runs the heuristics of onReport, and adds the
|
||||||
|
// request to its client's history, the anomaly counters and the metrics, with
|
||||||
|
// the status and bytes the report gives. A denied request's counting was
|
||||||
|
// finished at its decision. ctx is the report's own context.
|
||||||
|
func (h *handler) finishWith(
|
||||||
|
ctx context.Context,
|
||||||
|
held *decision,
|
||||||
|
report wafprotocol.Report,
|
||||||
|
) {
|
||||||
|
rq := &request{
|
||||||
|
decision: *held,
|
||||||
|
h: h,
|
||||||
|
out: &responseWriter{
|
||||||
|
status: report.Status,
|
||||||
|
bytes: report.ResponseBytes,
|
||||||
|
},
|
||||||
|
requestBytes: report.RequestBytes,
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.line.WAFLine = requestlog.WAFLineReport
|
||||||
|
rq.line.Status = report.Status
|
||||||
|
rq.line.RequestBytes = report.RequestBytes
|
||||||
|
rq.line.ResponseBytes = report.ResponseBytes
|
||||||
|
rq.line.DurationTotalMS = report.DurationTotalMS
|
||||||
|
|
||||||
|
if rq.allowed {
|
||||||
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
|
rq.banAndCount(ctx, rq.h.onReport)
|
||||||
|
|
||||||
|
rq.h.metrics.RequestReported(
|
||||||
|
&rq.line,
|
||||||
|
time.Duration(report.DurationTotalMS*float64(time.Millisecond)),
|
||||||
|
time.Duration(
|
||||||
|
report.DurationUpstreamTotalMS*float64(time.Millisecond),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.writeLine()
|
||||||
|
}
|
||||||
@@ -405,7 +405,7 @@ func (rq *request) addToHistory() {
|
|||||||
// SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in
|
// SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in
|
||||||
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its
|
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its
|
||||||
// client's AS number when answerAtTheEnd gives one. With every anomaly
|
// client's AS number when answerAtTheEnd gives one. With every anomaly
|
||||||
// threshold off, the default, it does nothing.
|
// threshold off, it does nothing.
|
||||||
func (rq *request) countAnomalies() {
|
func (rq *request) countAnomalies() {
|
||||||
if !anomalyThresholdsSet(rq.h.config) {
|
if !anomalyThresholdsSet(rq.h.config) {
|
||||||
return
|
return
|
||||||
|
|||||||
+321
-246
@@ -1,18 +1,48 @@
|
|||||||
|
// Package waf is the server the waf command runs. It answers
|
||||||
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
||||||
|
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
||||||
|
// blacklist each time the reputation package fetches it.
|
||||||
package waf
|
package waf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
lru "github.com/hashicorp/golang-lru/v2"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MetricsPath is where the metrics are, for a request that carries
|
||||||
|
// SWW_WAF_METRICS_TOKEN.
|
||||||
|
const MetricsPath = "/_sww/metrics"
|
||||||
|
|
||||||
|
// BansPath is where an admin lists and adds bans, and, followed by / and
|
||||||
|
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
||||||
|
const BansPath = "/_sww/bans"
|
||||||
|
|
||||||
|
// ClientsPath is where an admin asks what waf knows of a client, by
|
||||||
|
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
||||||
|
const ClientsPath = "/_sww/clients/"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// maxAwaitingReport is the most answered requests held until their
|
// maxAwaitingReport is the most answered requests held until their
|
||||||
// reports come. Past it, the request answered longest ago is dropped.
|
// reports come. Past it, the request answered longest ago is dropped.
|
||||||
@@ -24,6 +54,79 @@ const (
|
|||||||
wafIdleTimeout = 120 * time.Second
|
wafIdleTimeout = 120 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Params are what New needs.
|
||||||
|
type Params struct {
|
||||||
|
Config *config.WAF
|
||||||
|
// RequestLog receives two JSON lines per question, one at its decision and
|
||||||
|
// one when its report comes, and one per request to waf's own
|
||||||
|
// endpoints.
|
||||||
|
RequestLog io.Writer
|
||||||
|
// ProcessLog receives the process's own messages.
|
||||||
|
ProcessLog *slog.Logger
|
||||||
|
// Metrics count what waf does. Whoever builds them adds the metrics
|
||||||
|
// read from the other parts.
|
||||||
|
Metrics *metrics.Metrics
|
||||||
|
// Limiter is the table of clients the rate limits count, with each
|
||||||
|
// client's history.
|
||||||
|
Limiter *ratelimit.Limiter
|
||||||
|
// Ledger holds the bans.
|
||||||
|
Ledger *bans.Ledger
|
||||||
|
// Anomalies are the counters of the anomaly thresholds.
|
||||||
|
Anomalies *anomaly.Counters
|
||||||
|
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
||||||
|
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
||||||
|
Lists *reputation.Lists
|
||||||
|
DNSBL *reputation.DNSBL
|
||||||
|
AbuseIPDB *reputation.AbuseIPDB
|
||||||
|
// RouteWatch asks routewatch about clients, and is nil while
|
||||||
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
|
RouteWatch *routewatch.Client
|
||||||
|
// GeoJS looks clients' AS numbers and countries up while
|
||||||
|
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
||||||
|
// and GeoJSFailed as its Failed.
|
||||||
|
GeoJS *geojs.GeoJS
|
||||||
|
// LookupFile is the lookup database they are looked up in while
|
||||||
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||||
|
LookupFile *lookup.File
|
||||||
|
// CoreRuleSet is what inspects requests while
|
||||||
|
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||||
|
CoreRuleSet CoreRuleSetInspector
|
||||||
|
// Now tells the time by which requests are counted for the rate
|
||||||
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||||
|
// normally time.Now in UTC, the time the state files give.
|
||||||
|
Now func() time.Time
|
||||||
|
// Rules are the rule files' rules, which each request is checked
|
||||||
|
// against.
|
||||||
|
Rules *rules.Files
|
||||||
|
// Alerts receive the alert for each ban waf makes or makes
|
||||||
|
// permanent, for each count over an anomaly threshold, for each request
|
||||||
|
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
||||||
|
// request the Core Rule Set scores at or over
|
||||||
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
||||||
|
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
||||||
|
// or the day's AbuseIPDB checks used up.
|
||||||
|
Alerts *alerts.Queue
|
||||||
|
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
|
||||||
|
// for a request. Zero means besideLookupTimeLimit. Tests that run the
|
||||||
|
// Core Rule Set lengthen it, since the race detector and a busy host can
|
||||||
|
// slow an inspection past besideLookupTimeLimit.
|
||||||
|
BesideLookupTimeLimit time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
||||||
|
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
||||||
|
type CoreRuleSetInspector interface {
|
||||||
|
Inspect(
|
||||||
|
r *http.Request, client netip.Addr, body io.Reader,
|
||||||
|
) (coreruleset.Result, []byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server is the server waf runs.
|
||||||
|
type Server struct {
|
||||||
|
httpServer *http.Server
|
||||||
|
h *handler
|
||||||
|
}
|
||||||
|
|
||||||
// New returns the server `sww waf` runs. It answers
|
// New returns the server `sww waf` runs. It answers
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
||||||
// passes nothing to an app. rproxy applies the size and time limits to the
|
// passes nothing to an app. rproxy applies the size and time limits to the
|
||||||
@@ -70,6 +173,221 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
s.h.ServeHTTP(w, r)
|
s.h.ServeHTTP(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newHandler returns the handler with what every request shares, the parts
|
||||||
|
// in params among it. It builds the heuristics, each with the settings and
|
||||||
|
// parts it reads, and the lists they run in.
|
||||||
|
//
|
||||||
|
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
||||||
|
func newHandler(params Params) *handler {
|
||||||
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||||
|
if err != nil {
|
||||||
|
panic(err) // New fails only for a size below one
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := params.Config
|
||||||
|
h := &handler{
|
||||||
|
config: cfg,
|
||||||
|
requestLog: params.RequestLog,
|
||||||
|
processLog: params.ProcessLog,
|
||||||
|
now: params.Now,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
awaitingReport: awaitingReport,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
ledger: params.Ledger,
|
||||||
|
geojs: params.GeoJS,
|
||||||
|
anomalies: params.Anomalies,
|
||||||
|
lookupFile: params.LookupFile,
|
||||||
|
// The country list, the headers, the tag settings that name an asn=
|
||||||
|
// or country= tag and the biased thresholds act on the answer before
|
||||||
|
// the request goes on.
|
||||||
|
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
||||||
|
cfg.AddLookupHeaders ||
|
||||||
|
cfg.TagsNeedLookup() ||
|
||||||
|
biasedThresholdsSet(cfg),
|
||||||
|
besideLookupLimit: cmp.Or(
|
||||||
|
params.BesideLookupTimeLimit,
|
||||||
|
besideLookupTimeLimit,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
sharedBanMaker := banMaker{
|
||||||
|
ledger: params.Ledger,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
hitNoter := reputationHitNoter{
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
||||||
|
h.ruleFiles = ruleFiles{
|
||||||
|
rules: params.Rules,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
}
|
||||||
|
h.coreRuleSet = coreRuleSet{
|
||||||
|
inspector: params.CoreRuleSet,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
mode: cfg.CoreRuleSetMode,
|
||||||
|
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
||||||
|
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.beforeLookup = []heuristic{
|
||||||
|
headerLimits{},
|
||||||
|
tagLists{lists: params.Lists},
|
||||||
|
routeWatchTags{
|
||||||
|
routeWatch: params.RouteWatch,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
},
|
||||||
|
denyNets{netblocks: cfg.DenyNets, banResponse: cfg.BanResponse},
|
||||||
|
activeBan{banMaker: sharedBanMaker},
|
||||||
|
}
|
||||||
|
h.afterLookup = []heuristic{
|
||||||
|
countries{
|
||||||
|
allowed: cfg.ExclusivelyAllowedCountries,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
},
|
||||||
|
blocklists{
|
||||||
|
lists: params.Lists,
|
||||||
|
action: cfg.BlocklistAction,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
dnsbl{
|
||||||
|
dnsbl: params.DNSBL,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
abuseIPDB{
|
||||||
|
abuseIPDB: params.AbuseIPDB,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
keySet: cfg.AbuseIPDBKey != "",
|
||||||
|
minScore: cfg.AbuseIPDBMinScore,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
||||||
|
rateLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
exemptNets: cfg.RateLimitExemptNets,
|
||||||
|
exemptPaths: cfg.RateLimitExemptPaths,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
biasedThresholds: biasedThresholds{
|
||||||
|
lists: params.Lists,
|
||||||
|
tagLimitPercent: cfg.TagLimitPercent,
|
||||||
|
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
||||||
|
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
||||||
|
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
||||||
|
blocklistAction: cfg.BlocklistAction,
|
||||||
|
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
||||||
|
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
|
},
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||||
|
h.afterBesideLookup = []heuristic{
|
||||||
|
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
||||||
|
}
|
||||||
|
h.afterAnswer = []heuristic{
|
||||||
|
refusalLimit{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
perMinute: cfg.RefusalsLimitPerMinute,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
h.onReport = []heuristic{
|
||||||
|
byteLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
bytesCounted: cfg.BytesCounted,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
h.ownEndpoints = h.routes()
|
||||||
|
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
|
// belongs to one request is in a request.
|
||||||
|
type handler struct {
|
||||||
|
config *config.WAF
|
||||||
|
requestLog io.Writer
|
||||||
|
processLog *slog.Logger
|
||||||
|
now func() time.Time
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
ledger *bans.Ledger
|
||||||
|
geojs *geojs.GeoJS
|
||||||
|
anomalies *anomaly.Counters
|
||||||
|
lookupFile *lookup.File
|
||||||
|
// waitForGeoJS is true when a setting needs each request's answer from
|
||||||
|
// GeoJS before the request goes on.
|
||||||
|
waitForGeoJS bool
|
||||||
|
// besideLookupLimit is how long the heuristics of besideLookup have for a
|
||||||
|
// request, as Params.BesideLookupTimeLimit says.
|
||||||
|
besideLookupLimit time.Duration
|
||||||
|
// awaitingReport holds the decision on each answered question by its id
|
||||||
|
// until its report comes, and nil for a question still being handled.
|
||||||
|
awaitingReport *lru.Cache[string, *decision]
|
||||||
|
// ownEndpoints answers every request that is not a question, as routes
|
||||||
|
// lists the endpoints.
|
||||||
|
ownEndpoints http.Handler
|
||||||
|
|
||||||
|
// The lists of heuristics, each in the order its heuristics run. A client
|
||||||
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||||
|
|
||||||
|
// beforeLookup are the heuristics that run before the client's AS number
|
||||||
|
// and country are looked up, so that a client they refuse is not looked
|
||||||
|
// up. header_limits comes first: it only measures and counts, and a
|
||||||
|
// request it refuses causes no other work.
|
||||||
|
beforeLookup []heuristic
|
||||||
|
// afterLookup are the heuristics that run once the lookup has been made.
|
||||||
|
afterLookup []heuristic
|
||||||
|
// besideLookup are the heuristics that only read the request. Their work
|
||||||
|
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
||||||
|
// changes nothing of the request. Their check then applies what that work
|
||||||
|
// found, and runs only once waf takes it.
|
||||||
|
besideLookup []heuristic
|
||||||
|
// afterBesideLookup are the heuristics that run once afterLookup and
|
||||||
|
// besideLookup have both let the request through. tag_deny runs a second
|
||||||
|
// time, on the tag rule_files and core_rule_set can add.
|
||||||
|
afterBesideLookup []heuristic
|
||||||
|
// afterAnswer are the heuristics that run on a request once waf has
|
||||||
|
// answered it, before its decision line is written. They ban the client,
|
||||||
|
// and do not change what the request got: their verdict's action is not
|
||||||
|
// used.
|
||||||
|
afterAnswer []heuristic
|
||||||
|
// onReport are the heuristics that run on an allowed request when
|
||||||
|
// rproxy's report of it comes, before its report line is written. Like
|
||||||
|
// afterAnswer, they only ban.
|
||||||
|
onReport []heuristic
|
||||||
|
|
||||||
|
// The heuristics of besideLookup, whose work findBesideLookup does.
|
||||||
|
trapPaths trapPaths
|
||||||
|
ruleFiles ruleFiles
|
||||||
|
coreRuleSet coreRuleSet
|
||||||
|
}
|
||||||
|
|
||||||
// ServeHTTP handles one request to waf: a request carrying
|
// ServeHTTP handles one request to waf: a request carrying
|
||||||
// wafprotocol.ClientHeader is rproxy's question, and any other is for one of
|
// wafprotocol.ClientHeader is rproxy's question, and any other is for one of
|
||||||
// waf's own endpoints.
|
// waf's own endpoints.
|
||||||
@@ -82,246 +400,3 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
h.ownEndpoints.ServeHTTP(w, r)
|
h.ownEndpoints.ServeHTTP(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
|
||||||
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
|
||||||
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
|
|
||||||
// says it will refuse it. In observe mode, which refuses nothing, none is.
|
|
||||||
// rproxy lets a request through when waf answers too late.
|
|
||||||
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
|
||||||
if rq.h.config.Observe {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, each := range rq.h.afterLookup {
|
|
||||||
knows, ok := each.(refusesWithoutLookup)
|
|
||||||
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerQuestion runs the checks on the visitor's request a question
|
|
||||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
|
||||||
// is finished here, and an allowed one's by its report.
|
|
||||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|
||||||
question, ok := h.readQuestion(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq := h.startRequest(
|
|
||||||
w,
|
|
||||||
question.Visitor,
|
|
||||||
question.Client,
|
|
||||||
question.Peer,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
// These run even when a check panics. finish runs first, so that what is
|
|
||||||
// held has the finished log line. Go's server sends the answer only once
|
|
||||||
// this function has returned, so the report cannot come before it is held.
|
|
||||||
defer func() {
|
|
||||||
rq.finish()
|
|
||||||
rq.holdForReport()
|
|
||||||
}()
|
|
||||||
|
|
||||||
rq.site = r.Host
|
|
||||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
|
||||||
refused := rq.check(r.Context())
|
|
||||||
rq.checked = time.Now()
|
|
||||||
underOwnPrefix := strings.HasPrefix(
|
|
||||||
r.URL.Path,
|
|
||||||
wafprotocol.OwnEndpointPrefix,
|
|
||||||
)
|
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
|
||||||
rq.allow()
|
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
|
||||||
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if refused != nil {
|
|
||||||
rq.deny(*refused)
|
|
||||||
} else {
|
|
||||||
// The app never sees this prefix. waf's own endpoints are at its
|
|
||||||
// own address, so this answers as a path with no endpoint does.
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
wafprotocol.SetAnswerHeaders(
|
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
|
||||||
)
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusNotFound),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.banAndCount(r.Context(), rq.h.afterAnswer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readQuestion reads a question, and reports whether waf takes it. It answers
|
|
||||||
// a question it does not take with 400 and no disposition. rproxy passes that
|
|
||||||
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
|
|
||||||
// not apply. The id of a question waf takes is held from here on, without a
|
|
||||||
// decision yet, so that another question with that id is refused while this
|
|
||||||
// one is being handled.
|
|
||||||
//
|
|
||||||
// A question whose body stops arriving gets no answer, and is not looked at.
|
|
||||||
func (h *handler) readQuestion(
|
|
||||||
w http.ResponseWriter, r *http.Request,
|
|
||||||
) (wafprotocol.Question, bool) {
|
|
||||||
question, err := wafprotocol.ReadQuestion(w, r)
|
|
||||||
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
|
|
||||||
h.processLog.Warn(
|
|
||||||
"a question's body stopped arriving",
|
|
||||||
"error",
|
|
||||||
err.Error(),
|
|
||||||
)
|
|
||||||
// This panic has Go's server close the connection without an answer,
|
|
||||||
// and log nothing.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
h.refuseQuestion(w, err.Error())
|
|
||||||
|
|
||||||
return question, false
|
|
||||||
}
|
|
||||||
|
|
||||||
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
|
|
||||||
if held {
|
|
||||||
h.refuseQuestion(
|
|
||||||
w,
|
|
||||||
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
|
|
||||||
)
|
|
||||||
|
|
||||||
return question, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return question, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// refuseQuestion answers a question waf does not take, for reason.
|
|
||||||
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
|
||||||
h.processLog.Warn("a question is refused", "reason", reason)
|
|
||||||
http.Error(w, reason, http.StatusBadRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// allow answers a question with wafprotocol.DispositionAllow and, while
|
|
||||||
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
|
||||||
func (rq *request) allow() {
|
|
||||||
rq.allowed = true
|
|
||||||
|
|
||||||
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
|
||||||
if rq.h.config.AddLookupHeaders {
|
|
||||||
answer.ASN = rq.lookupAnswer.ASN
|
|
||||||
answer.Country = rq.lookupAnswer.Country
|
|
||||||
}
|
|
||||||
|
|
||||||
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
|
||||||
rq.out.WriteHeader(http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdForReport holds a copy of what the report's steps need, by the request's
|
|
||||||
// id. A report only ever reaches that copy, never the request being handled.
|
|
||||||
func (rq *request) holdForReport() {
|
|
||||||
held := rq.decision
|
|
||||||
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deny answers a question with what the visitor is to get for r.
|
|
||||||
func (rq *request) deny(r refusal) {
|
|
||||||
rq.refused = &r
|
|
||||||
|
|
||||||
if r.status == 0 {
|
|
||||||
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
|
||||||
// status stays 0: the visitor is sent nothing.
|
|
||||||
wafprotocol.SetAnswerHeaders(
|
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
|
|
||||||
)
|
|
||||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
wafprotocol.SetAnswerHeaders(
|
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
|
||||||
)
|
|
||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
|
|
||||||
// the report names, if it holds that request's decision, and ignores the
|
|
||||||
// report otherwise.
|
|
||||||
func (h *handler) handleReport() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
report, err := wafprotocol.ReadReport(w, r)
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
|
||||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
held, found := h.awaitingReport.Peek(report.RequestID)
|
|
||||||
// Remove reports true to one report only, of several that name the
|
|
||||||
// id. A question still being handled is held as nil.
|
|
||||||
if found && held != nil &&
|
|
||||||
h.awaitingReport.Remove(report.RequestID) {
|
|
||||||
h.finishWith(r.Context(), held, report)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
|
||||||
// request's last log line, which holds its decision's fields and the report's.
|
|
||||||
// For an allowed request it first runs the heuristics of onReport, and adds the
|
|
||||||
// request to its client's history, the anomaly counters and the metrics, with
|
|
||||||
// the status and bytes the report gives. A denied request's counting was
|
|
||||||
// finished at its decision. ctx is the report's own context.
|
|
||||||
func (h *handler) finishWith(
|
|
||||||
ctx context.Context,
|
|
||||||
held *decision,
|
|
||||||
report wafprotocol.Report,
|
|
||||||
) {
|
|
||||||
rq := &request{
|
|
||||||
decision: *held,
|
|
||||||
h: h,
|
|
||||||
out: &responseWriter{
|
|
||||||
status: report.Status,
|
|
||||||
bytes: report.ResponseBytes,
|
|
||||||
},
|
|
||||||
requestBytes: report.RequestBytes,
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.line.WAFLine = requestlog.WAFLineReport
|
|
||||||
rq.line.Status = report.Status
|
|
||||||
rq.line.RequestBytes = report.RequestBytes
|
|
||||||
rq.line.ResponseBytes = report.ResponseBytes
|
|
||||||
rq.line.DurationTotalMS = report.DurationTotalMS
|
|
||||||
|
|
||||||
if rq.allowed {
|
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
|
||||||
|
|
||||||
rq.banAndCount(ctx, rq.h.onReport)
|
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
|
||||||
&rq.line,
|
|
||||||
time.Duration(report.DurationTotalMS*float64(time.Millisecond)),
|
|
||||||
time.Duration(
|
|
||||||
report.DurationUpstreamTotalMS*float64(time.Millisecond),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.writeLine()
|
|
||||||
}
|
|
||||||
+8
-3
@@ -8,6 +8,8 @@ set -eu
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
# Own line: a failing command substitution inside an argument does
|
# Own line: a failing command substitution inside an argument does
|
||||||
@@ -18,12 +20,15 @@ main() {
|
|||||||
[ -n "$version" ] || version="unknown"
|
[ -n "$version" ] || version="unknown"
|
||||||
revision="$(git rev-parse HEAD 2>/dev/null || true)"
|
revision="$(git rev-parse HEAD 2>/dev/null || true)"
|
||||||
[ -n "$revision" ] || revision="unknown"
|
[ -n "$revision" ] || revision="unknown"
|
||||||
created="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
created="$(date -u +%Y-%m-%d)"
|
||||||
docker build --no-cache \
|
docker build \
|
||||||
--build-arg VERSION="$version" \
|
--build-arg VERSION="$version" \
|
||||||
--build-arg REVISION="$revision" \
|
--build-arg REVISION="$revision" \
|
||||||
--build-arg CREATED="$created" \
|
--build-arg CREATED="$created" \
|
||||||
-t "$("$SCRIPT_DIR/projectname")" .
|
-t "git.eeqj.de/sneak/$("$SCRIPT_DIR/projectname")" \
|
||||||
|
-t "git.eeqj.de/sneak/$("$SCRIPT_DIR/projectname"):$version" \
|
||||||
|
-t "git.eeqj.de/sneak/$("$SCRIPT_DIR/projectname"):$(date -u +%Y-%m-%d)" \
|
||||||
|
.
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
+2
-9
@@ -14,16 +14,9 @@ set -eu
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
PUBLISH_IMAGE="git.eeqj.de/sneak/smallwebwaf"
|
||||||
if [ -z "${DOCKER_HOST:-}" ]; then
|
|
||||||
echo "script/publish: DOCKER_HOST is not set; set it to the Docker host that builds the image, such as ssh://user@build.example.com" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ -z "${PUBLISH_IMAGE:-}" ]; then
|
|
||||||
echo "script/publish: PUBLISH_IMAGE is not set; set it to the image name in the registry, such as registry.example.com/smallwebwaf" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
# Own line: a failing command substitution inside an argument does
|
# Own line: a failing command substitution inside an argument does
|
||||||
|
|||||||
Reference in new issue
Block a user