Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e52edb9f62 | ||
|
|
5667661dd8 | ||
|
|
5ce35505bc | ||
|
|
e7cc0adbdb | ||
|
|
805c64a256 | ||
|
|
967ccdf75e | ||
|
|
809d2664d7 | ||
|
|
01e6b5d47e | ||
|
|
5099448f79 | ||
|
|
eb48796f27 |
No files matched your search
@@ -424,8 +424,8 @@ full, and the limits that are fixed rather than settings.
|
|||||||
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
||||||
`make docker` gives the image; on another host, use the name you pushed it
|
`make docker` gives the image; on another host, use the name you pushed it
|
||||||
under, by digest. Docker Compose puts the services on a private network of its
|
under, by digest. Docker Compose puts the services on a private network of its
|
||||||
own, where each reaches the others by service name. Each example publishes
|
own, where each reaches the others by service name. Each example but "On the
|
||||||
`rproxy`'s port on the host, for trying it out.
|
open internet" publishes `rproxy`'s port on the host, for trying it out.
|
||||||
|
|
||||||
### Smallest setup
|
### Smallest setup
|
||||||
|
|
||||||
@@ -457,6 +457,142 @@ volumes:
|
|||||||
waf-state:
|
waf-state:
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### On the open internet
|
||||||
|
|
||||||
|
One gitea at `git.example.com`, with traefik in front of `rproxy`. traefik gets
|
||||||
|
the certificate from Let's Encrypt and is the only service with published ports.
|
||||||
|
`waf` sets what most sites set. More `waf` settings follow, commented out, each
|
||||||
|
with what it does and when to set it.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
traefik:
|
||||||
|
# traefik v3.7.19, 2026-11-13
|
||||||
|
image: traefik@sha256:a6c0194dda77bcb804824580f36fa7edc56e315b1acfe933366c8b52844a78a4
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- --providers.docker=true
|
||||||
|
- --providers.docker.exposedByDefault=false
|
||||||
|
- --entryPoints.web.address=:80
|
||||||
|
- --entryPoints.websecure.address=:443
|
||||||
|
# Redirect plain HTTP to HTTPS.
|
||||||
|
- --entryPoints.web.http.redirections.entryPoint.to=websecure
|
||||||
|
- --entryPoints.web.http.redirections.entryPoint.scheme=https
|
||||||
|
# A Let's Encrypt certificate for each site, by the TLS-ALPN
|
||||||
|
# challenge on port 443.
|
||||||
|
- --entryPoints.websecure.http.tls.certResolver=le
|
||||||
|
- --certificatesResolvers.le.acme.email=hostmaster@example.com
|
||||||
|
- --certificatesResolvers.le.acme.storage=/letsencrypt/acme.json
|
||||||
|
- --certificatesResolvers.le.acme.tlsChallenge=true
|
||||||
|
# Give a request 10 minutes to arrive, rather than 1, for the
|
||||||
|
# large git pushes the rproxy lines below accept.
|
||||||
|
# - --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s
|
||||||
|
ports:
|
||||||
|
- "80:80"
|
||||||
|
- "443:443"
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
- acme:/letsencrypt
|
||||||
|
|
||||||
|
app:
|
||||||
|
# Your app: here a gitea, which listens on port 3000.
|
||||||
|
image: registry.example/app@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
rproxy:
|
||||||
|
image: smallwebwaf
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
||||||
|
SWW_RPROXY_UPSTREAM_URL: http://app:3000
|
||||||
|
# Accept git pushes up to 1 GiB, rather than 100 MiB, sent in
|
||||||
|
# up to 10 minutes, rather than 1, with traefik's readTimeout
|
||||||
|
# line above.
|
||||||
|
# SWW_RPROXY_MAX_REQUEST_SIZE: 1G
|
||||||
|
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
|
||||||
|
labels:
|
||||||
|
- traefik.enable=true
|
||||||
|
- traefik.http.routers.git.rule=Host(`git.example.com`)
|
||||||
|
- traefik.http.routers.git.entryPoints=websecure
|
||||||
|
- traefik.http.services.git.loadBalancer.server.port=8080
|
||||||
|
|
||||||
|
waf:
|
||||||
|
image: smallwebwaf
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
SWW_COMMAND: waf
|
||||||
|
# The name in every log line, alert and metric. The default,
|
||||||
|
# the container's host name, is a random id under Compose.
|
||||||
|
SWW_WAF_INSTANCE_NAME: host1/waf
|
||||||
|
# The app each hostname serves. Only that app's fixes to the
|
||||||
|
# Core Rule Set apply there.
|
||||||
|
SWW_WAF_APPS: git.example.com:gitea
|
||||||
|
# Monitoring skips every check.
|
||||||
|
SWW_WAF_ALLOW_NETS: 192.0.2.10
|
||||||
|
# The office and the CI runner, which would exceed the rate
|
||||||
|
# limits, are never rate limited.
|
||||||
|
SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
|
||||||
|
# WordPress paths, which only scanners ask a gitea for. One
|
||||||
|
# request for one bans the client.
|
||||||
|
SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php
|
||||||
|
# Spamhaus's list of netblocks hijacked or leased by
|
||||||
|
# criminals. Their clients are refused.
|
||||||
|
SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt
|
||||||
|
# The token for the ban endpoints, to lift a ban without
|
||||||
|
# editing bans.json.
|
||||||
|
SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
|
||||||
|
# Alerts, such as one for each ban, to an ntfy topic.
|
||||||
|
SWW_WAF_ALERT_NTFY_URL: https://ntfy.example.com/smallwebwaf
|
||||||
|
|
||||||
|
# Refuse nothing and ban no one, for the first days. Each log
|
||||||
|
# line says what enforce mode would have done.
|
||||||
|
# SWW_WAF_MODE: observe
|
||||||
|
# The access token for an ntfy topic that is not public.
|
||||||
|
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
|
||||||
|
# The token for /_sww/metrics, for Prometheus. Unset, the
|
||||||
|
# metrics are off.
|
||||||
|
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
|
||||||
|
# 600 requests an hour rather than 1500, to ban slower
|
||||||
|
# scrapers. A fast human reader can reach it too.
|
||||||
|
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
|
||||||
|
# A quarter of every limit for clients in two hosting
|
||||||
|
# networks, where scrapers run rather than people.
|
||||||
|
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:25,asn=16276:25
|
||||||
|
# For a site only for people in a few countries. Clients
|
||||||
|
# elsewhere are refused, and so are any the lookup cannot place.
|
||||||
|
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: de,at,ch
|
||||||
|
# Close a refused request's connection rather than answer
|
||||||
|
# 403. traefik then answers 502, so a scraper cannot tell a
|
||||||
|
# ban from a broken site.
|
||||||
|
# SWW_WAF_BAN_RESPONSE: close
|
||||||
|
# A copy of every log line to a syslog server.
|
||||||
|
# SWW_WAF_LOG_REMOTE_URL: syslog+tls://logs.example.com:6514
|
||||||
|
volumes:
|
||||||
|
- waf-state:/var/lib/sww
|
||||||
|
- /srv/sww/tokens:/etc/sww/tokens:ro
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
acme:
|
||||||
|
waf-state:
|
||||||
|
```
|
||||||
|
|
||||||
|
Change these before the first start:
|
||||||
|
|
||||||
|
- `git.example.com`, in the router rule and in `SWW_WAF_APPS`;
|
||||||
|
- the email address for Let's Encrypt;
|
||||||
|
- the ntfy topic's URL;
|
||||||
|
- the app's image and port;
|
||||||
|
- the netblocks, which are documentation addresses.
|
||||||
|
|
||||||
|
Make the admin token file first, as for any token file:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
mkdir -p /srv/sww/tokens
|
||||||
|
openssl rand -hex 32 > /srv/sww/tokens/admin
|
||||||
|
chown 65532:65532 /srv/sww/tokens/admin
|
||||||
|
chmod 0400 /srv/sww/tokens/admin
|
||||||
|
```
|
||||||
|
|
||||||
### Custom per-address limits
|
### Custom per-address limits
|
||||||
|
|
||||||
For a site with no API:
|
For a site with no API:
|
||||||
@@ -760,6 +896,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
|
|
||||||
- `cmd/sww`: the binary, which only calls `internal/sww`.
|
- `cmd/sww`: the binary, which only calls `internal/sww`.
|
||||||
- `internal/sww`: the process, as the command it is given, and the health check.
|
- `internal/sww`: the process, as the command it is given, and the health check.
|
||||||
|
It is the only package that imports `go.uber.org/fx`, which builds the parts
|
||||||
|
of `waf` and `rproxy`.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app.
|
- `internal/rproxy`: the reverse proxy in front of one app.
|
||||||
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
||||||
@@ -768,9 +906,11 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
- `internal/state`: the state and cache files.
|
||||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
- `internal/requestlog`: the request log.
|
||||||
sender and the metrics.
|
- `internal/processlog`: the process's own messages.
|
||||||
|
- `internal/remotelog`: the syslog sender.
|
||||||
|
- `internal/metrics`: the metrics.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -69,7 +69,9 @@ never as a time.
|
|||||||
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
|
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
|
||||||
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
|
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
|
||||||
With no command, or one it does not know, the binary prints the commands
|
With no command, or one it does not know, the binary prints the commands
|
||||||
and exits with status 2.
|
and exits with status 2. `internal/sww` is the only package that imports
|
||||||
|
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||||
|
and stops their background work.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
|
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
|
||||||
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
|
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
|
||||||
@@ -102,8 +104,10 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
||||||
sending them to the syslog server.
|
- `internal/processlog`: the process lines on stdout, which hold the
|
||||||
|
process's own messages.
|
||||||
|
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ require (
|
|||||||
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/sys v0.48.0
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,6 +45,9 @@ require (
|
|||||||
github.com/tidwall/match v1.1.1 // indirect
|
github.com/tidwall/match v1.1.1 // indirect
|
||||||
github.com/tidwall/pretty v1.2.1 // indirect
|
github.com/tidwall/pretty v1.2.1 // indirect
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
|
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
|
||||||
|
go.uber.org/dig v1.19.0 // indirect
|
||||||
|
go.uber.org/multierr v1.10.0 // indirect
|
||||||
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
golang.org/x/net v0.60.0 // indirect
|
golang.org/x/net v0.60.0 // indirect
|
||||||
|
|||||||
@@ -97,8 +97,16 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
|||||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
|
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
|
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
|
||||||
|
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
||||||
|
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
||||||
|
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
||||||
|
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
|
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||||
|
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||||
|
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||||
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
|
|||||||
+93
-32
@@ -75,7 +75,7 @@ type WAF struct {
|
|||||||
BytesLimitPerMinute int64
|
BytesLimitPerMinute int64
|
||||||
BytesLimitPerHour int64
|
BytesLimitPerHour int64
|
||||||
BytesLimitPerDay int64
|
BytesLimitPerDay int64
|
||||||
BytesCounted string
|
BytesCounted BytesCounted
|
||||||
// LookupSource is where each client's AS number and country are looked
|
// LookupSource is where each client's AS number and country are looked
|
||||||
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
||||||
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
||||||
@@ -86,7 +86,7 @@ type WAF struct {
|
|||||||
// AddLookupHeaders is true when the app is passed the client's AS number
|
// AddLookupHeaders is true when the app is passed the client's AS number
|
||||||
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
||||||
// LookupSource is off.
|
// LookupSource is off.
|
||||||
LookupSource string
|
LookupSource LookupSource
|
||||||
LookupDBPath string
|
LookupDBPath string
|
||||||
ASNAndCountryLookupTimeout time.Duration
|
ASNAndCountryLookupTimeout time.Duration
|
||||||
AddLookupHeaders bool
|
AddLookupHeaders bool
|
||||||
@@ -131,7 +131,7 @@ type WAF struct {
|
|||||||
// BlocklistLimitPercent is the percentage of every limit it gets.
|
// BlocklistLimitPercent is the percentage of every limit it gets.
|
||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
BlocklistRefresh time.Duration
|
BlocklistRefresh time.Duration
|
||||||
BlocklistAction string
|
BlocklistAction ReputationAction
|
||||||
BlocklistLimitPercent int64
|
BlocklistLimitPercent int64
|
||||||
// DNSBLZones are the DNSBL zones clients are asked about
|
// DNSBLZones are the DNSBL zones clients are asked about
|
||||||
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
||||||
@@ -154,7 +154,7 @@ type WAF struct {
|
|||||||
AbuseIPDBKey string
|
AbuseIPDBKey string
|
||||||
AbuseIPDBMinScore int64
|
AbuseIPDBMinScore int64
|
||||||
AbuseIPDBDailyBudget int
|
AbuseIPDBDailyBudget int
|
||||||
DNSBLAbuseIPDBAction string
|
DNSBLAbuseIPDBAction ReputationAction
|
||||||
DNSBLAbuseIPDBLimitPercent int64
|
DNSBLAbuseIPDBLimitPercent int64
|
||||||
IPReputationCacheTTL time.Duration
|
IPReputationCacheTTL time.Duration
|
||||||
DNSBLAbuseIPDBTimeout time.Duration
|
DNSBLAbuseIPDBTimeout time.Duration
|
||||||
@@ -237,7 +237,7 @@ type WAF struct {
|
|||||||
// reads none.
|
// reads none.
|
||||||
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
||||||
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
||||||
CoreRuleSetMode string
|
CoreRuleSetMode CoreRuleSetMode
|
||||||
CoreRuleSetParanoiaLevel int
|
CoreRuleSetParanoiaLevel int
|
||||||
CoreRuleSetScoreThreshold int
|
CoreRuleSetScoreThreshold int
|
||||||
CoreRuleSetDisabledRules []int
|
CoreRuleSetDisabledRules []int
|
||||||
@@ -316,20 +316,66 @@ type WAF struct {
|
|||||||
settings []slog.Attr
|
settings []slog.Attr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BytesCounted is which body bytes count toward the byte limits and the
|
||||||
|
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
|
||||||
|
type BytesCounted string
|
||||||
|
|
||||||
|
// The values of SWW_WAF_BYTES_COUNTED.
|
||||||
|
const (
|
||||||
|
// BytesCountedResponse counts the response's body bytes.
|
||||||
|
BytesCountedResponse BytesCounted = "response"
|
||||||
|
// BytesCountedRequest counts the request's body bytes.
|
||||||
|
BytesCountedRequest BytesCounted = "request"
|
||||||
|
// BytesCountedBoth counts the body bytes of both.
|
||||||
|
BytesCountedBoth BytesCounted = "both"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LookupSource is where each client's AS number and country are looked up,
|
||||||
|
// a value of SWW_WAF_LOOKUP_SOURCE.
|
||||||
|
type LookupSource string
|
||||||
|
|
||||||
|
// The values of SWW_WAF_LOOKUP_SOURCE.
|
||||||
|
const (
|
||||||
|
// LookupSourceGeoJS asks GeoJS.
|
||||||
|
LookupSourceGeoJS LookupSource = "geojs"
|
||||||
|
// LookupSourceFile looks clients up in the lookup database, the file
|
||||||
|
// SWW_WAF_LOOKUP_DB_PATH names.
|
||||||
|
LookupSourceFile LookupSource = "file"
|
||||||
|
// LookupSourceOff looks no client up.
|
||||||
|
LookupSourceOff LookupSource = "off"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReputationAction is what is done with a client a reputation source
|
||||||
|
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
|
||||||
|
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
||||||
|
type ReputationAction string
|
||||||
|
|
||||||
|
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
||||||
|
const (
|
||||||
|
// ReputationActionDeny refuses the client's requests.
|
||||||
|
ReputationActionDeny ReputationAction = "deny"
|
||||||
|
// ReputationActionLimit gives the client a percentage of every limit,
|
||||||
|
// written limit:<percent> in the setting.
|
||||||
|
ReputationActionLimit ReputationAction = "limit"
|
||||||
|
// ReputationActionLog only notes the hit in the log line and raises an
|
||||||
|
// alert.
|
||||||
|
ReputationActionLog ReputationAction = "log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CoreRuleSetMode is what the Core Rule Set does, a value of
|
||||||
|
// SWW_WAF_CORE_RULE_SET_MODE.
|
||||||
|
type CoreRuleSetMode string
|
||||||
|
|
||||||
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
||||||
const (
|
const (
|
||||||
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
||||||
CoreRuleSetModeOff = off
|
CoreRuleSetModeOff CoreRuleSetMode = "off"
|
||||||
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
||||||
CoreRuleSetModeDetect = "detect"
|
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
|
||||||
// CoreRuleSetModeBlock refuses a match with 403.
|
// CoreRuleSetModeBlock refuses a match with 403.
|
||||||
CoreRuleSetModeBlock = "block"
|
CoreRuleSetModeBlock CoreRuleSetMode = "block"
|
||||||
)
|
)
|
||||||
|
|
||||||
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
|
|
||||||
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
|
|
||||||
const fileSource = "file"
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
day = 24 * time.Hour
|
day = 24 * time.Hour
|
||||||
ipv4Bits = 32
|
ipv4Bits = 32
|
||||||
@@ -558,10 +604,13 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
),
|
),
|
||||||
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||||
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||||
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
|
BytesCounted: env.bytesCounted(
|
||||||
|
"SWW_WAF_BYTES_COUNTED",
|
||||||
|
BytesCountedBoth,
|
||||||
|
),
|
||||||
LookupSource: env.lookupSource(
|
LookupSource: env.lookupSource(
|
||||||
"SWW_WAF_LOOKUP_SOURCE",
|
"SWW_WAF_LOOKUP_SOURCE",
|
||||||
"geojs",
|
LookupSourceGeoJS,
|
||||||
),
|
),
|
||||||
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
||||||
ASNAndCountryLookupTimeout: env.milliseconds(
|
ASNAndCountryLookupTimeout: env.milliseconds(
|
||||||
@@ -758,9 +807,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||||
// The app gets the headers by default, unless no client is looked up.
|
// The app gets the headers by default, unless no client is looked up.
|
||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != off))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", "deny")
|
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||||
|
|
||||||
@@ -858,9 +907,12 @@ func (e *environment) count(name, defaultValue string) int64 {
|
|||||||
|
|
||||||
// bytesCounted reads the setting that is which body bytes count toward the
|
// bytesCounted reads the setting that is which body bytes count toward the
|
||||||
// byte limits and the byte anomaly thresholds: response, request or both.
|
// byte limits and the byte anomaly thresholds: response, request or both.
|
||||||
func (e *environment) bytesCounted(name, defaultValue string) string {
|
func (e *environment) bytesCounted(
|
||||||
value := e.value(name, defaultValue)
|
name string, defaultValue BytesCounted,
|
||||||
if value != "response" && value != "request" && value != "both" {
|
) BytesCounted {
|
||||||
|
value := BytesCounted(e.value(name, string(defaultValue)))
|
||||||
|
if value != BytesCountedResponse && value != BytesCountedRequest &&
|
||||||
|
value != BytesCountedBoth {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -905,8 +957,10 @@ func (e *environment) trapPaths(name string) []string {
|
|||||||
|
|
||||||
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
||||||
// detect or block.
|
// detect or block.
|
||||||
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
|
func (e *environment) coreRuleSetMode(
|
||||||
mode := e.value(name, defaultValue)
|
name string, defaultValue CoreRuleSetMode,
|
||||||
|
) CoreRuleSetMode {
|
||||||
|
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
|
||||||
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
||||||
mode != CoreRuleSetModeBlock {
|
mode != CoreRuleSetModeBlock {
|
||||||
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
||||||
@@ -1056,20 +1110,24 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
|||||||
// action reads a setting that is what is done with a client a blocklist
|
// action reads a setting that is what is done with a client a blocklist
|
||||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||||
// as limit and the percentage.
|
// as limit and the percentage.
|
||||||
func (e *environment) action(name, defaultValue string) (string, int64) {
|
func (e *environment) action(
|
||||||
value := e.value(name, defaultValue)
|
name, defaultValue string,
|
||||||
if value == "deny" || value == "log" {
|
) (ReputationAction, int64) {
|
||||||
|
value := ReputationAction(e.value(name, defaultValue))
|
||||||
|
if value == ReputationActionDeny || value == ReputationActionLog {
|
||||||
return value, 0
|
return value, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
percentText, isLimit := strings.CutPrefix(value, "limit:")
|
percentText, isLimit := strings.CutPrefix(
|
||||||
|
string(value), string(ReputationActionLimit)+":",
|
||||||
|
)
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
|
|
||||||
return "limit", percent
|
return ReputationActionLimit, percent
|
||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
@@ -1120,9 +1178,12 @@ func (e *environment) routeWatchURL(name string) string {
|
|||||||
|
|
||||||
// lookupSource reads the setting that is where clients are looked up:
|
// lookupSource reads the setting that is where clients are looked up:
|
||||||
// geojs, file, or off.
|
// geojs, file, or off.
|
||||||
func (e *environment) lookupSource(name, defaultValue string) string {
|
func (e *environment) lookupSource(
|
||||||
source := e.value(name, defaultValue)
|
name string, defaultValue LookupSource,
|
||||||
if source != "geojs" && source != fileSource && source != off {
|
) LookupSource {
|
||||||
|
source := LookupSource(e.value(name, string(defaultValue)))
|
||||||
|
if source != LookupSourceGeoJS && source != LookupSourceFile &&
|
||||||
|
source != LookupSourceOff {
|
||||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1134,12 +1195,12 @@ func (e *environment) lookupSource(name, defaultValue string) string {
|
|||||||
// one source at a time.
|
// one source at a time.
|
||||||
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||||
switch {
|
switch {
|
||||||
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
|
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
|
||||||
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
||||||
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
||||||
errNeedsDBPath,
|
errNeedsDBPath,
|
||||||
))
|
))
|
||||||
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
|
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
|
||||||
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
||||||
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
||||||
cfg.LookupSource,
|
cfg.LookupSource,
|
||||||
@@ -1164,7 +1225,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.LookupSource != off {
|
if cfg.LookupSource != LookupSourceOff {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -614,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
environment{
|
environment{
|
||||||
coreRuleSetMode: config.CoreRuleSetModeDetect,
|
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
|
||||||
coreRuleSetParanoiaLevel: "4",
|
coreRuleSetParanoiaLevel: "4",
|
||||||
coreRuleSetScoreThreshold: "10",
|
coreRuleSetScoreThreshold: "10",
|
||||||
coreRuleSetDisabledRules: "942100, 920350",
|
coreRuleSetDisabledRules: "942100, 920350",
|
||||||
@@ -2033,8 +2033,9 @@ func TestReputationSettingsAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
value, action string
|
value string
|
||||||
percent int64
|
action config.ReputationAction
|
||||||
|
percent int64
|
||||||
}{
|
}{
|
||||||
{actionDeny, actionDeny, 0},
|
{actionDeny, actionDeny, 0},
|
||||||
{actionLog, actionLog, 0},
|
{actionLog, actionLog, 0},
|
||||||
@@ -2167,7 +2168,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
|
|||||||
// want are the zones, resolver, action and percent config.WAF gives.
|
// want are the zones, resolver, action and percent config.WAF gives.
|
||||||
want []string
|
want []string
|
||||||
wantResolver string
|
wantResolver string
|
||||||
wantAction string
|
wantAction config.ReputationAction
|
||||||
wantPercent int64
|
wantPercent int64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -3049,7 +3050,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
logLevel: "info",
|
logLevel: "info",
|
||||||
ruleFilesDir: "/etc/sww/rules.d",
|
ruleFilesDir: "/etc/sww/rules.d",
|
||||||
ruleFilesEnabled: "true",
|
ruleFilesEnabled: "true",
|
||||||
coreRuleSetMode: config.CoreRuleSetModeBlock,
|
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
|
||||||
coreRuleSetParanoiaLevel: "1",
|
coreRuleSetParanoiaLevel: "1",
|
||||||
coreRuleSetScoreThreshold: "5",
|
coreRuleSetScoreThreshold: "5",
|
||||||
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
||||||
|
|||||||
+60
-37
@@ -84,12 +84,7 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
url string
|
params Params
|
||||||
answered func(Answer)
|
|
||||||
now func() time.Time
|
|
||||||
asked func()
|
|
||||||
failed func(err error, askingAgainIn time.Duration)
|
|
||||||
wentWithoutAnswer func()
|
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -99,8 +94,13 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way.
|
// asking is true while a request to GeoJS is under way, and endAsking
|
||||||
asking bool
|
// ends it. stopped is true once StopAsking has been called, and
|
||||||
|
// askGoroutines waits for the goroutine that makes the requests.
|
||||||
|
asking bool
|
||||||
|
endAsking context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
askGoroutines sync.WaitGroup
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -140,12 +140,7 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
url: params.URL,
|
params: params,
|
||||||
answered: params.Answered,
|
|
||||||
now: params.Now,
|
|
||||||
asked: params.Asked,
|
|
||||||
failed: params.Failed,
|
|
||||||
wentWithoutAnswer: params.WentWithoutAnswer,
|
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -188,7 +183,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -210,6 +205,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||||
|
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||||
|
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||||
|
// asked nothing.
|
||||||
|
func (g *GeoJS) StopAsking() {
|
||||||
|
g.mu.Lock()
|
||||||
|
|
||||||
|
g.stopped = true
|
||||||
|
if g.asking {
|
||||||
|
g.endAsking()
|
||||||
|
}
|
||||||
|
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
g.askGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -251,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -288,18 +300,19 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, so no answer can come.
|
// GeoJS is left alone after a failure, or asked nothing after
|
||||||
|
// StopAsking, so no answer can come.
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -310,7 +323,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -323,22 +336,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way or it is left alone after a failure. The requests to
|
// it is under way, it is left alone after a failure, or StopAsking has
|
||||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// ctx is given stops waiting.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
g.asking = true
|
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
g.asking, g.endAsking = true, end
|
||||||
|
|
||||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
g.askGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
g.askAboutWaiting(askCtx)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||||
// Answered, outside the lock, since Answered takes locks of its own.
|
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||||
|
// locks of its own.
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -347,11 +366,15 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
continue // ended by StopAsking, which is no failure
|
||||||
|
}
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.params.Answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.answered(answer)
|
g.params.Answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,13 +385,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
// none is waiting, or StopAsking has been called, it returns none and
|
||||||
// under way.
|
// notes that no request to GeoJS is under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 {
|
if len(g.waiting) == 0 || g.stopped {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -400,7 +423,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -444,7 +467,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.failed(err, g.retryDelay)
|
g.params.Failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -474,7 +497,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.url,
|
g.params.URL,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -483,7 +506,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.asked()
|
g.params.Asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
service, _, g, reports := startWithReports()
|
||||||
|
clients := newClients()
|
||||||
|
|
||||||
|
service.set(hanging)
|
||||||
|
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
|
||||||
|
// The request is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
g.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
reports.wantFailures(t)
|
||||||
|
|
||||||
|
// A new client is then not asked about, and its lookup does not
|
||||||
|
// wait.
|
||||||
|
service.set(answering)
|
||||||
|
wantCountry(t, g, clients(), "")
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
reports.wantFailures(t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
// Package processlog builds the logger for the process's own messages,
|
||||||
|
// which it writes as JSON lines marked "type":"process". The request log
|
||||||
|
// is package requestlog's.
|
||||||
|
package processlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params are what New needs.
|
||||||
|
type Params struct {
|
||||||
|
// Output is where the lines are written.
|
||||||
|
Output io.Writer
|
||||||
|
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
||||||
|
InstanceName string
|
||||||
|
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
||||||
|
Level slog.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns the logger for the process's own messages: JSON lines on
|
||||||
|
// params.Output, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's. It writes only the messages at params.Level or more
|
||||||
|
// severe; the request lines requestlog.Write writes are never held back.
|
||||||
|
func New(params Params) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
||||||
|
Level: params.Level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
requestlog.FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", params.InstanceName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package processlog_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// instanceName is the instance the tests' loggers are built with.
|
||||||
|
const instanceName = "host1/blog"
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: slog.LevelInfo,
|
||||||
|
}).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: level,
|
||||||
|
})
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,16 +156,28 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a Limiter for limits, with no client counted yet, whose
|
// Params are what New needs.
|
||||||
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
type Params struct {
|
||||||
// it, the least recently seen client is dropped, with its history, and
|
// Limits are what each client's requests and bytes are counted against.
|
||||||
// starts afresh if it comes back.
|
Limits Limits
|
||||||
func New(limits Limits, maxClients int) *Limiter {
|
// MaxClients is the most clients the table holds,
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
||||||
|
// is dropped, with its history, and starts afresh if it comes back.
|
||||||
|
MaxClients int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Limiter with no client counted yet.
|
||||||
|
func New(params Params) *Limiter {
|
||||||
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
||||||
|
params.MaxClients,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
|
limits := params.Limits
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
Limits: ratelimit.Limits{
|
||||||
tableSize,
|
PerMinute: 5,
|
||||||
)
|
BytesPerDay: math.MaxInt64,
|
||||||
|
},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: 1},
|
||||||
|
MaxClients: maxClients,
|
||||||
|
})
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
before := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
after := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way.
|
// checking are the clients whose check is under way, each with the
|
||||||
checking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopChecking has been
|
||||||
|
// called, and checkGoroutines waits for the goroutine of each check.
|
||||||
|
checking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
checkGoroutines sync.WaitGroup
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]bool{},
|
checking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, or the day's checks
|
// way, a check failed less than failureDelay ago, the day's checks have
|
||||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
// for a check. The check that uses the budget up is logged and raised as a
|
||||||
// is the context of the client's request, and a check goes on after the
|
// source_failure alert. ctx is the context of the client's request, and a
|
||||||
// request ends.
|
// check goes on after the request ends, until StopChecking.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
_, checking := a.checking[client]
|
||||||
a.spent < a.params.DailyBudget
|
|
||||||
|
check := offender && !checking && !a.stopped &&
|
||||||
|
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||||
if check {
|
if check {
|
||||||
a.checking[client] = true
|
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
a.checking[client] = end
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
a.checkGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
a.check(checkCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopChecking ends the checks under way, and returns once their goroutines
|
||||||
|
// have. A check it ends gives no score and is no failure: it is neither
|
||||||
|
// counted, logged nor raised as an alert, and checks are not left alone
|
||||||
|
// after it. The ended check stays spent from the day's budget. After
|
||||||
|
// StopChecking, Hit checks no client.
|
||||||
|
func (a *AbuseIPDB) StopChecking() {
|
||||||
|
a.mu.Lock()
|
||||||
|
|
||||||
|
a.stopped = true
|
||||||
|
for _, end := range a.checking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
a.checkGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay.
|
// source_failure alert, and no client is checked for failureDelay. A check
|
||||||
|
// ended as ctx ends, by StopChecking, is no failure.
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
} else {
|
case isFailure:
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := abuseIPDBParams()
|
||||||
|
p.Alerts = queue
|
||||||
|
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||||
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
|
|
||||||
|
wantScore(t, checker, suspect, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The check is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
checker.StopChecking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFailures(t, checker, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An offender without a score is then not checked.
|
||||||
|
wantScore(t, checker, other, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
wantChecked(t, abuseIPDB, suspect)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -82,8 +82,12 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way.
|
// asking are the queries under way, each with the function that ends
|
||||||
asking map[query]bool
|
// it. stopped is true once StopAsking has been called, and
|
||||||
|
// queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[query]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -120,7 +124,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]bool{},
|
asking: map[query]context.CancelFunc{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
@@ -138,9 +142,10 @@ func (d *DNSBL) Zones() []string {
|
|||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, or maxQueries are under way;
|
// zone is left alone after a failure, maxQueries are under way, or
|
||||||
// ListedBy never waits for a query. ctx is the context of the client's
|
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||||
// request, and a query goes on after the request ends.
|
// context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -153,23 +158,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
|
_, asking := d.asking[q]
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||||
d.asking[q] = true
|
len(d.asking) < maxQueries:
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
d.asking[q] = end
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
go d.ask(context.WithoutCancel(ctx), q)
|
d.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
d.ask(queryCtx, q)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their
|
||||||
|
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||||
|
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||||
|
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||||
|
func (d *DNSBL) StopAsking() {
|
||||||
|
d.mu.Lock()
|
||||||
|
|
||||||
|
d.stopped = true
|
||||||
|
for _, end := range d.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
d.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -238,27 +267,29 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay.
|
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := config.MaskZoneKey(q.zone)
|
||||||
|
|||||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := dnsblParams(zone)
|
||||||
|
p.Alerts = queue
|
||||||
|
resolver := &resolverStandIn{hanging: true}
|
||||||
|
dnsbl := newDNSBL(resolver, p)
|
||||||
|
|
||||||
|
wantZones(t, dnsbl, listed)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
dnsbl.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client without a verdict is then asked about in no zone.
|
||||||
|
wantZones(t, dnsbl, unlisted)
|
||||||
|
synctest.Wait()
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
wantAsked(t, resolver, listedName)
|
||||||
|
waitForTheResolver()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the lines sww prints on stdout: one
|
// Package requestlog writes the request log sww prints on stdout: JSON
|
||||||
// JSON object per request, two from waf and a short one from rproxy,
|
// lines marked "type":"request", at most two from waf and a short one
|
||||||
// marked "type":"request", and the process's own messages as JSON lines marked
|
// from rproxy for each request. Package processlog writes the process's
|
||||||
// "type":"process".
|
// own messages.
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewProcessLogger returns the logger for the process's own messages:
|
|
||||||
// JSON lines on w, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
|
||||||
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
|
||||||
// the request lines Write writes are never held back.
|
|
||||||
func NewProcessLogger(
|
|
||||||
w io.Writer,
|
|
||||||
instanceName string,
|
|
||||||
level slog.Level,
|
|
||||||
) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
|
||||||
Level: level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", instanceName)
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := requestlog.NewProcessLogger(
|
|
||||||
&out,
|
|
||||||
"host1/blog",
|
|
||||||
level,
|
|
||||||
)
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -99,8 +99,12 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way.
|
// asking are the clients whose query is under way, each with the
|
||||||
asking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopAsking has been
|
||||||
|
// called, and queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]bool{},
|
asking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||||
// client's request, and a query goes on after the request ends.
|
// is the context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
_, asking := c.asking[client]
|
||||||
c.asking[client] = true
|
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
c.asking[client] = end
|
||||||
|
|
||||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
c.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
c.ask(queryCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their goroutines
|
||||||
|
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||||
|
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||||
|
// routewatch nothing.
|
||||||
|
func (c *Client) StopAsking() {
|
||||||
|
c.mu.Lock()
|
||||||
|
|
||||||
|
c.stopped = true
|
||||||
|
for _, end := range c.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
c.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged.
|
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||||
|
// failure.
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||||
|
// says.
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
f := &fixture{t: t, log: &syncBuffer{}}
|
||||||
|
f.client = routewatch.New(routewatch.Params{
|
||||||
|
URL: silentURL, Now: time.Now,
|
||||||
|
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||||
|
})
|
||||||
|
|
||||||
|
f.route()
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
f.client.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A query started now would fail within waitLimit. None is, and the
|
||||||
|
// one ended is no failure: nothing is logged, and routewatch is not
|
||||||
|
// left alone.
|
||||||
|
for range 2 {
|
||||||
|
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||||
|
t.Errorf(
|
||||||
|
"Route gave %q, want %q",
|
||||||
|
missing,
|
||||||
|
routewatch.NoAnswerYet,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(waitLimit)
|
||||||
|
synctest.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.log.String() != "" {
|
||||||
|
t.Errorf("process log %q, want nothing", f.log.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -324,7 +324,8 @@ func greet(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
||||||
// the settings in env on top of the defaults. It returns where rproxy listens
|
// the settings in env on top of the defaults. It returns where rproxy listens
|
||||||
// and what it writes.
|
// and what it writes. SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS is a minute unless env
|
||||||
|
// sets it, so that a busy host cannot make rproxy drop a question or a report.
|
||||||
func startRProxy(
|
func startRProxy(
|
||||||
t *testing.T, backend, appURL string, env map[string]string,
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
) (string, *output) {
|
) (string, *output) {
|
||||||
@@ -357,6 +358,7 @@ func startServerLogging(
|
|||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
|
wafBackendTimeout: "60000",
|
||||||
}
|
}
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
|
|
||||||
@@ -368,11 +370,11 @@ func startServerLogging(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
processLog,
|
Output: processLog,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
|
|||||||
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
package sww
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/fx"
|
||||||
|
"go.uber.org/fx/fxevent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// shutdownTimeout is how long requests in progress may take to finish
|
||||||
|
// once sww is told to stop, before their connections are closed.
|
||||||
|
// runit and docker wait a little longer before they kill the process.
|
||||||
|
const shutdownTimeout = 5 * time.Second
|
||||||
|
|
||||||
|
// requestLog is where a daemon's server writes the request log: stdout, and
|
||||||
|
// the remote log sender while there is one. fx tells parts apart by type, so
|
||||||
|
// it has a type of its own.
|
||||||
|
type requestLog struct {
|
||||||
|
io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
// servingFailed gets why serving failed, for run, which then stops the
|
||||||
|
// daemon. fx tells parts apart by type, so it has a type of its own.
|
||||||
|
type servingFailed chan error
|
||||||
|
|
||||||
|
// utcNow is the time the state files, the alerts and the rate limits go by.
|
||||||
|
func utcNow() time.Time {
|
||||||
|
return time.Now().UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
// startError is why a daemon cannot start: the message its process log gives,
|
||||||
|
// and the error it logs with it.
|
||||||
|
type startError struct {
|
||||||
|
message string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e startError) Error() string {
|
||||||
|
return e.message + ": " + e.err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e startError) Unwrap() error {
|
||||||
|
return e.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// cannotStart logs why a daemon cannot start, with the message of the
|
||||||
|
// startError in err, and returns its exit status.
|
||||||
|
func cannotStart(processLog *slog.Logger, err error) int {
|
||||||
|
message, cause := "cannot start", err
|
||||||
|
|
||||||
|
failed, ok := errors.AsType[startError](err)
|
||||||
|
if ok {
|
||||||
|
message, cause = failed.message, failed.err
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Error(message, "error", cause.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// listen listens on address, which the setting names.
|
||||||
|
func listen(address, setting string) (net.Listener, error) {
|
||||||
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
|
context.Background(), "tcp", address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, startError{"cannot listen on " + setting, err}
|
||||||
|
}
|
||||||
|
|
||||||
|
return listener, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// background returns the hook that runs loop on a goroutine of its own from
|
||||||
|
// the start, and at the stop ends it and waits for it to return.
|
||||||
|
func background(loop func(context.Context)) fx.Hook {
|
||||||
|
running, stop := context.WithCancel(context.Background())
|
||||||
|
ended := make(chan struct{})
|
||||||
|
|
||||||
|
return fx.StartStopHook(
|
||||||
|
func() {
|
||||||
|
go func() {
|
||||||
|
loop(running)
|
||||||
|
close(ended)
|
||||||
|
}()
|
||||||
|
},
|
||||||
|
func() {
|
||||||
|
stop()
|
||||||
|
<-ended
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve serves requests on listener from the start. At the stop it gives the
|
||||||
|
// requests in progress shutdownTimeout to finish, then closes their
|
||||||
|
// connections. When serving fails, it logs why and sends the error to
|
||||||
|
// failed.
|
||||||
|
func serve(
|
||||||
|
lc fx.Lifecycle, server *http.Server, listener net.Listener,
|
||||||
|
processLog *slog.Logger, failed servingFailed,
|
||||||
|
) {
|
||||||
|
served := make(chan error, 1)
|
||||||
|
|
||||||
|
lc.Append(fx.Hook{
|
||||||
|
OnStart: func(context.Context) error {
|
||||||
|
go func() {
|
||||||
|
err := server.Serve(listener)
|
||||||
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
processLog.Error("serving failed", "error", err.Error())
|
||||||
|
|
||||||
|
failed <- err
|
||||||
|
}
|
||||||
|
|
||||||
|
served <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
OnStop: func(ctx context.Context) error {
|
||||||
|
processLog.Info("stopping")
|
||||||
|
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(ctx, shutdownTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := server.Shutdown(shutdownCtx)
|
||||||
|
if err != nil {
|
||||||
|
processLog.Warn("requests still in progress were cut off",
|
||||||
|
"error", err.Error())
|
||||||
|
|
||||||
|
_ = server.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
err = <-served
|
||||||
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// run starts app, which serves until ctx is done or serving fails, as
|
||||||
|
// failed tells. Then it stops app and takes the daemon's last step. It
|
||||||
|
// returns the process's exit status, 1 when serving failed, a part could
|
||||||
|
// not stop in order or the last step failed; each logs why.
|
||||||
|
func run(
|
||||||
|
ctx context.Context, app *fx.App, failed servingFailed,
|
||||||
|
processLog *slog.Logger, lastStep func() error,
|
||||||
|
) int {
|
||||||
|
// The hooks only start goroutines, and end them at the stop, so neither
|
||||||
|
// is cut short by a deadline or by ctx.
|
||||||
|
err := app.Start(context.WithoutCancel(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := 0
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-failed:
|
||||||
|
status = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
err = app.Stop(context.WithoutCancel(ctx))
|
||||||
|
if err != nil {
|
||||||
|
status = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
err = lastStep()
|
||||||
|
if err != nil {
|
||||||
|
status = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if status == 0 {
|
||||||
|
processLog.Info("stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
|
// fxEventLogger returns what fx.WithLogger takes to write fx's own events to
|
||||||
|
// processLog at debug, which the daemons' log levels normally hold back. Its
|
||||||
|
// errors are at debug too: the daemon logs each one itself.
|
||||||
|
func fxEventLogger(processLog *slog.Logger) func() fxevent.Logger {
|
||||||
|
return func() fxevent.Logger {
|
||||||
|
logger := &fxevent.SlogLogger{Logger: processLog}
|
||||||
|
logger.UseLogLevel(slog.LevelDebug)
|
||||||
|
logger.UseErrorLevel(slog.LevelDebug)
|
||||||
|
|
||||||
|
return logger
|
||||||
|
}
|
||||||
|
}
|
||||||
+48
-69
@@ -2,14 +2,15 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,34 +23,39 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
params.Stdout,
|
Output: params.Stdout,
|
||||||
hostname,
|
InstanceName: hostname,
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
)
|
})
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
processLog.Error("invalid setting", "error", err.Error())
|
return cannotStart(processLog, startError{"invalid setting", err})
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
var (
|
||||||
Config: cfg,
|
server *rproxy.Server
|
||||||
RequestLog: params.Stdout,
|
listener net.Listener
|
||||||
ProcessLog: processLog,
|
)
|
||||||
})
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
|
failed := make(servingFailed, 1)
|
||||||
|
|
||||||
|
app := fx.New(
|
||||||
|
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
|
||||||
|
fx.Provide(
|
||||||
|
newRProxyServer,
|
||||||
|
func(server *rproxy.Server) *http.Server { return server.Server },
|
||||||
|
newRProxyListener,
|
||||||
|
),
|
||||||
|
fx.Invoke(serve),
|
||||||
|
fx.Populate(&server, &listener),
|
||||||
|
fx.WithLogger(fxEventLogger(processLog)),
|
||||||
|
)
|
||||||
|
|
||||||
|
err = app.Err()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
processLog.Error(
|
return cannotStart(processLog, err)
|
||||||
"cannot listen on SWW_RPROXY_LISTEN_ADDR",
|
|
||||||
"error",
|
|
||||||
err.Error(),
|
|
||||||
)
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog.Info("starting",
|
processLog.Info("starting",
|
||||||
@@ -57,52 +63,25 @@ func RunRProxy(ctx context.Context, params Params) int {
|
|||||||
"address", listener.Addr().String(),
|
"address", listener.Addr().String(),
|
||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
return serveRProxy(ctx, server, listener, processLog)
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
|
server.WaitForReports()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// serveRProxy serves requests on listener until ctx is done, and then stops as
|
// newRProxyServer returns the server rproxy runs.
|
||||||
// RunRProxy says.
|
func newRProxyServer(
|
||||||
func serveRProxy(
|
cfg *config.RProxy, out requestLog, processLog *slog.Logger,
|
||||||
ctx context.Context, server *rproxy.Server, listener net.Listener,
|
) *rproxy.Server {
|
||||||
processLog *slog.Logger,
|
return rproxy.New(rproxy.Params{
|
||||||
) int {
|
Config: cfg,
|
||||||
served := make(chan error, 1)
|
RequestLog: out,
|
||||||
|
ProcessLog: processLog,
|
||||||
go func() {
|
})
|
||||||
served <- server.Serve(listener)
|
}
|
||||||
}()
|
|
||||||
|
// newRProxyListener listens on SWW_RPROXY_LISTEN_ADDR.
|
||||||
select {
|
func newRProxyListener(cfg *config.RProxy) (net.Listener, error) {
|
||||||
case err := <-served:
|
return listen(cfg.ListenAddr, "SWW_RPROXY_LISTEN_ADDR")
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("stopping")
|
|
||||||
|
|
||||||
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
|
|
||||||
shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := server.Shutdown(shutdownCtx)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Warn("requests still in progress were cut off",
|
|
||||||
"error", err.Error())
|
|
||||||
|
|
||||||
_ = server.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
err = <-served
|
|
||||||
if !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
server.WaitForReports()
|
|
||||||
processLog.Info("stopped")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
}
|
||||||
@@ -552,7 +552,9 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
|
|||||||
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
||||||
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
||||||
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
||||||
// status 0 within waitLimit.
|
// status 0 within waitLimit. It waits a minute for waf's answers unless env
|
||||||
|
// sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, so that a busy host cannot make it
|
||||||
|
// drop a question.
|
||||||
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -568,7 +570,8 @@ func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
|||||||
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
commandForMain+"=rproxy",
|
commandForMain+"=rproxy",
|
||||||
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0")
|
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0",
|
||||||
|
wafBackendTimeout+"=60000")
|
||||||
cmd.Env = append(cmd.Env, env...)
|
cmd.Env = append(cmd.Env, env...)
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
|
|||||||
+2
-403
@@ -4,17 +4,14 @@
|
|||||||
// stop, and then stops in an orderly way, writing the state and cache files.
|
// stop, and then stops in an orderly way, writing the state and cache files.
|
||||||
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
|
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
|
||||||
// front of one app that asks waf about each request. `sww healthcheck` is the
|
// front of one app that asks waf about each request. `sww healthcheck` is the
|
||||||
// container's health check.
|
// container's health check. fx builds the parts of each daemon, starts their
|
||||||
|
// background work and stops it.
|
||||||
package sww
|
package sww
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -22,29 +19,9 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// shutdownTimeout is how long requests in progress may take to finish
|
|
||||||
// once sww is told to stop, before their connections are closed.
|
|
||||||
// runit and docker wait a little longer before they kill the process.
|
|
||||||
const shutdownTimeout = 5 * time.Second
|
|
||||||
|
|
||||||
// remoteLogStopTimeout is how long, as waf stops, the log lines still
|
|
||||||
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
|
|
||||||
// carried them.
|
|
||||||
const remoteLogStopTimeout = 2 * time.Second
|
|
||||||
|
|
||||||
// usageStatus is the exit status with no command, or one sww does not
|
// usageStatus is the exit status with no command, or one sww does not
|
||||||
// know.
|
// know.
|
||||||
const usageStatus = 2
|
const usageStatus = 2
|
||||||
@@ -110,381 +87,3 @@ func printUsage(w io.Writer) int {
|
|||||||
|
|
||||||
return usageStatus
|
return usageStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunWAF is `sww waf`: it reads the settings, the rule files,
|
|
||||||
// the lookup database and the state and cache files, then serves requests until
|
|
||||||
// ctx is done. It returns the process's exit status, 1 when waf cannot start.
|
|
||||||
func RunWAF(ctx context.Context, params Params) int {
|
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
|
||||||
config.InstanceName(params.Viper), slog.LevelError)
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, "invalid setting", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
|
|
||||||
// is sent there too.
|
|
||||||
stdout := params.Stdout
|
|
||||||
|
|
||||||
var remote *remotelog.Sender
|
|
||||||
|
|
||||||
if cfg.LogRemoteURL != nil {
|
|
||||||
remote = newRemoteLogSender(cfg)
|
|
||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog = requestlog.NewProcessLogger(
|
|
||||||
stdout,
|
|
||||||
cfg.InstanceName,
|
|
||||||
cfg.LogLevel,
|
|
||||||
)
|
|
||||||
|
|
||||||
if remote != nil {
|
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
|
||||||
defer stopSending()
|
|
||||||
}
|
|
||||||
|
|
||||||
return runWAFWithSettings(
|
|
||||||
ctx,
|
|
||||||
params,
|
|
||||||
cfg,
|
|
||||||
stdout,
|
|
||||||
remote,
|
|
||||||
processLog,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runWAFWithSettings is the rest of RunWAF once the settings are read and
|
|
||||||
// the process log goes where they say. remote is nil unless
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL is set.
|
|
||||||
func runWAFWithSettings(
|
|
||||||
ctx context.Context,
|
|
||||||
params Params,
|
|
||||||
cfg *config.WAF,
|
|
||||||
stdout io.Writer,
|
|
||||||
remote *remotelog.Sender,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) int {
|
|
||||||
// The state files and the alerts give times in UTC.
|
|
||||||
now := func() time.Time { return time.Now().UTC() }
|
|
||||||
|
|
||||||
alertQueue := newAlertQueue(cfg, now, processLog)
|
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
|
||||||
Dir: cfg.RuleFilesDir,
|
|
||||||
Enabled: cfg.RuleFilesEnabled,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, "cannot use the rule files", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
crs, err := waf.NewCoreRuleSet(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, "cannot use the Core Rule Set", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
server, err := newServer(
|
|
||||||
cfg,
|
|
||||||
stdout,
|
|
||||||
processLog,
|
|
||||||
now,
|
|
||||||
ruleFiles,
|
|
||||||
alertQueue,
|
|
||||||
crs,
|
|
||||||
params.BesideLookupTimeLimit,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, "cannot use the lookup database", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if remote != nil {
|
|
||||||
server.Metrics.AddRemoteLog(remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(
|
|
||||||
processLog,
|
|
||||||
"cannot use the state and cache files",
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(
|
|
||||||
processLog,
|
|
||||||
"cannot listen on SWW_WAF_LISTEN_ADDR",
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("starting",
|
|
||||||
"version", params.Version,
|
|
||||||
"address", listener.Addr().String(),
|
|
||||||
"settings", cfg)
|
|
||||||
|
|
||||||
return serve(
|
|
||||||
ctx,
|
|
||||||
server,
|
|
||||||
listener,
|
|
||||||
files,
|
|
||||||
ruleFiles,
|
|
||||||
alertQueue,
|
|
||||||
processLog,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// cannotStart logs why waf cannot start, and returns its exit status.
|
|
||||||
func cannotStart(processLog *slog.Logger, message string, err error) int {
|
|
||||||
processLog.Error(message, "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServer returns the server waf runs, with the metrics of the alerts,
|
|
||||||
// after reading the lookup database while SWW_WAF_LOOKUP_SOURCE is file. A lookup
|
|
||||||
// database that cannot be read is an error.
|
|
||||||
func newServer(
|
|
||||||
cfg *config.WAF, stdout io.Writer, processLog *slog.Logger,
|
|
||||||
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
|
||||||
crs *coreruleset.CoreRuleSet, besideLookupTimeLimit time.Duration,
|
|
||||||
) (*waf.Server, error) {
|
|
||||||
var lookupFile *lookup.File
|
|
||||||
|
|
||||||
if cfg.LookupSource == "file" {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
lookupFile, err = lookup.OpenFile(lookup.FileParams{
|
|
||||||
Path: cfg.LookupDBPath,
|
|
||||||
Now: now,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
server := waf.New(waf.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: stdout,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
GeoJSURL: geojs.URL,
|
|
||||||
AbuseIPDBURL: reputation.AbuseIPDBURL,
|
|
||||||
LookupFile: lookupFile,
|
|
||||||
CoreRuleSet: crs,
|
|
||||||
Now: now,
|
|
||||||
Rules: ruleFiles,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
|
|
||||||
BesideLookupTimeLimit: besideLookupTimeLimit,
|
|
||||||
AbuseIPDBBlacklistURL: reputation.AbuseIPDBBlacklistURL,
|
|
||||||
})
|
|
||||||
server.Metrics.AddAlerts(alertQueue)
|
|
||||||
|
|
||||||
if lookupFile != nil {
|
|
||||||
server.Metrics.AddLookupFile(
|
|
||||||
lookupFile.LastRead,
|
|
||||||
lookupFile.ReadFailures,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return server, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadStateFiles reads the state and cache files into the parts of server and
|
|
||||||
// into alertQueue, as state.Load does.
|
|
||||||
func loadStateFiles(
|
|
||||||
cfg *config.WAF, server *waf.Server, alertQueue *alerts.Queue,
|
|
||||||
now func() time.Time, processLog *slog.Logger,
|
|
||||||
) (*state.Files, error) {
|
|
||||||
return state.Load(state.Params{
|
|
||||||
Dir: cfg.StateDir,
|
|
||||||
CacheDir: cfg.CacheDir,
|
|
||||||
BansWriteDelay: cfg.BansWriteDelay,
|
|
||||||
WriteInterval: cfg.StateWriteInterval,
|
|
||||||
Ledger: server.Ledger,
|
|
||||||
Limiter: server.Limiter,
|
|
||||||
GeoJS: server.GeoJS,
|
|
||||||
Lists: server.Lists,
|
|
||||||
DNSBL: server.DNSBL,
|
|
||||||
AbuseIPDB: server.AbuseIPDB,
|
|
||||||
RouteWatch: server.RouteWatch,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
Anomalies: server.Anomalies,
|
|
||||||
Now: now,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Metrics: server.Metrics,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
|
|
||||||
// ntfy, with the settings for them.
|
|
||||||
func newAlertQueue(
|
|
||||||
cfg *config.WAF, now func() time.Time, processLog *slog.Logger,
|
|
||||||
) *alerts.Queue {
|
|
||||||
return alerts.New(alerts.Params{
|
|
||||||
WebhookURL: cfg.AlertWebhookURL,
|
|
||||||
WebhookHeaders: cfg.AlertWebhookHeaders,
|
|
||||||
SlackURL: cfg.AlertSlackWebhookURL,
|
|
||||||
NtfyURL: cfg.AlertNtfyURL,
|
|
||||||
NtfyToken: cfg.AlertNtfyToken,
|
|
||||||
Events: cfg.AlertEvents,
|
|
||||||
Cooldown: cfg.AlertCooldown,
|
|
||||||
MaxPerHour: cfg.AlertMaxPerHour,
|
|
||||||
Instance: cfg.InstanceName,
|
|
||||||
Now: now,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRemoteLogSender returns a sender of the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
|
|
||||||
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
|
|
||||||
return remotelog.New(remotelog.Params{
|
|
||||||
URL: cfg.LogRemoteURL,
|
|
||||||
RootCAs: cfg.LogRemoteTLSCAs,
|
|
||||||
BufferLines: cfg.LogRemoteBufferLines,
|
|
||||||
Facility: cfg.LogRemoteFacility,
|
|
||||||
AppName: cfg.LogRemoteAppName,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// startSending runs remote until the function it returns is called, which
|
|
||||||
// then waits at most remoteLogStopTimeout for the lines still waiting to
|
|
||||||
// be sent. Sending goes on after ctx is done, so that the lines written
|
|
||||||
// while sww stops are sent too.
|
|
||||||
func startSending(
|
|
||||||
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
|
|
||||||
) func() {
|
|
||||||
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
sent := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
remote.Run(sending, processLog)
|
|
||||||
close(sent)
|
|
||||||
}()
|
|
||||||
|
|
||||||
return func() {
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-sent:
|
|
||||||
case <-time.After(remoteLogStopTimeout):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// serve serves requests on listener, writes the state and cache files as they
|
|
||||||
// are due, takes in an admin's edits of them, reads the rule files again as
|
|
||||||
// they change, and the lookup database when it is replaced, fetches the
|
|
||||||
// lists the settings name by URL as they are due, bans the clients on
|
|
||||||
// AbuseIPDB's blacklist as it is fetched, and sends the alerts,
|
|
||||||
// until ctx is done. Then it gives the requests in progress
|
|
||||||
// shutdownTimeout to finish, and writes every state and cache file,
|
|
||||||
// alerts.json with the alerts still waiting.
|
|
||||||
func serve(
|
|
||||||
ctx context.Context, server *waf.Server, listener net.Listener,
|
|
||||||
files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) int {
|
|
||||||
served := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
served <- server.Serve(listener)
|
|
||||||
}()
|
|
||||||
|
|
||||||
writing, stopWriting := context.WithCancel(ctx)
|
|
||||||
defer stopWriting()
|
|
||||||
|
|
||||||
written := inBackground(func() { files.Run(writing) })
|
|
||||||
watched := inBackground(func() { files.Watch(writing) })
|
|
||||||
rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
|
|
||||||
lookupFileWatched := inBackground(func() {
|
|
||||||
if server.LookupFile != nil {
|
|
||||||
server.LookupFile.Watch(writing)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
listsFetched := inBackground(func() { server.Lists.Run(writing) })
|
|
||||||
blacklistFetched := inBackground(func() {
|
|
||||||
server.AbuseIPDB.RunBlacklist(writing)
|
|
||||||
})
|
|
||||||
alertsSent := inBackground(func() { alertQueue.Run(writing) })
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-served:
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("stopping")
|
|
||||||
|
|
||||||
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
|
|
||||||
shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := server.Shutdown(shutdownCtx)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Warn("requests still in progress were cut off",
|
|
||||||
"error", err.Error())
|
|
||||||
|
|
||||||
_ = server.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
err = <-served
|
|
||||||
if !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run and Watch have ended, so nothing else reads or writes the
|
|
||||||
// files, and no alert is being sent, so that alerts.json keeps every
|
|
||||||
// alert not yet sent. Every request has ended too, but for one cut off
|
|
||||||
// because Shutdown timed out, which adds to its client's history only as
|
|
||||||
// it ends. That can be after this write, and then that request is missing
|
|
||||||
// from clients.json.
|
|
||||||
<-written
|
|
||||||
<-watched
|
|
||||||
<-rulesWatched
|
|
||||||
<-lookupFileWatched
|
|
||||||
<-listsFetched
|
|
||||||
<-blacklistFetched
|
|
||||||
<-alertsSent
|
|
||||||
|
|
||||||
err = files.WriteAll()
|
|
||||||
if err != nil {
|
|
||||||
processLog.Error(
|
|
||||||
"writing the state and cache files failed",
|
|
||||||
"error",
|
|
||||||
err.Error(),
|
|
||||||
)
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("stopped")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// inBackground runs task on a goroutine of its own, and returns a channel
|
|
||||||
// that is closed once task has returned.
|
|
||||||
func inBackground(task func()) <-chan struct{} {
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
task()
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
return done
|
|
||||||
}
|
|
||||||
@@ -660,7 +660,9 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
|||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
|
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
||||||
|
config.CoreRuleSetModeOff,
|
||||||
|
),
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
|
|||||||
@@ -0,0 +1,560 @@
|
|||||||
|
package sww
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/fx"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
)
|
||||||
|
|
||||||
|
// remoteLogStopTimeout is how long, as waf stops, the log lines still
|
||||||
|
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
|
||||||
|
// carried them.
|
||||||
|
const remoteLogStopTimeout = 2 * time.Second
|
||||||
|
|
||||||
|
// RunWAF is `sww waf`: it reads the settings, the rule files,
|
||||||
|
// the lookup database and the state and cache files, then serves requests until
|
||||||
|
// ctx is done. Then it gives the requests in progress shutdownTimeout to
|
||||||
|
// finish, ends its background work and writes every state and cache file,
|
||||||
|
// alerts.json with the alerts still waiting. It returns the process's exit
|
||||||
|
// status, 1 when waf cannot start or stop in order.
|
||||||
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
|
processLog := processlog.New(processlog.Params{
|
||||||
|
Output: params.Stdout,
|
||||||
|
InstanceName: config.InstanceName(params.Viper),
|
||||||
|
Level: slog.LevelError,
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, startError{"invalid setting", err})
|
||||||
|
}
|
||||||
|
|
||||||
|
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
|
||||||
|
// is sent there too. The sender runs before any part is built, so that it
|
||||||
|
// sends why waf cannot start.
|
||||||
|
stdout := params.Stdout
|
||||||
|
|
||||||
|
var remote *remotelog.Sender
|
||||||
|
|
||||||
|
if cfg.LogRemoteURL != nil {
|
||||||
|
remote = newRemoteLogSender(cfg)
|
||||||
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog = processlog.New(processlog.Params{
|
||||||
|
Output: stdout,
|
||||||
|
InstanceName: cfg.InstanceName,
|
||||||
|
Level: cfg.LogLevel,
|
||||||
|
})
|
||||||
|
|
||||||
|
if remote != nil {
|
||||||
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
|
defer stopSending()
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
files *state.Files
|
||||||
|
listener net.Listener
|
||||||
|
)
|
||||||
|
|
||||||
|
failed := make(servingFailed, 1)
|
||||||
|
|
||||||
|
app := fx.New(
|
||||||
|
fx.Supply(
|
||||||
|
params,
|
||||||
|
cfg,
|
||||||
|
processLog,
|
||||||
|
requestLog{stdout},
|
||||||
|
remote,
|
||||||
|
failed,
|
||||||
|
),
|
||||||
|
fx.Provide(
|
||||||
|
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
|
||||||
|
newLimiter, newLedger, newAnomalyCounters,
|
||||||
|
newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
|
||||||
|
newMetrics, newGeoJS, newWAFServer,
|
||||||
|
func(server *waf.Server) *http.Server { return server.Server },
|
||||||
|
newWAFListener, newStateFiles,
|
||||||
|
),
|
||||||
|
// fx stops hooks in the reverse of the order they were added in. The
|
||||||
|
// state files are read before serving is set up, so that serving
|
||||||
|
// stops before their loops do.
|
||||||
|
fx.Populate(&files),
|
||||||
|
fx.Invoke(serve),
|
||||||
|
fx.Populate(&listener),
|
||||||
|
fx.WithLogger(fxEventLogger(processLog)),
|
||||||
|
)
|
||||||
|
|
||||||
|
err = app.Err()
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Info("starting",
|
||||||
|
"version", params.Version,
|
||||||
|
"address", listener.Addr().String(),
|
||||||
|
"settings", cfg)
|
||||||
|
|
||||||
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
|
// loop that changes the state or raises an alert, and of every ask of
|
||||||
|
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||||
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
|
err := files.WriteAll()
|
||||||
|
if err != nil {
|
||||||
|
processLog.Error("writing the state and cache files failed",
|
||||||
|
"error", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
|
||||||
|
// ntfy, with the settings for them. Until waf stops, it sends them.
|
||||||
|
func newAlertQueue(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
) *alerts.Queue {
|
||||||
|
alertQueue := alerts.New(alerts.Params{
|
||||||
|
WebhookURL: cfg.AlertWebhookURL,
|
||||||
|
WebhookHeaders: cfg.AlertWebhookHeaders,
|
||||||
|
SlackURL: cfg.AlertSlackWebhookURL,
|
||||||
|
NtfyURL: cfg.AlertNtfyURL,
|
||||||
|
NtfyToken: cfg.AlertNtfyToken,
|
||||||
|
Events: cfg.AlertEvents,
|
||||||
|
Cooldown: cfg.AlertCooldown,
|
||||||
|
MaxPerHour: cfg.AlertMaxPerHour,
|
||||||
|
Instance: cfg.InstanceName,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(background(alertQueue.Run))
|
||||||
|
|
||||||
|
return alertQueue
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRuleFiles reads the rule files, and reads them again as they change
|
||||||
|
// until waf stops.
|
||||||
|
func newRuleFiles(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
) (*rules.Files, error) {
|
||||||
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
|
Dir: cfg.RuleFilesDir,
|
||||||
|
Enabled: cfg.RuleFilesEnabled,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, startError{"cannot use the rule files", err}
|
||||||
|
}
|
||||||
|
|
||||||
|
lc.Append(background(ruleFiles.Watch))
|
||||||
|
|
||||||
|
return ruleFiles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||||
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||||
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||||
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||||
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||||
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||||
|
// set file is loaded with the Core Rule Set only to check it.
|
||||||
|
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||||
|
cfg.CoreRuleSetExclusionsFile == "" {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
crs, err := coreruleset.New(coreruleset.Params{
|
||||||
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||||
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||||
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||||
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||||
|
Apps: cfg.Apps,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, startError{
|
||||||
|
"cannot use the Core Rule Set",
|
||||||
|
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
return crs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newLookupFile reads the lookup database while SWW_WAF_LOOKUP_SOURCE is
|
||||||
|
// file, and reads it again when it is replaced until waf stops. It returns
|
||||||
|
// nil while the source is anything else.
|
||||||
|
func newLookupFile(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
) (*lookup.File, error) {
|
||||||
|
if cfg.LookupSource != config.LookupSourceFile {
|
||||||
|
return nil, nil //nolint:nilnil // no lookup database, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
lookupFile, err := lookup.OpenFile(lookup.FileParams{
|
||||||
|
Path: cfg.LookupDBPath,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, startError{"cannot use the lookup database", err}
|
||||||
|
}
|
||||||
|
|
||||||
|
lc.Append(background(lookupFile.Watch))
|
||||||
|
|
||||||
|
return lookupFile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
|
return ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{
|
||||||
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
|
PerHour: cfg.RateLimitPerHour,
|
||||||
|
PerDay: cfg.RateLimitPerDay,
|
||||||
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
|
func newLedger(cfg *config.WAF) *bans.Ledger {
|
||||||
|
return bans.New(bans.Params{
|
||||||
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
|
||||||
|
AttackBanDuration: cfg.AttackBanDuration,
|
||||||
|
MaxBans: cfg.MaxBans,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newAnomalyCounters returns the counters of the anomaly thresholds.
|
||||||
|
func newAnomalyCounters(
|
||||||
|
cfg *config.WAF, alertQueue *alerts.Queue,
|
||||||
|
) *anomaly.Counters {
|
||||||
|
return anomaly.New(anomaly.Params{
|
||||||
|
Client: cfg.AnomalyClient,
|
||||||
|
Net: cfg.AnomalyNet,
|
||||||
|
ASN: cfg.AnomalyASN,
|
||||||
|
Total: cfg.AnomalyTotal,
|
||||||
|
WatchedNet: cfg.AnomalyWatchedNet,
|
||||||
|
NetV4Prefix: cfg.AnomalyNetV4Prefix,
|
||||||
|
NetV6Prefix: cfg.AnomalyNetV6Prefix,
|
||||||
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newLists returns the lists the settings name by URL, none fetched yet.
|
||||||
|
// Until waf stops, it fetches them as they are due.
|
||||||
|
func newLists(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
) *reputation.Lists {
|
||||||
|
lists := reputation.New(reputation.Params{
|
||||||
|
BlocklistURLs: cfg.BlocklistURLs,
|
||||||
|
Refresh: cfg.BlocklistRefresh,
|
||||||
|
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
||||||
|
TagLists: cfg.TagLists,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(background(lists.Run))
|
||||||
|
|
||||||
|
return lists
|
||||||
|
}
|
||||||
|
|
||||||
|
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||||
|
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||||
|
func newDNSBL(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
) *reputation.DNSBL {
|
||||||
|
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
|
Zones: cfg.DNSBLZones,
|
||||||
|
Resolver: cfg.DNSBLResolver,
|
||||||
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||||
|
|
||||||
|
return dnsbl
|
||||||
|
}
|
||||||
|
|
||||||
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||||
|
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||||
|
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||||
|
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||||
|
func newAbuseIPDB(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||||
|
) *reputation.AbuseIPDB {
|
||||||
|
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||||
|
URL: reputation.AbuseIPDBURL,
|
||||||
|
Key: cfg.AbuseIPDBKey,
|
||||||
|
MinScore: cfg.AbuseIPDBMinScore,
|
||||||
|
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||||
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
|
BlacklistURL: reputation.AbuseIPDBBlacklistURL,
|
||||||
|
Threshold: cfg.AbuseIPDBThreshold,
|
||||||
|
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
||||||
|
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
||||||
|
waf.BanAbuseIPDBBlacklist(
|
||||||
|
cfg, ledger, processLog, utcNow(), listed,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||||
|
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||||
|
|
||||||
|
return abuseIPDB
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
|
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||||
|
// way, as routewatch.Client.StopAsking says.
|
||||||
|
func newRouteWatch(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
) *routewatch.Client {
|
||||||
|
if cfg.RouteWatchURL == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
routeWatch := routewatch.New(routewatch.Params{
|
||||||
|
URL: cfg.RouteWatchURL,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||||
|
|
||||||
|
return routeWatch
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
|
// the metrics are asked for: AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set,
|
||||||
|
// the lookup database's while there is one, and the remote log sender's
|
||||||
|
// while there is one.
|
||||||
|
func newMetrics(
|
||||||
|
cfg *config.WAF, ledger *bans.Ledger, limiter *ratelimit.Limiter,
|
||||||
|
ruleFiles *rules.Files, lists *reputation.Lists,
|
||||||
|
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
|
||||||
|
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||||
|
remote *remotelog.Sender,
|
||||||
|
) *metrics.Metrics {
|
||||||
|
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
||||||
|
m.AddBansAndClients(ledger, limiter, utcNow)
|
||||||
|
m.AddRules(ruleFiles)
|
||||||
|
m.AddReputation(lists, dnsbl)
|
||||||
|
|
||||||
|
if cfg.AbuseIPDBKey != "" {
|
||||||
|
m.AddAbuseIPDB(abuseIPDB)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.AddAlerts(alertQueue)
|
||||||
|
|
||||||
|
if lookupFile != nil {
|
||||||
|
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
||||||
|
}
|
||||||
|
|
||||||
|
if remote != nil {
|
||||||
|
m.AddRemoteLog(remote)
|
||||||
|
}
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
|
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||||
|
// under way, as geojs.GeoJS.StopAsking says.
|
||||||
|
func newGeoJS(
|
||||||
|
lc fx.Lifecycle,
|
||||||
|
cfg *config.WAF,
|
||||||
|
m *metrics.Metrics,
|
||||||
|
limiter *ratelimit.Limiter,
|
||||||
|
ledger *bans.Ledger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
processLog *slog.Logger,
|
||||||
|
) *geojs.GeoJS {
|
||||||
|
geoJS := geojs.New(geojs.Params{
|
||||||
|
URL: geojs.URL,
|
||||||
|
Answered: func(answer geojs.Answer) {
|
||||||
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
|
},
|
||||||
|
Now: utcNow,
|
||||||
|
Asked: m.GeoJSRequests.Inc,
|
||||||
|
Failed: func(err error, askingAgainIn time.Duration) {
|
||||||
|
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
|
||||||
|
},
|
||||||
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||||
|
|
||||||
|
return geoJS
|
||||||
|
}
|
||||||
|
|
||||||
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
// limit params gives.
|
||||||
|
func newWAFServer(
|
||||||
|
params Params, cfg *config.WAF, out requestLog, processLog *slog.Logger,
|
||||||
|
m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger,
|
||||||
|
anomalies *anomaly.Counters, lists *reputation.Lists,
|
||||||
|
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
|
||||||
|
routeWatch *routewatch.Client, geoJS *geojs.GeoJS,
|
||||||
|
lookupFile *lookup.File, crs *coreruleset.CoreRuleSet,
|
||||||
|
ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
||||||
|
) *waf.Server {
|
||||||
|
return waf.New(waf.Params{
|
||||||
|
Config: cfg,
|
||||||
|
RequestLog: out,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Metrics: m,
|
||||||
|
Limiter: limiter,
|
||||||
|
Ledger: ledger,
|
||||||
|
Anomalies: anomalies,
|
||||||
|
Lists: lists,
|
||||||
|
DNSBL: dnsbl,
|
||||||
|
AbuseIPDB: abuseIPDB,
|
||||||
|
RouteWatch: routeWatch,
|
||||||
|
GeoJS: geoJS,
|
||||||
|
LookupFile: lookupFile,
|
||||||
|
CoreRuleSet: crs,
|
||||||
|
Now: utcNow,
|
||||||
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
|
||||||
|
BesideLookupTimeLimit: params.BesideLookupTimeLimit,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newWAFListener listens on SWW_WAF_LISTEN_ADDR.
|
||||||
|
func newWAFListener(cfg *config.WAF) (net.Listener, error) {
|
||||||
|
return listen(cfg.ListenAddr, "SWW_WAF_LISTEN_ADDR")
|
||||||
|
}
|
||||||
|
|
||||||
|
// newStateFiles reads the state and cache files into the parts and the
|
||||||
|
// alert queue, as state.Load does. Until waf stops, it writes them as they
|
||||||
|
// are due and takes in an admin's edits of them.
|
||||||
|
func newStateFiles(
|
||||||
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
m *metrics.Metrics, ledger *bans.Ledger, limiter *ratelimit.Limiter,
|
||||||
|
geoJS *geojs.GeoJS, lists *reputation.Lists, dnsbl *reputation.DNSBL,
|
||||||
|
abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client,
|
||||||
|
alertQueue *alerts.Queue, anomalies *anomaly.Counters,
|
||||||
|
) (*state.Files, error) {
|
||||||
|
files, err := state.Load(state.Params{
|
||||||
|
Dir: cfg.StateDir,
|
||||||
|
CacheDir: cfg.CacheDir,
|
||||||
|
BansWriteDelay: cfg.BansWriteDelay,
|
||||||
|
WriteInterval: cfg.StateWriteInterval,
|
||||||
|
Ledger: ledger,
|
||||||
|
Limiter: limiter,
|
||||||
|
GeoJS: geoJS,
|
||||||
|
Lists: lists,
|
||||||
|
DNSBL: dnsbl,
|
||||||
|
AbuseIPDB: abuseIPDB,
|
||||||
|
RouteWatch: routeWatch,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
Anomalies: anomalies,
|
||||||
|
Now: utcNow,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Metrics: m,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, startError{"cannot use the state and cache files", err}
|
||||||
|
}
|
||||||
|
|
||||||
|
lc.Append(background(files.Run))
|
||||||
|
lc.Append(background(files.Watch))
|
||||||
|
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRemoteLogSender returns a sender of the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
|
||||||
|
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
|
||||||
|
return remotelog.New(remotelog.Params{
|
||||||
|
URL: cfg.LogRemoteURL,
|
||||||
|
RootCAs: cfg.LogRemoteTLSCAs,
|
||||||
|
BufferLines: cfg.LogRemoteBufferLines,
|
||||||
|
Facility: cfg.LogRemoteFacility,
|
||||||
|
AppName: cfg.LogRemoteAppName,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSending runs remote until the function it returns is called, which
|
||||||
|
// then waits at most remoteLogStopTimeout for the lines still waiting to
|
||||||
|
// be sent. Sending goes on after ctx is done, so that the lines written
|
||||||
|
// while sww stops are sent too.
|
||||||
|
func startSending(
|
||||||
|
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
|
||||||
|
) func() {
|
||||||
|
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
sent := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
remote.Run(sending, processLog)
|
||||||
|
close(sent)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
stop()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-sent:
|
||||||
|
case <-time.After(remoteLogStopTimeout):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -90,6 +90,8 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
cmd := exec.CommandContext(t.Context(), binary)
|
cmd := exec.CommandContext(t.Context(), binary)
|
||||||
cmd.Stdout = out
|
cmd.Stdout = out
|
||||||
|
|
||||||
|
// The Core Rule Set is off. Main keeps waf's own limit for its checks,
|
||||||
|
// and on a busy host the Core Rule Set's inspection can pass it.
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
"TZ="+zone,
|
"TZ="+zone,
|
||||||
commandForMain+"="+wafCommand,
|
commandForMain+"="+wafCommand,
|
||||||
@@ -97,7 +99,8 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
stateDir+"="+dir,
|
stateDir+"="+dir,
|
||||||
cacheDir+"="+dir,
|
cacheDir+"="+dir,
|
||||||
ruleFilesDir+"="+t.TempDir(),
|
ruleFilesDir+"="+t.TempDir(),
|
||||||
lookupSource+"=off")
|
lookupSource+"=off",
|
||||||
|
"SWW_WAF_CORE_RULE_SET_MODE=off")
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func (abuseIPDB) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
client := rq.h.clientGroup(rq.client)
|
client := clientGroup(rq.h.config, rq.client)
|
||||||
held, _ := rq.h.limiter.Client(client)
|
held, _ := rq.h.limiter.Client(client)
|
||||||
offender := held.History.Offences != ratelimit.Offences{}
|
offender := held.History.Offences != ratelimit.Offences{}
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,34 +4,41 @@ package waf
|
|||||||
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// banAbuseIPDBBlacklist bans the netblock of each client on AbuseIPDB's
|
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
|
||||||
// blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS, as the ledger's
|
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
|
||||||
// BanForAbuseIPDB describes, and logs how many clients were listed and how
|
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
|
||||||
// many bans were made. Clients in one netblock make one ban, with the
|
// clients were listed and how many bans were made. Clients in one netblock
|
||||||
// highest of their scores. In observe mode it makes no ban.
|
// make one ban, with the highest of their scores. In observe mode it makes
|
||||||
func (h *handler) banAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
// no ban.
|
||||||
|
func BanAbuseIPDBBlacklist(
|
||||||
|
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
|
||||||
|
now time.Time, listed []reputation.BlacklistEntry,
|
||||||
|
) {
|
||||||
scores := make(map[netip.Prefix]int64, len(listed))
|
scores := make(map[netip.Prefix]int64, len(listed))
|
||||||
|
|
||||||
for _, client := range listed {
|
for _, client := range listed {
|
||||||
netblock := h.netblock(client.Addr)
|
netblock := banNetblock(cfg, client.Addr)
|
||||||
scores[netblock] = max(scores[netblock], client.Score)
|
scores[netblock] = max(scores[netblock], client.Score)
|
||||||
}
|
}
|
||||||
|
|
||||||
made := 0
|
made := 0
|
||||||
|
|
||||||
if !h.config.Observe {
|
if !cfg.Observe {
|
||||||
now := h.now()
|
made = ledger.BanForAbuseIPDB(
|
||||||
made = h.ledger.BanForAbuseIPDB(
|
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
|
||||||
now, now.Add(h.config.AbuseIPDBBanDuration), scores,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
h.processLog.Info(
|
processLog.Info(
|
||||||
"fetched AbuseIPDB's blacklist",
|
"fetched AbuseIPDB's blacklist",
|
||||||
"clients", len(listed),
|
"clients", len(listed),
|
||||||
"bans_made", made,
|
"bans_made", made,
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ func (rq *request) addBan() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
netblock, err = rq.h.banNetblock(toAdd.Netblock)
|
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -190,12 +190,12 @@ func (rq *request) readBanToAdd() (banToAdd, error) {
|
|||||||
return toAdd, nil
|
return toAdd, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// banNetblock reads value, a netblock such as 203.0.113.0/24, or a
|
// parseBanNetblock reads value, a netblock such as 203.0.113.0/24, or a
|
||||||
// client's address, which stands for the netblock a ban on that client
|
// client's address, which stands for the netblock a ban on that client
|
||||||
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
|
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
|
||||||
// refused, since a client's address is looked up as IPv4 and a ban on it
|
// refused, since a client's address is looked up as IPv4 and a ban on it
|
||||||
// would refuse nothing, and so is a value with a zone.
|
// would refuse nothing, and so is a value with a zone.
|
||||||
func (h *handler) banNetblock(value string) (netip.Prefix, error) {
|
func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
||||||
netblock, err := netip.ParsePrefix(value)
|
netblock, err := netip.ParsePrefix(value)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if netblock.Addr().Is4In6() {
|
if netblock.Addr().Is4In6() {
|
||||||
@@ -224,7 +224,7 @@ func (h *handler) banNetblock(value string) (netip.Prefix, error) {
|
|||||||
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
||||||
}
|
}
|
||||||
|
|
||||||
return h.netblock(addr), nil
|
return banNetblock(h.config, addr), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// expiry returns when a ban made at now for duration ends: duration
|
// expiry returns when a ban made at now for duration ends: duration
|
||||||
@@ -295,7 +295,7 @@ func (rq *request) showClient() {
|
|||||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||||
}
|
}
|
||||||
|
|
||||||
client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr))
|
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
|
||||||
if seen {
|
if seen {
|
||||||
answer.Client = &client
|
answer.Client = &client
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -877,18 +877,13 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
|
|||||||
held, _, _ := startHeldGeoJS(t)
|
held, _, _ := startHeldGeoJS(t)
|
||||||
resolver, releaseResolver := startHeldResolver(t)
|
resolver, releaseResolver := startHeldResolver(t)
|
||||||
|
|
||||||
server, out, _ := newWAFWith(t, func(
|
server, out, _ := newWAFWith(t, waf.New,
|
||||||
params waf.Params,
|
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
|
||||||
) *waf.Server {
|
tagDeny: "country=KP", coreRuleSetMode: block,
|
||||||
params.AbuseIPDBURL = silentAbuseIPDBURL
|
dnsblZones: dnsblZone, dnsblResolver: resolver,
|
||||||
|
abuseIPDBKey: accountKey,
|
||||||
return waf.New(params)
|
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
||||||
}, silentGeoJSURL, time.Now, map[string]string{
|
})
|
||||||
tagDeny: "country=KP", coreRuleSetMode: block,
|
|
||||||
dnsblZones: dnsblZone, dnsblResolver: resolver,
|
|
||||||
abuseIPDBKey: accountKey,
|
|
||||||
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
|
||||||
})
|
|
||||||
|
|
||||||
// The lists are fetched outside the synctest bubble below: a wait on the
|
// The lists are fetched outside the synctest bubble below: a wait on the
|
||||||
// network inside it would keep its clock from moving on.
|
// network inside it would keep its clock from moving on.
|
||||||
|
|||||||
+13
-11
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
@@ -85,13 +86,14 @@ func (rq *request) countedBytes() int64 {
|
|||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch rq.h.config.BytesCounted {
|
||||||
case "response":
|
case config.BytesCountedResponse:
|
||||||
return response
|
return response
|
||||||
case "request":
|
case config.BytesCountedRequest:
|
||||||
return request
|
return request
|
||||||
default: // both
|
case config.BytesCountedBoth:
|
||||||
return response + request
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||||
@@ -121,7 +123,7 @@ func (rq *request) banForLimit(
|
|||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
rq.h.metrics.LimitHit(hit)
|
rq.h.metrics.LimitHit(hit)
|
||||||
|
|
||||||
netblock := rq.h.netblock(rq.client)
|
netblock := banNetblock(rq.h.config, rq.client)
|
||||||
if rq.h.config.Observe &&
|
if rq.h.config.Observe &&
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
||||||
return
|
return
|
||||||
@@ -157,7 +159,7 @@ func (rq *request) banForLimit(
|
|||||||
}
|
}
|
||||||
|
|
||||||
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
||||||
rq.h.limiter.Reset(rq.h.clientGroup(rq.client))
|
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
||||||
rq.line.BanExpires = banExpires(ban)
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
if made {
|
if made {
|
||||||
@@ -171,7 +173,7 @@ func (rq *request) banForLimit(
|
|||||||
// no ban, and raises the alert for the ban it would have made, if that
|
// no ban, and raises the alert for the ban it would have made, if that
|
||||||
// alert would be sent. It returns the action for that ban.
|
// alert would be sent. It returns the action for that ban.
|
||||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
||||||
netblock := rq.h.netblock(rq.client)
|
netblock := banNetblock(rq.h.config, rq.client)
|
||||||
if rq.h.config.Observe &&
|
if rq.h.config.Observe &&
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
||||||
return banAction(
|
return banAction(
|
||||||
@@ -273,16 +275,16 @@ func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
|
|||||||
return rq.h.limiter.Requests(netblock) + 1
|
return rq.h.limiter.Requests(netblock) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
// netblock is the netblock a ban on client covers: its IPv4 address,
|
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
||||||
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
|
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
|
||||||
// counts it in.
|
// counts it in.
|
||||||
func (h *handler) netblock(client netip.Addr) netip.Prefix {
|
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
|
||||||
addr := client.Unmap()
|
addr := client.Unmap()
|
||||||
if addr.Is4() {
|
if addr.Is4() {
|
||||||
return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked()
|
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return h.clientGroup(addr)
|
return clientGroup(cfg, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// banExpires is when ban ends, as the log line gives it: a time, or
|
// banExpires is when ban ends, as the log line gives it: a time, or
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ func newWAFHoldingTheCoreRuleSet(
|
|||||||
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
|
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
|
||||||
|
|
||||||
return waf.New(params)
|
return waf.New(params)
|
||||||
}, silentGeoJSURL, time.Now, settings)
|
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
|
||||||
|
|
||||||
return server, out, queue
|
return server, out, queue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,7 +63,8 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
if rq.blocklisted &&
|
||||||
|
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
"SWW_WAF_BLOCKLIST_ACTION",
|
||||||
@@ -72,7 +73,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == "limit" {
|
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -41,5 +42,6 @@ func (blocklists) check(
|
|||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
||||||
|
config.ReputationActionDeny
|
||||||
}
|
}
|
||||||
@@ -30,7 +30,7 @@ func (byteLimits) check(
|
|||||||
}
|
}
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.CountBytes(
|
counts, hit, over := rq.h.limiter.CountBytes(
|
||||||
rq.h.clientGroup(rq.client),
|
clientGroup(rq.h.config, rq.client),
|
||||||
now,
|
now,
|
||||||
rq.countedBytes(),
|
rq.countedBytes(),
|
||||||
rq.bytesLimitPercent.percent,
|
rq.bytesLimitPercent.percent,
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer.
|
// peerAddress is the address of the request's TCP peer.
|
||||||
@@ -45,10 +47,10 @@ func scheme(r *http.Request, isQuestion bool) string {
|
|||||||
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
||||||
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
|
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
|
||||||
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
|
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
|
||||||
func (h *handler) clientGroup(addr netip.Addr) netip.Prefix {
|
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
|
||||||
addr = addr.Unmap()
|
addr = addr.Unmap()
|
||||||
if addr.Is6() {
|
if addr.Is6() {
|
||||||
return netip.PrefixFrom(addr, h.config.IPv6GroupPrefix).Masked()
|
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen())
|
return netip.PrefixFrom(addr, addr.BitLen())
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := cfg.CoreRuleSetScoreThreshold
|
||||||
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = config.CoreRuleSetModeDetect
|
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
@@ -108,7 +108,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
Event: alerts.EventCoreRuleSetMatch,
|
Event: alerts.EventCoreRuleSetMatch,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: rq.h.clientGroup(rq.client),
|
Netblock: clientGroup(rq.h.config, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
@@ -32,7 +32,8 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
if !rq.dnsblListed ||
|
||||||
|
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,5 +9,5 @@ const BesideLookupTimeLimit = besideLookupTimeLimit
|
|||||||
// the clients on listed does.
|
// the clients on listed does.
|
||||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
||||||
h.banAbuseIPDBBlacklist(listed)
|
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
|
||||||
}
|
}
|
||||||
+28
-15
@@ -2,14 +2,18 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The headers in which an answer of allow gives rproxy the client's AS number
|
// The headers in which an answer of allow gives rproxy the client's AS number
|
||||||
@@ -45,15 +49,16 @@ func (rq *request) startLookUp(
|
|||||||
) <-chan lookupResult {
|
) <-chan lookupResult {
|
||||||
result := make(chan lookupResult, 1)
|
result := make(chan lookupResult, 1)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
if rq.h.config.LookupSource == config.LookupSourceOff ||
|
||||||
|
!canBePlaced(rq.client) {
|
||||||
result <- lookupResult{}
|
result <- lookupResult{}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
client := rq.h.clientGroup(rq.client)
|
client := clientGroup(rq.h.config, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == "file" {
|
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
||||||
}
|
}
|
||||||
@@ -128,22 +133,30 @@ func countryTag(country string) string {
|
|||||||
return config.CountryTagPrefix + country
|
return config.CountryTagPrefix + country
|
||||||
}
|
}
|
||||||
|
|
||||||
// addLookup adds answer, an answer about a client from the lookup
|
// AddLookup adds answer, an answer about a client from the lookup database
|
||||||
// database or GeoJS, to the client's history, and to the notes of the bans
|
// or GeoJS, to the client's history in limiter, and to the notes of the bans
|
||||||
// on its netblock that have no AS number, AS name or country yet.
|
// in ledger on its netblock that have no AS number, AS name or country yet.
|
||||||
func (h *handler) addLookup(answer geojs.Answer) {
|
// cfg sets the netblock's size.
|
||||||
h.limiter.AddLookup(answer.Client, answer.Answered,
|
func AddLookup(
|
||||||
|
cfg *config.WAF, limiter *ratelimit.Limiter, ledger *bans.Ledger,
|
||||||
|
answer geojs.Answer,
|
||||||
|
) {
|
||||||
|
limiter.AddLookup(answer.Client, answer.Answered,
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
answer.ASN, answer.ASName, answer.Country)
|
||||||
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
|
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
answer.ASN, answer.ASName, answer.Country)
|
||||||
}
|
}
|
||||||
|
|
||||||
// geojsFailed logs a failure of GeoJS, raises a source_failure alert for it
|
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
||||||
// and counts it. askingAgainIn is how long GeoJS is left alone after it.
|
// source_failure alert for it on alertQueue and counts it in m.
|
||||||
func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
|
// askingAgainIn is how long GeoJS is left alone after it.
|
||||||
h.processLog.Warn("asking GeoJS failed",
|
func GeoJSFailed(
|
||||||
|
processLog *slog.Logger, alertQueue *alerts.Queue, m *metrics.Metrics,
|
||||||
|
err error, askingAgainIn time.Duration,
|
||||||
|
) {
|
||||||
|
processLog.Warn("asking GeoJS failed",
|
||||||
"error", err.Error(), "asking_again_in", askingAgainIn.String())
|
"error", err.Error(), "asking_again_in", askingAgainIn.String())
|
||||||
h.alerts.Raise(alerts.Alert{
|
alertQueue.Raise(alerts.Alert{
|
||||||
Event: alerts.EventSourceFailure,
|
Event: alerts.EventSourceFailure,
|
||||||
Reason: "asking GeoJS failed",
|
Reason: "asking GeoJS failed",
|
||||||
Detail: map[string]any{
|
Detail: map[string]any{
|
||||||
@@ -151,7 +164,7 @@ func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
|
|||||||
"asking_again_in": askingAgainIn.String(),
|
"asking_again_in": askingAgainIn.String(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
h.metrics.GeoJSFailures.Inc()
|
m.GeoJSFailures.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// setLookupHeaders sets the headers that give the client's AS number and
|
// setLookupHeaders sets the headers that give the client's AS number and
|
||||||
|
|||||||
+49
-163
@@ -6,7 +6,6 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -58,21 +57,33 @@ type Params struct {
|
|||||||
RequestLog io.Writer
|
RequestLog io.Writer
|
||||||
// ProcessLog receives the process's own messages.
|
// ProcessLog receives the process's own messages.
|
||||||
ProcessLog *slog.Logger
|
ProcessLog *slog.Logger
|
||||||
// GeoJSURL is where clients' AS numbers and countries are looked up
|
// Metrics count what waf does. Whoever builds them adds the metrics
|
||||||
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally geojs.URL.
|
// read from the other parts.
|
||||||
GeoJSURL string
|
Metrics *metrics.Metrics
|
||||||
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
|
// Limiter is the table of clients the rate limits count, with each
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
|
// client's history.
|
||||||
AbuseIPDBURL string
|
Limiter *ratelimit.Limiter
|
||||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from
|
// Ledger holds the bans.
|
||||||
// while SWW_WAF_ABUSEIPDB_KEY is set, normally
|
Ledger *bans.Ledger
|
||||||
// reputation.AbuseIPDBBlacklistURL.
|
// Anomalies are the counters of the anomaly thresholds.
|
||||||
AbuseIPDBBlacklistURL string
|
Anomalies *anomaly.Counters
|
||||||
|
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
||||||
|
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
||||||
|
Lists *reputation.Lists
|
||||||
|
DNSBL *reputation.DNSBL
|
||||||
|
AbuseIPDB *reputation.AbuseIPDB
|
||||||
|
// RouteWatch asks routewatch about clients, and is nil while
|
||||||
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
|
RouteWatch *routewatch.Client
|
||||||
|
// GeoJS looks clients' AS numbers and countries up while
|
||||||
|
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
||||||
|
// and GeoJSFailed as its Failed.
|
||||||
|
GeoJS *geojs.GeoJS
|
||||||
// LookupFile is the lookup database they are looked up in while
|
// LookupFile is the lookup database they are looked up in while
|
||||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||||
LookupFile *lookup.File
|
LookupFile *lookup.File
|
||||||
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
// CoreRuleSet is what inspects requests while
|
||||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||||
CoreRuleSet CoreRuleSetInspector
|
CoreRuleSet CoreRuleSetInspector
|
||||||
// Now tells the time by which requests are counted for the rate
|
// Now tells the time by which requests are counted for the rate
|
||||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||||
@@ -104,11 +115,8 @@ type CoreRuleSetInspector interface {
|
|||||||
) (coreruleset.Result, []byte, error)
|
) (coreruleset.Result, []byte, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Server is the server waf runs, with the parts whose state the state
|
// Server is the server waf runs, with the parts of its Params. The tests
|
||||||
// files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
|
// read them.
|
||||||
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
|
|
||||||
// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
|
|
||||||
// RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
|
|
||||||
type Server struct {
|
type Server struct {
|
||||||
*http.Server
|
*http.Server
|
||||||
|
|
||||||
@@ -124,167 +132,45 @@ type Server struct {
|
|||||||
Metrics *metrics.Metrics
|
Metrics *metrics.Metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// newHandler returns the handler with what every request shares, as the
|
// newHandler returns the handler with what every request shares, the parts
|
||||||
// settings in params make it.
|
// in params among it.
|
||||||
func newHandler(params Params) *handler {
|
func newHandler(params Params) *handler {
|
||||||
m := metrics.New(
|
|
||||||
params.Config.MetricsBusiestCountriesAndASNs,
|
|
||||||
params.Config.InstanceName,
|
|
||||||
)
|
|
||||||
|
|
||||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // New fails only for a size below one
|
panic(err) // New fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
h := &handler{
|
return &handler{
|
||||||
config: params.Config,
|
config: params.Config,
|
||||||
requestLog: params.RequestLog,
|
requestLog: params.RequestLog,
|
||||||
processLog: params.ProcessLog,
|
processLog: params.ProcessLog,
|
||||||
now: params.Now,
|
now: params.Now,
|
||||||
metrics: m,
|
metrics: params.Metrics,
|
||||||
awaitingReport: awaitingReport,
|
awaitingReport: awaitingReport,
|
||||||
limiter: ratelimit.New(ratelimit.Limits{
|
limiter: params.Limiter,
|
||||||
PerMinute: params.Config.RateLimitPerMinute,
|
ledger: params.Ledger,
|
||||||
PerHour: params.Config.RateLimitPerHour,
|
geojs: params.GeoJS,
|
||||||
PerDay: params.Config.RateLimitPerDay,
|
anomalies: params.Anomalies,
|
||||||
BytesPerMinute: params.Config.BytesLimitPerMinute,
|
lookupFile: params.LookupFile,
|
||||||
BytesPerHour: params.Config.BytesLimitPerHour,
|
lists: params.Lists,
|
||||||
BytesPerDay: params.Config.BytesLimitPerDay,
|
dnsbl: params.DNSBL,
|
||||||
}, params.Config.MaxTrackedClients),
|
abuseIPDB: params.AbuseIPDB,
|
||||||
ledger: bans.New(bans.Params{
|
routeWatch: params.RouteWatch,
|
||||||
LimitBanDuration: params.Config.LimitBanDuration,
|
rules: params.Rules,
|
||||||
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
|
coreRuleSet: params.CoreRuleSet,
|
||||||
LimitBanPermanentAbove: params.Config.LimitBanPermanentAbove,
|
alerts: params.Alerts,
|
||||||
AttackBanDuration: params.Config.AttackBanDuration,
|
// The country list, the headers, the tag settings that name an asn=
|
||||||
MaxBans: params.Config.MaxBans,
|
// or country= tag and the biased thresholds act on the answer before
|
||||||
}),
|
// the request goes on.
|
||||||
anomalies: anomaly.New(anomaly.Params{
|
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||||
Client: params.Config.AnomalyClient,
|
params.Config.AddLookupHeaders ||
|
||||||
Net: params.Config.AnomalyNet,
|
params.Config.TagsNeedLookup() ||
|
||||||
ASN: params.Config.AnomalyASN,
|
biasedThresholdsSet(params.Config),
|
||||||
Total: params.Config.AnomalyTotal,
|
|
||||||
WatchedNet: params.Config.AnomalyWatchedNet,
|
|
||||||
NetV4Prefix: params.Config.AnomalyNetV4Prefix,
|
|
||||||
NetV6Prefix: params.Config.AnomalyNetV6Prefix,
|
|
||||||
NamedNetblocks: params.Config.AnomalyWatchedNets,
|
|
||||||
Alerts: params.Alerts,
|
|
||||||
}),
|
|
||||||
lookupFile: params.LookupFile,
|
|
||||||
rules: params.Rules,
|
|
||||||
coreRuleSet: params.CoreRuleSet,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
|
|
||||||
besideLookupLimit: cmp.Or(
|
besideLookupLimit: cmp.Or(
|
||||||
params.BesideLookupTimeLimit,
|
params.BesideLookupTimeLimit,
|
||||||
besideLookupTimeLimit,
|
besideLookupTimeLimit,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
h.lists, h.dnsbl, h.abuseIPDB = newReputation(
|
|
||||||
params, m, h.banAbuseIPDBBlacklist,
|
|
||||||
)
|
|
||||||
|
|
||||||
if params.Config.RouteWatchURL != "" {
|
|
||||||
h.routeWatch = routewatch.New(routewatch.Params{
|
|
||||||
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The country list, the headers, the tag settings that name an asn= or
|
|
||||||
// country= tag and the biased thresholds act on the answer before the
|
|
||||||
// request goes on.
|
|
||||||
h.waitForGeoJS = len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
|
||||||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
|
|
||||||
biasedThresholdsSet(params.Config)
|
|
||||||
h.geojs = geojs.New(geojs.Params{
|
|
||||||
URL: params.GeoJSURL,
|
|
||||||
Answered: h.addLookup,
|
|
||||||
Now: params.Now,
|
|
||||||
Asked: m.GeoJSRequests.Inc,
|
|
||||||
Failed: h.geojsFailed,
|
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
|
||||||
})
|
|
||||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
|
||||||
m.AddRules(params.Rules)
|
|
||||||
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
|
||||||
// verdicts and AbuseIPDB's scores, as the settings in params name them,
|
|
||||||
// with none fetched, asked for or checked yet, and adds their metrics to
|
|
||||||
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set. blacklisted receives
|
|
||||||
// the clients on each AbuseIPDB blacklist fetched.
|
|
||||||
func newReputation(
|
|
||||||
params Params, m *metrics.Metrics,
|
|
||||||
blacklisted func([]reputation.BlacklistEntry),
|
|
||||||
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
|
|
||||||
cfg := params.Config
|
|
||||||
lists := reputation.New(reputation.Params{
|
|
||||||
BlocklistURLs: cfg.BlocklistURLs,
|
|
||||||
Refresh: cfg.BlocklistRefresh,
|
|
||||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
|
||||||
TagLists: cfg.TagLists,
|
|
||||||
Now: params.Now,
|
|
||||||
ProcessLog: params.ProcessLog,
|
|
||||||
Alerts: params.Alerts,
|
|
||||||
})
|
|
||||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
|
||||||
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver,
|
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
||||||
Alerts: params.Alerts,
|
|
||||||
})
|
|
||||||
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
|
||||||
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
|
|
||||||
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
|
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout, BlacklistURL: params.AbuseIPDBBlacklistURL,
|
|
||||||
Threshold: cfg.AbuseIPDBThreshold, BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
|
||||||
Blacklisted: blacklisted, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
||||||
Alerts: params.Alerts,
|
|
||||||
})
|
|
||||||
|
|
||||||
m.AddReputation(lists, dnsbl)
|
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
|
||||||
m.AddAbuseIPDB(abuseIPDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
return lists, dnsbl, abuseIPDB
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
|
||||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
||||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
||||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
||||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
||||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
||||||
// set file is loaded with the Core Rule Set only to check it.
|
|
||||||
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
|
||||||
cfg.CoreRuleSetExclusionsFile == "" {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
return coreRuleSet, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
|
|||||||
+189
-30
@@ -16,10 +16,19 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
@@ -390,11 +399,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"rproxy",
|
InstanceName: "rproxy",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -436,13 +445,15 @@ func newWAF(
|
|||||||
) (*waf.Server, *output, *alerts.Queue) {
|
) (*waf.Server, *output, *alerts.Queue) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return newWAFWith(t, waf.New, geojsURL, now, env)
|
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFWith is newWAF for the server build makes.
|
// newWAFWith is newWAF for the server build makes, with clients checked with
|
||||||
|
// AbuseIPDB at abuseIPDBCheckURL.
|
||||||
func newWAFWith(
|
func newWAFWith(
|
||||||
t *testing.T, build func(waf.Params) *waf.Server,
|
t *testing.T, build func(waf.Params) *waf.Server,
|
||||||
geojsURL string, now func() time.Time, env map[string]string,
|
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
|
||||||
|
env map[string]string,
|
||||||
) (*waf.Server, *output, *alerts.Queue) {
|
) (*waf.Server, *output, *alerts.Queue) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -461,11 +472,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
out,
|
Output: out,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -495,29 +506,177 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
|
params := waf.Params{
|
||||||
|
Config: cfg,
|
||||||
|
RequestLog: out,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
LookupFile: lookupFile,
|
||||||
|
Now: now,
|
||||||
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
// The tests of the limit set it back, and hold the Core Rule Set
|
||||||
|
// with a stand-in.
|
||||||
|
BesideLookupTimeLimit: time.Hour,
|
||||||
|
}
|
||||||
|
|
||||||
|
addCoreRuleSet(t, ¶ms)
|
||||||
|
addCounters(¶ms)
|
||||||
|
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||||
|
|
||||||
|
server := build(params)
|
||||||
|
|
||||||
|
return server, out, alertQueue
|
||||||
|
}
|
||||||
|
|
||||||
|
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
|
||||||
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
|
||||||
|
func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg := params.Config
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||||
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||||
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||||
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||||
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||||
|
Apps: cfg.Apps,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Core Rule Set: %v", err)
|
t.Fatalf("Core Rule Set: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
server := build(waf.Params{
|
params.CoreRuleSet = coreRuleSet
|
||||||
Config: cfg,
|
}
|
||||||
RequestLog: out,
|
|
||||||
ProcessLog: processLog,
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
GeoJSURL: geojsURL,
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
AbuseIPDBURL: abuseIPDBURL,
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
LookupFile: lookupFile,
|
func addCounters(params *waf.Params) {
|
||||||
CoreRuleSet: coreRuleSet,
|
cfg := params.Config
|
||||||
Now: now,
|
|
||||||
Rules: ruleFiles,
|
params.Metrics = metrics.New(
|
||||||
Alerts: alertQueue,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
// The tests of the limit set it back, and hold the Core Rule Set
|
cfg.InstanceName,
|
||||||
// with a stand-in.
|
)
|
||||||
BesideLookupTimeLimit: time.Hour,
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
AbuseIPDBBlacklistURL: abuseIPDBBlacklistURL,
|
Limits: ratelimit.Limits{
|
||||||
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
|
PerHour: cfg.RateLimitPerHour,
|
||||||
|
PerDay: cfg.RateLimitPerDay,
|
||||||
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
|
params.Ledger = bans.New(bans.Params{
|
||||||
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
|
||||||
|
AttackBanDuration: cfg.AttackBanDuration,
|
||||||
|
MaxBans: cfg.MaxBans,
|
||||||
|
})
|
||||||
|
params.Anomalies = anomaly.New(anomaly.Params{
|
||||||
|
Client: cfg.AnomalyClient,
|
||||||
|
Net: cfg.AnomalyNet,
|
||||||
|
ASN: cfg.AnomalyASN,
|
||||||
|
Total: cfg.AnomalyTotal,
|
||||||
|
WatchedNet: cfg.AnomalyWatchedNet,
|
||||||
|
NetV4Prefix: cfg.AnomalyNetV4Prefix,
|
||||||
|
NetV6Prefix: cfg.AnomalyNetV6Prefix,
|
||||||
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
|
Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|
||||||
return server, out, alertQueue
|
params.Metrics.AddBansAndClients(
|
||||||
|
params.Ledger,
|
||||||
|
params.Limiter,
|
||||||
|
params.Now,
|
||||||
|
)
|
||||||
|
params.Metrics.AddRules(params.Rules)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||||
|
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||||
|
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||||
|
// them from params' settings and the parts addCounters added. The metrics
|
||||||
|
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
|
func addOutsideSources(
|
||||||
|
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||||
|
) {
|
||||||
|
cfg := params.Config
|
||||||
|
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
||||||
|
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||||
|
|
||||||
|
params.Lists = reputation.New(reputation.Params{
|
||||||
|
BlocklistURLs: cfg.BlocklistURLs,
|
||||||
|
Refresh: cfg.BlocklistRefresh,
|
||||||
|
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
||||||
|
TagLists: cfg.TagLists,
|
||||||
|
Now: params.Now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
params.DNSBL = reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
|
Zones: cfg.DNSBLZones,
|
||||||
|
Resolver: cfg.DNSBLResolver,
|
||||||
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
|
Now: params.Now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||||
|
URL: abuseIPDBCheckURL,
|
||||||
|
Key: cfg.AbuseIPDBKey,
|
||||||
|
MinScore: cfg.AbuseIPDBMinScore,
|
||||||
|
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||||
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
|
BlacklistURL: abuseIPDBBlacklistURL,
|
||||||
|
Threshold: cfg.AbuseIPDBThreshold,
|
||||||
|
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
||||||
|
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
||||||
|
waf.BanAbuseIPDBBlacklist(
|
||||||
|
cfg, ledger, processLog, params.Now(), listed,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
Now: params.Now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
|
||||||
|
if cfg.RouteWatchURL != "" {
|
||||||
|
params.RouteWatch = routewatch.New(routewatch.Params{
|
||||||
|
URL: cfg.RouteWatchURL,
|
||||||
|
Now: params.Now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
params.GeoJS = geojs.New(geojs.Params{
|
||||||
|
URL: geojsURL,
|
||||||
|
Answered: func(answer geojs.Answer) {
|
||||||
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
|
},
|
||||||
|
Now: params.Now,
|
||||||
|
Asked: m.GeoJSRequests.Inc,
|
||||||
|
Failed: func(err error, askingAgainIn time.Duration) {
|
||||||
|
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
|
||||||
|
},
|
||||||
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
|
})
|
||||||
|
|
||||||
|
m.AddReputation(params.Lists, params.DNSBL)
|
||||||
|
|
||||||
|
if cfg.AbuseIPDBKey != "" {
|
||||||
|
m.AddAbuseIPDB(params.AbuseIPDB)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// newClient returns an HTTP client that sends requests as they are made,
|
// newClient returns an HTTP client that sends requests as they are made,
|
||||||
|
|||||||
@@ -38,8 +38,11 @@ func (rateLimits) check(
|
|||||||
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
||||||
rq.bytesLimitPercent.logged()
|
rq.bytesLimitPercent.logged()
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
|
counts, hit, over := rq.h.limiter.Count(
|
||||||
rq.rateLimitPercent.percent)
|
clientGroup(rq.h.config, rq.client),
|
||||||
|
now,
|
||||||
|
rq.rateLimitPercent.percent,
|
||||||
|
)
|
||||||
rq.line.Counts = counts
|
rq.line.Counts = counts
|
||||||
|
|
||||||
if !over {
|
if !over {
|
||||||
@@ -59,7 +62,7 @@ func (rateLimits) check(
|
|||||||
// limit. It counts nothing.
|
// limit. It counts nothing.
|
||||||
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
||||||
return rq.countedByRateLimits() &&
|
return rq.countedByRateLimits() &&
|
||||||
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
|
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
|
||||||
}
|
}
|
||||||
|
|
||||||
// countedByRateLimits reports whether the rate limits count the request: its
|
// countedByRateLimits reports whether the rate limits count the request: its
|
||||||
|
|||||||
@@ -36,8 +36,11 @@ func (refusalLimit) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
|
hit, over := rq.h.limiter.CountRefusal(
|
||||||
cfg.RefusalsLimitPerMinute)
|
clientGroup(rq.h.config, rq.client),
|
||||||
|
now,
|
||||||
|
cfg.RefusalsLimitPerMinute,
|
||||||
|
)
|
||||||
if !over {
|
if !over {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,10 +7,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
)
|
)
|
||||||
|
|
||||||
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
|
||||||
// refuses the requests of a client a source lists.
|
|
||||||
const deny = "deny"
|
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
@@ -37,7 +33,7 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
|||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: rq.h.clientGroup(rq.client),
|
Netblock: clientGroup(rq.h.config, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ func (h *handler) startRequest(
|
|||||||
RequestID: requestID(r, isQuestion),
|
RequestID: requestID(r, isQuestion),
|
||||||
PeerIP: peer.String(),
|
PeerIP: peer.String(),
|
||||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||||
ClientGroup: h.clientGroup(client).String(),
|
ClientGroup: clientGroup(h.config, client).String(),
|
||||||
ContentType: r.Header.Get("Content-Type"),
|
ContentType: r.Header.Get("Content-Type"),
|
||||||
RequestHeaders: requestHeaders(
|
RequestHeaders: requestHeaders(
|
||||||
r,
|
r,
|
||||||
@@ -365,7 +365,7 @@ func (rq *request) addToHistory() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.h.limiter.AddToHistory(
|
rq.h.limiter.AddToHistory(
|
||||||
rq.h.clientGroup(rq.client),
|
clientGroup(rq.h.config, rq.client),
|
||||||
rq.h.now(),
|
rq.h.now(),
|
||||||
request,
|
request,
|
||||||
)
|
)
|
||||||
@@ -373,7 +373,7 @@ func (rq *request) addToHistory() {
|
|||||||
|
|
||||||
answer, found := rq.answerAtTheEnd()
|
answer, found := rq.answerAtTheEnd()
|
||||||
if found {
|
if found {
|
||||||
rq.h.addLookup(answer)
|
AddLookup(rq.h.config, rq.h.limiter, rq.h.ledger, answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,7 +394,7 @@ func (rq *request) countAnomalies() {
|
|||||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
ClientGroup: rq.h.clientGroup(rq.client),
|
ClientGroup: clientGroup(rq.h.config, rq.client),
|
||||||
ASN: answer.ASN,
|
ASN: answer.ASN,
|
||||||
ASName: answer.ASName,
|
ASName: answer.ASName,
|
||||||
Country: answer.Country,
|
Country: answer.Country,
|
||||||
@@ -421,11 +421,11 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return geojs.Answer{}, false
|
return geojs.Answer{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.LookupSource == "file" {
|
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
|
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseWriter is waf's answer to a request. It notes the status and
|
// responseWriter is waf's answer to a request. It notes the status and
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ func (routeWatchTags) check(
|
|||||||
|
|
||||||
route, missing := rq.h.routeWatch.Route(
|
route, missing := rq.h.routeWatch.Route(
|
||||||
ctx,
|
ctx,
|
||||||
rq.h.clientGroup(rq.client),
|
clientGroup(rq.h.config, rq.client),
|
||||||
rq.client,
|
rq.client,
|
||||||
)
|
)
|
||||||
if missing != "" {
|
if missing != "" {
|
||||||
|
|||||||
Reference in new issue
Block a user