Compare commits

..
Author SHA1 Message Date
clawbot 89f2918d25 bans: one ledger method makes a request's ban, one place decides observe mode (closes #444)
check / check (push) Waiting to run
The ledger's BanForLimit, WouldBanForLimit, BanForAttack and
WouldBanForAttack become Ban and WouldBan, which take the cause and the
notes in one bans.Grounds. The waf's banForLimit and banForAttack fill in
what differs and call banClient, the one function that tests observe
mode on the way to a ban. Behaviour is unchanged; tests change only
where they called the four methods.

Model: opus-5-5
2026-11-19 17:39:32 +00:00
66 changed files with 985 additions and 1883 deletions

No files matched your search

+3 -5
View File
@@ -906,11 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each. `internal/alerts`: one check or source each.
- `internal/state`: the state and cache files. - `internal/state`, `internal/requestlog`, `internal/remotelog` and
- `internal/requestlog`: the request log. `internal/metrics`: the state and cache files, the log lines, the syslog
- `internal/processlog`: the process's own messages. sender and the metrics.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
## Rationale ## Rationale
+3 -7
View File
@@ -69,8 +69,7 @@ never as a time.
that command's settings, and runs `internal/rproxy` until `SIGTERM` or that command's settings, and runs `internal/rproxy` until `SIGTERM` or
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check. `SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
With no command, or one it does not know, the binary prints the commands With no command, or one it does not know, the binary prints the commands
and exits with status 2. `sww help` and `sww --help` print them and exit and exits with status 2. `internal/sww` is the only package that imports
with status 0. `internal/sww` is the only package that imports
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts `go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work. and stops their background work.
- `internal/config`: reads the settings, the one place they are read. - `internal/config`: reads the settings, the one place they are read.
@@ -105,10 +104,8 @@ never as a time.
its own. its own.
- `internal/state`: reads the state and cache files at start, takes in edits - `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop. while running, and writes the files when due and at the stop.
- `internal/requestlog`: the request log, as JSON lines on stdout. - `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
- `internal/processlog`: the process lines on stdout, which hold the sending them to the syslog server.
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format. - `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image. - `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its `deploy/example-app` is an app built on the image, with `rproxy` in its
@@ -127,7 +124,6 @@ never as a time.
- `github.com/fsnotify/fsnotify` to notice edited files; - `github.com/fsnotify/fsnotify` to notice edited files;
- `github.com/prometheus/client_golang` for the metrics; - `github.com/prometheus/client_golang` for the metrics;
- `github.com/spf13/viper` to read the settings from the environment; - `github.com/spf13/viper` to read the settings from the environment;
- `github.com/spf13/cobra` for the commands and their help text;
- `github.com/maxmind/mmdbwriter`, in the tests, to write lookup databases; - `github.com/maxmind/mmdbwriter`, in the tests, to write lookup databases;
- remote log sending written in this project, since the standard library's - remote log sending written in this project, since the standard library's
`log/syslog` is frozen and writes only the older syslog format. `log/syslog` is frozen and writes only the older syslog format.
+1 -3
View File
@@ -11,10 +11,8 @@ require (
github.com/maxmind/mmdbwriter v1.2.0 github.com/maxmind/mmdbwriter v1.2.0
github.com/oschwald/maxminddb-golang/v2 v2.7.0 github.com/oschwald/maxminddb-golang/v2 v2.7.0
github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_golang v1.24.1
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0 github.com/spf13/viper v1.21.0
go.uber.org/fx v1.24.0 go.uber.org/fx v1.24.0
golang.org/x/net v0.60.0
golang.org/x/sys v0.48.0 golang.org/x/sys v0.48.0
) )
@@ -27,7 +25,6 @@ require (
github.com/goccy/go-yaml v1.19.2 // indirect github.com/goccy/go-yaml v1.19.2 // indirect
github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kaptinlin/go-i18n v0.1.4 // indirect github.com/kaptinlin/go-i18n v0.1.4 // indirect
github.com/kaptinlin/jsonschema v0.4.6 // indirect github.com/kaptinlin/jsonschema v0.4.6 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect
@@ -53,6 +50,7 @@ require (
go.uber.org/zap v1.26.0 // indirect go.uber.org/zap v1.26.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect golang.org/x/sync v0.23.0 // indirect
golang.org/x/text v0.42.0 // indirect golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect google.golang.org/protobuf v1.36.11 // indirect
-9
View File
@@ -14,7 +14,6 @@ github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 h1:KLEjvBBSw
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68/go.mod h1:9XocblQEwK6EYylRth7PrPIifF2l0iSoYo4FAXhXU/8= github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68/go.mod h1:9XocblQEwK6EYylRth7PrPIifF2l0iSoYo4FAXhXU/8=
github.com/corazawaf/libinjection-go v0.3.4 h1:jX2Tit2xZxYJJyVryNXucYBZtQqS7473x9Nc6lgRRbk= github.com/corazawaf/libinjection-go v0.3.4 h1:jX2Tit2xZxYJJyVryNXucYBZtQqS7473x9Nc6lgRRbk=
github.com/corazawaf/libinjection-go v0.3.4/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk= github.com/corazawaf/libinjection-go v0.3.4/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0= github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0=
github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -35,8 +34,6 @@ github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 h1:c7gcN
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU= github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y= github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y=
github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk= github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs= github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs=
@@ -75,7 +72,6 @@ github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
@@ -84,9 +80,6 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
@@ -116,7 +109,6 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
@@ -135,7 +127,6 @@ golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
+25 -4
View File
@@ -19,8 +19,6 @@ import (
"time" "time"
"github.com/spf13/viper" "github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// off is the value that switches a timeout, a size limit or a rate limit // off is the value that switches a timeout, a size limit or a rate limit
@@ -42,7 +40,9 @@ var (
errNotPositive = errors.New("must be more than zero, or off") errNotPositive = errors.New("must be more than zero, or off")
errNotMilliseconds = errors.New( errNotMilliseconds = errors.New(
"is not a whole number of milliseconds above zero, such as 2000") "is not a whole number of milliseconds above zero, such as 2000")
errEmptyItem = errors.New("has an empty item in its list") errEmptyItem = errors.New("has an empty item in its list")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotListenAddr = errors.New( errNotListenAddr = errors.New(
"is not an address to listen on, such as :8080") "is not an address to listen on, such as :8080")
errNotCommand = errors.New("is not rproxy or waf") errNotCommand = errors.New("is not rproxy or waf")
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
netblocks := make([]netip.Prefix, 0, len(items)) netblocks := make([]netip.Prefix, 0, len(items))
for _, item := range items { for _, item := range items {
netblock, err := reputation.ParseNetblock(item) netblock, err := ParseNetblock(item)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -264,6 +264,27 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
return netblocks, nil return netblocks, nil
} }
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// parseListenAddr checks an address to listen on: an optional host and a // parseListenAddr checks an address to listen on: an optional host and a
// port number. // port number.
func parseListenAddr(value string) (string, error) { func parseListenAddr(value string) (string, error) {
+121 -77
View File
@@ -20,26 +20,11 @@ import (
"unicode/utf8" "unicode/utf8"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The names of the settings that waf writes into log lines and ban notes,
// as the setting that refused a request or lowered a client's limits.
const (
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
) )
// WAF is the settings of `sww waf`. A time or size limit of zero is off. // WAF is the settings of `sww waf`. A time or size limit of zero is off.
@@ -115,7 +100,7 @@ type WAF struct {
// with a tag that percentage of every rate limit and byte limit, from 1 // with a tag that percentage of every rate limit and byte limit, from 1
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in // to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
// lower case, a country code in capitals. // lower case, a country code in capitals.
TagLists []reputation.TagList TagLists []TagList
TagDeny []string TagDeny []string
TagLimitPercent map[string]int64 TagLimitPercent map[string]int64
// RouteWatchURL is where the routewatch instance is that adds the asn= // RouteWatchURL is where the routewatch instance is that adds the asn=
@@ -512,8 +497,12 @@ var (
"is not a whole number above zero, such as 60, or off", "is not a whole number above zero, such as 60, or off",
) )
errNotUTF8 = errors.New("is not valid UTF-8") errNotUTF8 = errors.New("is not valid UTF-8")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercentItem = errors.New( errNotPercentItem = errors.New(
"is not a code, : and a percentage, such as AS64496:50 or cn:25") "is not a code, : and a percentage, such as AS64496:50 or cn:25")
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
errNotAbuseIPDBThreshold = errors.New( errNotAbuseIPDBThreshold = errors.New(
"is not a whole number from 25 to 100, the scores AbuseIPDB's " + "is not a whole number from 25 to 100, the scores AbuseIPDB's " +
"blacklist can be cut at", "blacklist can be cut at",
@@ -629,24 +618,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"800", "800",
), ),
ExclusivelyAllowedCountries: env.countries( ExclusivelyAllowedCountries: env.countries(
ExclusivelyAllowedCountriesSetting, ""), "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"), TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
TagDeny: env.tags("SWW_WAF_TAG_DENY"), TagDeny: env.tags("SWW_WAF_TAG_DENY"),
TagLimitPercent: env.tagPercents(TagLimitPercentSetting), TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"), RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents( ASNBytesLimitPercent: env.percents(
ASNBytesLimitPercentSetting, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
reputation.ParseASN, ParseASN,
), ),
CountryBytesLimitPercent: env.percents( CountryBytesLimitPercent: env.percents(
CountryBytesLimitPercentSetting, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
parseCountry, parseCountry,
), ),
UnknownCountryLimitPercent: env.percent( UnknownCountryLimitPercent: env.percent(
UnknownCountryLimitPercentSetting, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
"100", "100",
), ),
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting), ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"), BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"), BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"), DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
@@ -820,9 +809,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff)) strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action( cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
BlocklistActionSetting, string(ReputationActionDeny)) "SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action( cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
DNSBLAbuseIPDBActionSetting, "limit:25") "SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg) env.checkLookupDBPath(cfg)
@@ -1022,7 +1011,7 @@ func (e *environment) percents(
} }
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default. // tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
func (e *environment) tagLists(name string) []reputation.TagList { func (e *environment) tagLists(name string) []TagList {
lists, err := parseTagLists(e.value(name, "")) lists, err := parseTagLists(e.value(name, ""))
e.check(name, err) e.check(name, err)
@@ -1048,7 +1037,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
// percent reads a setting that is a percentage, from 0 to 100. // percent reads a setting that is a percentage, from 0 to 100.
func (e *environment) percent(name, defaultValue string) int64 { func (e *environment) percent(name, defaultValue string) int64 {
percent, err := reputation.ParsePercent(e.value(name, defaultValue)) percent, err := ParsePercent(e.value(name, defaultValue))
e.check(name, err) e.check(name, err)
return percent return percent
@@ -1133,7 +1122,7 @@ func (e *environment) action(
string(value), string(ReputationActionLimit)+":", string(value), string(ReputationActionLimit)+":",
) )
percent, err := reputation.ParsePercent(percentText) percent, err := ParsePercent(percentText)
if !isLimit || err != nil { if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction)) e.check(name, fmt.Errorf("%q %w", value, errNotAction))
} }
@@ -1142,8 +1131,8 @@ func (e *environment) action(
} }
// zones reads the setting that is the list of DNSBL zones. It is empty by // zones reads the setting that is the list of DNSBL zones. It is empty by
// default. The log shows each zone with its key masked, as // default. The log shows each zone with its key masked, as MaskZoneKey
// reputation.MaskZoneKey masks it. // masks it.
func (e *environment) zones(name string) []string { func (e *environment) zones(name string) []string {
value, _ := e.lookup(name) value, _ := e.lookup(name)
zones, err := parseZones(value) zones, err := parseZones(value)
@@ -1151,7 +1140,7 @@ func (e *environment) zones(name string) []string {
logged := make([]string, len(zones)) logged := make([]string, len(zones))
for i, zone := range zones { for i, zone := range zones {
logged[i] = reputation.MaskZoneKey(zone) logged[i] = MaskZoneKey(zone)
} }
e.settings = append( e.settings = append(
@@ -1231,7 +1220,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
func (e *environment) checkCountriesAndLookups(cfg *WAF) { func (e *environment) checkCountriesAndLookups(cfg *WAF) {
for _, country := range cfg.ExclusivelyAllowedCountries { for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) { if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
e.check(ExclusivelyAllowedCountriesSetting, e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
fmt.Errorf("%q %w", country, errOnBothLists)) fmt.Errorf("%q %w", country, errOnBothLists))
} }
} }
@@ -1250,15 +1239,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
name string name string
set bool set bool
}{ }{
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0}, {"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders}, {"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)}, {"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
{TagLimitPercentSetting, slices.ContainsFunc( {"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)}, slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0}, {"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0}, {"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100}, {"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""}, {"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0}, {"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0}, {"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0}, {"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -1278,7 +1267,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
// blocklist too: the file at a URL is fetched as one list or the other. // blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *WAF) { func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) { if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check(ASNLimitPercentURLSetting, e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs)) fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
} }
} }
@@ -1305,18 +1294,14 @@ func (e *environment) checkTags(cfg *WAF) {
} }
for tag := range cfg.TagLimitPercent { for tag := range cfg.TagLimitPercent {
e.check(TagLimitPercentSetting, checkTag(cfg, tag)) e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
} }
} }
// checkTag returns an error when tag would never apply, as checkTags says. // checkTag returns an error when tag would never apply, as checkTags says.
func checkTag(cfg *WAF, tag string) error { func checkTag(cfg *WAF, tag string) error {
if !requestCanGetTag(cfg, tag) { if !requestCanGetTag(cfg, tag) {
return fmt.Errorf( return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
"%q %w",
reputation.MaskZoneKey(tag),
errNoRequestGetsTag,
)
} }
return nil return nil
@@ -1324,7 +1309,7 @@ func checkTag(cfg *WAF, tag string) error {
// requestCanGetTag reports whether anything cfg sets up can give a request tag. // requestCanGetTag reports whether anything cfg sets up can give a request tag.
func requestCanGetTag(cfg *WAF, tag string) bool { func requestCanGetTag(cfg *WAF, tag string) bool {
isListName := func(list reputation.TagList) bool { return list.Name == tag } isListName := func(list TagList) bool { return list.Name == tag }
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) { if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
return true return true
} }
@@ -1348,7 +1333,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
return isZoneTag && return isZoneTag &&
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool { slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
return strings.ToLower(reputation.MaskZoneKey(each)) == zone return strings.ToLower(MaskZoneKey(each)) == zone
}) })
} }
@@ -1846,9 +1831,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock) return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
} }
netblock, err := reputation.ParseNetblock( netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
strings.TrimSpace(netblockText),
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -2054,6 +2037,21 @@ func parseCountry(value string) (string, error) {
return country, nil return country, nil
} }
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// The tags a request can get that the settings can name, as waf // The tags a request can get that the settings can name, as waf
// writes them. The prefixes are followed by a value. // writes them. The prefixes are followed by a value.
const ( const (
@@ -2067,17 +2065,24 @@ const (
ExploitSignatureTag = "exploitsignature" ExploitSignatureTag = "exploitsignature"
) )
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// parseTagLists reads a comma-separated list of name=URL pairs, such as // parseTagLists reads a comma-separated list of name=URL pairs, such as
// torexit=https://example.com/tor.txt, and returns each list with its name // torexit=https://example.com/tor.txt, and returns each list with its name
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the // in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
// = before http. A name or a URL listed twice is an error. // = before http. A name or a URL listed twice is an error.
func parseTagLists(value string) ([]reputation.TagList, error) { func parseTagLists(value string) ([]TagList, error) {
items, err := parseList(value) items, err := parseList(value)
if err != nil { if err != nil {
return nil, err return nil, err
} }
lists := make([]reputation.TagList, 0, len(items)) lists := make([]TagList, 0, len(items))
for _, item := range items { for _, item := range items {
end := strings.Index(item, "=http") end := strings.Index(item, "=http")
@@ -2085,12 +2090,9 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
return nil, fmt.Errorf("%q %w", item, errNotTagList) return nil, fmt.Errorf("%q %w", item, errNotTagList)
} }
list := reputation.TagList{ list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
Name: strings.ToLower(item[:end]),
URL: item[end+1:],
}
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool { if slices.ContainsFunc(lists, func(earlier TagList) bool {
return earlier.Name == list.Name || earlier.URL == list.URL return earlier.Name == list.Name || earlier.URL == list.URL
}) { }) {
return nil, fmt.Errorf("%q %w", item, errListedTwice) return nil, fmt.Errorf("%q %w", item, errListedTwice)
@@ -2103,8 +2105,7 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
} }
// parseTags reads a comma-separated list of tags, which parseTag reads. A // parseTags reads a comma-separated list of tags, which parseTag reads. A
// tag listed twice is an error, which shows the tag as // tag listed twice is an error, which shows the tag as MaskZoneKey does.
// reputation.MaskZoneKey does.
func parseTags(value string) ([]string, error) { func parseTags(value string) ([]string, error) {
items, err := parseList(value) items, err := parseList(value)
if err != nil { if err != nil {
@@ -2124,7 +2125,7 @@ func parseTags(value string) ([]string, error) {
if slices.Contains(tags, tag) { if slices.Contains(tags, tag) {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item), MaskZoneKey(item),
errListedTwice, errListedTwice,
) )
} }
@@ -2193,7 +2194,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if colon <= 0 { if colon <= 0 {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item), MaskZoneKey(item),
errNotTagPercentItem, errNotTagPercentItem,
) )
} }
@@ -2207,7 +2208,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if err != nil || percent < 1 || percent > 99 { if err != nil || percent < 1 || percent > 99 {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item[colon+1:]), MaskZoneKey(item[colon+1:]),
errNotTagPercent, errNotTagPercent,
) )
} }
@@ -2219,11 +2220,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if _, listed := percents[tag]; listed { if _, listed := percents[tag]; listed {
// tag, since item[:colon] may end in a space, which MaskZoneKey // tag, since item[:colon] may end in a space, which MaskZoneKey
// would not mask. // would not mask.
return nil, fmt.Errorf( return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
"%q %w",
reputation.MaskZoneKey(tag),
errListedTwice,
)
} }
percents[tag] = percent percents[tag] = percent
@@ -2257,7 +2254,7 @@ func parsePercents(
return nil, err return nil, err
} }
percent, err := reputation.ParsePercent(percentText) percent, err := ParsePercent(percentText)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -2272,6 +2269,38 @@ func parsePercents(
return percents, nil return percents, nil
} }
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// headerNameChars are the characters RFC 9110 allows in a header name:
// letters, digits and these marks.
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
"0123456789!#$%&'*+-.^_`|~"
// IsHeaderName reports whether name can be a header name: one or more of
// the characters RFC 9110 allows in one.
func IsHeaderName(name string) bool {
if name == "" {
return false
}
for _, char := range name {
if !strings.ContainsRune(headerNameChars, char) {
return false
}
}
return true
}
// parseHeaderNames reads a comma-separated list of header names in either // parseHeaderNames reads a comma-separated list of header names in either
// case, and returns them in lower case. Host and Transfer-Encoding are // case, and returns them in lower case. Host and Transfer-Encoding are
// refused: Go's HTTP server takes them out of the request's headers. // refused: Go's HTTP server takes them out of the request's headers.
@@ -2284,7 +2313,7 @@ func parseHeaderNames(value string) ([]string, error) {
headers := make([]string, 0, len(items)) headers := make([]string, 0, len(items))
for _, item := range items { for _, item := range items {
if !httpguts.ValidHeaderFieldName(item) { if !IsHeaderName(item) {
return nil, fmt.Errorf("%q %w", item, errNotHeaderName) return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
} }
@@ -2440,8 +2469,8 @@ const (
// other name for one that does not exist, so that the zone would list no // other name for one that does not exist, so that the zone would list no
// client. A zone listed twice is an error, whatever the case of its // client. A zone listed twice is an error, whatever the case of its
// letters, which DNS names ignore, and whatever its key, since // letters, which DNS names ignore, and whatever its key, since
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a // MaskZoneKey shows two keys of one zone alike. An error shows a zone as
// zone as it does. // MaskZoneKey does.
func parseZones(value string) ([]string, error) { func parseZones(value string) ([]string, error) {
zones, err := parseList(value) zones, err := parseList(value)
if err != nil { if err != nil {
@@ -2451,14 +2480,11 @@ func parseZones(value string) ([]string, error) {
} }
for i, zone := range zones { for i, zone := range zones {
shown := reputation.MaskZoneKey(zone) shown := MaskZoneKey(zone)
listedBefore := slices.ContainsFunc( listedBefore := slices.ContainsFunc(
zones[:i], zones[:i],
func(earlier string) bool { func(earlier string) bool {
return strings.EqualFold( return strings.EqualFold(MaskZoneKey(earlier), shown)
reputation.MaskZoneKey(earlier),
shown,
)
}, },
) )
@@ -2475,6 +2501,24 @@ func parseZones(value string) ([]string, error) {
return zones, nil return zones, nil
} }
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return masked + "." + rest
}
// isZone reports whether each label of zone is as parseZones takes it. It // isZone reports whether each label of zone is as parseZones takes it. It
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too. // also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
func isZone(zone string) bool { func isZone(zone string) bool {
@@ -2530,7 +2574,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
name, headerValue, found := strings.Cut(item, ":") name, headerValue, found := strings.Cut(item, ":")
name = strings.TrimSpace(name) name = strings.TrimSpace(name)
if !found || !httpguts.ValidHeaderFieldName(name) || if !found || !IsHeaderName(name) ||
strings.ContainsAny(headerValue, "\r\n\x00") { strings.ContainsAny(headerValue, "\r\n\x00") {
return nil, "", fmt.Errorf( return nil, "", fmt.Errorf(
"item %d %w", "item %d %w",
+20 -29
View File
@@ -18,12 +18,10 @@ import (
"time" "time"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// The settings, by name. // The settings, by name.
@@ -866,32 +864,6 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
} }
} }
// config and rules check header names with httpguts.ValidHeaderFieldName.
// It takes a name of one or more of the characters RFC 9110 allows in one,
// as the check it replaced did: letters, digits and these marks.
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
t.Parallel()
const marks = "!#$%&'*+-.^_`|~"
if httpguts.ValidHeaderFieldName("") {
t.Error("the empty name is taken")
}
for b := range 256 {
char := byte(b)
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
// for a byte above 0x7f.
name := string([]byte{char})
if got := httpguts.ValidHeaderFieldName(name); got != want {
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
}
}
}
func TestRemoteLogSettingsDefaults(t *testing.T) { func TestRemoteLogSettingsDefaults(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1722,7 +1694,7 @@ func TestTagSettingsAsSet(t *testing.T) {
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99", tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
}) })
wantLists := []reputation.TagList{ wantLists := []config.TagList{
{Name: torExitTag, URL: torURL}, {Name: torExitTag, URL: torURL},
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"}, {Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
} }
@@ -2316,6 +2288,25 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
} }
} }
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := config.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) { func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
t.Parallel() t.Parallel()
+37 -60
View File
@@ -84,7 +84,12 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every // one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use. // client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct { type GeoJS struct {
params Params url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to // httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure. // GeoJS alone: a redirect is a failure.
httpClient *http.Client httpClient *http.Client
@@ -94,13 +99,8 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about, // waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about. // and those it is being asked about.
waiting map[netip.Prefix]*wait waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way, and endAsking // asking is true while a request to GeoJS is under way.
// ends it. stopped is true once StopAsking has been called, and asking bool
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// retryDelay is how long GeoJS is left alone after its last failure, // retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again. // zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration retryDelay time.Duration
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
} }
return &GeoJS{ return &GeoJS{
params: params, url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{ httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client) answer, found := g.kept(client)
if !found { if !found {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
} }
w, waiting := g.waiting[client] w, waiting := g.waiting[client]
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer return answer
} }
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS. // Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock() g.mu.Lock()
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge() g.answers.Purge()
now := g.params.Now() now := g.now()
for _, answer := range answers { for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor { if now.Sub(answer.Answered) < keepFor {
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
} }
if w == nil { if w == nil {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already return Answer{}, nil // too many clients wait already
} }
if !g.asking { if !g.asking {
// GeoJS is left alone after a failure, or asked nothing after // GeoJS is left alone after a failure, so no answer can come.
// StopAsking, so no answer can come.
w.late = true w.late = true
} }
if w.late { if w.late {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
return Answer{}, nil return Answer{}, nil
} }
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago, // kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used. // and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.params.Now() now := g.now()
kept, found := g.answers.Get(client) kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor { if !found || now.Sub(kept.Answered) >= keepFor {
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
} }
// ask starts asking GeoJS about the waiting clients, unless a request to // ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way, it is left alone after a failure, or StopAsking has // it is under way or it is left alone after a failure. The requests to
// been called. The requests to GeoJS are for every client waiting, so they // GeoJS are for every client waiting, so they go on when the caller whose
// go on when the caller whose ctx is given stops waiting, until StopAsking. // ctx is given stops waiting.
func (g *GeoJS) ask(ctx context.Context) { func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) { if g.asking || g.now().Before(g.retryAt) {
return return
} }
askCtx, end := context.WithCancel(context.WithoutCancel(ctx)) g.asking = true
g.asking, g.endAsking = true, end
g.askGoroutines.Go(func() { go g.askAboutWaiting(context.WithoutCancel(ctx))
defer end()
g.askAboutWaiting(askCtx)
})
} }
// askAboutWaiting asks GeoJS about the waiting clients, one request at a // askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each // time, until none is left or GeoJS fails. Each answer kept is given to
// answer kept is given to Answered, outside the lock, since Answered takes // Answered, outside the lock, since Answered takes locks of its own.
// locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) { func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for { for {
clients := g.nextClients() clients := g.nextClients()
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
given, err := g.request(ctx, clients) given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err) kept, answered := g.keep(clients, given, err)
if g.params.Answered != nil { if g.answered != nil {
for _, answer := range kept { for _, answer := range kept {
g.params.Answered(answer) g.answered(answer)
} }
} }
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
// nextClients returns up to maxPerRequest of the waiting clients. When // nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, or StopAsking has been called, it returns none and // none is waiting, it returns none and notes that no request to GeoJS is
// notes that no request to GeoJS is under way. // under way.
func (g *GeoJS) nextClients() []netip.Prefix { func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
if len(g.waiting) == 0 || g.stopped { if len(g.waiting) == 0 {
g.asking = false g.asking = false
return nil return nil
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
now := g.params.Now() now := g.now()
kept := make([]Answer, 0, len(clients)) kept := make([]Answer, 0, len(clients))
leftOut := 0 leftOut := 0
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
w.late = true w.late = true
} }
g.params.Failed(err, g.retryDelay) g.failed(err, g.retryDelay)
return kept, false return kept, false
} }
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
http.MethodGet, http.MethodGet,
g.params.URL, g.url,
http.NoBody, http.NoBody,
) )
if err != nil { if err != nil {
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.params.Asked() g.asked()
res, err := g.httpClient.Do(req) res, err := g.httpClient.Do(req)
if err != nil { if err != nil {
-38
View File
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
service, _, g, reports := startWithReports()
clients := newClients()
service.set(hanging)
g.LookUpWithoutWaiting(t.Context(), clients())
waitForRequests(t, service, 1)
// The request is ended, not waited out, and is no failure.
began := time.Now()
g.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
reports.wantFailures(t)
// A new client is then not asked about, and its lookup does not
// wait.
service.set(answering)
wantCountry(t, g, clients(), "")
if waited := time.Since(began); waited != 0 {
t.Errorf("the lookup returned after %s, want at once", waited)
}
waitForRequests(t, service, 1)
reports.wantFailures(t)
})
}
func TestRequestOver10SecondsIsAFailure(t *testing.T) { func TestRequestOver10SecondsIsAFailure(t *testing.T) {
t.Parallel() t.Parallel()
+3 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -300,7 +301,7 @@ const sourceLabel = "source"
// AddReputation adds the metrics of the lists fetched from URLs and of the // AddReputation adds the metrics of the lists fetched from URLs and of the
// DNSBL zones, by source, each list's URL or each zone, its key masked as // DNSBL zones, by source, each list's URL or each zone, its key masked as
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a // config.MaskZoneKey masks it: the requests whose client a blocklist, a
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, // zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
// and, read from lists and dnsbl as the metrics are asked for, for a list, // and, read from lists and dnsbl as the metrics are asked for, for a list,
// the fetches that failed and when the copy in use was fetched, and for a // the fetches that failed and when the copy in use was fetched, and for a
@@ -319,9 +320,7 @@ func (m *Metrics) AddReputation(
m.registry.MustRegister(m.reputationHits) m.registry.MustRegister(m.reputationHits)
for _, zone := range dnsbl.Zones() { for _, zone := range dnsbl.Zones() {
source := prometheus.Labels{ source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
sourceLabel: reputation.MaskZoneKey(zone),
}
m.addReputationQueries( m.addReputationQueries(
source, source,
-44
View File
@@ -1,44 +0,0 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
-105
View File
@@ -1,105 +0,0 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+4 -7
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) { func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32") other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight() start := midnight()
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for client, requests := range map[string]int{ for client, requests := range map[string]int{
"198.51.100.9/32": 2, "198.51.100.9/32": 2,
+6 -18
View File
@@ -156,28 +156,16 @@ type Request struct {
BlockedByCoreRuleSet bool BlockedByCoreRuleSet bool
} }
// Params are what New needs. // New returns a Limiter for limits, with no client counted yet, whose
type Params struct { // table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
// Limits are what each client's requests and bytes are counted against. // it, the least recently seen client is dropped, with its history, and
Limits Limits // starts afresh if it comes back.
// MaxClients is the most clients the table holds, func New(limits Limits, maxClients int) *Limiter {
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
if err != nil { if err != nil {
panic(err) // NewLRU fails only for a size below one panic(err) // NewLRU fails only for a size below one
} }
limits := params.Limits
return &Limiter{ return &Limiter{
windows: [3]window{ windows: [3]window{
{ {
+39 -63
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight() now := midnight()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
MaxClients: tableSize, tableSize,
}) )
if limiter.OverARateLimit(client, now) { if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit") t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute) wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over. // A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) off := ratelimit.New(ratelimit.Limits{}, tableSize)
off.Count(client, now, whole) off.Count(client, now, whole)
if off.OverARateLimit(client, now) { if off.OverARateLimit(client, now) {
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(tc.limits, tableSize)
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
quarter := tc.length / 4 quarter := tc.length / 4
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit}, ratelimit.Limits{PerMinute: limit, PerHour: limit},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{ ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
PerMinute: 5, tableSize,
BytesPerDay: math.MaxInt64, )
},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel() t.Parallel()
// Only the hour has limits: the minute's and the day's are off. // Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(tc.limits, tableSize)
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it. // 600 bytes are within the limit, 600 more over it.
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32") other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) { ) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit}, ratelimit.Limits{PerMinute: limit, PerDay: limit},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
MaxClients: tableSize, tableSize,
}) )
refused := netip.MustParsePrefix("203.0.113.9/32") refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32") within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3 const maxClients = 3
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
now := midnight() now := midnight()
clients := make([]netip.Prefix, maxClients+1) clients := make([]netip.Prefix, maxClients+1)
+6 -12
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64", "2001:db8::/64",
} }
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for _, i := range []int{2, 3, 0, 1} { for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
} }
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
before := ratelimit.New(ratelimit.Params{ before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
for range limit { for range limit {
wantCount(t, before, client, start, "") wantCount(t, before, client, start, "")
} }
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no // Loaded into a new limiter, as across a restart, the client has no
// fresh allowance. // fresh allowance.
later := start.Add(time.Minute) later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Params{ after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after.Load(before.Snapshot(), later) after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour) wantCount(t, after, client, later, hour)
} }
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
limiter.Count(client, start, whole) limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole) limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit) limiter.CountRefusal(client, start, limit)
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client { loaded := func(now time.Time) ratelimit.Client {
t.Helper() t.Helper()
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) after := ratelimit.New(ratelimit.Limits{}, tableSize)
after.Load(limiter.Snapshot(), now) after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0] return after.Snapshot()[0]
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next() addr = addr.Next()
} }
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients}) limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
limiter.Load(clients, midnight()) limiter.Load(clients, midnight())
got := limiter.Snapshot() got := limiter.Snapshot()
+16 -48
View File
@@ -110,12 +110,8 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved // scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score] scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way, each with the // checking are the clients whose check is under way.
// function that ends it. stopped is true once StopChecking has been checking map[netip.Prefix]bool
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// day is the day, in UTC, of the checks spent counts. // day is the day, in UTC, of the checks spent counts.
day civil.Date day civil.Date
spent int spent int
@@ -138,7 +134,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
scores: scores, scores: scores,
checking: map[netip.Prefix]context.CancelFunc{}, checking: map[netip.Prefix]bool{},
} }
} }
@@ -148,11 +144,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in // addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if // the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under // offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, the day's checks have // way, a check failed less than failureDelay ago, or the day's checks
// used up DailyBudget, or StopChecking has been called; Hit never waits // have used up DailyBudget; Hit never waits for a check. The check that
// for a check. The check that uses the budget up is logged and raised as a // uses the budget up is logged and raised as a source_failure alert. ctx
// source_failure alert. ctx is the context of the client's request, and a // is the context of the client's request, and a check goes on after the
// check goes on after the request ends, until StopChecking. // request ends.
func (a *AbuseIPDB) Hit( func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool, ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) { ) (int64, bool) {
@@ -171,21 +167,14 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0 a.day, a.spent = today, 0
} }
_, checking := a.checking[client] check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
if check { if check {
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx)) a.checking[client] = true
a.checking[client] = end
a.checks++ a.checks++
a.spent++ a.spent++
a.checkGoroutines.Go(func() { go a.check(context.WithoutCancel(ctx), client, addr)
defer end()
a.check(checkCtx, client, addr)
})
} }
usedUp := check && a.spent == a.params.DailyBudget usedUp := check && a.spent == a.params.DailyBudget
@@ -200,24 +189,6 @@ func (a *AbuseIPDB) Hit(
return 0, false return 0, false
} }
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made. // Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int { func (a *AbuseIPDB) Checked() int {
a.mu.Lock() a.mu.Lock()
@@ -344,8 +315,7 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
// check checks client with AbuseIPDB by addr, one of its addresses, keeps // check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A // the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a // check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay. A check // source_failure alert, and no client is checked for failureDelay.
// ended as ctx ends, by StopChecking, is no failure.
func (a *AbuseIPDB) check( func (a *AbuseIPDB) check(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -353,26 +323,24 @@ func (a *AbuseIPDB) check(
) { ) {
score, err := a.ask(ctx, addr) score, err := a.ask(ctx, addr)
now := a.params.Now() now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock() a.mu.Lock()
delete(a.checking, client) delete(a.checking, client)
switch { if err == nil {
case err == nil:
a.scores.Add( a.scores.Add(
client, client,
Score{Client: client, Score: score, Fetched: now}, Score{Client: client, Score: score, Fetched: now},
) )
case isFailure: } else {
a.failures++ a.failures++
a.retryAt = now.Add(failureDelay) a.retryAt = now.Add(failureDelay)
} }
a.mu.Unlock() a.mu.Unlock()
if isFailure { if err != nil {
a.alert("checking a client with AbuseIPDB failed", err) a.alert("checking a client with AbuseIPDB failed", err)
} }
} }
-37
View File
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
}) })
} }
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) { func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel() t.Parallel()
+17 -67
View File
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/golang-lru/v2/simplelru" "github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
const ( const (
@@ -31,8 +32,6 @@ const (
// it fails, and no client is checked with AbuseIPDB after a check // it fails, and no client is checked with AbuseIPDB after a check
// fails, so that a source refusing them is not asked on every request. // fails, so that a source refusing them is not asked on every request.
failureDelay = time.Minute failureDelay = time.Minute
// maskedKey is what MaskZoneKey shows in place of a zone's key.
maskedKey = "********"
) )
var ( var (
@@ -83,12 +82,8 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved // verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict] verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way, each with the function that ends // asking are the queries under way.
// it. stopped is true once StopAsking has been called, and asking map[query]bool
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// queries and failures count, by zone, the queries made and those that // queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be // failed, and retryAt is when a zone whose last query failed may be
// asked again. // asked again.
@@ -125,31 +120,13 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params, params: params,
resolver: resolver, resolver: resolver,
verdicts: verdicts, verdicts: verdicts,
asking: map[query]context.CancelFunc{}, asking: map[query]bool{},
queries: map[string]int{}, queries: map[string]int{},
failures: map[string]int{}, failures: map[string]int{},
retryAt: map[string]time.Time{}, retryAt: map[string]time.Time{},
} }
} }
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return maskedKey + "." + rest
}
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them. // Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
func (d *DNSBL) Zones() []string { func (d *DNSBL) Zones() []string {
return slices.Clone(d.params.Zones) return slices.Clone(d.params.Zones)
@@ -157,14 +134,13 @@ func (d *DNSBL) Zones() []string {
// ListedBy returns the zones whose verdict on addr, a client's address, // ListedBy returns the zones whose verdict on addr, a client's address,
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key // lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
// masked, as MaskZoneKey masks it, since they go to the request // masked, as config.MaskZoneKey masks it, since they go to the request
// log, the alerts and the metrics. A verdict is used until CacheTTL has // log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr // passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the // in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, maxQueries are under way, or // zone is left alone after a failure, or maxQueries are under way;
// StopAsking has been called; ListedBy never waits for a query. ctx is the // ListedBy never waits for a query. ctx is the context of the client's
// context of the client's request, and a query goes on after the request // request, and a query goes on after the request ends.
// ends, until StopAsking.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string { func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock() d.mu.Lock()
defer d.mu.Unlock() defer d.mu.Unlock()
@@ -177,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr} q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q) kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch { switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL: case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed { if kept.Listed {
listedBy = append(listedBy, MaskZoneKey(zone)) listedBy = append(listedBy, config.MaskZoneKey(zone))
} }
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) && case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
len(d.asking) < maxQueries: d.asking[q] = true
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
d.queries[zone]++ d.queries[zone]++
d.queryGoroutines.Go(func() { go d.ask(context.WithoutCancel(ctx), q)
defer end()
d.ask(queryCtx, q)
})
} }
} }
return listedBy return listedBy
} }
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone. // Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int { func (d *DNSBL) Queries(zone string) int {
d.mu.Lock() d.mu.Lock()
@@ -286,32 +238,30 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it // query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the // is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for // zone with its key masked, and the zone is not asked again for
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure. // failureDelay.
func (d *DNSBL) ask(ctx context.Context, q query) { func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q) listed, err := d.lookUp(ctx, q)
now := d.params.Now() now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock() d.mu.Lock()
delete(d.asking, q) delete(d.asking, q)
switch { if err == nil {
case err == nil:
d.verdicts.Add(q, Verdict{ d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now, Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
}) })
case isFailure: } else {
d.failures[q.zone]++ d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay) d.retryAt[q.zone] = now.Add(failureDelay)
} }
d.mu.Unlock() d.mu.Unlock()
if isFailure { if err != nil {
const failed = "asking a DNSBL zone failed" const failed = "asking a DNSBL zone failed"
shown := MaskZoneKey(q.zone) shown := config.MaskZoneKey(q.zone)
// Raised before it is logged, so that the alert is there once the // Raised before it is logged, so that the alert is there once the
// log line is. // log line is.
-64
View File
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
}) })
} }
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
// //
//nolint:paralleltest // one at a time, as the comment at the top of this file says //nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute( func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
@@ -860,28 +821,3 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
t.Errorf("asked about %v, want %v", got, want) t.Errorf("asked about %v, want %v", got, want)
} }
} }
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
const (
dronebl = "dnsbl.dronebl.org"
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
spamhausMasked = "********.xbl.dq.spamhaus.net"
)
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := reputation.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
+11 -77
View File
@@ -7,10 +7,8 @@
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent // clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to files and reads them // today. The state package writes all of these to files and reads them
// from there, so that a restart keeps them too. It fetches AbuseIPDB's // from there, so that a restart keeps them too. Last, it fetches
// blacklist, which it does not keep. Last, it reads the netblocks, AS // AbuseIPDB's blacklist, which it does not keep.
// numbers and percentages that both the settings and the lists hold, and
// masks the keys of DNSBL zones.
package reputation package reputation
import ( import (
@@ -22,12 +20,12 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"slices" "slices"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
const ( const (
@@ -42,28 +40,15 @@ const (
) )
var ( var (
errStatus = errors.New("the server answered") errStatus = errors.New("the server answered")
errTooLong = errors.New("the list is longer than 16 MiB") errTooLong = errors.New("the list is longer than 16 MiB")
errNotNetblockLine = errors.New( errNotNetblock = errors.New(
"is not an address or a netblock, such as 192.0.2.0/24", "is not an address or a netblock, such as 192.0.2.0/24",
) )
errNotASNPercent = errors.New( errNotASNPercent = errors.New(
"is not an AS number, : and a percentage, such as AS64496:50") "is not an AS number, : and a percentage, such as AS64496:50")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
) )
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// List is a list as lists.json holds it: the URL it is fetched from, // List is a list as lists.json holds it: the URL it is fetched from,
// when it was last tried, the fetch failed or not, and its last good copy: // when it was last tried, the fetch failed or not, and its last good copy:
// when that was fetched, and its lines, as fetched, comment lines // when that was fetched, and its lines, as fetched, comment lines
@@ -83,7 +68,7 @@ type Params struct {
BlocklistURLs []string BlocklistURLs []string
ASNLimitPercentURL string ASNLimitPercentURL string
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS). // TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
TagLists []TagList TagLists []config.TagList
// Refresh is how long after a list was last fetched or tried it is // Refresh is how long after a list was last fetched or tried it is
// fetched again (SWW_WAF_BLOCKLIST_REFRESH). // fetched again (SWW_WAF_BLOCKLIST_REFRESH).
Refresh time.Duration Refresh time.Duration
@@ -480,11 +465,7 @@ func parseNetblocks(lines []string) (entries, error) {
netblock, ok := parseNetblock(text) netblock, ok := parseNetblock(text)
if !ok { if !ok {
return entries{}, fmt.Errorf( return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
"line %d %w",
i+1,
errNotNetblockLine,
)
} }
found.netblocks[netblock] = true found.netblocks[netblock] = true
@@ -504,7 +485,7 @@ func parseNetblocks(lines []string) (entries, error) {
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which // stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
// stands for none, is refused. // stands for none, is refused.
func parseNetblock(text string) (netip.Prefix, bool) { func parseNetblock(text string) (netip.Prefix, bool) {
netblock, err := ParseNetblock(text) netblock, err := config.ParseNetblock(text)
if err != nil { if err != nil {
return netip.Prefix{}, false return netip.Prefix{}, false
} }
@@ -539,9 +520,9 @@ func parsePercents(lines []string) (entries, error) {
} }
asnText, percentText, _ := strings.Cut(text, ":") asnText, percentText, _ := strings.Cut(text, ":")
asn, asnErr := ParseASN(asnText) asn, asnErr := config.ParseASN(asnText)
percent, percentErr := ParsePercent(percentText) percent, percentErr := config.ParsePercent(percentText)
if asnErr != nil || percentErr != nil { if asnErr != nil || percentErr != nil {
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent) return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
} }
@@ -555,53 +536,6 @@ func parsePercents(lines []string) (entries, error) {
return found, nil return found, nil
} }
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// withoutComment returns line without anything after a ; or a #, and // withoutComment returns line without anything after a ; or a #, and
// without the spaces around what is left. // without the spaces around what is left.
func withoutComment(line string) string { func withoutComment(line string) string {
+33 -4
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the request log sww prints on stdout: JSON // Package requestlog writes the lines sww prints on stdout: one
// lines marked "type":"request", at most two from waf and a short one // JSON object per request, two from waf and a short one from rproxy,
// from rproxy for each request. Package processlog writes the process's // marked "type":"request", and the process's own messages as JSON lines marked
// own messages. // "type":"process".
package requestlog package requestlog
import ( import (
@@ -9,6 +9,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log/slog"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond, time.Millisecond/time.Microsecond,
) )
} }
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
+87
View File
@@ -3,6 +3,8 @@ package requestlog_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"log/slog"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want) t.Errorf("wrote %s, want %s", out.String(), want)
} }
} }
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+12 -43
View File
@@ -99,12 +99,8 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up, // answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped. // so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer] answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way, each with the // asking are the clients whose query is under way.
// function that ends it. stopped is true once StopAsking has been asking map[netip.Prefix]bool
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// retryAt is when routewatch may be asked again after a query failed. // retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time retryAt time.Time
} }
@@ -120,7 +116,7 @@ func New(params Params) *Client {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
answers: answers, answers: answers,
asking: map[netip.Prefix]context.CancelFunc{}, asking: map[netip.Prefix]bool{},
} }
} }
@@ -128,9 +124,8 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client // IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its // without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than // request came from, unless its query is under way, a query failed less than
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx // failureKept ago, or maxQueries are under way. ctx is the context of the
// is the context of the client's request, and a query goes on after the request // client's request, and a query goes on after the request ends.
// ends, until StopAsking.
func (c *Client) Route( func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr, ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) { ) (Route, Missing) {
@@ -152,38 +147,15 @@ func (c *Client) Route(
return Route{}, RequestFailed return Route{}, RequestFailed
} }
_, asking := c.asking[client] if !c.asking[client] && len(c.asking) < maxQueries {
if !asking && !c.stopped && len(c.asking) < maxQueries { c.asking[client] = true
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
c.queryGoroutines.Go(func() { go c.ask(context.WithoutCancel(ctx), client, addr)
defer end()
c.ask(queryCtx, client, addr)
})
} }
return Route{}, NoAnswerYet return Route{}, NoAnswerYet
} }
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as // Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them. // routewatch.json lists them.
func (c *Client) Snapshot() []Answer { func (c *Client) Snapshot() []Answer {
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the // ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and // query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of // routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged. A query ended as ctx ends, by StopAsking, is no // those under way is logged.
// failure.
func (c *Client) ask( func (c *Client) ask(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -242,20 +213,18 @@ func (c *Client) ask(
) { ) {
route, err := c.query(ctx, addr) route, err := c.query(ctx, addr)
now := c.params.Now() now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock() c.mu.Lock()
delete(c.asking, client) delete(c.asking, client)
firstFailure := isFailure && !now.Before(c.retryAt) firstFailure := err != nil && !now.Before(c.retryAt)
switch { if err == nil {
case err == nil:
c.answers.Add(client, Answer{ c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now, Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
}) })
case isFailure: } else {
c.retryAt = now.Add(failureKept) c.retryAt = now.Add(failureKept)
} }
-48
View File
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) { func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel() t.Parallel()
-6
View File
@@ -2,12 +2,6 @@ package rproxy
import "time" import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time. // Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct { type Breaker struct {
b breaker b breaker
+10 -19
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: io.Discard, io.Discard,
InstanceName: "test", "test",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
@@ -870,14 +870,10 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) { func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
t.Parallel() t.Parallel()
// rproxy is waiting for the rest of the body well within this limit, even on
// a busy host, and answers when it runs out, well before dial's deadline.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, greet) app := startApp(t, greet)
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "3000",
}) })
conn := dial(t, addr) conn := dial(t, addr)
@@ -1266,9 +1262,7 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
) { ) {
t.Parallel() t.Parallel()
// The app's answer arrives well within this limit, even on a busy host, and const limit = 2 * time.Second
// the connection closes when it runs out, well before dial's deadline.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, answerBeforeTheBody) app := startApp(t, answerBeforeTheBody)
@@ -1433,20 +1427,17 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
const answer = "the end of a long answer" const answer = "the end of a long answer"
// The request reaches the app well within this limit, even on a busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, func(w http.ResponseWriter, r *http.Request) { app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body) _, _ = io.Copy(io.Discard, r.Body)
_ = http.NewResponseController(w).Flush() _ = http.NewResponseController(w).Flush()
time.Sleep(limit + 100*time.Millisecond) time.Sleep(2100 * time.Millisecond)
_, _ = io.WriteString(w, answer) _, _ = io.WriteString(w, answer)
}) })
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "2000",
}) })
got := visit( got := visit(
+2 -5
View File
@@ -94,7 +94,6 @@ func New(params Params) *Server {
reportURL: (&url.URL{ reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(), }).String(),
breakerClock: time.Now,
} }
return &Server{ return &Server{
@@ -134,10 +133,8 @@ type handler struct {
wafTransport http.RoundTripper wafTransport http.RoundTripper
// reportURL is where the reports go. // reportURL is where the reports go.
reportURL string reportURL string
// breaker is the circuit breaker for asks to waf, and breakerClock gives it // breaker is the circuit breaker for asks to waf.
// the time. breaker breaker
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one // reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another. // against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup reports sync.WaitGroup
+21 -54
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -356,19 +356,6 @@ func startServerLogging(
) (string, *rproxy.Server) { ) (string, *rproxy.Server) {
t.Helper() t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{ settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000", wafBackendTimeout: "60000",
@@ -380,20 +367,15 @@ func newServer(
t.Fatalf("settings %v: %v", settings, err) t.Fatalf("settings %v: %v", settings, err)
} }
return rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: processLog, processLog,
InstanceName: "test", "test",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
t.Context(), t.Context(),
@@ -414,7 +396,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
server.WaitForReports() server.WaitForReports()
}) })
return listener.Addr().String() return listener.Addr().String(), server
} }
// answered is the visitor's answer. // answered is the visitor's answer.
@@ -1452,18 +1434,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
greet(w, r) greet(w, r)
}) })
out := &output{} addr, out := startRProxy(
server := newServer( t,
t, standIn.URL, app.URL, standIn.URL,
map[string]string{wafBackendTimeout: "200"}, app.URL,
out, out, map[string]string{
wafBackendTimeout: "200",
},
) )
// The breaker's clock stands still, as in
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
// failures could take more than five seconds, and the next request could
// come more than a second later.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
// An ask can time out before it reaches the stand-in, so these // An ask can time out before it reaches the stand-in, so these
// may leave fewer than five questions, or leave some late. // may leave fewer than five questions, or leave some late.
@@ -1554,13 +1532,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r) tooLarge(w, r)
}) })
app := startApp(t, greet) app := startApp(t, greet)
out := &output{} addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
server := newServer(t, standIn.URL, app.URL, nil, out, out) wafBackendTimeout: "60000",
// The breaker's clock stands still. On a slow host, the five failures could })
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 { for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway { if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
@@ -1835,15 +1809,11 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) { func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
t.Parallel() t.Parallel()
// The app's answer to the upgrade arrives well within these limits, even on a
// busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, echoAfterUpgrade) app := startApp(t, echoAfterUpgrade)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "2000",
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10), maxResponseTime: "2000",
}) })
conn := dial(t, addr) conn := dial(t, addr)
@@ -1868,10 +1838,7 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
} }
// Past both time limits, the connection still carries what is sent. // Past both time limits, the connection still carries what is sent.
time.Sleep(limit + 100*time.Millisecond) time.Sleep(2100 * time.Millisecond)
// dial's deadline ran from before the upgrade and this wait.
_ = conn.SetDeadline(time.Now().Add(waitLimit))
const message = "still here\n" const message = "still here\n"
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"maps" "maps"
"net/http" "net/http"
"strings" "strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -96,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context, ctx context.Context,
) (*wafAnswer, error) { ) (*wafAnswer, error) {
b := &rq.h.breaker b := &rq.h.breaker
if !b.mayAsk(rq.h.breakerClock()) { if !b.mayAsk(time.Now()) {
return nil, errBreakerOpen return nil, errBreakerOpen
} }
@@ -106,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil: case err == nil:
b.noteAnswer() b.noteAnswer()
case ctx.Err() == nil: case ctx.Err() == nil:
b.noteFailure(rq.h.breakerClock()) b.noteFailure(time.Now())
} }
return answer, err return answer, err
+2 -2
View File
@@ -21,9 +21,9 @@ import (
"time" "time"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
// The actions a rule takes when it matches. // The actions a rule takes when it matches.
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
rule.Target, rule.Target,
errNotTarget, errNotTarget,
) )
case isHeader && !httpguts.ValidHeaderFieldName(headerName): case isHeader && !config.IsHeaderName(headerName):
return Rule{}, false, fmt.Errorf( return Rule{}, false, fmt.Errorf(
"the target %q %w", "the target %q %w",
rule.Target, rule.Target,
+1 -1
View File
@@ -2452,7 +2452,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour, AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000, MaxBans: 5000,
}), }),
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}), Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}), GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{ Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{ BlocklistURLs: []string{
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
) )
@@ -23,11 +23,11 @@ import (
// setting for either. // setting for either.
func RunRProxy(ctx context.Context, params Params) int { func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := processlog.New(processlog.Params{ processLog := requestlog.NewProcessLogger(
Output: params.Stdout, params.Stdout,
InstanceName: hostname, hostname,
Level: slog.LevelInfo, slog.LevelInfo,
}) )
cfg, err := config.RProxyFromEnvironment(params.Viper) cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil { if err != nil {
+7 -4
View File
@@ -755,7 +755,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
} }
} }
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. // sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper() t.Helper()
@@ -768,6 +769,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close() _ = conn.Close()
}) })
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request) _, err = io.WriteString(conn, request)
if err != nil { if err != nil {
t.Fatalf("send: %v", err) t.Fatalf("send: %v", err)
@@ -832,9 +838,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop() stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn) text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 { if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err) t.Errorf("read %q (%v), want the connection cut off", text, err)
+30 -52
View File
@@ -10,14 +10,13 @@ package sww
import ( import (
"context" "context"
"errors" "fmt"
"io" "io"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
"github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
@@ -27,7 +26,14 @@ import (
// know. // know.
const usageStatus = 2 const usageStatus = 2
var errNoCommand = errors.New("no command given") // usage is what sww prints then.
const usage = `usage: sww <command>
commands:
rproxy the reverse proxy in front of one app
waf decides for rproxy, and keeps bans, counters and history
healthcheck the container's health check
`
// Params are what RunWAF and RunRProxy need from the process. // Params are what RunWAF and RunRProxy need from the process.
type Params struct { type Params struct {
@@ -45,9 +51,14 @@ type Params struct {
// Main runs the command os.Args names, and returns the process's exit status. // Main runs the command os.Args names, and returns the process's exit status.
// `sww rproxy` and `sww waf` run until SIGTERM or SIGINT. // `sww rproxy` and `sww waf` run until SIGTERM or SIGINT.
// With no command, or one it does not know, cobra prints the error and the // With no command, or one it does not know, it prints the commands and returns
// commands to stderr, and Main returns usageStatus. // usageStatus.
func Main(version string) int { func Main(version string) int {
args := os.Args[1:]
if len(args) == 0 {
return printUsage(os.Stderr)
}
ctx, stop := signal.NotifyContext(context.Background(), ctx, stop := signal.NotifyContext(context.Background(),
syscall.SIGTERM, os.Interrupt) syscall.SIGTERM, os.Interrupt)
defer stop() defer stop()
@@ -58,54 +69,21 @@ func Main(version string) int {
Stdout: os.Stdout, Stdout: os.Stdout,
} }
// status is what the command that ran returns. switch args[0] {
status := 0 case "rproxy":
return RunRProxy(ctx, params)
root := &cobra.Command{ case "waf":
Use: "sww", return RunWAF(ctx, params)
Short: "a web application firewall", case "healthcheck":
Args: cobra.NoArgs, return HealthCheck(ctx, args[1:], params.Viper, os.Stderr)
RunE: func(*cobra.Command, []string) error { default:
return errNoCommand return printUsage(os.Stderr)
},
} }
root.CompletionOptions.DisableDefaultCmd = true }
root.AddCommand( // printUsage writes the commands to w, and returns usageStatus.
&cobra.Command{ func printUsage(w io.Writer) int {
Use: "rproxy", _, _ = fmt.Fprint(w, usage)
Short: "the reverse proxy in front of one app",
Run: func(cmd *cobra.Command, _ []string) {
status = RunRProxy(cmd.Context(), params)
},
},
&cobra.Command{
Use: "waf",
Short: "decides for rproxy, and keeps bans, counters and history",
Run: func(cmd *cobra.Command, _ []string) {
status = RunWAF(cmd.Context(), params)
},
},
&cobra.Command{
Use: "healthcheck",
Short: "the container's health check",
// HealthCheck refuses every argument itself, flags included.
DisableFlagParsing: true,
Run: func(cmd *cobra.Command, args []string) {
status = HealthCheck(
cmd.Context(),
args,
params.Viper,
cmd.ErrOrStderr(),
)
},
},
)
err := root.ExecuteContext(ctx) return usageStatus
if err != nil {
return usageStatus
}
return status
} }
+28 -58
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state" "sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,11 +43,8 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int { func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's // Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through. // error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := processlog.New(processlog.Params{ processLog := requestlog.NewProcessLogger(params.Stdout,
Output: params.Stdout, config.InstanceName(params.Viper), slog.LevelError)
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
cfg, err := config.WAFFromEnvironment(params.Viper) cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil { if err != nil {
@@ -66,11 +63,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote) stdout = io.MultiWriter(params.Stdout, remote)
} }
processLog = processlog.New(processlog.Params{ processLog = requestlog.NewProcessLogger(
Output: stdout, stdout,
InstanceName: cfg.InstanceName, cfg.InstanceName,
Level: cfg.LogLevel, cfg.LogLevel,
}) )
if remote != nil { if remote != nil {
stopSending := startSending(ctx, remote, processLog) stopSending := startSending(ctx, remote, processLog)
@@ -121,8 +118,7 @@ func RunWAF(ctx context.Context, params Params) int {
"settings", cfg) "settings", cfg)
// The last write follows the stop of every hook, and so the end of every // The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert, and of every ask of // loop that changes the state or raises an alert. A hook of the state
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// files would stop before the loops of the parts built ahead of them. // files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error { return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll() err := files.WriteAll()
@@ -242,17 +238,14 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits. // newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter { func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Params{ return ratelimit.New(ratelimit.Limits{
Limits: ratelimit.Limits{ PerMinute: cfg.RateLimitPerMinute,
PerMinute: cfg.RateLimitPerMinute, PerHour: cfg.RateLimitPerHour,
PerHour: cfg.RateLimitPerHour, PerDay: cfg.RateLimitPerDay,
PerDay: cfg.RateLimitPerDay, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerMinute: cfg.BytesLimitPerMinute, BytesPerHour: cfg.BytesLimitPerHour,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerDay: cfg.BytesLimitPerDay,
BytesPerDay: cfg.BytesLimitPerDay, }, cfg.MaxTrackedClients)
},
MaxClients: cfg.MaxTrackedClients,
})
} }
// newLedger returns the ledger of the bans, with the settings for them. // newLedger returns the ledger of the bans, with the settings for them.
@@ -304,13 +297,11 @@ func newLists(
return lists return lists
} }
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it // newDNSBL returns what asks the DNSBL zones, none asked yet.
// ends the queries under way, as reputation.DNSBL.StopAsking says.
func newDNSBL( func newDNSBL(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
alertQueue *alerts.Queue,
) *reputation.DNSBL { ) *reputation.DNSBL {
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{ return reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones, Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver, Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL, CacheTTL: cfg.IPReputationCacheTTL,
@@ -319,17 +310,12 @@ func newDNSBL(
ProcessLog: processLog, ProcessLog: processLog,
Alerts: alertQueue, Alerts: alertQueue,
}) })
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
} }
// newAbuseIPDB returns what checks clients with AbuseIPDB while // newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it // SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in // fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks // ledger, as waf.BanAbuseIPDBBlacklist says.
// under way, as reputation.AbuseIPDB.StopChecking says.
func newAbuseIPDB( func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger, alertQueue *alerts.Queue, ledger *bans.Ledger,
@@ -355,30 +341,24 @@ func newAbuseIPDB(
}) })
lc.Append(background(abuseIPDB.RunBlacklist)) lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB return abuseIPDB
} }
// newRouteWatch returns what asks routewatch about clients, or nil while // newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under // SWW_WAF_ROUTEWATCH_URL is unset.
// way, as routewatch.Client.StopAsking says.
func newRouteWatch( func newRouteWatch(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client { ) *routewatch.Client {
if cfg.RouteWatchURL == "" { if cfg.RouteWatchURL == "" {
return nil return nil
} }
routeWatch := routewatch.New(routewatch.Params{ return routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL, URL: cfg.RouteWatchURL,
Now: utcNow, Now: utcNow,
ProcessLog: processLog, ProcessLog: processLog,
}) })
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
} }
// newMetrics returns the metrics, with those read from the other parts as // newMetrics returns the metrics, with those read from the other parts as
@@ -416,18 +396,12 @@ func newMetrics(
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each // newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each // answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request // failure is handled as waf.GeoJSFailed says.
// under way, as geojs.GeoJS.StopAsking says.
func newGeoJS( func newGeoJS(
lc fx.Lifecycle, cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
cfg *config.WAF, ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
) *geojs.GeoJS { ) *geojs.GeoJS {
geoJS := geojs.New(geojs.Params{ return geojs.New(geojs.Params{
URL: geojs.URL, URL: geojs.URL,
Answered: func(answer geojs.Answer) { Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer) waf.AddLookup(cfg, limiter, ledger, answer)
@@ -439,10 +413,6 @@ func newGeoJS(
}, },
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc, WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
}) })
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
} }
// newWAFServer returns the server waf runs, with its parts and the time // newWAFServer returns the server waf runs, with its parts and the time
+29 -88
View File
@@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"context" "context"
"crypto/rand" "crypto/rand"
"errors"
"io" "io"
"net/http" "net/http"
"os" "os"
@@ -165,104 +164,46 @@ func wantUTC(t *testing.T, where string, value any) {
func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) { func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
t.Parallel() t.Parallel()
// completion is cobra's own command, which sww turns off.
for _, command := range []string{"", "serve", "completion"} {
t.Run("command "+command, func(t *testing.T) {
t.Parallel()
code, stdout, stderr := runMain(t, command)
if code != 2 {
t.Errorf("exit status %d, want 2", code)
}
wantTheCommands(t, "stderr", stderr)
if stdout != "" {
t.Errorf("stdout %q, want nothing", stdout)
}
})
}
}
func TestMainHelpPrintsTheCommands(t *testing.T) {
t.Parallel()
for _, command := range []string{"help", "--help"} {
t.Run("command "+command, func(t *testing.T) {
t.Parallel()
code, stdout, _ := runMain(t, command)
if code != 0 {
t.Errorf("exit status %d, want 0", code)
}
wantTheCommands(t, "stdout", stdout)
})
}
}
// wantTheCommands checks that text, what Main wrote to where, names each
// command.
func wantTheCommands(t *testing.T, where, text string) {
t.Helper()
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
if !strings.Contains(text, name) {
t.Errorf("%s %q does not name %s", where, text, name)
}
}
}
func TestMainHealthCheckRefusesAnArgument(t *testing.T) {
t.Parallel()
for _, argument := range []string{"now", "--x", "-h"} {
t.Run("argument "+argument, func(t *testing.T) {
t.Parallel()
code, stdout, stderr := runMain(t, "healthcheck "+argument)
want := "sww healthcheck: unexpected argument \"" +
argument + "\"\n"
if code != 1 || stderr != want || stdout != "" {
t.Errorf("exit status %d, stdout %q and stderr %q, "+
"want 1, nothing and %q", code, stdout, stderr, want)
}
})
}
}
// runMain runs Main with command in a process of its own, and returns its
// exit status and what it wrote to stdout and stderr.
func runMain(t *testing.T, command string) (int, string, string) {
t.Helper()
binary, err := os.Executable() binary, err := os.Executable()
if err != nil { if err != nil {
t.Fatalf("find the test binary: %v", err) t.Fatalf("find the test binary: %v", err)
} }
ctx, cancel := context.WithTimeout(t.Context(), waitLimit) for _, command := range []string{"", "serve"} {
defer cancel() t.Run("command "+command, func(t *testing.T) {
t.Parallel()
var stdout, stderr bytes.Buffer ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
defer cancel()
//nolint:gosec // this test binary var stdout, stderr bytes.Buffer
cmd := exec.CommandContext(ctx, binary)
cmd.Stdout, cmd.Stderr = &stdout, &stderr
cmd.Env = append(os.Environ(), commandForMain+"="+command) cmd := exec.CommandContext(ctx, binary)
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err = cmd.Run() cmd.Env = append(os.Environ(), commandForMain+"="+command)
var exitErr *exec.ExitError err := cmd.Run()
if err != nil && !errors.As(err, &exitErr) {
t.Fatalf("run: %v", err) if code := cmd.ProcessState.ExitCode(); code != 2 {
t.Errorf("exit status %d (%v), want 2", code, err)
}
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
if !strings.Contains(stderr.String(), name) {
t.Errorf(
"stderr %q does not name %s",
stderr.String(),
name,
)
}
}
if stdout.Len() != 0 {
t.Errorf("stdout %q, want nothing", stdout.String())
}
})
} }
return cmd.ProcessState.ExitCode(), stdout.String(), stderr.String()
} }
// ask sends waf at addr rproxy's question about a request for path from // ask sends waf at addr rproxy's question about a request for path from
+9 -20
View File
@@ -22,42 +22,31 @@ import (
// an offence, and the request does not wait for the answer. The score is then // an offence, and the request does not wait for the answer. The score is then
// used for each address of the client. Its figures are the score and // used for each address of the client. Its figures are the score and
// SWW_WAF_ABUSEIPDB_MIN_SCORE. // SWW_WAF_ABUSEIPDB_MIN_SCORE.
type abuseIPDB struct { type abuseIPDB struct{}
abuseIPDB *reputation.AbuseIPDB
// limiter is the table of clients the rate limits count, whose histories
// say which clients are offenders.
limiter *ratelimit.Limiter
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
hitNoter reputationHitNoter
}
func (abuseIPDB) name() string { return "abuseipdb" } func (abuseIPDB) name() string { return "abuseipdb" }
func (a abuseIPDB) check( func (abuseIPDB) check(
ctx context.Context, ctx context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
if !a.keySet { if rq.h.config.AbuseIPDBKey == "" {
return nil return nil
} }
client := clientGroup(a.ipv6GroupPrefix, rq.client) client := clientGroup(rq.h.config, rq.client)
held, _ := a.limiter.Client(client) held, _ := rq.h.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{} offender := held.History.Offences != ratelimit.Offences{}
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender) score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
if !hit { if !hit {
return nil return nil
} }
rq.abuseIPDBHit = true rq.abuseIPDBHit = true
rq.addTag(config.AbuseIPDBTag) rq.addTag(config.AbuseIPDBTag)
a.hitNoter.noteHit( rq.noteHit(
rq,
bans.ReputationHit{ bans.ReputationHit{
Source: reputation.AbuseIPDBSource, Source: reputation.AbuseIPDBSource,
Score: &score, Score: &score,
@@ -65,14 +54,14 @@ func (a abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE", "scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
) )
if a.action != config.ReputationActionDeny { if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
return nil return nil
} }
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
figures: map[string]any{ figures: map[string]any{
figureScore: score, "min_score": a.minScore, figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
}, },
} }
} }
+1 -3
View File
@@ -26,9 +26,7 @@ func BanAbuseIPDBBlacklist(
scores := make(map[netip.Prefix]int64, len(listed)) scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed { for _, client := range listed {
netblock := banNetblock( netblock := banNetblock(cfg, client.Addr)
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
)
scores[netblock] = max(scores[netblock], client.Score) scores[netblock] = max(scores[netblock], client.Score)
} }
+2 -8
View File
@@ -224,11 +224,7 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
} }
return banNetblock( return banNetblock(h.config, addr), nil
h.config.BanScopeV4Prefix,
h.config.IPv6GroupPrefix,
addr,
), nil
} }
// expiry returns when a ban made at now for duration ends: duration // expiry returns when a ban made at now for duration ends: duration
@@ -299,9 +295,7 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)), Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
} }
client, seen := rq.h.limiter.Client( client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
)
if seen { if seen {
answer.Client = &client answer.Client = &client
} }
+61 -105
View File
@@ -1,7 +1,7 @@
package waf package waf
// The heuristic active_ban, and banMaker, which makes the bans of several // The heuristic active_ban, and the code that makes bans, which several
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. // heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
import ( import (
"context" "context"
@@ -11,7 +11,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -26,22 +25,18 @@ func (rq *request) banResponse(action string) *refusal {
// in, in force at now. It notes for the log line when that ban ends. A request // in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert // that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends. // for it. Its figures are the ban's netblock and when it ends.
type activeBan struct { type activeBan struct{}
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" } func (activeBan) name() string { return "active_ban" }
func (a activeBan) check( func (activeBan) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
ledger := a.banMaker.ledger check := rq.h.ledger.Check
if rq.h.config.Observe {
check := ledger.Check check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
} }
ban, banned, madePermanent := check(rq.client, now) ban, banned, madePermanent := check(rq.client, now)
@@ -51,7 +46,7 @@ func (a activeBan) check(
if madePermanent { if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
a.banMaker.alertBan(rq, ban) rq.alertBan(ban)
} }
if !banned { if !banned {
@@ -86,12 +81,11 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
// countedBytes returns the request's bytes, once it has ended, as the // countedBytes returns the request's bytes, once it has ended, as the
// byte limits and the anomaly thresholds count them: the response's body // byte limits and the anomaly thresholds count them: the response's body
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED, // bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
// says. func (rq *request) countedBytes() int64 {
func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
response, request := rq.out.bytes, rq.requestBytes response, request := rq.out.bytes, rq.requestBytes
switch bytesCounted { switch rq.h.config.BytesCounted {
case config.BytesCountedResponse: case config.BytesCountedResponse:
return response return response
case config.BytesCountedRequest: case config.BytesCountedRequest:
@@ -102,29 +96,7 @@ func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
return response + request return response + request
} }
// banMaker makes the bans of the heuristics that ban a client, in the ban // banForLimit bans the client's netblock at now for a broken limit, the
// ledger, and raises the alert for each. In observe mode it makes no ban, and
// raises the alert for the ban it would have made.
type banMaker struct {
ledger *bans.Ledger
// limiter is the table of clients the rate limits count. A ban for a limit
// sets the client's counters back to zero, and a ban's notes give how many
// requests its netblock has sent.
limiter *ratelimit.Limiter
metrics *metrics.Metrics
alerts *alerts.Queue
// observe is true in observe mode (SWW_WAF_MODE).
observe bool
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
// clear sign of attack give.
banResponse int
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
banScopeV4Prefix int
ipv6GroupPrefix int
}
// banForLimit bans the netblock of rq's client at now for a broken limit, the
// one hit names, notes the offence for the log line and counts the hit in // one hit names, notes the offence for the log line and counts the hit in
// the metrics. status is what the client was sent, or is sent: // the metrics. status is what the client was sent, or is sent:
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for // SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
@@ -133,8 +105,7 @@ type banMaker struct {
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets // rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what // the client's counters back to zero. In observe mode, banClient says what
// happens instead. // happens instead.
func (b banMaker) banForLimit( func (rq *request) banForLimit(
rq *request,
now time.Time, now time.Time,
hit ratelimit.Hit, hit ratelimit.Hit,
status int, status int,
@@ -149,8 +120,7 @@ func (b banMaker) banForLimit(
} }
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
b.metrics.LimitHit(hit)
notes := bans.Notes{ notes := bans.Notes{
Kind: hit.Kind, Kind: hit.Kind,
@@ -166,94 +136,81 @@ func (b banMaker) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
} }
b.banClient( rq.banClient(
rq,
now, now,
status, status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes}, bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
) )
} }
// banForAttack bans the netblock of rq's client at now for a clear sign of // banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path // attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient // asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead. // says what happens instead.
func (b banMaker) banForAttack( func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
rq *request, now time.Time, notes bans.Notes, return rq.banClient(
) string {
return b.banClient(
rq,
now, now,
b.banResponse, rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes}, bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
) )
} }
// banClient bans the netblock of rq's client at now on grounds, and returns // banClient bans the client's netblock at now on grounds, and returns the
// the action for that ban. It adds to the notes what the request tells of the // action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit // client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero. // sets the client's counters back to zero.
// //
// In observe mode it makes no ban and sets nothing back. It raises the // In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent. // alert for the ban it would have made, if that alert would be sent.
func (b banMaker) banClient( func (rq *request) banClient(
rq *request,
now time.Time, now time.Time,
status int, status int,
grounds bans.Grounds, grounds bans.Grounds,
) string { ) string {
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client) netblock := banNetblock(rq.h.config, rq.client)
if b.observe { if rq.h.config.Observe {
if !b.wouldAlertBan(netblock, now, grounds.Cause) { if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction( return banAction(
b.ledger.WouldBePermanent(netblock, now, grounds.Cause), rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
) )
} }
grounds.Notes = b.withClientNotes( grounds.Notes = rq.withClientNotes(
rq,
grounds.Notes, grounds.Notes,
now, now,
status, status,
netblock, netblock,
) )
ban, wouldBan := b.ledger.WouldBan(netblock, now, grounds) ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
if wouldBan { if wouldBan {
b.alertBan(rq, ban) rq.alertBan(ban)
} }
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
grounds.Notes = b.withClientNotes( grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
rq,
grounds.Notes,
now,
status,
netblock,
)
ban, made := b.ledger.Ban(netblock, now, grounds) ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit { if grounds.Cause == bans.CauseLimit {
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client)) rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
} }
rq.line.BanExpires = banExpires(ban) rq.line.BanExpires = banExpires(ban)
if made { if made {
b.alertBan(rq, ban) rq.alertBan(ban)
} }
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
// withClientNotes is notes with what rq tells of its client added: its AS, // withClientNotes is notes with what the request tells of the client
// country and reputation, the request at now with status, and how many // added: its AS, country and reputation, the request at now with status,
// requests netblock, the client's, has sent. // and how many requests netblock, the client's, has sent.
func (b banMaker) withClientNotes( func (rq *request) withClientNotes(
rq *request,
notes bans.Notes, notes bans.Notes,
now time.Time, now time.Time,
status int, status int,
@@ -264,7 +221,7 @@ func (b banMaker) withClientNotes(
notes.Country = rq.line.Country notes.Country = rq.line.Country
notes.Reputation = rq.reputation notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status) notes.Request = rq.noted(now, status)
notes.Requests = b.netblockRequests(netblock) notes.Requests = rq.netblockRequests(netblock)
return notes return notes
} }
@@ -274,22 +231,23 @@ func (b banMaker) withClientNotes(
// have made is worked out only then, at most once per // have made is worked out only then, at most once per
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the // SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
// netblock's requests, which can mean going through every client. // netblock's requests, which can mean going through every client.
func (b banMaker) wouldAlertBan( func (rq *request) wouldAlertBan(
netblock netip.Prefix, now time.Time, cause string, netblock netip.Prefix, now time.Time, cause string,
) bool { ) bool {
event := alerts.EventBan event := alerts.EventBan
if b.ledger.WouldBePermanent(netblock, now, cause) { if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
event = alerts.EventPermanentBan event = alerts.EventPermanentBan
} }
return b.alerts.WouldSend(event, netblock) return rq.h.alerts.WouldSend(event, netblock)
} }
// alertBan raises the alert for ban, which rq made, or made permanent: // alertBan raises the alert for ban, which the request made, or made
// permanent_ban for a permanent ban, ban for another. Its detail gives the // permanent: permanent_ban for a permanent ban, ban for another. Its
// ban's cause, when it ends, and its notes, and in observe mode, where ban is // detail gives the ban's cause, when it ends, and its notes, and in
// the ban that would have been made, or made permanent, mode, observe. // observe mode, where ban is the ban that would have been made, or made
func (b banMaker) alertBan(rq *request, ban bans.Ban) { // permanent, mode, observe.
func (rq *request) alertBan(ban bans.Ban) {
event := alerts.EventBan event := alerts.EventBan
if ban.Permanent() { if ban.Permanent() {
event = alerts.EventPermanentBan event = alerts.EventPermanentBan
@@ -298,11 +256,11 @@ func (b banMaker) alertBan(rq *request, ban bans.Ban) {
detail := map[string]any{ detail := map[string]any{
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes, "cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
} }
if b.observe { if rq.h.config.Observe {
detail["mode"] = "observe" detail["mode"] = "observe"
} }
b.alerts.Raise(alerts.Alert{ rq.h.alerts.Raise(alerts.Alert{
Event: event, Event: event,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
@@ -315,13 +273,6 @@ func (b banMaker) alertBan(rq *request, ban bans.Ban) {
}) })
} }
// netblockRequests is how many requests netblock has sent since it was
// first seen, the request being banned for included: the histories count it
// only once it has ended.
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
return b.limiter.Requests(netblock) + 1
}
// noted is the request, at now, with status, what the client was sent, or // noted is the request, at now, with status, what the client was sent, or
// in observe mode would have been, as the notes of the ban it makes keep // in observe mode would have been, as the notes of the ban it makes keep
// it. // it.
@@ -336,18 +287,23 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
} }
} }
// netblockRequests is how many requests netblock has sent since it was
// first seen, this one included: the histories count it only once it has
// ended.
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// banNetblock is the netblock a ban on client covers: its IPv4 address, // banNetblock is the netblock a ban on client covers: its IPv4 address,
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group // widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// clientGroup counts it in, of length ipv6GroupPrefix. // counts it in.
func banNetblock( func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
) netip.Prefix {
addr := client.Unmap() addr := client.Unmap()
if addr.Is4() { if addr.Is4() {
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked() return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
} }
return clientGroup(ipv6GroupPrefix, addr) return clientGroup(cfg, addr)
} }
// banExpires is when ban ends, as the log line gives it: a time, or // banExpires is when ban ends, as the log line gives it: a time, or
+5 -4
View File
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
// body is sent as it is after the headers, header holding its // body is sent as it is after the headers, header holding its
// Content-Length or Transfer-Encoding. header may hold several lines, // Content-Length or Transfer-Encoding. header may hold several lines,
// separated by "\r\n". It returns the whole answer. // separated by "\r\n". It returns the whole answer.
//
// It waits for the answer as long as it takes, since a busy host can slow
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
// whose answer never comes.
func (s *sender) requestWithBody( func (s *sender) requestWithBody(
method, from, path, header, body string, status int, action string, method, from, path, header, body string, status int, action string,
) (logLine, answer) { ) (logLine, answer) {
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+ send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body) forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
if err != nil {
s.t.Fatalf("set read deadline: %v", err)
}
var got answer var got answer
res, err := http.ReadResponse(bufio.NewReader(conn), nil) res, err := http.ReadResponse(bufio.NewReader(conn), nil)
+12 -12
View File
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
) *verdict { ) *verdict {
var running atomic.Pointer[string] var running atomic.Pointer[string]
running.Store(new(rq.h.trapPaths.name())) running.Store(new(trapPaths{}.name()))
found := make(chan *besideLookupFindings, 1) found := make(chan *besideLookupFindings, 1)
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
return v return v
} }
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now()) return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
} }
// findBesideLookup does the work of the heuristics of besideLookup on the // findBesideLookup does the work of the heuristics of besideLookup on the
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
) *besideLookupFindings { ) *besideLookupFindings {
findings := &besideLookupFindings{} findings := &besideLookupFindings{}
findings.trapPath = rq.h.trapPaths.trapPath(rq) findings.trapPath = rq.trapPath()
if findings.trapPath != "" { if findings.trapPath != "" {
findings.refuses = true findings.refuses = true
return findings return findings
} }
running.Store(new(rq.h.ruleFiles.name())) running.Store(new(ruleFiles{}.name()))
findings.rules = rq.h.ruleFiles.rules.Match(rq.in) findings.rules = rq.h.rules.Match(rq.in)
if refusedByRule(findings.rules) { if refusedByRule(findings.rules) {
findings.refuses = true findings.refuses = true
return findings return findings
} }
coreRuleSet := rq.h.coreRuleSet running.Store(new(coreRuleSet{}.name()))
running.Store(new(coreRuleSet.name()))
if coreRuleSet.mode == config.CoreRuleSetModeOff || cfg := rq.h.config
pathExempt(rq.in, coreRuleSet.exemptPaths) { if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
return findings return findings
} }
start := time.Now() start := time.Now()
// A question's body has been read into memory, so that reading it here // A question's body has been read into memory, so that reading it here
// cannot fail. // cannot fail.
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect( findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
rq.in, rq.client, rq.in.Body, rq.in, rq.client, rq.in.Body,
) )
findings.inspectionTime = time.Since(start) findings.inspectionTime = time.Since(start)
findings.inspected = true findings.inspected = true
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection) findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
return findings return findings
} }
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
) *verdict { ) *verdict {
rq.findings = findings rq.findings = findings
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now()) return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
} }
// timedOut refuses the request for the heuristics of besideLookup not having // timedOut refuses the request for the heuristics of besideLookup not having
+22 -43
View File
@@ -2,7 +2,6 @@ package waf
import ( import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// whole is the percentage of each limit a client gets when no biased // whole is the percentage of each limit a client gets when no biased
@@ -30,25 +29,6 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
cfg.ASNLimitPercentURL != "" cfg.ASNLimitPercentURL != ""
} }
// biasedThresholds are the settings and the lists by which limitPercentages
// lowers a client's limits.
type biasedThresholds struct {
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
lists *reputation.Lists
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
// and its percentage.
blocklistAction config.ReputationAction
blocklistLimitPercent int64
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
dnsblAbuseIPDBAction config.ReputationAction
dnsblAbuseIPDBLimitPercent int64
}
// limitPercentages returns the client's limit percentages, for the rate // limitPercentages returns the client's limit percentages, for the rate
// limits and for the byte limits, by its tags, its AS number and country // limits and for the byte limits, by its tags, its AS number and country
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and // as looked up, each "" when unknown, and the blocklists, DNSBL zones and
@@ -65,50 +45,49 @@ type biasedThresholds struct {
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and // the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
// an AS number or a country they list. // an AS number or a country they list.
func (b biasedThresholds) limitPercentages( func (rq *request) limitPercentages() (percentage, percentage) {
rq *request, cfg := rq.h.config
) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country asn, country := rq.line.ASN, rq.line.Country
unknown := percentage{percent: whole} unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait { if country == "" && !rq.skippedLookupWait {
unknown = percentage{ unknown = percentage{
b.unknownCountryLimitPercent, cfg.UnknownCountryLimitPercent,
config.UnknownCountryLimitPercentSetting, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
} }
} }
fetched := percentage{percent: whole} fetched := percentage{percent: whole}
if percent, listed := b.lists.ASNLimitPercent(asn); listed { if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, config.ASNLimitPercentURLSetting} fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
} }
blocklisted := percentage{percent: whole} blocklisted := percentage{percent: whole}
if rq.blocklisted && if rq.blocklisted &&
b.blocklistAction == config.ReputationActionLimit { cfg.BlocklistAction == config.ReputationActionLimit {
blocklisted = percentage{ blocklisted = percentage{
b.blocklistLimitPercent, cfg.BlocklistLimitPercent,
config.BlocklistActionSetting, "SWW_WAF_BLOCKLIST_ACTION",
} }
} }
dnsblAbuseIPDBListed := percentage{percent: whole} dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) && if (rq.dnsblListed || rq.abuseIPDBHit) &&
b.dnsblAbuseIPDBAction == config.ReputationActionLimit { cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{ dnsblAbuseIPDBListed = percentage{
b.dnsblAbuseIPDBLimitPercent, cfg.DNSBLAbuseIPDBLimitPercent,
config.DNSBLAbuseIPDBActionSetting, "SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
} }
} }
const tagSetting = config.TagLimitPercentSetting const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
asnRequests := lowest( asnRequests := lowest(
given(b.tagLimitPercent, asnTag(asn), tagSetting), given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
fetched, fetched,
) )
countryRequests := given( countryRequests := given(
b.tagLimitPercent, cfg.TagLimitPercent,
countryTag(country), countryTag(country),
tagSetting, tagSetting,
) )
@@ -119,25 +98,25 @@ func (b biasedThresholds) limitPercentages(
if tag != asnTag(asn) && tag != countryTag(country) { if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest( otherTags = lowest(
otherTags, otherTags,
given(b.tagLimitPercent, tag, tagSetting), given(cfg.TagLimitPercent, tag, tagSetting),
) )
} }
} }
asnBytes, countryBytes := asnRequests, countryRequests asnBytes, countryBytes := asnRequests, countryRequests
if _, listed := b.asnBytesLimitPercent[asn]; listed { if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
asnBytes = given( asnBytes = given(
b.asnBytesLimitPercent, cfg.ASNBytesLimitPercent,
asn, asn,
config.ASNBytesLimitPercentSetting, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
) )
} }
if _, listed := b.countryBytesLimitPercent[country]; listed { if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
countryBytes = given( countryBytes = given(
b.countryBytesLimitPercent, cfg.CountryBytesLimitPercent,
country, country,
config.CountryBytesLimitPercentSetting, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
) )
} }
+8 -12
View File
@@ -8,7 +8,6 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -16,24 +15,20 @@ import (
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it // and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
// lowers the client's limits instead (see limitPercentages), and being log, it // lowers the client's limits instead (see limitPercentages), and being log, it
// does nothing more. Its figure is the blocklists that list the client. // does nothing more. Its figure is the blocklists that list the client.
type blocklists struct { type blocklists struct{}
lists *reputation.Lists
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
hitNoter reputationHitNoter
}
func (blocklists) name() string { return "blocklists" } func (blocklists) name() string { return "blocklists" }
func (b blocklists) check( func (blocklists) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
listedBy := b.lists.ListedBy(rq.client) listedBy := rq.h.lists.ListedBy(rq.client)
rq.blocklisted = len(listedBy) > 0 rq.blocklisted = len(listedBy) > 0
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist") rq.noteListed(listedBy, "listed by a blocklist")
if !b.refusedByBlocklists(listedBy) { if !rq.refusedByBlocklists(listedBy) {
return nil return nil
} }
@@ -46,6 +41,7 @@ func (b blocklists) check(
// refusedByBlocklists reports whether the heuristic blocklists refuses a // refusedByBlocklists reports whether the heuristic blocklists refuses a
// client, given listedBy, the blocklists that list it: it does when one lists // client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny. // it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (b blocklists) refusedByBlocklists(listedBy []string) bool { func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && b.action == config.ReputationActionDeny return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
} }
+6 -14
View File
@@ -6,9 +6,6 @@ package waf
import ( import (
"context" "context"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
) )
// byteLimits counts the request's bytes, as countedBytes gives them, for the // byteLimits counts the request's bytes, as countedBytes gives them, for the
@@ -19,16 +16,11 @@ import (
// the client over a byte limit, as its limit percentage for the byte limits // the client over a byte limit, as its limit percentage for the byte limits
// lowers it, break it, which bans the client; the response was passed on whole. // lowers it, break it, which bans the client; the response was passed on whole.
// Its figures are the window, the count and the limit. // Its figures are the window, the count and the limit.
type byteLimits struct { type byteLimits struct{}
limiter *ratelimit.Limiter
bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (byteLimits) name() string { return "byte_limits" } func (byteLimits) name() string { return "byte_limits" }
func (b byteLimits) check( func (byteLimits) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -37,10 +29,10 @@ func (b byteLimits) check(
return nil return nil
} }
counts, hit, over := b.limiter.CountBytes( counts, hit, over := rq.h.limiter.CountBytes(
clientGroup(b.ipv6GroupPrefix, rq.client), clientGroup(rq.h.config, rq.client),
now, now,
rq.countedBytes(b.bytesCounted), rq.countedBytes(),
rq.bytesLimitPercent.percent, rq.bytesLimitPercent.percent,
) )
rq.line.Counts.MinuteBytes = counts.MinuteBytes rq.line.Counts.MinuteBytes = counts.MinuteBytes
@@ -51,7 +43,7 @@ func (b byteLimits) check(
return nil return nil
} }
b.banMaker.banForLimit(rq, now, hit, rq.out.status) rq.banForLimit(now, hit, rq.out.status)
return &verdict{figures: limitFigures(hit)} return &verdict{figures: limitFigures(hit)}
} }
+6 -5
View File
@@ -5,6 +5,8 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"slices" "slices"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
// peerAddress is the address of the request's TCP peer. // peerAddress is the address of the request's TCP peer.
@@ -43,13 +45,12 @@ func scheme(r *http.Request, isQuestion bool) string {
// clientGroup is the client a request is counted toward: its IPv4 // clientGroup is the client a request is counted toward: its IPv4
// address, or its IPv6 group, the netblock its IPv6 address is in of the // address, or its IPv6 group, the netblock its IPv6 address is in of the
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since // length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as // usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
// IPv4. func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
addr = addr.Unmap() addr = addr.Unmap()
if addr.Is6() { if addr.Is6() {
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked() return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
} }
return netip.PrefixFrom(addr, addr.BitLen()) return netip.PrefixFrom(addr, addr.BitLen())
+18 -30
View File
@@ -10,7 +10,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -23,22 +22,11 @@ import (
// exploitsignature, raises the core_rule_set_match alert, and in block mode // exploitsignature, raises the core_rule_set_match alert, and in block mode
// refuses the request with 403, which is an offence its client's history // refuses the request with 403, which is an offence its client's history
// counts. Its figures are the score and the threshold. // counts. Its figures are the score and the threshold.
type coreRuleSet struct { type coreRuleSet struct{}
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
// off.
inspector CoreRuleSetInspector
metrics *metrics.Metrics
alerts *alerts.Queue
mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
observe bool // SWW_WAF_MODE is observe
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (coreRuleSet) name() string { return "core_rule_set" } func (coreRuleSet) name() string { return "core_rule_set" }
func (c coreRuleSet) check( func (coreRuleSet) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
@@ -48,6 +36,7 @@ func (c coreRuleSet) check(
return nil return nil
} }
cfg := rq.h.config
result := findings.inspection result := findings.inspection
rq.requestBytes = int64(len(findings.read)) rq.requestBytes = int64(len(findings.read))
@@ -58,18 +47,18 @@ func (c coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs { for _, id := range result.RuleIDs {
c.metrics.CoreRuleSetMatched(string(c.mode), id) rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
} }
threshold := c.scoreThreshold threshold := cfg.CoreRuleSetScoreThreshold
if threshold == 0 || result.Score < threshold { if threshold == 0 || result.Score < threshold {
return nil return nil
} }
rq.addTag(config.ExploitSignatureTag) rq.addTag(config.ExploitSignatureTag)
c.alertCoreRuleSetMatch(rq, result) rq.alertCoreRuleSetMatch(result)
if !c.refusedByCoreRuleSet(result) { if !rq.refusedByCoreRuleSet(result) {
return nil return nil
} }
@@ -87,21 +76,20 @@ func (c coreRuleSet) check(
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core // refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
// Rule Set found result of: whether the score is at or over // Rule Set found result of: whether the score is at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode. // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool { func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
threshold := c.scoreThreshold cfg := rq.h.config
threshold := cfg.CoreRuleSetScoreThreshold
return c.mode == config.CoreRuleSetModeBlock && return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
threshold != 0 && result.Score >= threshold threshold != 0 && result.Score >= threshold
} }
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the // alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
// Core Rule Set scored at result, at or over // which the Core Rule Set scored at result, at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
// score, the method and the path with the query, and, for a request that is // score, the method and the path with the query, and, for a request that is
// not refused for it, the mode: detect, or observe in observe mode. // not refused for it, the mode: detect, or observe in observe mode.
func (c coreRuleSet) alertCoreRuleSetMatch( func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
rq *request, result coreruleset.Result,
) {
detail := map[string]any{ detail := map[string]any{
"rule_ids": result.RuleIDs, "rule_ids": result.RuleIDs,
"score": result.Score, "score": result.Score,
@@ -110,17 +98,17 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
} }
switch { switch {
case c.mode == config.CoreRuleSetModeDetect: case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect) detail["mode"] = string(config.CoreRuleSetModeDetect)
case c.observe: case rq.h.config.Observe:
detail["mode"] = "observe" detail["mode"] = "observe"
} }
c.alerts.Raise(alerts.Alert{ rq.h.alerts.Raise(alerts.Alert{
Event: alerts.EventCoreRuleSetMatch, Event: alerts.EventCoreRuleSetMatch,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), Netblock: clientGroup(rq.h.config, rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+4 -8
View File
@@ -7,7 +7,6 @@ import (
"slices" "slices"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -15,18 +14,16 @@ import (
// does not hold the client's country as it was looked up. A client without a // does not hold the client's country as it was looked up. A client without a
// country, or whose country cannot be found, is refused. Its figures are the // country, or whose country cannot be found, is refused. Its figures are the
// country and the setting that refused it. // country and the setting that refused it.
type countries struct { type countries struct{}
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
}
func (countries) name() string { return "countries" } func (countries) name() string { return "countries" }
func (c countries) check( func (countries) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
allowed := c.allowed allowed := rq.h.config.ExclusivelyAllowedCountries
country := rq.line.Country country := rq.line.Country
if len(allowed) == 0 || slices.Contains(allowed, country) { if len(allowed) == 0 || slices.Contains(allowed, country) {
@@ -36,8 +33,7 @@ func (c countries) check(
return &verdict{ return &verdict{
action: requestlog.ActionCountryDenied, action: requestlog.ActionCountryDenied,
figures: map[string]any{ figures: map[string]any{
"country": country, "country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
"setting": config.ExclusivelyAllowedCountriesSetting,
}, },
} }
} }
+3 -10
View File
@@ -4,7 +4,6 @@ package waf
import ( import (
"context" "context"
"net/netip"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -12,18 +11,12 @@ import (
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of // denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
// the setting the client is in. // the setting the client is in.
type denyNets struct { type denyNets struct{}
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
}
func (denyNets) name() string { return "deny_nets" } func (denyNets) name() string { return "deny_nets" }
func (d denyNets) check( func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
_ context.Context, for _, netblock := range rq.h.config.DenyNets {
rq *request,
_ time.Time,
) *verdict {
for _, netblock := range d.netblocks {
if netblock.Contains(rq.client) { if netblock.Contains(rq.client) {
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
-47
View File
@@ -1,47 +0,0 @@
package waf
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
t.Parallel()
heuristic := denyNets{
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
}
// The request has no handler: the heuristic reads only what it holds.
inside := &request{
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
}
v := heuristic.check(t.Context(), inside, time.Time{})
if v == nil {
t.Fatal("a client inside the netblock was let through")
}
if v.action != requestlog.ActionClientDenied {
t.Errorf(
"action = %q, want %q",
v.action,
requestlog.ActionClientDenied,
)
}
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
}
outside := &request{
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
}
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
t.Errorf("a client outside the netblock was refused: %+v", v)
}
}
+6 -14
View File
@@ -9,7 +9,6 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -20,28 +19,21 @@ import (
// being log, it does nothing more. A zone without a verdict on the client is // being log, it does nothing more. A zone without a verdict on the client is
// asked about it in the background, and the request does not wait for the // asked about it in the background, and the request does not wait for the
// answer. Its figure is the zones that list the client, their keys masked. // answer. Its figure is the zones that list the client, their keys masked.
type dnsbl struct { type dnsbl struct{}
dnsbl *reputation.DNSBL
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
hitNoter reputationHitNoter
}
func (dnsbl) name() string { return "dnsbl" } func (dnsbl) name() string { return "dnsbl" }
func (d dnsbl) check( func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
ctx context.Context, listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
rq *request,
_ time.Time,
) *verdict {
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
rq.dnsblListed = len(listedBy) > 0 rq.dnsblListed = len(listedBy) > 0
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone") rq.noteListed(listedBy, "listed by a DNSBL zone")
for _, zone := range listedBy { for _, zone := range listedBy {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
} }
if !rq.dnsblListed || d.action != config.ReputationActionDeny { if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
return nil return nil
} }
+77 -4
View File
@@ -5,10 +5,9 @@ import (
"time" "time"
) )
// To add a heuristic, write one file with a struct that implements heuristic // To add a heuristic, write one file with a type that implements heuristic,
// and holds the settings and parts it reads. Add one line for it to one of the // and add one line for it to one of the lists below. Its settings go in
// lists that newHandler builds, which hands it those settings and parts. Its // config.WAF, and the comment at the top of its file names them.
// settings go in config.WAF, and the comment at the top of its file names them.
// heuristic is one of the checks that decide what waf does with a // heuristic is one of the checks that decide what waf does with a
// request. // request.
@@ -37,6 +36,80 @@ const (
figureNetblock = "netblock" figureNetblock = "netblock"
) )
// The lists of heuristics, each in the order its heuristics run. A client in
// SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number and
// country are looked up, so that a client they refuse is not looked up.
// header_limits comes first: it only measures and counts, and a request it
// refuses causes no other work.
func beforeLookup() []heuristic {
return []heuristic{
headerLimits{},
tagLists{},
routeWatchTags{},
denyNets{},
activeBan{},
}
}
// afterLookup are the heuristics that run once the lookup has been made.
func afterLookup() []heuristic {
return []heuristic{
countries{},
blocklists{},
dnsbl{},
abuseIPDB{},
tagDeny{},
rateLimits{},
}
}
// besideLookup are the heuristics that only read the request. Their work runs
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
// of the request. Their check then applies what that work found, and runs only
// once waf takes it.
func besideLookup() []heuristic {
return []heuristic{
trapPaths{},
ruleFiles{},
coreRuleSet{},
}
}
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second time,
// on the tag rule_files and core_rule_set can add.
func afterBesideLookup() []heuristic {
return []heuristic{
tagDeny{},
}
}
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client, and do
// not change what the request got: their verdict's action is not used.
func afterAnswer() []heuristic {
return []heuristic{
refusalLimit{},
}
}
// onReport are the heuristics that run on an allowed request when rproxy's
// report of it comes, before its report line is written. Like afterAnswer, they
// only ban.
func onReport() []heuristic {
return []heuristic{
byteLimits{},
}
}
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
}
// runHeuristics runs list, one of those checkClient runs, on the request at now, // runHeuristics runs list, one of those checkClient runs, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's // and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its // heuristic and heuristic_figures name the heuristic that gave it, and give its
+3 -5
View File
@@ -56,7 +56,7 @@ func (rq *request) startLookUp(
return result return result
} }
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client) client := clientGroup(rq.h.config, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile { if rq.h.config.LookupSource == config.LookupSourceFile {
result <- lookupResult{ result <- lookupResult{
@@ -143,10 +143,8 @@ func AddLookup(
) { ) {
limiter.AddLookup(answer.Client, answer.Answered, limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country) answer.ASN, answer.ASName, answer.Country)
netblock := banNetblock( ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(), answer.ASN, answer.ASName, answer.Country)
)
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
} }
// GeoJSFailed logs a failure of GeoJS to processLog, raises a // GeoJSFailed logs a failure of GeoJS to processLog, raises a
+31 -168
View File
@@ -133,19 +133,15 @@ type Server struct {
} }
// newHandler returns the handler with what every request shares, the parts // newHandler returns the handler with what every request shares, the parts
// in params among it. It builds the heuristics, each with the settings and // in params among it.
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler { func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil { if err != nil {
panic(err) // New fails only for a size below one panic(err) // New fails only for a size below one
} }
cfg := params.Config return &handler{
h := &handler{ config: params.Config,
config: cfg,
requestLog: params.RequestLog, requestLog: params.RequestLog,
processLog: params.ProcessLog, processLog: params.ProcessLog,
now: params.Now, now: params.Now,
@@ -156,142 +152,47 @@ func newHandler(params Params) *handler {
geojs: params.GeoJS, geojs: params.GeoJS,
anomalies: params.Anomalies, anomalies: params.Anomalies,
lookupFile: params.LookupFile, lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn= // The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before // or country= tag and the biased thresholds act on the answer before
// the request goes on. // the request goes on.
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 || waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
cfg.AddLookupHeaders || params.Config.AddLookupHeaders ||
cfg.TagsNeedLookup() || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(cfg), biasedThresholdsSet(params.Config),
besideLookupLimit: cmp.Or( besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit, params.BesideLookupTimeLimit,
besideLookupTimeLimit, besideLookupTimeLimit,
), ),
} }
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.blocklists = blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
}
h.rateLimits = rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
h.blocklists,
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
h.rateLimits,
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
return h
} }
// handler is waf's handler. It holds what every request shares; what // handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request. // belongs to one request is in a request.
type handler struct { type handler struct {
config *config.WAF config *config.WAF
requestLog io.Writer requestLog io.Writer
processLog *slog.Logger processLog *slog.Logger
now func() time.Time now func() time.Time
metrics *metrics.Metrics metrics *metrics.Metrics
limiter *ratelimit.Limiter limiter *ratelimit.Limiter
ledger *bans.Ledger ledger *bans.Ledger
geojs *geojs.GeoJS geojs *geojs.GeoJS
anomalies *anomaly.Counters anomalies *anomaly.Counters
lookupFile *lookup.File lookupFile *lookup.File
lists *reputation.Lists
dnsbl *reputation.DNSBL
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet CoreRuleSetInspector
alerts *alerts.Queue
// waitForGeoJS is true when a setting needs each request's answer from // waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on. // GeoJS before the request goes on.
waitForGeoJS bool waitForGeoJS bool
@@ -301,42 +202,4 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id // awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled. // until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision] awaitingReport *lru.Cache[string, *decision]
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of the lists that other steps of a request use too:
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
blocklists blocklists
rateLimits rateLimits
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
} }
+24 -24
View File
@@ -24,7 +24,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -194,11 +193,11 @@ func (o *output) requestLine(t *testing.T) logLine {
// endedLines waits for count requests that went through rproxy, or to waf's // endedLines waits for count requests that went through rproxy, or to waf's
// own endpoints, to have ended, and returns the last log line of each, in the // own endpoints, to have ended, and returns the last log line of each, in the
// order waf took them. A question's last line is the one its report writes. // order waf took them. A question's last line is the one its report writes.
// It waits as long as it takes, as requestWithBody does.
func (o *output) endedLines(t *testing.T, count int) []logLine { func (o *output) endedLines(t *testing.T, count int) []logLine {
t.Helper() t.Helper()
for { deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
found := o.lastLines(t) found := o.lastLines(t)
ended := len(found) >= count ended := len(found) >= count
@@ -214,6 +213,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
time.Sleep(pollInterval) time.Sleep(pollInterval)
} }
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
return nil
} }
// lastLines returns the last request log line written so far of each request, // lastLines returns the last request log line written so far of each request,
@@ -395,11 +398,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: io.Discard, io.Discard,
InstanceName: "rproxy", "rproxy",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
@@ -468,11 +471,11 @@ func newWAFWith(
} }
out := &output{} out := &output{}
processLog := processlog.New(processlog.Params{ processLog := requestlog.NewProcessLogger(
Output: out, out,
InstanceName: cfg.InstanceName, cfg.InstanceName,
Level: cfg.LogLevel, cfg.LogLevel,
}) )
ruleFiles, err := rules.Load(rules.Params{ ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -558,17 +561,14 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs, cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName, cfg.InstanceName,
) )
params.Limiter = ratelimit.New(ratelimit.Params{ params.Limiter = ratelimit.New(ratelimit.Limits{
Limits: ratelimit.Limits{ PerMinute: cfg.RateLimitPerMinute,
PerMinute: cfg.RateLimitPerMinute, PerHour: cfg.RateLimitPerHour,
PerHour: cfg.RateLimitPerHour, PerDay: cfg.RateLimitPerDay,
PerDay: cfg.RateLimitPerDay, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerMinute: cfg.BytesLimitPerMinute, BytesPerHour: cfg.BytesLimitPerHour,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerDay: cfg.BytesLimitPerDay,
BytesPerDay: cfg.BytesLimitPerDay, }, cfg.MaxTrackedClients)
},
MaxClients: cfg.MaxTrackedClients,
})
params.Ledger = bans.New(bans.Params{ params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration, LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
+22 -34
View File
@@ -6,10 +6,8 @@ package waf
import ( import (
"context" "context"
"net/netip"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -20,37 +18,28 @@ import (
// refused, and bans the client. A request exempt from the rate limits is exempt // refused, and bans the client. A request exempt from the rate limits is exempt
// from the byte limits too. Its figures are the window, the count and the // from the byte limits too. Its figures are the window, the count and the
// limit. // limit.
type rateLimits struct { type rateLimits struct{}
limiter *ratelimit.Limiter
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
biasedThresholds biasedThresholds
banMaker banMaker
}
func (rateLimits) name() string { return "rate_limits" } func (rateLimits) name() string { return "rate_limits" }
func (r rateLimits) check( func (rateLimits) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
rq.counted = r.countedByRateLimits(rq) rq.counted = rq.countedByRateLimits()
if !rq.counted { if !rq.counted {
return nil return nil
} }
rq.rateLimitPercent, rq.bytesLimitPercent = rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
r.biasedThresholds.limitPercentages(rq)
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting = rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
rq.rateLimitPercent.logged() rq.rateLimitPercent.logged()
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged() rq.bytesLimitPercent.logged()
counts, hit, over := r.limiter.Count( counts, hit, over := rq.h.limiter.Count(
clientGroup(r.ipv6GroupPrefix, rq.client), clientGroup(rq.h.config, rq.client),
now, now,
rq.rateLimitPercent.percent, rq.rateLimitPercent.percent,
) )
@@ -60,29 +49,28 @@ func (r rateLimits) check(
return nil return nil
} }
r.banMaker.banForLimit(rq, now, hit, r.banResponse) rq.banForLimit(now, hit, rq.h.config.BanResponse)
return &verdict{ return &verdict{
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit), action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
} }
} }
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq // overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
// at now, whatever its client's limit percentages: the rate limits count it, // request at now, whatever its client's limit percentages: the rate limits
// and one more request from its client takes it over a whole rate limit. It // count it, and one more request from its client takes it over a whole rate
// counts nothing. // limit. It counts nothing.
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool { func (rq *request) overAWholeRateLimit(now time.Time) bool {
return r.countedByRateLimits(rq) && return rq.countedByRateLimits() &&
r.limiter.OverARateLimit( rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
)
} }
// countedByRateLimits reports whether the rate limits count rq: its client is // countedByRateLimits reports whether the rate limits count the request: its
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under // client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. // under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (r rateLimits) countedByRateLimits(rq *request) bool { func (rq *request) countedByRateLimits() bool {
return !isInside(rq.client, r.exemptNets) && cfg := rq.h.config
!pathExempt(rq.in, r.exemptPaths)
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
} }
+9 -15
View File
@@ -8,7 +8,6 @@ import (
"net/http" "net/http"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -19,22 +18,17 @@ import (
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the // a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
// heuristics skip, is not counted, and nothing is while the limit is off. // heuristics skip, is not counted, and nothing is while the limit is off.
// Its figures are the window, the count and the limit. // Its figures are the window, the count and the limit.
type refusalLimit struct { type refusalLimit struct{}
limiter *ratelimit.Limiter
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (refusalLimit) name() string { return "refusal_limit" } func (refusalLimit) name() string { return "refusal_limit" }
func (r refusalLimit) check( func (refusalLimit) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
if r.perMinute == 0 { cfg := rq.h.config
if cfg.RefusalsLimitPerMinute == 0 {
return nil return nil
} }
@@ -42,10 +36,10 @@ func (r refusalLimit) check(
return nil return nil
} }
hit, over := r.limiter.CountRefusal( hit, over := rq.h.limiter.CountRefusal(
clientGroup(r.ipv6GroupPrefix, rq.client), clientGroup(rq.h.config, rq.client),
now, now,
r.perMinute, cfg.RefusalsLimitPerMinute,
) )
if !over { if !over {
return nil return nil
@@ -60,10 +54,10 @@ func (r refusalLimit) check(
status = http.StatusForbidden status = http.StatusForbidden
case requestlog.ActionTemporarilyBanned, case requestlog.ActionTemporarilyBanned,
requestlog.ActionPermanentlyBanned: requestlog.ActionPermanentlyBanned:
status = r.banResponse status = cfg.BanResponse
} }
r.banMaker.banForLimit(rq, now, hit, status) rq.banForLimit(now, hit, status)
return &verdict{figures: limitFigures(hit)} return &verdict{figures: limitFigures(hit)}
} }
+8 -21
View File
@@ -5,35 +5,22 @@ package waf
import ( import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/metrics"
) )
// reputationHitNoter notes the reputation sources that list a request's
// client, for the reputation heuristics.
type reputationHitNoter struct {
metrics *metrics.Metrics
alerts *alerts.Queue
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
// noteListed notes each of sources, the URLs of the blocklists or the // noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list rq's client, as noteHit does, // DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason. // with reason.
func (n reputationHitNoter) noteListed( func (rq *request) noteListed(sources []string, reason string) {
rq *request, sources []string, reason string,
) {
for _, source := range sources { for _, source := range sources {
n.noteHit(rq, bans.ReputationHit{Source: source}, reason) rq.noteHit(bans.ReputationHit{Source: source}, reason)
} }
} }
// noteHit adds hit's source, which lists rq's client, to the log line's // noteHit adds hit's source, which lists the client, to the log line's
// reputation, and hit to the notes of a ban the request makes, counts the // reputation, and hit to the notes of a ban the request makes, counts the
// source in the metrics, and raises a reputation_hit alert with reason, // source in the metrics, and raises a reputation_hit alert with reason,
// whose detail gives hit's source and score. // whose detail gives hit's source and score.
func (n reputationHitNoter) noteHit( func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
rq *request, hit bans.ReputationHit, reason string,
) {
detail := map[string]any{"source": hit.Source} detail := map[string]any{"source": hit.Source}
if hit.Score != nil { if hit.Score != nil {
detail["score"] = *hit.Score detail["score"] = *hit.Score
@@ -41,12 +28,12 @@ func (n reputationHitNoter) noteHit(
rq.line.Reputation = append(rq.line.Reputation, hit.Source) rq.line.Reputation = append(rq.line.Reputation, hit.Source)
rq.reputation = append(rq.reputation, hit) rq.reputation = append(rq.reputation, hit)
n.metrics.ReputationHit(hit.Source) rq.h.metrics.ReputationHit(hit.Source)
n.alerts.Raise(alerts.Alert{ rq.h.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit, Event: alerts.EventReputationHit,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), Netblock: clientGroup(rq.h.config, rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+8 -23
View File
@@ -122,11 +122,8 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion), RequestID: requestID(r, isQuestion),
PeerIP: peer.String(), PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "), ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup( ClientGroup: clientGroup(h.config, client).String(),
h.config.IPv6GroupPrefix, ContentType: r.Header.Get("Content-Type"),
client,
).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders( RequestHeaders: requestHeaders(
r, r,
h.config.LogRequestHeaders, h.config.LogRequestHeaders,
@@ -236,7 +233,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
now := rq.h.now() now := rq.h.now()
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now) v := rq.runHeuristics(ctx, beforeLookup(), now)
if v != nil { if v != nil {
return v return v
} }
@@ -268,7 +265,7 @@ func (rq *request) runAfterLookup(
// again: a request counted at the time it arrived could be counted // again: a request counted at the time it arrived could be counted
// after a later one of its client's, which the rate limits would take // after a later one of its client's, which the rate limits would take
// for the clock set back. // for the clock set back.
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now()) return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
} }
// pathExempt reports whether r is exempt under prefixes, // pathExempt reports whether r is exempt under prefixes,
@@ -348,16 +345,6 @@ func timing(start, end time.Time) *float64 {
return new(requestlog.Milliseconds(end.Sub(start))) return new(requestlog.Milliseconds(end.Sub(start)))
} }
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
// It then counts the request for the anomaly thresholds and adds it to its
// client's history. list runs first, so that a refusal limit it finds broken
// is in that history. ctx is the request's own context.
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
rq.runBanningHeuristics(ctx, list)
rq.countAnomalies()
rq.addToHistory()
}
// addToHistory adds the request, which has ended, to its client's // addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's // history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and // answer about the client, as answerAtTheEnd gives it, to that history and
@@ -378,7 +365,7 @@ func (rq *request) addToHistory() {
} }
rq.h.limiter.AddToHistory( rq.h.limiter.AddToHistory(
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), clientGroup(rq.h.config, rq.client),
rq.h.now(), rq.h.now(),
request, request,
) )
@@ -407,11 +394,11 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), ClientGroup: clientGroup(rq.h.config, rq.client),
ASN: answer.ASN, ASN: answer.ASN,
ASName: answer.ASName, ASName: answer.ASName,
Country: answer.Country, Country: answer.Country,
Bytes: rq.countedBytes(rq.h.config.BytesCounted), Bytes: rq.countedBytes(),
}) })
} }
@@ -438,9 +425,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return rq.lookupAnswer, true return rq.lookupAnswer, true
} }
return rq.h.geojs.Kept( return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
)
} }
// responseWriter is waf's answer to a request. It notes the status and // responseWriter is waf's answer to a request. It notes the status and
+5 -10
View File
@@ -8,7 +8,6 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
) )
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock> // routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
@@ -18,26 +17,22 @@ import (
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and // SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
// asked for by the request's address, and a route without an origin AS gives only // asked for by the request's address, and a route without an origin AS gives only
// the prefix= tag. // the prefix= tag.
type routeWatchTags struct { type routeWatchTags struct{}
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
routeWatch *routewatch.Client
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (routeWatchTags) name() string { return "routewatch_tags" } func (routeWatchTags) name() string { return "routewatch_tags" }
func (r routeWatchTags) check( func (routeWatchTags) check(
ctx context.Context, ctx context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
if r.routeWatch == nil { if rq.h.routeWatch == nil {
return nil return nil
} }
route, missing := r.routeWatch.Route( route, missing := rq.h.routeWatch.Route(
ctx, ctx,
clientGroup(r.ipv6GroupPrefix, rq.client), clientGroup(rq.h.config, rq.client),
rq.client, rq.client,
) )
if missing != "" { if missing != "" {
+4 -10
View File
@@ -9,7 +9,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
) )
@@ -23,15 +22,11 @@ import (
// mode raises the alert for the ban // mode raises the alert for the ban
// it would have made. Either rule's match is noted as an offence, for the // it would have made. Either rule's match is noted as an offence, for the
// client's history. Its figures are the rule's id and action. // client's history. Its figures are the rule's id and action.
type ruleFiles struct { type ruleFiles struct{}
rules *rules.Files
metrics *metrics.Metrics
banMaker banMaker
}
func (ruleFiles) name() string { return "rule_files" } func (ruleFiles) name() string { return "rule_files" }
func (r ruleFiles) check( func (ruleFiles) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -40,7 +35,7 @@ func (r ruleFiles) check(
for _, rule := range matched { for _, rule := range matched {
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID) rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
r.metrics.RuleMatched(rule.ID, rule.Action) rq.h.metrics.RuleMatched(rule.ID, rule.Action)
} }
if len(matched) == 0 { if len(matched) == 0 {
@@ -65,8 +60,7 @@ func (r ruleFiles) check(
rq.attack = true rq.attack = true
return &verdict{ return &verdict{
action: r.banMaker.banForAttack( action: rq.banForAttack(
rq,
now, now,
bans.Notes{RuleID: last.ID, Target: last.Target}, bans.Notes{RuleID: last.ID, Target: last.Target},
), ),
+3 -9
View File
@@ -12,19 +12,13 @@ import (
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is // tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
// the first such tag, in sorted order. // the first such tag, in sorted order.
type tagDeny struct { type tagDeny struct{}
tags []string // SWW_WAF_TAG_DENY
}
func (tagDeny) name() string { return "tag_deny" } func (tagDeny) name() string { return "tag_deny" }
func (t tagDeny) check( func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, tag := range rq.line.Tags { for _, tag := range rq.line.Tags {
if slices.Contains(t.tags, tag) { if slices.Contains(rq.h.config.TagDeny, tag) {
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
figures: map[string]any{"tag": tag}, figures: map[string]any{"tag": tag},
+3 -11
View File
@@ -6,25 +6,17 @@ package waf
import ( import (
"context" "context"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that // tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
// holds the client, the tag being the list's name. It refuses nothing, and runs // holds the client, the tag being the list's name. It refuses nothing, and runs
// first so that every refusal's log line carries these tags. // first so that every refusal's log line carries these tags.
type tagLists struct { type tagLists struct{}
lists *reputation.Lists
}
func (tagLists) name() string { return "tag_lists" } func (tagLists) name() string { return "tag_lists" }
func (t tagLists) check( func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
_ context.Context, for _, name := range rq.h.lists.TagsOf(rq.client) {
rq *request,
_ time.Time,
) *verdict {
for _, name := range t.lists.TagsOf(rq.client) {
rq.addTag(name) rq.addTag(name)
} }
+8 -15
View File
@@ -16,14 +16,11 @@ import (
// it bans the client's netblock, or in observe mode raises the alert for the // it bans the client's netblock, or in observe mode raises the alert for the
// ban it would have made. The ban's notes give the path as sent, and so does // ban it would have made. The ban's notes give the path as sent, and so does
// its figure. // its figure.
type trapPaths struct { type trapPaths struct{}
paths []string // SWW_WAF_TRAP_PATHS
banMaker banMaker
}
func (trapPaths) name() string { return "trap_paths" } func (trapPaths) name() string { return "trap_paths" }
func (t trapPaths) check( func (trapPaths) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -36,22 +33,18 @@ func (t trapPaths) check(
rq.attack = true rq.attack = true
return &verdict{ return &verdict{
action: t.banMaker.banForAttack( action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
rq,
now,
bans.Notes{TrapPath: path},
),
figures: map[string]any{"path": path}, figures: map[string]any{"path": path},
} }
} }
// trapPath returns rq's path as sent, when its path as a path rule sees it, // trapPath returns the request's path as sent, when its path as a path rule
// without the query, as sent or once percent-decoded, is in // sees it, without the query, as sent or once percent-decoded, is in
// SWW_WAF_TRAP_PATHS, and "" otherwise. // SWW_WAF_TRAP_PATHS, and "" otherwise.
func (t trapPaths) trapPath(rq *request) string { func (rq *request) trapPath() string {
path := rules.Path(rq.in) path := rules.Path(rq.in)
if !slices.Contains(t.paths, path) && if !slices.Contains(rq.h.config.TrapPaths, path) &&
!slices.Contains(t.paths, rules.DecodeOnce(path)) { !slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
return "" return ""
} }
+34 -29
View File
@@ -123,10 +123,10 @@ func New(params Params) *Server {
GeoJS: h.geojs, GeoJS: h.geojs,
Anomalies: h.anomalies, Anomalies: h.anomalies,
LookupFile: h.lookupFile, LookupFile: h.lookupFile,
Lists: params.Lists, Lists: h.lists,
DNSBL: params.DNSBL, DNSBL: h.dnsbl,
AbuseIPDB: params.AbuseIPDB, AbuseIPDB: h.abuseIPDB,
RouteWatch: params.RouteWatch, RouteWatch: h.routeWatch,
Metrics: h.metrics, Metrics: h.metrics,
} }
} }
@@ -154,10 +154,8 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
return false return false
} }
listedBy := rq.h.blocklists.lists.ListedBy(rq.client) return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
rq.overAWholeRateLimit(now)
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
} }
// answerQuestion runs the checks on the visitor's request a question // answerQuestion runs the checks on the visitor's request a question
@@ -179,13 +177,11 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
peer.Unmap(), peer.Unmap(),
true, true,
) )
// These run even when a check panics. finish runs first, so that what is // The last deferred runs first, so that what is held has the finished log
// held has the finished log line. Go's server sends the answer only once // line. Go's server sends the answer only once this function has
// this function has returned, so the report cannot come before it is held. // returned, so the report cannot come before it is held.
defer func() { defer rq.holdForReport()
rq.finish() defer rq.finish()
rq.holdForReport()
}()
rq.site = r.Host rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision rq.line.WAFLine = requestlog.WAFLineDecision
@@ -194,28 +190,35 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix) underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
if refused == nil && !underOwnPrefix { if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here. // In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer) defer rq.runAfterAnswer(r.Context())
rq.allow()
return return
} }
// The last deferred runs first: runAfterAnswer before addToHistory, so that
// a broken refusal limit is in the client's history.
defer rq.addToHistory()
defer rq.countAnomalies()
defer rq.runAfterAnswer(r.Context())
if refused != nil { if refused != nil {
rq.deny(*refused) rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its return
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
} }
rq.banAndCount(r.Context(), rq.h.afterAnswer) // The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
} }
// readQuestion returns the visitor's address and the body a question carries, // readQuestion returns the visitor's address and the body a question carries,
@@ -488,7 +491,9 @@ func (h *handler) finishWith(
if rq.allowed { if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, rq.h.onReport) rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.h.metrics.RequestReported( rq.h.metrics.RequestReported(
&rq.line, &rq.line,