Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
249d6a1142 | ||
|
|
0a0850f797 |
No files matched your search
@@ -908,8 +908,6 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
|
||||||
`internal/lookup` read their files again when they change.
|
|
||||||
- `internal/state`: the state and cache files.
|
- `internal/state`: the state and cache files.
|
||||||
- `internal/requestlog`: the request log.
|
- `internal/requestlog`: the request log.
|
||||||
- `internal/processlog`: the process's own messages.
|
- `internal/processlog`: the process's own messages.
|
||||||
|
|||||||
@@ -69,14 +69,13 @@ never as a time.
|
|||||||
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
|
that command's settings, and runs `internal/rproxy` until `SIGTERM` or
|
||||||
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
|
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
|
||||||
With no command, or one it does not know, the binary prints the commands
|
With no command, or one it does not know, the binary prints the commands
|
||||||
and exits with status 2. `sww help` and `sww --help` print them and exit
|
and exits with status 2. `internal/sww` is the only package that imports
|
||||||
with status 0. `internal/sww` is the only package that imports
|
|
||||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||||
and stops their background work.
|
and stops their background work.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`. Its
|
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
|
||||||
`routes.go` lists `waf`'s own endpoints. It counts bytes, refusals and
|
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
|
||||||
anomalies, and writes the log lines.
|
and anomalies, and writes the log lines.
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
||||||
about each request. It passes a request to the app with the standard
|
about each request. It passes a request to the app with the standard
|
||||||
library's `httputil.ReverseProxy`, within the time and size limits.
|
library's `httputil.ReverseProxy`, within the time and size limits.
|
||||||
@@ -97,9 +96,6 @@ never as a time.
|
|||||||
imports no other package of this repository: `waf` gives it what it needs
|
imports no other package of this repository: `waf` gives it what it needs
|
||||||
to count and report failures.
|
to count and report failures.
|
||||||
- `internal/lookup`: looks clients up in the lookup database.
|
- `internal/lookup`: looks clients up in the lookup database.
|
||||||
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
|
||||||
gone a short time without a change. `internal/rules` and `internal/lookup`
|
|
||||||
read their files again this way.
|
|
||||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||||
@@ -128,13 +124,11 @@ never as a time.
|
|||||||
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
|
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
|
||||||
version. They are checked again before the version changes, since rule
|
version. They are checked again before the version changes, since rule
|
||||||
ids, and what each rule matches, change between versions;
|
ids, and what each rule matches, change between versions;
|
||||||
- `github.com/go-chi/chi/v5` to route the requests to `waf`'s own endpoints;
|
|
||||||
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
|
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
|
||||||
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
|
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
|
||||||
- `github.com/fsnotify/fsnotify` to notice edited files;
|
- `github.com/fsnotify/fsnotify` to notice edited files;
|
||||||
- `github.com/prometheus/client_golang` for the metrics;
|
- `github.com/prometheus/client_golang` for the metrics;
|
||||||
- `github.com/spf13/viper` to read the settings from the environment;
|
- `github.com/spf13/viper` to read the settings from the environment;
|
||||||
- `github.com/spf13/cobra` for the commands and their help text;
|
|
||||||
- `github.com/maxmind/mmdbwriter`, in the tests, to write lookup databases;
|
- `github.com/maxmind/mmdbwriter`, in the tests, to write lookup databases;
|
||||||
- remote log sending written in this project, since the standard library's
|
- remote log sending written in this project, since the standard library's
|
||||||
`log/syslog` is frozen and writes only the older syslog format.
|
`log/syslog` is frozen and writes only the older syslog format.
|
||||||
|
|||||||
@@ -7,12 +7,10 @@ require (
|
|||||||
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
|
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
|
||||||
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
|
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
|
||||||
github.com/fsnotify/fsnotify v1.10.1
|
github.com/fsnotify/fsnotify v1.10.1
|
||||||
github.com/go-chi/chi/v5 v5.5.0
|
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||||
github.com/maxmind/mmdbwriter v1.2.0
|
github.com/maxmind/mmdbwriter v1.2.0
|
||||||
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/cobra v1.10.2
|
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/net v0.60.0
|
golang.org/x/net v0.60.0
|
||||||
@@ -28,7 +26,6 @@ require (
|
|||||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||||
github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect
|
github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect
|
||||||
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect
|
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
|
||||||
github.com/kaptinlin/go-i18n v0.1.4 // indirect
|
github.com/kaptinlin/go-i18n v0.1.4 // indirect
|
||||||
github.com/kaptinlin/jsonschema v0.4.6 // indirect
|
github.com/kaptinlin/jsonschema v0.4.6 // indirect
|
||||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||||
|
|||||||
@@ -14,15 +14,12 @@ github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68 h1:KLEjvBBSw
|
|||||||
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68/go.mod h1:9XocblQEwK6EYylRth7PrPIifF2l0iSoYo4FAXhXU/8=
|
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68/go.mod h1:9XocblQEwK6EYylRth7PrPIifF2l0iSoYo4FAXhXU/8=
|
||||||
github.com/corazawaf/libinjection-go v0.3.4 h1:jX2Tit2xZxYJJyVryNXucYBZtQqS7473x9Nc6lgRRbk=
|
github.com/corazawaf/libinjection-go v0.3.4 h1:jX2Tit2xZxYJJyVryNXucYBZtQqS7473x9Nc6lgRRbk=
|
||||||
github.com/corazawaf/libinjection-go v0.3.4/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk=
|
github.com/corazawaf/libinjection-go v0.3.4/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
|
||||||
github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0=
|
github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0=
|
||||||
github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
|
github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
|
|
||||||
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||||
@@ -37,8 +34,6 @@ github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 h1:c7gcN
|
|||||||
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU=
|
github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
|
||||||
github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y=
|
github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y=
|
||||||
github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
|
github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
|
||||||
github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs=
|
github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs=
|
||||||
@@ -77,7 +72,6 @@ github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+
|
|||||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
|
||||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||||
@@ -86,9 +80,6 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
|||||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
|
||||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
|
||||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
||||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||||
@@ -118,7 +109,6 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
|||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
|
||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
|
||||||
@@ -137,7 +127,6 @@ golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
|||||||
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
|
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
// Package dirwatch watches a directory, and calls a function once the
|
|
||||||
// directory has gone a short time without a change after one. The rule
|
|
||||||
// files and the lookup database are read again this way.
|
|
||||||
package dirwatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
)
|
|
||||||
|
|
||||||
// quietTime is how long the directory must go without a change before the
|
|
||||||
// function is called, so that a file still being written, such as one
|
|
||||||
// saved in place, appended to or copied in with scp, is read only once
|
|
||||||
// whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// Watcher watches one directory. The directory is watched, rather than a
|
|
||||||
// file in it, so that a file replaced by a rename is seen.
|
|
||||||
type Watcher struct {
|
|
||||||
watcher *fsnotify.Watcher
|
|
||||||
}
|
|
||||||
|
|
||||||
// New starts watching dir, or returns why it cannot.
|
|
||||||
func New(dir string) (*Watcher, error) {
|
|
||||||
watcher, err := fsnotify.NewWatcher()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = watcher.Add(dir)
|
|
||||||
if err != nil {
|
|
||||||
_ = watcher.Close()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Watcher{watcher: watcher}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close stops watching the directory.
|
|
||||||
func (w *Watcher) Close() error {
|
|
||||||
return w.watcher.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
|
||||||
// without a change, after a change to a path for which counts is true, until
|
|
||||||
// ctx is done. It also calls failed with each error the watch reports. The
|
|
||||||
// wait starts at once, as if for a change, so that a change made before the
|
|
||||||
// watch began is taken in too.
|
|
||||||
func (w *Watcher) CallWhenQuiet(
|
|
||||||
ctx context.Context,
|
|
||||||
counts func(path string) bool,
|
|
||||||
changed func(),
|
|
||||||
failed func(error),
|
|
||||||
) {
|
|
||||||
callWhenQuiet(
|
|
||||||
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
|
||||||
// errors from errs.
|
|
||||||
func callWhenQuiet(
|
|
||||||
ctx context.Context,
|
|
||||||
events <-chan fsnotify.Event,
|
|
||||||
errs <-chan error,
|
|
||||||
counts func(path string) bool,
|
|
||||||
changed func(),
|
|
||||||
failed func(error),
|
|
||||||
) {
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case event := <-events:
|
|
||||||
if counts(event.Name) {
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
}
|
|
||||||
case <-quiet.C:
|
|
||||||
changed()
|
|
||||||
case err := <-errs:
|
|
||||||
failed(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package dirwatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"path/filepath"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errWatch is an error the watch reports.
|
|
||||||
var errWatch = errors.New("too many changes")
|
|
||||||
|
|
||||||
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
|
||||||
if err == nil {
|
|
||||||
t.Error("a missing directory is watched, want an error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
|
||||||
// clock of the test's own: time.Sleep moves it on at once, and
|
|
||||||
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
|
||||||
// due by then is made. The test sends the changes itself, as the watch of
|
|
||||||
// a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 0)
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 0)
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
// An edit saved before the watch began sends no change, and is
|
|
||||||
// taken in by this call.
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
|
|
||||||
// Long after, it has been called only once.
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, func(path string) bool {
|
|
||||||
return path == "ipinfo_lite.mmdb"
|
|
||||||
})
|
|
||||||
|
|
||||||
// The wait that starts with the watch ends with a call.
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 1)
|
|
||||||
|
|
||||||
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
loop.wantCalls(t, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
loop := start(t, everyPath)
|
|
||||||
|
|
||||||
loop.errs <- errWatch
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
if !errors.Is(loop.failure, errWatch) {
|
|
||||||
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// loop is a callWhenQuiet that a test runs.
|
|
||||||
type loop struct {
|
|
||||||
// changes and errs send it changes and errors.
|
|
||||||
changes chan fsnotify.Event
|
|
||||||
errs chan error
|
|
||||||
// calls counts its calls of changed, and failure is the last error it
|
|
||||||
// handed to failed.
|
|
||||||
calls atomic.Int32
|
|
||||||
failure error
|
|
||||||
}
|
|
||||||
|
|
||||||
// start runs callWhenQuiet, with counts, until the test ends.
|
|
||||||
func start(t *testing.T, counts func(path string) bool) *loop {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
|
||||||
func() { l.calls.Add(1) },
|
|
||||||
func(err error) { l.failure = err },
|
|
||||||
)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
// everyPath counts a change to any path.
|
|
||||||
func everyPath(string) bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantCalls checks how many times changed has been called.
|
|
||||||
func (l *loop) wantCalls(t *testing.T, want int32) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := l.calls.Load()
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("changed called %d times, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+49
-18
@@ -13,13 +13,18 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/oschwald/maxminddb-golang/v2"
|
"github.com/oschwald/maxminddb-golang/v2"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// quietTime is how long the lookup database must go without a change
|
||||||
|
// before it is read again, so that a file still being copied in is read
|
||||||
|
// only once whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
// FileParams are what OpenFile needs.
|
// FileParams are what OpenFile needs.
|
||||||
type FileParams struct {
|
type FileParams struct {
|
||||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||||
@@ -117,13 +122,20 @@ func (f *File) ReadFailures() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches the directory of the lookup database until ctx is done,
|
// Watch watches the directory of the lookup database until ctx is done,
|
||||||
// and reads the file again once it has gone without a change for a short
|
// and reads the file again once it has gone without a change for
|
||||||
// time, after it is replaced, written or removed, and after Watch starts
|
// quietTime, after it is replaced, written or removed, and after Watch
|
||||||
// watching, as dirwatch describes. A change to another file in its
|
// starts watching. If the directory cannot be watched, that is logged, and
|
||||||
// directory does not count. If the directory cannot be watched, that is
|
// the database read at start stays in use.
|
||||||
// logged, and the database read at start stays in use.
|
|
||||||
func (f *File) Watch(ctx context.Context) {
|
func (f *File) Watch(ctx context.Context) {
|
||||||
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err == nil {
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = watcher.Add(filepath.Dir(f.params.Path))
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error(
|
f.params.ProcessLog.Error(
|
||||||
"cannot watch the lookup database for replacements",
|
"cannot watch the lookup database for replacements",
|
||||||
@@ -134,22 +146,41 @@ func (f *File) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||||
"file", f.params.Path)
|
"file", f.params.Path)
|
||||||
|
|
||||||
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
|
||||||
"error", err.Error())
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isDatabase reports whether path names the lookup database.
|
// readAfterChanges reads the lookup database again once quietTime has
|
||||||
func (f *File) isDatabase(path string) bool {
|
// passed without a change to it from events, until ctx is done, and logs
|
||||||
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
// the errors from errs. A change to another file in its directory does not
|
||||||
|
// count. The wait starts at once, as if for a change, so that a file
|
||||||
|
// replaced after OpenFile read it, and before its directory was watched,
|
||||||
|
// is read too.
|
||||||
|
func (f *File) readAfterChanges(
|
||||||
|
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||||
|
) {
|
||||||
|
path := filepath.Clean(f.params.Path)
|
||||||
|
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case event := <-events:
|
||||||
|
if filepath.Clean(event.Name) == path {
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
}
|
||||||
|
case <-quiet.C:
|
||||||
|
f.readAgain()
|
||||||
|
case err := <-errs:
|
||||||
|
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||||
|
"error", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the lookup database again, in place of the one in use,
|
// readAgain reads the lookup database again, in place of the one in use,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package lookup
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
@@ -131,48 +133,66 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||||
|
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||||
|
// reading due by then is done. The test sends the changes itself, as the
|
||||||
|
// watch of a directory cannot run in a bubble.
|
||||||
|
|
||||||
|
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
writeDatabase(t, path, "DE")
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
|
|
||||||
for changed, want := range map[string]bool{
|
|
||||||
path: true,
|
|
||||||
dir + "/./ipinfo_lite.mmdb": true,
|
|
||||||
filepath.Join(dir, "other.mmdb"): false,
|
|
||||||
} {
|
|
||||||
if f.isDatabase(changed) != want {
|
|
||||||
t.Errorf(
|
|
||||||
"a change to %s counts: %t, want %t",
|
|
||||||
changed,
|
|
||||||
!want,
|
|
||||||
want,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tests below run in a synctest bubble, where time is a clock of the
|
|
||||||
// test's own, so that the time a file is read can be checked. The wait for
|
|
||||||
// the directory to go quiet before a reading is tested in dirwatch.
|
|
||||||
|
|
||||||
func TestReplacementIsReadAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
|
||||||
|
|
||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
writeDatabase(t, path, "KP")
|
other := filepath.Join(dir, "replacement.mmdb")
|
||||||
time.Sleep(time.Hour)
|
writeDatabase(t, other, "KP")
|
||||||
f.readAgain()
|
|
||||||
|
//nolint:gosec // a file the test wrote
|
||||||
|
replacement, err := os.ReadFile(other)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", other, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The file in use is overwritten in place, and keeps giving what
|
||||||
|
// it gave. Its first part alone is not a .mmdb file.
|
||||||
|
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create %s: %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = file.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
half := len(replacement) / 2
|
||||||
|
write(t, file, replacement[:half])
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
write(t, file, replacement[half:])
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
wantCountry(t, f, "KP")
|
wantCountry(t, f, "KP")
|
||||||
|
|
||||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||||
@@ -199,10 +219,15 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
read := f.LastRead()
|
read := f.LastRead()
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
writeFile(t, path, "not a lookup database\n")
|
writeFile(t, path, "not a lookup database\n")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
// Long after, the replacement has been read once.
|
||||||
time.Sleep(time.Hour)
|
time.Sleep(time.Hour)
|
||||||
f.readAgain()
|
synctest.Wait()
|
||||||
wantCountry(t, f, "DE")
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||||
@@ -215,7 +240,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
}
|
}
|
||||||
|
|
||||||
wantAlerts(t, queue, alerts.Alert{
|
wantAlerts(t, queue, alerts.Alert{
|
||||||
Time: read.Add(time.Hour),
|
Time: read.Add(quietTime),
|
||||||
Event: alerts.EventFileError,
|
Event: alerts.EventFileError,
|
||||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||||
Detail: map[string]any{
|
Detail: map[string]any{
|
||||||
@@ -227,6 +252,55 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
changes := watch(t, f)
|
||||||
|
|
||||||
|
// The wait that starts with the watch ends with a reading.
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
writeDatabase(t, path, "KP")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "KP")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||||
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
|
||||||
|
// Saved after OpenFile read the file, and before its directory was
|
||||||
|
// watched, so that no change is seen for it.
|
||||||
|
writeDatabase(t, path, "KP")
|
||||||
|
watch(t, f)
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantCountry(t, f, "KP")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||||
// them, so that they wait in it for the test to look at.
|
// them, so that they wait in it for the test to look at.
|
||||||
func newQueue() *alerts.Queue {
|
func newQueue() *alerts.Queue {
|
||||||
@@ -270,6 +344,28 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// watch runs f's readAfterChanges until the test ends, and returns the
|
||||||
|
// channel that sends it changes.
|
||||||
|
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
changes := make(chan fsnotify.Event)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
f.readAfterChanges(ctx, changes, nil)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return changes
|
||||||
|
}
|
||||||
|
|
||||||
// wantCountry checks the country f gives testClient.
|
// wantCountry checks the country f gives testClient.
|
||||||
func wantCountry(t *testing.T, f *File, want string) {
|
func wantCountry(t *testing.T, f *File, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -305,3 +401,13 @@ func writeFile(t *testing.T, path, content string) {
|
|||||||
t.Fatalf("write %s: %v", path, err)
|
t.Fatalf("write %s: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// write writes data to the end of file.
|
||||||
|
func write(t *testing.T, file *os.File, data []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := file.Write(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+264
-389
@@ -15,33 +15,12 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Params are what New builds the metrics from. Each part is nil while it is
|
|
||||||
// off, and a part that is nil adds no metric of its own.
|
|
||||||
type Params struct {
|
|
||||||
// BusiestCount is how many countries and how many AS numbers get series
|
|
||||||
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
|
||||||
BusiestCount int
|
|
||||||
// InstanceName is the label instance every metric carries
|
|
||||||
// (SWW_WAF_INSTANCE_NAME).
|
|
||||||
InstanceName string
|
|
||||||
// Now returns the time at which Bans counts the bans active.
|
|
||||||
Now func() time.Time
|
|
||||||
|
|
||||||
Bans banLedger
|
|
||||||
Clients sized
|
|
||||||
Rules sized
|
|
||||||
Lists fetchedLists
|
|
||||||
DNSBL dnsblZones
|
|
||||||
AbuseIPDB abuseIPDBChecker
|
|
||||||
Alerts alertQueue
|
|
||||||
LookupFile lookupDatabase
|
|
||||||
RemoteLog remoteLogSender
|
|
||||||
}
|
|
||||||
|
|
||||||
// Metrics are sww's metrics. They are safe for concurrent use.
|
// Metrics are sww's metrics. They are safe for concurrent use.
|
||||||
type Metrics struct {
|
type Metrics struct {
|
||||||
// registry gives every metric registered with it the label instance.
|
// registry gives every metric registered with it the label instance.
|
||||||
@@ -57,6 +36,8 @@ type Metrics struct {
|
|||||||
rateLimitHits *prometheus.CounterVec
|
rateLimitHits *prometheus.CounterVec
|
||||||
offences *prometheus.CounterVec
|
offences *prometheus.CounterVec
|
||||||
coreRuleSetMatches *prometheus.CounterVec
|
coreRuleSetMatches *prometheus.CounterVec
|
||||||
|
// ruleMatches are made by AddRules, and reputationHits by
|
||||||
|
// AddReputation.
|
||||||
ruleMatches *prometheus.CounterVec
|
ruleMatches *prometheus.CounterVec
|
||||||
reputationHits *prometheus.CounterVec
|
reputationHits *prometheus.CounterVec
|
||||||
countries *busiest
|
countries *busiest
|
||||||
@@ -78,18 +59,20 @@ type Metrics struct {
|
|||||||
stateFileEditsSetAside *prometheus.CounterVec
|
stateFileEditsSetAside *prometheus.CounterVec
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the metrics, with the Go runtime's and the process's own, and
|
// New returns the metrics, with the Go runtime's and the process's own.
|
||||||
// those read from each part in params as the metrics are asked for.
|
// busiestCount is how many countries and how many AS numbers get series of
|
||||||
|
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
||||||
|
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
||||||
//
|
//
|
||||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||||
func New(params Params) *Metrics {
|
func New(busiestCount int, instanceName string) *Metrics {
|
||||||
byStatus := []string{"status_class", "action"}
|
byStatus := []string{"status_class", "action"}
|
||||||
byFile := []string{"file"}
|
byFile := []string{"file"}
|
||||||
registry := prometheus.NewRegistry()
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
m := &Metrics{
|
m := &Metrics{
|
||||||
registry: prometheus.WrapRegistererWith(
|
registry: prometheus.WrapRegistererWith(
|
||||||
prometheus.Labels{"instance": params.InstanceName}, registry),
|
prometheus.Labels{"instance": instanceName}, registry),
|
||||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
Name: "sww_requests_in_flight",
|
Name: "sww_requests_in_flight",
|
||||||
@@ -130,19 +113,8 @@ func New(params Params) *Metrics {
|
|||||||
"and the rule's id.",
|
"and the rule's id.",
|
||||||
[]string{"mode", "rule_id"},
|
[]string{"mode", "rule_id"},
|
||||||
),
|
),
|
||||||
ruleMatches: counterVec(
|
countries: newCountries(busiestCount),
|
||||||
"sww_rule_matches_total",
|
asns: newASNs(busiestCount),
|
||||||
"Requests that matched a rule of the rule files, by its id and action.",
|
|
||||||
[]string{"rule_id", "action"},
|
|
||||||
),
|
|
||||||
reputationHits: counterVec(
|
|
||||||
"sww_reputation_hits_total",
|
|
||||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
|
||||||
"by the list's URL, the zone, or abuseipdb.",
|
|
||||||
[]string{sourceLabel},
|
|
||||||
),
|
|
||||||
countries: newCountries(params.BusiestCount),
|
|
||||||
asns: newASNs(params.BusiestCount),
|
|
||||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "sww_geojs_requests_total",
|
Name: "sww_geojs_requests_total",
|
||||||
Help: "Requests to GeoJS.",
|
Help: "Requests to GeoJS.",
|
||||||
@@ -194,7 +166,6 @@ func New(params Params) *Metrics {
|
|||||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||||
m.requestDuration, m.upstreamDuration,
|
m.requestDuration, m.upstreamDuration,
|
||||||
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
||||||
m.ruleMatches, m.reputationHits,
|
|
||||||
m.countries, m.asns,
|
m.countries, m.asns,
|
||||||
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
||||||
m.stateFileWrites, m.stateFileWriteFailures,
|
m.stateFileWrites, m.stateFileWriteFailures,
|
||||||
@@ -202,45 +173,261 @@ func New(params Params) *Metrics {
|
|||||||
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
||||||
)
|
)
|
||||||
|
|
||||||
if params.Bans != nil {
|
|
||||||
m.addBans(params.Bans, params.Now)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Clients != nil {
|
|
||||||
m.addClients(params.Clients)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Rules != nil {
|
|
||||||
m.addRules(params.Rules)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Lists != nil {
|
|
||||||
m.addLists(params.Lists)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.DNSBL != nil {
|
|
||||||
m.addDNSBL(params.DNSBL)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.AbuseIPDB != nil {
|
|
||||||
m.addAbuseIPDB(params.AbuseIPDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Alerts != nil {
|
|
||||||
m.addAlerts(params.Alerts)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.LookupFile != nil {
|
|
||||||
m.addLookupFile(params.LookupFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.RemoteLog != nil {
|
|
||||||
m.addRemoteLog(params.RemoteLog)
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddBansAndClients adds the metrics read from the ledger and the table
|
||||||
|
// of clients as the metrics are asked for: the bans made since the start,
|
||||||
|
// by cause, the bans active and permanent at now, and the clients in the
|
||||||
|
// table.
|
||||||
|
func (m *Metrics) AddBansAndClients(
|
||||||
|
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||||
|
) {
|
||||||
|
for _, cause := range []string{
|
||||||
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||||
|
} {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_bans_made_total",
|
||||||
|
Help: "Bans made, by cause.",
|
||||||
|
ConstLabels: prometheus.Labels{"cause": cause},
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ledger.Made(cause))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_active_bans",
|
||||||
|
Help: "Bans active now, the permanent ones included.",
|
||||||
|
}, func() float64 {
|
||||||
|
active, _ := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(active)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_permanent_bans",
|
||||||
|
Help: "Permanent bans not lifted.",
|
||||||
|
}, func() float64 {
|
||||||
|
_, permanent := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(permanent)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_tracked_clients",
|
||||||
|
Help: "Clients in the table of clients.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(limiter.Len())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRules adds the metrics of the rule files: the requests that matched
|
||||||
|
// each rule, which RuleMatched counts, and the rules loaded from
|
||||||
|
// ruleFiles, read as the metrics are asked for. It is called once, before
|
||||||
|
// RuleMatched.
|
||||||
|
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
|
||||||
|
m.ruleMatches = counterVec(
|
||||||
|
"sww_rule_matches_total",
|
||||||
|
"Requests that matched a rule of the rule files, by its id and action.",
|
||||||
|
[]string{"rule_id", "action"},
|
||||||
|
)
|
||||||
|
|
||||||
|
m.registry.MustRegister(m.ruleMatches,
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_rules_loaded",
|
||||||
|
Help: "Rules loaded from the rule files.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ruleFiles.Len())
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRemoteLog adds the metrics of sending the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||||
|
// lines sent, those dropped, and those waiting in the buffer.
|
||||||
|
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_sent_total",
|
||||||
|
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Sent())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_dropped_total",
|
||||||
|
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||||
|
"whose sending failed.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Dropped())
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_remote_log_lines_waiting",
|
||||||
|
Help: "Log lines in the buffer, waiting to be sent.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Depth())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddLookupFile adds the metrics of the lookup database, read as the
|
||||||
|
// metrics are asked for: when the file in use was read, which lastRead
|
||||||
|
// returns, and the replacements of it that could not be read, which
|
||||||
|
// readFailures returns.
|
||||||
|
func (m *Metrics) AddLookupFile(
|
||||||
|
lastRead func() time.Time,
|
||||||
|
readFailures func() int,
|
||||||
|
) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||||
|
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(lastRead().Unix())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_lookup_database_read_failures_total",
|
||||||
|
Help: "Replacements of the lookup database that could not be read.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(readFailures())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||||
|
// DNSBL zone, its key masked, or abuseipdb.
|
||||||
|
const sourceLabel = "source"
|
||||||
|
|
||||||
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
|
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
|
// zone, the queries made and those that failed. It is called once, before
|
||||||
|
// ReputationHit.
|
||||||
|
func (m *Metrics) AddReputation(
|
||||||
|
lists *reputation.Lists,
|
||||||
|
dnsbl *reputation.DNSBL,
|
||||||
|
) {
|
||||||
|
m.reputationHits = counterVec(
|
||||||
|
"sww_reputation_hits_total",
|
||||||
|
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||||
|
"by the list's URL, the zone, or abuseipdb.",
|
||||||
|
[]string{sourceLabel},
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
|
for _, zone := range dnsbl.Zones() {
|
||||||
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
|
m.addReputationQueries(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Queries(zone) },
|
||||||
|
)
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Failures(zone) },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, listURL := range lists.URLs() {
|
||||||
|
source := prometheus.Labels{sourceLabel: listURL}
|
||||||
|
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return lists.Failures(listURL) },
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||||
|
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||||
|
"1970, or 0 while there is none.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
fetched := lists.Fetched(listURL)
|
||||||
|
if fetched.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return float64(fetched.Unix())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||||
|
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||||
|
// that failed, and how many checks the day's budget has left. It is
|
||||||
|
// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set.
|
||||||
|
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
||||||
|
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||||
|
|
||||||
|
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||||
|
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_daily_budget_remaining",
|
||||||
|
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(abuseIPDB.BudgetLeft())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||||
|
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||||
|
func (m *Metrics) ReputationHit(source string) {
|
||||||
|
m.reputationHits.WithLabelValues(source).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAlerts adds the metrics of the alerts sent to each destination set,
|
||||||
|
// read from queue as the metrics are asked for, by destination: the
|
||||||
|
// alerts sent, the requests to the destination that failed, the alerts
|
||||||
|
// held back, which are the same for every destination, and those
|
||||||
|
// dropped. With no destination set, it adds none.
|
||||||
|
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
|
||||||
|
for _, name := range queue.DestinationsSet() {
|
||||||
|
destination := prometheus.Labels{"destination": name}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_sent_total",
|
||||||
|
Help: "Alerts the destination took.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Sent)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alert_send_failures_total",
|
||||||
|
Help: "Requests to the destination that failed.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).SendFailures)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_suppressed_total",
|
||||||
|
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||||
|
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Suppressed())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_dropped_total",
|
||||||
|
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||||
|
"alerts given up as the destination refused them.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Dropped)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ServeHTTP answers with the metrics in the Prometheus text format.
|
// ServeHTTP answers with the metrics in the Prometheus text format.
|
||||||
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
m.handler.ServeHTTP(w, r)
|
m.handler.ServeHTTP(w, r)
|
||||||
@@ -310,12 +497,6 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
|||||||
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
|
||||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
|
||||||
func (m *Metrics) ReputationHit(source string) {
|
|
||||||
m.reputationHits.WithLabelValues(source).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// StateFileWritten counts a write of the state or cache file name, of size
|
// StateFileWritten counts a write of the state or cache file name, of size
|
||||||
// bytes, that ended with err.
|
// bytes, that ended with err.
|
||||||
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
||||||
@@ -347,312 +528,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
|||||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// banLedger is what the metrics read from the ledger of bans.
|
|
||||||
type banLedger interface {
|
|
||||||
// Made returns the bans made since the start with cause.
|
|
||||||
Made(cause string) int
|
|
||||||
// Count returns the bans active at now, the permanent ones included,
|
|
||||||
// and the permanent ones.
|
|
||||||
Count(now time.Time) (active, permanent int)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addBans adds the metrics read from ledger as the metrics are asked for:
|
|
||||||
// the bans made since the start, by cause, and the bans active and
|
|
||||||
// permanent at now.
|
|
||||||
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
|
||||||
for _, cause := range []string{
|
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
|
||||||
} {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_bans_made_total",
|
|
||||||
Help: "Bans made, by cause.",
|
|
||||||
ConstLabels: prometheus.Labels{"cause": cause},
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(ledger.Made(cause))
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_active_bans",
|
|
||||||
Help: "Bans active now, the permanent ones included.",
|
|
||||||
}, func() float64 {
|
|
||||||
active, _ := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(active)
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_permanent_bans",
|
|
||||||
Help: "Permanent bans not lifted.",
|
|
||||||
}, func() float64 {
|
|
||||||
_, permanent := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(permanent)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sized is what the metrics read from the table of clients and from the
|
|
||||||
// rule files.
|
|
||||||
type sized interface {
|
|
||||||
// Len returns how many clients the table holds, or how many rules were
|
|
||||||
// loaded from the rule files.
|
|
||||||
Len() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addClients adds the metric of the clients in table, read as the metrics
|
|
||||||
// are asked for.
|
|
||||||
func (m *Metrics) addClients(table sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_tracked_clients",
|
|
||||||
Help: "Clients in the table of clients.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(table.Len())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRules adds the metric of the rules loaded from files, read as the
|
|
||||||
// metrics are asked for.
|
|
||||||
func (m *Metrics) addRules(files sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_rules_loaded",
|
|
||||||
Help: "Rules loaded from the rule files.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(files.Len())
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// remoteLogSender is what the metrics read from what sends the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL.
|
|
||||||
type remoteLogSender interface {
|
|
||||||
// Sent returns the lines sent.
|
|
||||||
Sent() int64
|
|
||||||
// Dropped returns the lines dropped.
|
|
||||||
Dropped() int64
|
|
||||||
// Depth returns the lines waiting in the buffer.
|
|
||||||
Depth() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRemoteLog adds the metrics of sending the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
|
||||||
// lines sent, those dropped, and those waiting in the buffer.
|
|
||||||
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_sent_total",
|
|
||||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Sent())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_dropped_total",
|
|
||||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
|
||||||
"whose sending failed.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Dropped())
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_remote_log_lines_waiting",
|
|
||||||
Help: "Log lines in the buffer, waiting to be sent.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Depth())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookupDatabase is what the metrics read from the lookup database.
|
|
||||||
type lookupDatabase interface {
|
|
||||||
// LastRead returns when the file in use was read.
|
|
||||||
LastRead() time.Time
|
|
||||||
// ReadFailures returns the replacements of the file that could not be
|
|
||||||
// read.
|
|
||||||
ReadFailures() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLookupFile adds the metrics of the lookup database, read from file as
|
|
||||||
// the metrics are asked for: when the file in use was read, and the
|
|
||||||
// replacements of it that could not be read.
|
|
||||||
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
|
||||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.LastRead().Unix())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_lookup_database_read_failures_total",
|
|
||||||
Help: "Replacements of the lookup database that could not be read.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.ReadFailures())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
|
||||||
// DNSBL zone, its key masked, or abuseipdb.
|
|
||||||
const sourceLabel = "source"
|
|
||||||
|
|
||||||
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
|
||||||
type dnsblZones interface {
|
|
||||||
// Zones returns the zones asked, their keys unmasked.
|
|
||||||
Zones() []string
|
|
||||||
// Queries returns the queries made to zone.
|
|
||||||
Queries(zone string) int
|
|
||||||
// Failures returns the queries to zone that failed.
|
|
||||||
Failures(zone string) int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
|
||||||
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
|
||||||
// asked for: the queries made and those that failed.
|
|
||||||
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
|
||||||
source := prometheus.Labels{
|
|
||||||
sourceLabel: reputation.MaskZoneKey(zone),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.addReputationQueries(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Queries(zone) },
|
|
||||||
)
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Failures(zone) },
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
|
||||||
type fetchedLists interface {
|
|
||||||
// URLs returns the lists' URLs.
|
|
||||||
URLs() []string
|
|
||||||
// Failures returns the fetches of the list at listURL that failed.
|
|
||||||
Failures(listURL string) int
|
|
||||||
// Fetched returns when the copy in use of the list at listURL was
|
|
||||||
// fetched, or the zero time while there is none.
|
|
||||||
Fetched(listURL string) time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLists adds the metrics of each list fetched from a URL, by source, the
|
|
||||||
// list's URL, read from lists as the metrics are asked for: the fetches
|
|
||||||
// that failed and when the copy in use was fetched.
|
|
||||||
func (m *Metrics) addLists(lists fetchedLists) {
|
|
||||||
for _, listURL := range lists.URLs() {
|
|
||||||
source := prometheus.Labels{sourceLabel: listURL}
|
|
||||||
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return lists.Failures(listURL) },
|
|
||||||
)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
|
||||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
|
||||||
"1970, or 0 while there is none.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
fetched := lists.Fetched(listURL)
|
|
||||||
if fetched.IsZero() {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return float64(fetched.Unix())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// abuseIPDBChecker is what the metrics read from what checks clients with
|
|
||||||
// AbuseIPDB.
|
|
||||||
type abuseIPDBChecker interface {
|
|
||||||
// Checked returns the checks made.
|
|
||||||
Checked() int
|
|
||||||
// Failures returns the checks that failed.
|
|
||||||
Failures() int
|
|
||||||
// BudgetLeft returns how many checks the day's budget has left.
|
|
||||||
BudgetLeft() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
|
||||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
|
||||||
// that failed, and how many checks the day's budget has left.
|
|
||||||
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
|
||||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
|
||||||
|
|
||||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
|
||||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_daily_budget_remaining",
|
|
||||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(abuseIPDB.BudgetLeft())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// alertQueue is what the metrics read from the queue of alerts.
|
|
||||||
type alertQueue interface {
|
|
||||||
// DestinationsSet returns the names of the destinations set.
|
|
||||||
DestinationsSet() []string
|
|
||||||
// Counts returns the alerts sent to the destination name, the requests
|
|
||||||
// to it that failed, and the alerts dropped.
|
|
||||||
Counts(name string) alerts.Counts
|
|
||||||
// Suppressed returns the alerts held back.
|
|
||||||
Suppressed() int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAlerts adds the metrics of the alerts sent to each destination set,
|
|
||||||
// read from queue as the metrics are asked for, by destination: the
|
|
||||||
// alerts sent, the requests to the destination that failed, the alerts
|
|
||||||
// held back, which are the same for every destination, and those
|
|
||||||
// dropped. With no destination set, it adds none.
|
|
||||||
func (m *Metrics) addAlerts(queue alertQueue) {
|
|
||||||
for _, name := range queue.DestinationsSet() {
|
|
||||||
destination := prometheus.Labels{"destination": name}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_sent_total",
|
|
||||||
Help: "Alerts the destination took.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Sent)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alert_send_failures_total",
|
|
||||||
Help: "Requests to the destination that failed.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).SendFailures)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_suppressed_total",
|
|
||||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
|
||||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Suppressed())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_dropped_total",
|
|
||||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
|
||||||
"alerts given up as the destination refused them.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Dropped)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
// count counts a request that has ended, as RequestEnded and RequestReported
|
||||||
// describe.
|
// describe.
|
||||||
func (m *Metrics) count(
|
func (m *Metrics) count(
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
package metrics_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
listURL = "https://lists.example/block.txt"
|
|
||||||
zone = "dnsbl.example"
|
|
||||||
destination = "webhook"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Each stand-in answers with a number of its own, so that each metric
|
|
||||||
// shows which part it was read from.
|
|
||||||
|
|
||||||
type ledgerStandIn struct{}
|
|
||||||
|
|
||||||
func (ledgerStandIn) Made(string) int { return 1 }
|
|
||||||
|
|
||||||
func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 }
|
|
||||||
|
|
||||||
type clientTableStandIn struct{}
|
|
||||||
|
|
||||||
func (clientTableStandIn) Len() int { return 3 }
|
|
||||||
|
|
||||||
type ruleFilesStandIn struct{}
|
|
||||||
|
|
||||||
func (ruleFilesStandIn) Len() int { return 4 }
|
|
||||||
|
|
||||||
type listsStandIn struct{}
|
|
||||||
|
|
||||||
func (listsStandIn) URLs() []string { return []string{listURL} }
|
|
||||||
|
|
||||||
func (listsStandIn) Failures(string) int { return 5 }
|
|
||||||
|
|
||||||
func (listsStandIn) Fetched(string) time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
type dnsblStandIn struct{}
|
|
||||||
|
|
||||||
func (dnsblStandIn) Zones() []string { return []string{zone} }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Queries(string) int { return 6 }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Failures(string) int { return 0 }
|
|
||||||
|
|
||||||
type abuseIPDBStandIn struct{}
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Checked() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Failures() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) BudgetLeft() int { return 7 }
|
|
||||||
|
|
||||||
type alertQueueStandIn struct{}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) DestinationsSet() []string {
|
|
||||||
return []string{destination}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Counts(string) alerts.Counts {
|
|
||||||
return alerts.Counts{Sent: 8}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Suppressed() int64 { return 0 }
|
|
||||||
|
|
||||||
type lookupDatabaseStandIn struct{}
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) ReadFailures() int { return 9 }
|
|
||||||
|
|
||||||
type remoteLogStandIn struct{}
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Sent() int64 { return 10 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Dropped() int64 { return 0 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Depth() int { return 0 }
|
|
||||||
|
|
||||||
func TestMetricsAreReadFromEachPart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
Now: func() time.Time { return time.Time{} },
|
|
||||||
Bans: ledgerStandIn{},
|
|
||||||
Clients: clientTableStandIn{},
|
|
||||||
Rules: ruleFilesStandIn{},
|
|
||||||
Lists: listsStandIn{},
|
|
||||||
DNSBL: dnsblStandIn{},
|
|
||||||
AbuseIPDB: abuseIPDBStandIn{},
|
|
||||||
Alerts: alertQueueStandIn{},
|
|
||||||
LookupFile: lookupDatabaseStandIn{},
|
|
||||||
RemoteLog: remoteLogStandIn{},
|
|
||||||
})
|
|
||||||
scraped := scrape(t, m)
|
|
||||||
|
|
||||||
for _, line := range []string{
|
|
||||||
`sww_bans_made_total{cause="limit",instance="app"} 1`,
|
|
||||||
`sww_active_bans{instance="app"} 2`,
|
|
||||||
`sww_tracked_clients{instance="app"} 3`,
|
|
||||||
`sww_rules_loaded{instance="app"} 4`,
|
|
||||||
`sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`,
|
|
||||||
`sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`,
|
|
||||||
`sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`,
|
|
||||||
`sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`,
|
|
||||||
`sww_lookup_database_read_failures_total{instance="app"} 9`,
|
|
||||||
`sww_remote_log_lines_sent_total{instance="app"} 10`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(scraped, "\n"+line+"\n") {
|
|
||||||
t.Errorf("metrics\n%s\nwant\n%s", scraped, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPartThatIsOffAddsNoMetric(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
scraped := scrape(t, metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
}))
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"sww_bans_made_total",
|
|
||||||
"sww_tracked_clients",
|
|
||||||
"sww_rules_loaded",
|
|
||||||
"sww_reputation_",
|
|
||||||
"sww_alert",
|
|
||||||
"sww_lookup_database_",
|
|
||||||
"sww_remote_log_",
|
|
||||||
} {
|
|
||||||
if strings.Contains(scraped, name) {
|
|
||||||
t.Errorf("metrics\n%s\nshow %s", scraped, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// scrape returns the metrics m serves.
|
|
||||||
func scrape(t *testing.T, m *metrics.Metrics) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
scraped := httptest.NewRecorder()
|
|
||||||
m.ServeHTTP(
|
|
||||||
scraped,
|
|
||||||
httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, "/", http.NoBody,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
return scraped.Body.String()
|
|
||||||
}
|
|
||||||
@@ -445,9 +445,12 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
|||||||
scores: map[string]int64{suspect: 100},
|
scores: map[string]int64{suspect: 100},
|
||||||
}
|
}
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one it refuses with an answer
|
// One check that AbuseIPDB answers, and one it refuses with an answer
|
||||||
// that names the key.
|
// that names the key.
|
||||||
@@ -500,9 +503,12 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
|
|||||||
p := abuseIPDBParams()
|
p := abuseIPDBParams()
|
||||||
p.DailyBudget = 5
|
p.DailyBudget = 5
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one that fails.
|
// One check that AbuseIPDB answers, and one that fails.
|
||||||
hitFrom(t, checker, suspect, true)
|
hitFrom(t, checker, suspect, true)
|
||||||
@@ -805,9 +811,6 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// instanceName is the label instance of the metrics the tests build.
|
|
||||||
const instanceName = "app"
|
|
||||||
|
|
||||||
// scrapeMetrics returns the metrics m serves.
|
// scrapeMetrics returns the metrics m serves.
|
||||||
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -358,9 +358,8 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
|||||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||||
}}
|
}}
|
||||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
wantZones(t, dnsbl, listed)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
@@ -405,9 +404,8 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) {
|
|||||||
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
||||||
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
||||||
}}, p)
|
}}, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
// Both clients are asked about before either answer comes, so that
|
// Both clients are asked about before either answer comes, so that
|
||||||
// the failure does not keep the zone from the other query.
|
// the failure does not keep the zone from the other query.
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package rproxy
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
@@ -23,9 +22,7 @@ type requestBody struct {
|
|||||||
// body is the visitor's body, ending in an *http.MaxBytesError past
|
// body is the visitor's body, ending in an *http.MaxBytesError past
|
||||||
// SWW_RPROXY_MAX_REQUEST_SIZE.
|
// SWW_RPROXY_MAX_REQUEST_SIZE.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
// refuse refuses the request with a status, as the request's own refuse
|
rq *request
|
||||||
// does.
|
|
||||||
refuse func(status int)
|
|
||||||
// held is the part read before the request goes to the app, which Read gives
|
// held is the part read before the request goes to the app, which Read gives
|
||||||
// first.
|
// first.
|
||||||
held []byte
|
held []byte
|
||||||
@@ -71,7 +68,7 @@ func (b *requestBody) Read(p []byte) (int, error) {
|
|||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.received.Store(true)
|
b.received.Store(true)
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.refuse(http.StatusRequestEntityTooLarge)
|
b.rq.refuse(http.StatusRequestEntityTooLarge)
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
@@ -89,13 +86,7 @@ type responseBody struct {
|
|||||||
// body is the app's body, ending in an *http.MaxBytesError past
|
// body is the app's body, ending in an *http.MaxBytesError past
|
||||||
// SWW_RPROXY_MAX_RESPONSE_SIZE.
|
// SWW_RPROXY_MAX_RESPONSE_SIZE.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
// refuse refuses the request with a status, as the request's own refuse
|
rq *request
|
||||||
// does.
|
|
||||||
refuse func(status int)
|
|
||||||
// responseReceived is called once the app's whole answer has arrived.
|
|
||||||
responseReceived func()
|
|
||||||
// ctx is the visitor's request's context.
|
|
||||||
ctx context.Context //nolint:containedctx // the body ends with the request
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read reads from the app's body.
|
// Read reads from the app's body.
|
||||||
@@ -109,15 +100,15 @@ func (b *responseBody) Read(p []byte) (int, error) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.responseReceived()
|
b.rq.responseReceived()
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.refuse(http.StatusBadGateway)
|
b.rq.refuse(http.StatusBadGateway)
|
||||||
|
|
||||||
return n, errResponseTooLarge
|
return n, errResponseTooLarge
|
||||||
case b.ctx.Err() == nil:
|
case b.rq.in.Context().Err() == nil:
|
||||||
// The answer broke off, not because the visitor went away. If a time
|
// The answer broke off, not because the visitor went away. If a time
|
||||||
// limit cut it, that refusal came first and is the one kept.
|
// limit cut it, that refusal came first and is the one kept.
|
||||||
b.refuse(http.StatusBadGateway)
|
b.rq.refuse(http.StatusBadGateway)
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ func (h *handler) newRequest(
|
|||||||
if r.Body != http.NoBody {
|
if r.Body != http.NoBody {
|
||||||
rq.body = &requestBody{
|
rq.body = &requestBody{
|
||||||
body: limitBody(r.Body, h.config.MaxRequestSize),
|
body: limitBody(r.Body, h.config.MaxRequestSize),
|
||||||
refuse: rq.refuse,
|
rq: rq,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,12 +291,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
|||||||
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
||||||
rq.startResponseTimer(requestEnd)
|
rq.startResponseTimer(requestEnd)
|
||||||
|
|
||||||
res.Body = &responseBody{
|
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
||||||
body: limitBody(res.Body, maxBytes),
|
|
||||||
refuse: rq.refuse,
|
|
||||||
responseReceived: rq.responseReceived,
|
|
||||||
ctx: rq.in.Context(),
|
|
||||||
}
|
|
||||||
rq.out.afterStatus = rq.setResponseDeadline
|
rq.out.afterStatus = rq.setResponseDeadline
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
+48
-19
@@ -18,11 +18,12 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
"golang.org/x/net/http/httpguts"
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -39,6 +40,12 @@ const (
|
|||||||
// extension ends the name of every rule file.
|
// extension ends the name of every rule file.
|
||||||
const extension = ".rules"
|
const extension = ".rules"
|
||||||
|
|
||||||
|
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
||||||
|
// the rule files are read again, so that a file still being written, such
|
||||||
|
// as one saved in place, appended to or copied in with scp, is read only
|
||||||
|
// once whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
// headerTarget starts the target that is one request header,
|
// headerTarget starts the target that is one request header,
|
||||||
// header:<Name>.
|
// header:<Name>.
|
||||||
const headerTarget = "header:"
|
const headerTarget = "header:"
|
||||||
@@ -169,18 +176,26 @@ func (f *Files) Len() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||||
// once Dir has had no change for a short time, after one is edited, added
|
// once Dir has had no change for quietTime, after one is edited, added or
|
||||||
// or removed, and after Watch starts watching, as dirwatch describes. If
|
// removed, and after Watch starts watching. If they then hold an error,
|
||||||
// they then hold an error, the rules stay as they were, the error is
|
// the rules stay as they were, the error is logged with its file and
|
||||||
// logged with its file and line, and the files are read again after the
|
// line, and the files are read again after the next change. If Dir cannot
|
||||||
// next change. If Dir cannot be watched, that is logged, and the rules
|
// be watched, that is logged, and the rules stay as they were loaded.
|
||||||
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
// While Enabled is false, Watch returns at once.
|
||||||
func (f *Files) Watch(ctx context.Context) {
|
func (f *Files) Watch(ctx context.Context) {
|
||||||
if !f.params.Enabled {
|
if !f.params.Enabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
watcher, err := dirwatch.New(f.params.Dir)
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err == nil {
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = watcher.Add(f.params.Dir)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
@@ -188,22 +203,36 @@ func (f *Files) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||||
"directory", f.params.Dir)
|
"directory", f.params.Dir)
|
||||||
|
|
||||||
// A change to any file in Dir counts.
|
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
||||||
watcher.CallWhenQuiet(ctx,
|
}
|
||||||
func(string) bool { return true },
|
|
||||||
f.readAgain,
|
// readAfterChanges reads the rule files again once quietTime has passed
|
||||||
func(err error) {
|
// without a change from events, until ctx is done, and logs the errors
|
||||||
|
// from errs. The wait starts at once, as if for a change, so that an edit
|
||||||
|
// saved after Load read the files, and before Dir was watched, is taken
|
||||||
|
// in too.
|
||||||
|
func (f *Files) readAfterChanges(
|
||||||
|
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
||||||
|
) {
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-events:
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
case <-quiet.C:
|
||||||
|
f.readAgain()
|
||||||
|
case err := <-errs:
|
||||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
},
|
}
|
||||||
)
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package rules
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The tests below run readAfterChanges in a synctest bubble, where time is
|
||||||
|
// a clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once readAfterChanges waits again, so that every
|
||||||
|
// reading due by then is done. The test sends the changes itself, as the
|
||||||
|
// watch of a directory cannot run in a bubble.
|
||||||
|
|
||||||
|
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "50-app.rules")
|
||||||
|
writeFile(t, path, "first path block ^/first\n")
|
||||||
|
files := load(t, dir)
|
||||||
|
changes := run(t, files)
|
||||||
|
|
||||||
|
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = file.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
// The first part ends in the middle of a ban rule's regex, which,
|
||||||
|
// read then, would ban every request.
|
||||||
|
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/anything")
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
write(t, file, `\.env$`+"\n")
|
||||||
|
|
||||||
|
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/.env")
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/.env", "probe")
|
||||||
|
wantMatched(t, files, "/anything")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "50-app.rules")
|
||||||
|
writeFile(t, path, "first path block ^/first\n")
|
||||||
|
files := load(t, dir)
|
||||||
|
|
||||||
|
// Saved after Load read the files, and before the directory was
|
||||||
|
// watched, so that no change is seen for it.
|
||||||
|
writeFile(t, path, "first path block ^/edited\n")
|
||||||
|
run(t, files)
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
wantMatched(t, files, "/edited", "first")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// load loads the rules in dir.
|
||||||
|
func load(t *testing.T, dir string) *Files {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
files, err := Load(Params{
|
||||||
|
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
||||||
|
Alerts: alerts.New(alerts.Params{}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
// run runs files' readAfterChanges until the test ends, and returns the
|
||||||
|
// channel that sends it changes.
|
||||||
|
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
changes := make(chan fsnotify.Event)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
files.readAfterChanges(ctx, changes, nil)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return changes
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFile writes content to the file at path.
|
||||||
|
func writeFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.WriteFile(path, []byte(content), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// write writes text to the end of file.
|
||||||
|
func write(t *testing.T, file *os.File, text string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := file.WriteString(text)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantMatched checks the ids of the rules that a GET request for path
|
||||||
|
// matches, in order.
|
||||||
|
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||||
|
"http://app.example"+path, nil)
|
||||||
|
|
||||||
|
matched := files.Match(r)
|
||||||
|
|
||||||
|
got := make([]string, 0, len(matched))
|
||||||
|
for _, rule := range matched {
|
||||||
|
got = append(got, rule.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("%s matched %v, want %v", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2430,7 +2430,7 @@ func midnight() time.Time {
|
|||||||
// counts, with thresholds fill does not reach.
|
// counts, with thresholds fill does not reach.
|
||||||
func newParams(dir string) state.Params {
|
func newParams(dir string) state.Params {
|
||||||
discard := slog.New(slog.DiscardHandler)
|
discard := slog.New(slog.DiscardHandler)
|
||||||
m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"})
|
m := metrics.New(1, "app")
|
||||||
queue := alerts.New(alerts.Params{
|
queue := alerts.New(alerts.Params{
|
||||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||||
Events: alerts.Events(),
|
Events: alerts.Events(),
|
||||||
|
|||||||
+29
-51
@@ -10,14 +10,13 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
@@ -27,7 +26,14 @@ import (
|
|||||||
// know.
|
// know.
|
||||||
const usageStatus = 2
|
const usageStatus = 2
|
||||||
|
|
||||||
var errNoCommand = errors.New("no command given")
|
// usage is what sww prints then.
|
||||||
|
const usage = `usage: sww <command>
|
||||||
|
|
||||||
|
commands:
|
||||||
|
rproxy the reverse proxy in front of one app
|
||||||
|
waf decides for rproxy, and keeps bans, counters and history
|
||||||
|
healthcheck the container's health check
|
||||||
|
`
|
||||||
|
|
||||||
// Params are what RunWAF and RunRProxy need from the process.
|
// Params are what RunWAF and RunRProxy need from the process.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
@@ -45,9 +51,14 @@ type Params struct {
|
|||||||
|
|
||||||
// Main runs the command os.Args names, and returns the process's exit status.
|
// Main runs the command os.Args names, and returns the process's exit status.
|
||||||
// `sww rproxy` and `sww waf` run until SIGTERM or SIGINT.
|
// `sww rproxy` and `sww waf` run until SIGTERM or SIGINT.
|
||||||
// With no command, or one it does not know, cobra prints the error and the
|
// With no command, or one it does not know, it prints the commands and returns
|
||||||
// commands to stderr, and Main returns usageStatus.
|
// usageStatus.
|
||||||
func Main(version string) int {
|
func Main(version string) int {
|
||||||
|
args := os.Args[1:]
|
||||||
|
if len(args) == 0 {
|
||||||
|
return printUsage(os.Stderr)
|
||||||
|
}
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(),
|
ctx, stop := signal.NotifyContext(context.Background(),
|
||||||
syscall.SIGTERM, os.Interrupt)
|
syscall.SIGTERM, os.Interrupt)
|
||||||
defer stop()
|
defer stop()
|
||||||
@@ -58,54 +69,21 @@ func Main(version string) int {
|
|||||||
Stdout: os.Stdout,
|
Stdout: os.Stdout,
|
||||||
}
|
}
|
||||||
|
|
||||||
// status is what the command that ran returns.
|
switch args[0] {
|
||||||
status := 0
|
case "rproxy":
|
||||||
|
return RunRProxy(ctx, params)
|
||||||
root := &cobra.Command{
|
case "waf":
|
||||||
Use: "sww",
|
return RunWAF(ctx, params)
|
||||||
Short: "a web application firewall",
|
case "healthcheck":
|
||||||
Args: cobra.NoArgs,
|
return HealthCheck(ctx, args[1:], params.Viper, os.Stderr)
|
||||||
RunE: func(*cobra.Command, []string) error {
|
default:
|
||||||
return errNoCommand
|
return printUsage(os.Stderr)
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
root.CompletionOptions.DisableDefaultCmd = true
|
|
||||||
|
|
||||||
root.AddCommand(
|
// printUsage writes the commands to w, and returns usageStatus.
|
||||||
&cobra.Command{
|
func printUsage(w io.Writer) int {
|
||||||
Use: "rproxy",
|
_, _ = fmt.Fprint(w, usage)
|
||||||
Short: "the reverse proxy in front of one app",
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
status = RunRProxy(cmd.Context(), params)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
&cobra.Command{
|
|
||||||
Use: "waf",
|
|
||||||
Short: "decides for rproxy, and keeps bans, counters and history",
|
|
||||||
Run: func(cmd *cobra.Command, _ []string) {
|
|
||||||
status = RunWAF(cmd.Context(), params)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
&cobra.Command{
|
|
||||||
Use: "healthcheck",
|
|
||||||
Short: "the container's health check",
|
|
||||||
// HealthCheck refuses every argument itself, flags included.
|
|
||||||
DisableFlagParsing: true,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
status = HealthCheck(
|
|
||||||
cmd.Context(),
|
|
||||||
args,
|
|
||||||
params.Viper,
|
|
||||||
cmd.ErrOrStderr(),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
err := root.ExecuteContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return usageStatus
|
return usageStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
return status
|
|
||||||
}
|
|
||||||
+11
-18
@@ -392,33 +392,26 @@ func newMetrics(
|
|||||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||||
remote *remotelog.Sender,
|
remote *remotelog.Sender,
|
||||||
) *metrics.Metrics {
|
) *metrics.Metrics {
|
||||||
params := metrics.Params{
|
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
m.AddBansAndClients(ledger, limiter, utcNow)
|
||||||
InstanceName: cfg.InstanceName,
|
m.AddRules(ruleFiles)
|
||||||
Now: utcNow,
|
m.AddReputation(lists, dnsbl)
|
||||||
Bans: ledger,
|
|
||||||
Clients: limiter,
|
if cfg.AbuseIPDBKey != "" {
|
||||||
Rules: ruleFiles,
|
m.AddAbuseIPDB(abuseIPDB)
|
||||||
Lists: lists,
|
|
||||||
DNSBL: dnsbl,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A nil pointer in an interface field is not a nil interface, so a part
|
m.AddAlerts(alertQueue)
|
||||||
// that is off is left out rather than set to nil.
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
|
||||||
params.AbuseIPDB = abuseIPDB
|
|
||||||
}
|
|
||||||
|
|
||||||
if lookupFile != nil {
|
if lookupFile != nil {
|
||||||
params.LookupFile = lookupFile
|
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
||||||
}
|
}
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
params.RemoteLog = remote
|
m.AddRemoteLog(remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(params)
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
|
|||||||
+22
-81
@@ -4,7 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
@@ -165,104 +164,46 @@ func wantUTC(t *testing.T, where string, value any) {
|
|||||||
func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
|
func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// completion is cobra's own command, which sww turns off.
|
|
||||||
for _, command := range []string{"", "serve", "completion"} {
|
|
||||||
t.Run("command "+command, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
code, stdout, stderr := runMain(t, command)
|
|
||||||
|
|
||||||
if code != 2 {
|
|
||||||
t.Errorf("exit status %d, want 2", code)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantTheCommands(t, "stderr", stderr)
|
|
||||||
|
|
||||||
if stdout != "" {
|
|
||||||
t.Errorf("stdout %q, want nothing", stdout)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMainHelpPrintsTheCommands(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, command := range []string{"help", "--help"} {
|
|
||||||
t.Run("command "+command, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
code, stdout, _ := runMain(t, command)
|
|
||||||
|
|
||||||
if code != 0 {
|
|
||||||
t.Errorf("exit status %d, want 0", code)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantTheCommands(t, "stdout", stdout)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantTheCommands checks that text, what Main wrote to where, names each
|
|
||||||
// command.
|
|
||||||
func wantTheCommands(t *testing.T, where, text string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
|
|
||||||
if !strings.Contains(text, name) {
|
|
||||||
t.Errorf("%s %q does not name %s", where, text, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMainHealthCheckRefusesAnArgument(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, argument := range []string{"now", "--x", "-h"} {
|
|
||||||
t.Run("argument "+argument, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
code, stdout, stderr := runMain(t, "healthcheck "+argument)
|
|
||||||
|
|
||||||
want := "sww healthcheck: unexpected argument \"" +
|
|
||||||
argument + "\"\n"
|
|
||||||
if code != 1 || stderr != want || stdout != "" {
|
|
||||||
t.Errorf("exit status %d, stdout %q and stderr %q, "+
|
|
||||||
"want 1, nothing and %q", code, stdout, stderr, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runMain runs Main with command in a process of its own, and returns its
|
|
||||||
// exit status and what it wrote to stdout and stderr.
|
|
||||||
func runMain(t *testing.T, command string) (int, string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
binary, err := os.Executable()
|
binary, err := os.Executable()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("find the test binary: %v", err)
|
t.Fatalf("find the test binary: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, command := range []string{"", "serve"} {
|
||||||
|
t.Run("command "+command, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
|
ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
//nolint:gosec // this test binary
|
|
||||||
cmd := exec.CommandContext(ctx, binary)
|
cmd := exec.CommandContext(ctx, binary)
|
||||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), commandForMain+"="+command)
|
cmd.Env = append(os.Environ(), commandForMain+"="+command)
|
||||||
|
|
||||||
err = cmd.Run()
|
err := cmd.Run()
|
||||||
|
|
||||||
var exitErr *exec.ExitError
|
if code := cmd.ProcessState.ExitCode(); code != 2 {
|
||||||
if err != nil && !errors.As(err, &exitErr) {
|
t.Errorf("exit status %d (%v), want 2", code, err)
|
||||||
t.Fatalf("run: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return cmd.ProcessState.ExitCode(), stdout.String(), stderr.String()
|
for _, name := range []string{"rproxy", wafCommand, "healthcheck"} {
|
||||||
|
if !strings.Contains(stderr.String(), name) {
|
||||||
|
t.Errorf(
|
||||||
|
"stderr %q does not name %s",
|
||||||
|
stderr.String(),
|
||||||
|
name,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Errorf("stdout %q, want nothing", stdout.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ask sends waf at addr rproxy's question about a request for path from
|
// ask sends waf at addr rproxy's question about a request for path from
|
||||||
|
|||||||
+107
-52
@@ -2,6 +2,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"crypto/subtle"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -14,6 +15,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,23 +41,80 @@ var (
|
|||||||
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
||||||
)
|
)
|
||||||
|
|
||||||
// handleHealth answers GET wafprotocol.HealthPath.
|
// answerHealth answers a request for the health endpoint.
|
||||||
func handleHealth() http.HandlerFunc {
|
func (rq *request) answerHealth() {
|
||||||
return func(w http.ResponseWriter, _ *http.Request) {
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
// Set here rather than left to Go's server, which would set it only
|
// Set here rather than left to Go's server, which would set it only
|
||||||
// after the log line has taken the response's headers.
|
// after the log line has taken the response's headers.
|
||||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
_, _ = io.WriteString(w, "ok\n")
|
_, _ = io.WriteString(rq.out, "ok\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerAdmin answers a request for one of waf's own endpoints. Each
|
||||||
|
// needs a token, sent as Authorization: Bearer <token>: the metrics
|
||||||
|
// SWW_WAF_METRICS_TOKEN, the others SWW_WAF_ADMIN_TOKEN. A request without it is
|
||||||
|
// refused with 401. An endpoint whose token is unset answers 404, as any other
|
||||||
|
// path does.
|
||||||
|
func (rq *request) answerAdmin() {
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
|
||||||
|
token, answer := rq.endpoint()
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case token == "":
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusNotFound),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
|
case !hasToken(rq.in, token):
|
||||||
|
rq.out.Header().Set("WWW-Authenticate", "Bearer")
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusUnauthorized),
|
||||||
|
http.StatusUnauthorized,
|
||||||
|
)
|
||||||
|
default:
|
||||||
|
answer()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleListBans answers GET BansPath with every ban held.
|
// endpoint returns the token the request's endpoint needs, and what
|
||||||
func (h *handler) handleListBans() http.HandlerFunc {
|
// answers the request there; "" when there is no such endpoint.
|
||||||
return func(w http.ResponseWriter, _ *http.Request) {
|
func (rq *request) endpoint() (string, func()) {
|
||||||
h.answerBans(w, h.ledger.Snapshot())
|
cfg := rq.h.config
|
||||||
|
method, path := rq.in.Method, rq.in.URL.Path
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case method == http.MethodGet && path == MetricsPath:
|
||||||
|
return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) }
|
||||||
|
case method == http.MethodGet && path == BansPath:
|
||||||
|
return cfg.AdminToken, rq.listBans
|
||||||
|
case method == http.MethodPost && path == BansPath:
|
||||||
|
return cfg.AdminToken, rq.addBan
|
||||||
|
case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"):
|
||||||
|
return cfg.AdminToken, rq.liftBans
|
||||||
|
case method == http.MethodGet && strings.HasPrefix(path, ClientsPath):
|
||||||
|
return cfg.AdminToken, rq.showClient
|
||||||
|
default:
|
||||||
|
return "", nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hasToken reports whether r carries token, as Authorization: Bearer
|
||||||
|
// <token>.
|
||||||
|
func hasToken(r *http.Request, token string) bool {
|
||||||
|
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||||
|
|
||||||
|
return strings.EqualFold(scheme, "Bearer") &&
|
||||||
|
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// listBans answers GET BansPath with every ban held.
|
||||||
|
func (rq *request) listBans() {
|
||||||
|
rq.answerBans(rq.h.ledger.Snapshot())
|
||||||
|
}
|
||||||
|
|
||||||
// banToAdd is the body of POST BansPath.
|
// banToAdd is the body of POST BansPath.
|
||||||
type banToAdd struct {
|
type banToAdd struct {
|
||||||
// Netblock is a netblock, or a client's address, which stands for the
|
// Netblock is a netblock, or a client's address, which stands for the
|
||||||
@@ -67,21 +126,20 @@ type banToAdd struct {
|
|||||||
Reason string `json:"reason"`
|
Reason string `json:"reason"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleAddBan answers POST BansPath: it bans the netblock the body names,
|
// addBan answers POST BansPath: it bans the netblock the body names, as
|
||||||
// as an admin, from now for the duration the body gives, with its reason,
|
// an admin, from now for the duration the body gives, with its reason,
|
||||||
// and answers with that ban.
|
// and answers with that ban.
|
||||||
func (h *handler) handleAddBan() http.HandlerFunc {
|
func (rq *request) addBan() {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
toAdd, err := rq.readBanToAdd()
|
||||||
toAdd, err := readBanToAdd(r)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
netblock netip.Prefix
|
netblock netip.Prefix
|
||||||
expires time.Time
|
expires time.Time
|
||||||
now = h.now()
|
now = rq.h.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
netblock, err = h.parseBanNetblock(toAdd.Netblock)
|
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -89,22 +147,23 @@ func (h *handler) handleAddBan() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ban := h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
|
ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
|
||||||
h.answerBans(w, []bans.Ban{ban})
|
rq.answerBans([]bans.Ban{ban})
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
||||||
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
||||||
func readBanToAdd(r *http.Request) (banToAdd, error) {
|
func (rq *request) readBanToAdd() (banToAdd, error) {
|
||||||
data, err := io.ReadAll(
|
data, err := io.ReadAll(
|
||||||
http.MaxBytesReader(nil, r.Body, banBodyMaxBytes),
|
http.MaxBytesReader(nil, rq.in.Body, banBodyMaxBytes),
|
||||||
)
|
)
|
||||||
|
rq.requestBytes = int64(len(data))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
||||||
}
|
}
|
||||||
@@ -191,22 +250,21 @@ func expiry(duration string, now time.Time) (time.Time, error) {
|
|||||||
return now.Add(length), nil
|
return now.Add(length), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleLiftBans answers DELETE BansPath/<client>: it lifts every ban
|
// liftBans answers DELETE BansPath/<client>: it lifts every ban active on
|
||||||
// active on a netblock the client's address is in, and answers with those
|
// a netblock the client's address is in, and answers with those bans, or
|
||||||
// bans, or with 404 when none is active.
|
// with 404 when none is active.
|
||||||
func (h *handler) handleLiftBans() http.HandlerFunc {
|
func (rq *request) liftBans() {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
client, err := pathAddress(rq.in.URL.Path, BansPath+"/")
|
||||||
client, err := pathAddress(r.URL.Path, BansPath+"/")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
lifted := h.ledger.Lift(client, h.now())
|
lifted := rq.h.ledger.Lift(client, rq.h.now())
|
||||||
if len(lifted) == 0 {
|
if len(lifted) == 0 {
|
||||||
http.Error(
|
http.Error(
|
||||||
w,
|
rq.out,
|
||||||
"no ban is active on "+client.String(),
|
"no ban is active on "+client.String(),
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
)
|
)
|
||||||
@@ -214,8 +272,7 @@ func (h *handler) handleLiftBans() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.answerBans(w, lifted)
|
rq.answerBans(lifted)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
|
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
|
||||||
@@ -227,31 +284,29 @@ type clientAnswer struct {
|
|||||||
Bans []state.BanEntry `json:"bans"`
|
Bans []state.BanEntry `json:"bans"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleShowClient answers GET ClientsPath<ip> with what waf knows of the
|
// showClient answers GET ClientsPath<ip> with what waf knows of the client:
|
||||||
// client: its counters, its history, which holds its country as last looked
|
// its counters, its history, which holds its country as last looked up and its
|
||||||
// up and its offences, and its bans with their notes.
|
// offences, and its bans with their notes.
|
||||||
func (h *handler) handleShowClient() http.HandlerFunc {
|
func (rq *request) showClient() {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
|
||||||
addr, err := pathAddress(r.URL.Path, ClientsPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
answer := clientAnswer{
|
answer := clientAnswer{
|
||||||
Bans: state.BanEntries(h.ledger.Covering(addr)),
|
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||||
}
|
}
|
||||||
|
|
||||||
client, seen := h.limiter.Client(
|
client, seen := rq.h.limiter.Client(
|
||||||
clientGroup(h.config.IPv6GroupPrefix, addr),
|
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
||||||
)
|
)
|
||||||
if seen {
|
if seen {
|
||||||
answer.Client = &client
|
answer.Client = &client
|
||||||
}
|
}
|
||||||
|
|
||||||
h.answerJSON(w, answer)
|
rq.answerJSON(answer)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathAddress reads the client's address that follows prefix in path.
|
// pathAddress reads the client's address that follows prefix in path.
|
||||||
@@ -267,27 +322,27 @@ func pathAddress(path, prefix string) (netip.Addr, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// answerBans answers with held under bans, as bans.json lists them.
|
// answerBans answers with held under bans, as bans.json lists them.
|
||||||
func (h *handler) answerBans(w http.ResponseWriter, held []bans.Ban) {
|
func (rq *request) answerBans(held []bans.Ban) {
|
||||||
h.answerJSON(w, struct {
|
rq.answerJSON(struct {
|
||||||
Bans []state.BanEntry `json:"bans"`
|
Bans []state.BanEntry `json:"bans"`
|
||||||
}{state.BanEntries(held)})
|
}{state.BanEntries(held)})
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerJSON answers with value as indented JSON.
|
// answerJSON answers with value as indented JSON.
|
||||||
func (h *handler) answerJSON(w http.ResponseWriter, value any) {
|
func (rq *request) answerJSON(value any) {
|
||||||
body, err := json.MarshalIndent(value, "", " ")
|
body, err := json.MarshalIndent(value, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.processLog.Error(
|
rq.h.processLog.Error(
|
||||||
"encoding an answer failed",
|
"encoding an answer failed",
|
||||||
"error",
|
"error",
|
||||||
err.Error(),
|
err.Error(),
|
||||||
)
|
)
|
||||||
http.Error(w, http.StatusText(http.StatusInternalServerError),
|
http.Error(rq.out, http.StatusText(http.StatusInternalServerError),
|
||||||
http.StatusInternalServerError)
|
http.StatusInternalServerError)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
rq.out.Header().Set("Content-Type", "application/json")
|
||||||
_, _ = w.Write(append(body, '\n'))
|
_, _ = rq.out.Write(append(body, '\n'))
|
||||||
}
|
}
|
||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -334,39 +333,6 @@ func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBanToAddIsLoggedWithTheBytesOfItsBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, _, _ := startWithClock(
|
|
||||||
t,
|
|
||||||
"",
|
|
||||||
map[string]string{adminToken: adminSecret},
|
|
||||||
)
|
|
||||||
|
|
||||||
s.admin(http.MethodPost, waf.BansPath, banOtherClient, http.StatusOK)
|
|
||||||
|
|
||||||
line := s.out.endedLines(t, s.sent)[s.sent-1]
|
|
||||||
if line.RequestBytes != int64(len(banOtherClient)) {
|
|
||||||
t.Errorf(
|
|
||||||
"the ban to add was logged with request_bytes %d, want %d",
|
|
||||||
line.RequestBytes,
|
|
||||||
len(banOtherClient),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The health endpoint reads none of a body sent to it.
|
|
||||||
s.ownEndpointRequest("", http.MethodGet, wafprotocol.HealthPath, "ok",
|
|
||||||
http.StatusOK)
|
|
||||||
|
|
||||||
line = s.out.endedLines(t, s.sent)[s.sent-1]
|
|
||||||
if line.RequestBytes != 0 {
|
|
||||||
t.Errorf(
|
|
||||||
"the health request was logged with request_bytes %d, want 0",
|
|
||||||
line.RequestBytes,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -58,14 +58,3 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHealthEndpointIsFoundAtItsPathPercentEncoded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// %68 is h. An endpoint is found by the decoded path.
|
|
||||||
s, _, _ := startWithClock(t, "", nil)
|
|
||||||
|
|
||||||
got := s.ownEndpointRequest("", http.MethodGet, "/_sww/%68ealthz", "",
|
|
||||||
http.StatusOK)
|
|
||||||
wantBody(t, got, "ok\n")
|
|
||||||
}
|
|
||||||
@@ -268,8 +268,6 @@ func newHandler(params Params) *handler {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
h.ownEndpoints = h.routes()
|
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -295,9 +293,6 @@ type handler struct {
|
|||||||
// awaitingReport holds the decision on each answered question by its id
|
// awaitingReport holds the decision on each answered question by its id
|
||||||
// until its report comes, and nil for a question still being handled.
|
// until its report comes, and nil for a question still being handled.
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
awaitingReport *lru.Cache[string, *decision]
|
||||||
// ownEndpoints answers every request that is not a question, as routes
|
|
||||||
// lists the endpoints.
|
|
||||||
ownEndpoints http.Handler
|
|
||||||
|
|
||||||
// The lists of heuristics, each in the order its heuristics run. A client
|
// The lists of heuristics, each in the order its heuristics run. A client
|
||||||
// in SWW_WAF_ALLOW_NETS skips them all.
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||||
|
|||||||
+20
-27
@@ -548,11 +548,16 @@ func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
|||||||
params.CoreRuleSet = coreRuleSet
|
params.CoreRuleSet = coreRuleSet
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCounters adds to params the table of clients, the ledger and the
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
// anomaly counters, as sww builds them from params' settings.
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
func addCounters(params *waf.Params) {
|
func addCounters(params *waf.Params) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
|
|
||||||
|
params.Metrics = metrics.New(
|
||||||
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
|
cfg.InstanceName,
|
||||||
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
Limits: ratelimit.Limits{
|
Limits: ratelimit.Limits{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
@@ -582,18 +587,26 @@ func addCounters(params *waf.Params) {
|
|||||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
Alerts: params.Alerts,
|
Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
params.Metrics.AddBansAndClients(
|
||||||
|
params.Ledger,
|
||||||
|
params.Limiter,
|
||||||
|
params.Now,
|
||||||
|
)
|
||||||
|
params.Metrics.AddRules(params.Rules)
|
||||||
}
|
}
|
||||||
|
|
||||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||||
// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as
|
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||||
// sww builds them from params' settings and the parts addCounters added.
|
// them from params' settings and the parts addCounters added. The metrics
|
||||||
|
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
func addOutsideSources(
|
func addOutsideSources(
|
||||||
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||||
) {
|
) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
limiter, ledger := params.Limiter, params.Ledger
|
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
||||||
processLog, alertQueue := params.ProcessLog, params.Alerts
|
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||||
|
|
||||||
params.Lists = reputation.New(reputation.Params{
|
params.Lists = reputation.New(reputation.Params{
|
||||||
@@ -642,9 +655,6 @@ func addOutsideSources(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
m := newMetrics(params)
|
|
||||||
params.Metrics = m
|
|
||||||
|
|
||||||
params.GeoJS = geojs.New(geojs.Params{
|
params.GeoJS = geojs.New(geojs.Params{
|
||||||
URL: geojsURL,
|
URL: geojsURL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
@@ -657,29 +667,12 @@ func addOutsideSources(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
// newMetrics returns the metrics, as sww builds them from params' settings,
|
m.AddReputation(params.Lists, params.DNSBL)
|
||||||
// read from the ledger, the table of clients, the rules, the lists, the
|
|
||||||
// zones' verdicts, and AbuseIPDB's while its key is set.
|
|
||||||
func newMetrics(params *waf.Params) *metrics.Metrics {
|
|
||||||
cfg := params.Config
|
|
||||||
metricsParams := metrics.Params{
|
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
|
||||||
InstanceName: cfg.InstanceName,
|
|
||||||
Now: params.Now,
|
|
||||||
Bans: params.Ledger,
|
|
||||||
Clients: params.Limiter,
|
|
||||||
Rules: params.Rules,
|
|
||||||
Lists: params.Lists,
|
|
||||||
DNSBL: params.DNSBL,
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
if cfg.AbuseIPDBKey != "" {
|
||||||
metricsParams.AbuseIPDB = params.AbuseIPDB
|
m.AddAbuseIPDB(params.AbuseIPDB)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(metricsParams)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newClient returns an HTTP client that sends requests as they are made,
|
// newClient returns an HTTP client that sends requests as they are made,
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
package waf
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/subtle"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
|
||||||
|
|
||||||
// routes returns the router of waf's own endpoints. A question never reaches
|
|
||||||
// it, since ServeHTTP answers a question first. No check runs on a request
|
|
||||||
// here: waf's address is on the private network, where the peers are rproxy
|
|
||||||
// and the admin's tools, not visitors.
|
|
||||||
func (h *handler) routes() http.Handler {
|
|
||||||
r := chi.NewRouter()
|
|
||||||
r.Use(matchDecodedPath)
|
|
||||||
|
|
||||||
// The report writes no log line of its own: it finishes the line of the
|
|
||||||
// request it reports.
|
|
||||||
r.Post(wafprotocol.ReportPath, h.handleReport())
|
|
||||||
|
|
||||||
r.Group(func(r chi.Router) {
|
|
||||||
r.Use(h.logOwnEndpoint)
|
|
||||||
// The answer to a path with no endpoint, and to a method the path has
|
|
||||||
// no endpoint for, is set in this group, so that it is logged too.
|
|
||||||
r.NotFound(answerNotFound)
|
|
||||||
r.MethodNotAllowed(answerNotFound)
|
|
||||||
|
|
||||||
r.Get(wafprotocol.HealthPath, handleHealth())
|
|
||||||
|
|
||||||
r.Group(func(r chi.Router) {
|
|
||||||
r.Use(requireToken(h.config.MetricsToken))
|
|
||||||
r.Get(MetricsPath, h.metrics.ServeHTTP)
|
|
||||||
})
|
|
||||||
|
|
||||||
// The handlers under BansPath/ and ClientsPath read the address from
|
|
||||||
// the path themselves, so that one that is not an address gets 400
|
|
||||||
// after the token check.
|
|
||||||
r.Group(func(r chi.Router) {
|
|
||||||
r.Use(requireToken(h.config.AdminToken))
|
|
||||||
r.Get(BansPath, h.handleListBans())
|
|
||||||
r.Post(BansPath, h.handleAddBan())
|
|
||||||
r.Delete(BansPath+"/*", h.handleLiftBans())
|
|
||||||
r.Get(ClientsPath+"*", h.handleShowClient())
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// matchDecodedPath has the router match the decoded path, which the handlers
|
|
||||||
// read. chi matches the path as sent when it is encoded in an unusual way,
|
|
||||||
// which would make /_sww/%68ealthz a path with no endpoint.
|
|
||||||
func matchDecodedPath(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// logOwnEndpoint writes one log line for each request, with the action
|
|
||||||
// own_endpoint. The line's request_bytes is how much of the body the handler
|
|
||||||
// read.
|
|
||||||
func (h *handler) logOwnEndpoint(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
peer := peerAddress(r)
|
|
||||||
|
|
||||||
rq := h.startRequest(w, r, peer, peer, false)
|
|
||||||
defer rq.finish()
|
|
||||||
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
|
|
||||||
counted := r.Clone(r.Context())
|
|
||||||
counted.Body = &countedBody{
|
|
||||||
ReadCloser: r.Body,
|
|
||||||
read: &rq.requestBytes,
|
|
||||||
}
|
|
||||||
|
|
||||||
next.ServeHTTP(rq.out, counted)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// countedBody is a request's body that adds the bytes read of it to read.
|
|
||||||
type countedBody struct {
|
|
||||||
io.ReadCloser
|
|
||||||
|
|
||||||
read *int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read reads part of the body.
|
|
||||||
func (b *countedBody) Read(p []byte) (int, error) {
|
|
||||||
n, err := b.ReadCloser.Read(p)
|
|
||||||
*b.read += int64(n)
|
|
||||||
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireToken returns middleware that lets a request through only when it
|
|
||||||
// carries token, as Authorization: Bearer <token>. It answers 401 to a request
|
|
||||||
// without it. While token is unset, it answers 404, as a path with no endpoint
|
|
||||||
// does.
|
|
||||||
func requireToken(token string) func(http.Handler) http.Handler {
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case token == "":
|
|
||||||
answerNotFound(w, r)
|
|
||||||
case !hasToken(r, token):
|
|
||||||
w.Header().Set("WWW-Authenticate", "Bearer")
|
|
||||||
http.Error(
|
|
||||||
w,
|
|
||||||
http.StatusText(http.StatusUnauthorized),
|
|
||||||
http.StatusUnauthorized,
|
|
||||||
)
|
|
||||||
default:
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// hasToken reports whether r carries token, as Authorization: Bearer
|
|
||||||
// <token>.
|
|
||||||
func hasToken(r *http.Request, token string) bool {
|
|
||||||
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
|
|
||||||
|
|
||||||
return strings.EqualFold(scheme, "Bearer") &&
|
|
||||||
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerNotFound answers 404, for a path with no endpoint, and for a method
|
|
||||||
// the path has no endpoint for.
|
|
||||||
func answerNotFound(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
}
|
|
||||||
+34
-13
@@ -58,8 +58,8 @@ func New(params Params) *Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP handles one request to waf: a request carrying
|
// ServeHTTP handles one request to waf: a request carrying
|
||||||
// wafprotocol.ClientHeader is rproxy's question, and any other is for one of
|
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
|
||||||
// waf's own endpoints.
|
// itself.
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
||||||
h.answerQuestion(w, r)
|
h.answerQuestion(w, r)
|
||||||
@@ -67,7 +67,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.ownEndpoints.ServeHTTP(w, r)
|
h.answerOwnEndpoint(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
||||||
@@ -240,11 +240,34 @@ func (rq *request) deny(r refusal) {
|
|||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
|
// answerOwnEndpoint answers a request to waf's own address. No check
|
||||||
// the report names, if it holds that request's decision, and ignores the
|
// runs on it: the address is on the private network, where the peers are
|
||||||
// report otherwise.
|
// rproxy and the admin's tools, not visitors.
|
||||||
func (h *handler) handleReport() http.HandlerFunc {
|
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
|
||||||
|
h.takeReport(w, r)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
peer := peerAddress(r)
|
||||||
|
|
||||||
|
rq := h.startRequest(w, r, peer, peer, false)
|
||||||
|
defer rq.finish()
|
||||||
|
|
||||||
|
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
||||||
|
rq.answerHealth()
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.answerAdmin()
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
|
||||||
|
// report names, if it holds that request's decision, and ignores the report
|
||||||
|
// otherwise.
|
||||||
|
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||||
report, err := wafprotocol.ReadReport(w, r)
|
report, err := wafprotocol.ReadReport(w, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||||
@@ -254,16 +277,14 @@ func (h *handler) handleReport() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
held, found := h.awaitingReport.Peek(report.RequestID)
|
held, found := h.awaitingReport.Peek(report.RequestID)
|
||||||
// Remove reports true to one report only, of several that name the
|
// Remove reports true to one report only, of several that name the id. A
|
||||||
// id. A question still being handled is held as nil.
|
// question still being handled is held as nil.
|
||||||
if found && held != nil &&
|
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
|
||||||
h.awaitingReport.Remove(report.RequestID) {
|
|
||||||
h.finishWith(r.Context(), held, report)
|
h.finishWith(r.Context(), held, report)
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
// finishWith finishes a request with what its report says, and writes the
|
||||||
// request's last log line, which holds its decision's fields and the report's.
|
// request's last log line, which holds its decision's fields and the report's.
|
||||||
|
|||||||
Reference in new issue
Block a user