Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17400f68b8 | ||
|
|
9d008ed31a | ||
|
|
86bb33bf07 | ||
|
|
91444a37b3 |
No files matched your search
@@ -74,9 +74,9 @@ never as a time.
|
|||||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||||
and stops their background work.
|
and stops their background work.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
|
- `internal/waf`: `waf`'s server. It runs the checks in `check`. Its
|
||||||
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
|
`routes.go` lists `waf`'s own endpoints. It counts bytes, refusals and
|
||||||
and anomalies, and writes the log lines.
|
anomalies, and writes the log lines.
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
||||||
about each request. It passes a request to the app with the standard
|
about each request. It passes a request to the app with the standard
|
||||||
library's `httputil.ReverseProxy`, within the time and size limits.
|
library's `httputil.ReverseProxy`, within the time and size limits.
|
||||||
@@ -128,6 +128,7 @@ never as a time.
|
|||||||
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
|
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
|
||||||
version. They are checked again before the version changes, since rule
|
version. They are checked again before the version changes, since rule
|
||||||
ids, and what each rule matches, change between versions;
|
ids, and what each rule matches, change between versions;
|
||||||
|
- `github.com/go-chi/chi/v5` to route the requests to `waf`'s own endpoints;
|
||||||
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
|
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
|
||||||
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
|
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
|
||||||
- `github.com/fsnotify/fsnotify` to notice edited files;
|
- `github.com/fsnotify/fsnotify` to notice edited files;
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ require (
|
|||||||
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
|
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
|
||||||
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
|
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
|
||||||
github.com/fsnotify/fsnotify v1.10.1
|
github.com/fsnotify/fsnotify v1.10.1
|
||||||
|
github.com/go-chi/chi/v5 v5.5.0
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||||
github.com/maxmind/mmdbwriter v1.2.0
|
github.com/maxmind/mmdbwriter v1.2.0
|
||||||
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
|
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
|
||||||
|
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||||
|
|||||||
+393
-268
@@ -15,12 +15,33 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Params are what New builds the metrics from. Each part is nil while it is
|
||||||
|
// off, and a part that is nil adds no metric of its own.
|
||||||
|
type Params struct {
|
||||||
|
// BusiestCount is how many countries and how many AS numbers get series
|
||||||
|
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
||||||
|
BusiestCount int
|
||||||
|
// InstanceName is the label instance every metric carries
|
||||||
|
// (SWW_WAF_INSTANCE_NAME).
|
||||||
|
InstanceName string
|
||||||
|
// Now returns the time at which Bans counts the bans active.
|
||||||
|
Now func() time.Time
|
||||||
|
|
||||||
|
Bans banLedger
|
||||||
|
Clients sized
|
||||||
|
Rules sized
|
||||||
|
Lists fetchedLists
|
||||||
|
DNSBL dnsblZones
|
||||||
|
AbuseIPDB abuseIPDBChecker
|
||||||
|
Alerts alertQueue
|
||||||
|
LookupFile lookupDatabase
|
||||||
|
RemoteLog remoteLogSender
|
||||||
|
}
|
||||||
|
|
||||||
// Metrics are sww's metrics. They are safe for concurrent use.
|
// Metrics are sww's metrics. They are safe for concurrent use.
|
||||||
type Metrics struct {
|
type Metrics struct {
|
||||||
// registry gives every metric registered with it the label instance.
|
// registry gives every metric registered with it the label instance.
|
||||||
@@ -36,12 +57,10 @@ type Metrics struct {
|
|||||||
rateLimitHits *prometheus.CounterVec
|
rateLimitHits *prometheus.CounterVec
|
||||||
offences *prometheus.CounterVec
|
offences *prometheus.CounterVec
|
||||||
coreRuleSetMatches *prometheus.CounterVec
|
coreRuleSetMatches *prometheus.CounterVec
|
||||||
// ruleMatches are made by AddRules, and reputationHits by
|
ruleMatches *prometheus.CounterVec
|
||||||
// AddReputation.
|
reputationHits *prometheus.CounterVec
|
||||||
ruleMatches *prometheus.CounterVec
|
countries *busiest
|
||||||
reputationHits *prometheus.CounterVec
|
asns *busiest
|
||||||
countries *busiest
|
|
||||||
asns *busiest
|
|
||||||
|
|
||||||
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
||||||
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
||||||
@@ -59,20 +78,18 @@ type Metrics struct {
|
|||||||
stateFileEditsSetAside *prometheus.CounterVec
|
stateFileEditsSetAside *prometheus.CounterVec
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the metrics, with the Go runtime's and the process's own.
|
// New returns the metrics, with the Go runtime's and the process's own, and
|
||||||
// busiestCount is how many countries and how many AS numbers get series of
|
// those read from each part in params as the metrics are asked for.
|
||||||
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
|
||||||
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
|
||||||
//
|
//
|
||||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||||
func New(busiestCount int, instanceName string) *Metrics {
|
func New(params Params) *Metrics {
|
||||||
byStatus := []string{"status_class", "action"}
|
byStatus := []string{"status_class", "action"}
|
||||||
byFile := []string{"file"}
|
byFile := []string{"file"}
|
||||||
registry := prometheus.NewRegistry()
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
m := &Metrics{
|
m := &Metrics{
|
||||||
registry: prometheus.WrapRegistererWith(
|
registry: prometheus.WrapRegistererWith(
|
||||||
prometheus.Labels{"instance": instanceName}, registry),
|
prometheus.Labels{"instance": params.InstanceName}, registry),
|
||||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
Name: "sww_requests_in_flight",
|
Name: "sww_requests_in_flight",
|
||||||
@@ -113,8 +130,19 @@ func New(busiestCount int, instanceName string) *Metrics {
|
|||||||
"and the rule's id.",
|
"and the rule's id.",
|
||||||
[]string{"mode", "rule_id"},
|
[]string{"mode", "rule_id"},
|
||||||
),
|
),
|
||||||
countries: newCountries(busiestCount),
|
ruleMatches: counterVec(
|
||||||
asns: newASNs(busiestCount),
|
"sww_rule_matches_total",
|
||||||
|
"Requests that matched a rule of the rule files, by its id and action.",
|
||||||
|
[]string{"rule_id", "action"},
|
||||||
|
),
|
||||||
|
reputationHits: counterVec(
|
||||||
|
"sww_reputation_hits_total",
|
||||||
|
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||||
|
"by the list's URL, the zone, or abuseipdb.",
|
||||||
|
[]string{sourceLabel},
|
||||||
|
),
|
||||||
|
countries: newCountries(params.BusiestCount),
|
||||||
|
asns: newASNs(params.BusiestCount),
|
||||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "sww_geojs_requests_total",
|
Name: "sww_geojs_requests_total",
|
||||||
Help: "Requests to GeoJS.",
|
Help: "Requests to GeoJS.",
|
||||||
@@ -166,6 +194,7 @@ func New(busiestCount int, instanceName string) *Metrics {
|
|||||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||||
m.requestDuration, m.upstreamDuration,
|
m.requestDuration, m.upstreamDuration,
|
||||||
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
||||||
|
m.ruleMatches, m.reputationHits,
|
||||||
m.countries, m.asns,
|
m.countries, m.asns,
|
||||||
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
||||||
m.stateFileWrites, m.stateFileWriteFailures,
|
m.stateFileWrites, m.stateFileWriteFailures,
|
||||||
@@ -173,261 +202,45 @@ func New(busiestCount int, instanceName string) *Metrics {
|
|||||||
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if params.Bans != nil {
|
||||||
|
m.addBans(params.Bans, params.Now)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.Clients != nil {
|
||||||
|
m.addClients(params.Clients)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.Rules != nil {
|
||||||
|
m.addRules(params.Rules)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.Lists != nil {
|
||||||
|
m.addLists(params.Lists)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.DNSBL != nil {
|
||||||
|
m.addDNSBL(params.DNSBL)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.AbuseIPDB != nil {
|
||||||
|
m.addAbuseIPDB(params.AbuseIPDB)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.Alerts != nil {
|
||||||
|
m.addAlerts(params.Alerts)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.LookupFile != nil {
|
||||||
|
m.addLookupFile(params.LookupFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
if params.RemoteLog != nil {
|
||||||
|
m.addRemoteLog(params.RemoteLog)
|
||||||
|
}
|
||||||
|
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddBansAndClients adds the metrics read from the ledger and the table
|
|
||||||
// of clients as the metrics are asked for: the bans made since the start,
|
|
||||||
// by cause, the bans active and permanent at now, and the clients in the
|
|
||||||
// table.
|
|
||||||
func (m *Metrics) AddBansAndClients(
|
|
||||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
|
||||||
) {
|
|
||||||
for _, cause := range []string{
|
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
|
||||||
} {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_bans_made_total",
|
|
||||||
Help: "Bans made, by cause.",
|
|
||||||
ConstLabels: prometheus.Labels{"cause": cause},
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(ledger.Made(cause))
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_active_bans",
|
|
||||||
Help: "Bans active now, the permanent ones included.",
|
|
||||||
}, func() float64 {
|
|
||||||
active, _ := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(active)
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_permanent_bans",
|
|
||||||
Help: "Permanent bans not lifted.",
|
|
||||||
}, func() float64 {
|
|
||||||
_, permanent := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(permanent)
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_tracked_clients",
|
|
||||||
Help: "Clients in the table of clients.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(limiter.Len())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddRules adds the metrics of the rule files: the requests that matched
|
|
||||||
// each rule, which RuleMatched counts, and the rules loaded from
|
|
||||||
// ruleFiles, read as the metrics are asked for. It is called once, before
|
|
||||||
// RuleMatched.
|
|
||||||
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
|
|
||||||
m.ruleMatches = counterVec(
|
|
||||||
"sww_rule_matches_total",
|
|
||||||
"Requests that matched a rule of the rule files, by its id and action.",
|
|
||||||
[]string{"rule_id", "action"},
|
|
||||||
)
|
|
||||||
|
|
||||||
m.registry.MustRegister(m.ruleMatches,
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_rules_loaded",
|
|
||||||
Help: "Rules loaded from the rule files.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(ruleFiles.Len())
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddRemoteLog adds the metrics of sending the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
|
||||||
// lines sent, those dropped, and those waiting in the buffer.
|
|
||||||
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_sent_total",
|
|
||||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Sent())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_dropped_total",
|
|
||||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
|
||||||
"whose sending failed.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Dropped())
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_remote_log_lines_waiting",
|
|
||||||
Help: "Log lines in the buffer, waiting to be sent.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Depth())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddLookupFile adds the metrics of the lookup database, read as the
|
|
||||||
// metrics are asked for: when the file in use was read, which lastRead
|
|
||||||
// returns, and the replacements of it that could not be read, which
|
|
||||||
// readFailures returns.
|
|
||||||
func (m *Metrics) AddLookupFile(
|
|
||||||
lastRead func() time.Time,
|
|
||||||
readFailures func() int,
|
|
||||||
) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
|
||||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(lastRead().Unix())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_lookup_database_read_failures_total",
|
|
||||||
Help: "Replacements of the lookup database that could not be read.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(readFailures())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
|
||||||
// DNSBL zone, its key masked, or abuseipdb.
|
|
||||||
const sourceLabel = "source"
|
|
||||||
|
|
||||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
|
||||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
|
||||||
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
|
||||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
|
||||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
|
||||||
// the fetches that failed and when the copy in use was fetched, and for a
|
|
||||||
// zone, the queries made and those that failed. It is called once, before
|
|
||||||
// ReputationHit.
|
|
||||||
func (m *Metrics) AddReputation(
|
|
||||||
lists *reputation.Lists,
|
|
||||||
dnsbl *reputation.DNSBL,
|
|
||||||
) {
|
|
||||||
m.reputationHits = counterVec(
|
|
||||||
"sww_reputation_hits_total",
|
|
||||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
|
||||||
"by the list's URL, the zone, or abuseipdb.",
|
|
||||||
[]string{sourceLabel},
|
|
||||||
)
|
|
||||||
m.registry.MustRegister(m.reputationHits)
|
|
||||||
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
|
||||||
source := prometheus.Labels{
|
|
||||||
sourceLabel: reputation.MaskZoneKey(zone),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.addReputationQueries(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Queries(zone) },
|
|
||||||
)
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Failures(zone) },
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, listURL := range lists.URLs() {
|
|
||||||
source := prometheus.Labels{sourceLabel: listURL}
|
|
||||||
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return lists.Failures(listURL) },
|
|
||||||
)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
|
||||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
|
||||||
"1970, or 0 while there is none.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
fetched := lists.Fetched(listURL)
|
|
||||||
if fetched.IsZero() {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return float64(fetched.Unix())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
|
||||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
|
||||||
// that failed, and how many checks the day's budget has left. It is
|
|
||||||
// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set.
|
|
||||||
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
|
||||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
|
||||||
|
|
||||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
|
||||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_daily_budget_remaining",
|
|
||||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(abuseIPDB.BudgetLeft())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
|
||||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
|
||||||
func (m *Metrics) ReputationHit(source string) {
|
|
||||||
m.reputationHits.WithLabelValues(source).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddAlerts adds the metrics of the alerts sent to each destination set,
|
|
||||||
// read from queue as the metrics are asked for, by destination: the
|
|
||||||
// alerts sent, the requests to the destination that failed, the alerts
|
|
||||||
// held back, which are the same for every destination, and those
|
|
||||||
// dropped. With no destination set, it adds none.
|
|
||||||
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
|
|
||||||
for _, name := range queue.DestinationsSet() {
|
|
||||||
destination := prometheus.Labels{"destination": name}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_sent_total",
|
|
||||||
Help: "Alerts the destination took.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Sent)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alert_send_failures_total",
|
|
||||||
Help: "Requests to the destination that failed.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).SendFailures)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_suppressed_total",
|
|
||||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
|
||||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Suppressed())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_dropped_total",
|
|
||||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
|
||||||
"alerts given up as the destination refused them.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Dropped)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeHTTP answers with the metrics in the Prometheus text format.
|
// ServeHTTP answers with the metrics in the Prometheus text format.
|
||||||
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
m.handler.ServeHTTP(w, r)
|
m.handler.ServeHTTP(w, r)
|
||||||
@@ -497,6 +310,12 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
|||||||
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||||
|
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||||
|
func (m *Metrics) ReputationHit(source string) {
|
||||||
|
m.reputationHits.WithLabelValues(source).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
// StateFileWritten counts a write of the state or cache file name, of size
|
// StateFileWritten counts a write of the state or cache file name, of size
|
||||||
// bytes, that ended with err.
|
// bytes, that ended with err.
|
||||||
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
||||||
@@ -528,6 +347,312 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
|||||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// banLedger is what the metrics read from the ledger of bans.
|
||||||
|
type banLedger interface {
|
||||||
|
// Made returns the bans made since the start with cause.
|
||||||
|
Made(cause string) int
|
||||||
|
// Count returns the bans active at now, the permanent ones included,
|
||||||
|
// and the permanent ones.
|
||||||
|
Count(now time.Time) (active, permanent int)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addBans adds the metrics read from ledger as the metrics are asked for:
|
||||||
|
// the bans made since the start, by cause, and the bans active and
|
||||||
|
// permanent at now.
|
||||||
|
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
||||||
|
for _, cause := range []string{
|
||||||
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||||
|
} {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_bans_made_total",
|
||||||
|
Help: "Bans made, by cause.",
|
||||||
|
ConstLabels: prometheus.Labels{"cause": cause},
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ledger.Made(cause))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_active_bans",
|
||||||
|
Help: "Bans active now, the permanent ones included.",
|
||||||
|
}, func() float64 {
|
||||||
|
active, _ := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(active)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_permanent_bans",
|
||||||
|
Help: "Permanent bans not lifted.",
|
||||||
|
}, func() float64 {
|
||||||
|
_, permanent := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(permanent)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sized is what the metrics read from the table of clients and from the
|
||||||
|
// rule files.
|
||||||
|
type sized interface {
|
||||||
|
// Len returns how many clients the table holds, or how many rules were
|
||||||
|
// loaded from the rule files.
|
||||||
|
Len() int
|
||||||
|
}
|
||||||
|
|
||||||
|
// addClients adds the metric of the clients in table, read as the metrics
|
||||||
|
// are asked for.
|
||||||
|
func (m *Metrics) addClients(table sized) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_tracked_clients",
|
||||||
|
Help: "Clients in the table of clients.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(table.Len())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addRules adds the metric of the rules loaded from files, read as the
|
||||||
|
// metrics are asked for.
|
||||||
|
func (m *Metrics) addRules(files sized) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_rules_loaded",
|
||||||
|
Help: "Rules loaded from the rule files.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(files.Len())
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// remoteLogSender is what the metrics read from what sends the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL.
|
||||||
|
type remoteLogSender interface {
|
||||||
|
// Sent returns the lines sent.
|
||||||
|
Sent() int64
|
||||||
|
// Dropped returns the lines dropped.
|
||||||
|
Dropped() int64
|
||||||
|
// Depth returns the lines waiting in the buffer.
|
||||||
|
Depth() int
|
||||||
|
}
|
||||||
|
|
||||||
|
// addRemoteLog adds the metrics of sending the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||||
|
// lines sent, those dropped, and those waiting in the buffer.
|
||||||
|
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_sent_total",
|
||||||
|
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Sent())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_dropped_total",
|
||||||
|
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||||
|
"whose sending failed.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Dropped())
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_remote_log_lines_waiting",
|
||||||
|
Help: "Log lines in the buffer, waiting to be sent.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Depth())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookupDatabase is what the metrics read from the lookup database.
|
||||||
|
type lookupDatabase interface {
|
||||||
|
// LastRead returns when the file in use was read.
|
||||||
|
LastRead() time.Time
|
||||||
|
// ReadFailures returns the replacements of the file that could not be
|
||||||
|
// read.
|
||||||
|
ReadFailures() int
|
||||||
|
}
|
||||||
|
|
||||||
|
// addLookupFile adds the metrics of the lookup database, read from file as
|
||||||
|
// the metrics are asked for: when the file in use was read, and the
|
||||||
|
// replacements of it that could not be read.
|
||||||
|
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||||
|
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(file.LastRead().Unix())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_lookup_database_read_failures_total",
|
||||||
|
Help: "Replacements of the lookup database that could not be read.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(file.ReadFailures())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||||
|
// DNSBL zone, its key masked, or abuseipdb.
|
||||||
|
const sourceLabel = "source"
|
||||||
|
|
||||||
|
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
||||||
|
type dnsblZones interface {
|
||||||
|
// Zones returns the zones asked, their keys unmasked.
|
||||||
|
Zones() []string
|
||||||
|
// Queries returns the queries made to zone.
|
||||||
|
Queries(zone string) int
|
||||||
|
// Failures returns the queries to zone that failed.
|
||||||
|
Failures(zone string) int
|
||||||
|
}
|
||||||
|
|
||||||
|
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
||||||
|
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
||||||
|
// asked for: the queries made and those that failed.
|
||||||
|
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
||||||
|
for _, zone := range dnsbl.Zones() {
|
||||||
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
|
m.addReputationQueries(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Queries(zone) },
|
||||||
|
)
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Failures(zone) },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
||||||
|
type fetchedLists interface {
|
||||||
|
// URLs returns the lists' URLs.
|
||||||
|
URLs() []string
|
||||||
|
// Failures returns the fetches of the list at listURL that failed.
|
||||||
|
Failures(listURL string) int
|
||||||
|
// Fetched returns when the copy in use of the list at listURL was
|
||||||
|
// fetched, or the zero time while there is none.
|
||||||
|
Fetched(listURL string) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// addLists adds the metrics of each list fetched from a URL, by source, the
|
||||||
|
// list's URL, read from lists as the metrics are asked for: the fetches
|
||||||
|
// that failed and when the copy in use was fetched.
|
||||||
|
func (m *Metrics) addLists(lists fetchedLists) {
|
||||||
|
for _, listURL := range lists.URLs() {
|
||||||
|
source := prometheus.Labels{sourceLabel: listURL}
|
||||||
|
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return lists.Failures(listURL) },
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||||
|
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||||
|
"1970, or 0 while there is none.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
fetched := lists.Fetched(listURL)
|
||||||
|
if fetched.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return float64(fetched.Unix())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// abuseIPDBChecker is what the metrics read from what checks clients with
|
||||||
|
// AbuseIPDB.
|
||||||
|
type abuseIPDBChecker interface {
|
||||||
|
// Checked returns the checks made.
|
||||||
|
Checked() int
|
||||||
|
// Failures returns the checks that failed.
|
||||||
|
Failures() int
|
||||||
|
// BudgetLeft returns how many checks the day's budget has left.
|
||||||
|
BudgetLeft() int
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||||
|
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||||
|
// that failed, and how many checks the day's budget has left.
|
||||||
|
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
||||||
|
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||||
|
|
||||||
|
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||||
|
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_daily_budget_remaining",
|
||||||
|
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(abuseIPDB.BudgetLeft())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// alertQueue is what the metrics read from the queue of alerts.
|
||||||
|
type alertQueue interface {
|
||||||
|
// DestinationsSet returns the names of the destinations set.
|
||||||
|
DestinationsSet() []string
|
||||||
|
// Counts returns the alerts sent to the destination name, the requests
|
||||||
|
// to it that failed, and the alerts dropped.
|
||||||
|
Counts(name string) alerts.Counts
|
||||||
|
// Suppressed returns the alerts held back.
|
||||||
|
Suppressed() int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAlerts adds the metrics of the alerts sent to each destination set,
|
||||||
|
// read from queue as the metrics are asked for, by destination: the
|
||||||
|
// alerts sent, the requests to the destination that failed, the alerts
|
||||||
|
// held back, which are the same for every destination, and those
|
||||||
|
// dropped. With no destination set, it adds none.
|
||||||
|
func (m *Metrics) addAlerts(queue alertQueue) {
|
||||||
|
for _, name := range queue.DestinationsSet() {
|
||||||
|
destination := prometheus.Labels{"destination": name}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_sent_total",
|
||||||
|
Help: "Alerts the destination took.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Sent)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alert_send_failures_total",
|
||||||
|
Help: "Requests to the destination that failed.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).SendFailures)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_suppressed_total",
|
||||||
|
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||||
|
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Suppressed())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_dropped_total",
|
||||||
|
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||||
|
"alerts given up as the destination refused them.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Dropped)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
// count counts a request that has ended, as RequestEnded and RequestReported
|
||||||
// describe.
|
// describe.
|
||||||
func (m *Metrics) count(
|
func (m *Metrics) count(
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package metrics_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
listURL = "https://lists.example/block.txt"
|
||||||
|
zone = "dnsbl.example"
|
||||||
|
destination = "webhook"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Each stand-in answers with a number of its own, so that each metric
|
||||||
|
// shows which part it was read from.
|
||||||
|
|
||||||
|
type ledgerStandIn struct{}
|
||||||
|
|
||||||
|
func (ledgerStandIn) Made(string) int { return 1 }
|
||||||
|
|
||||||
|
func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 }
|
||||||
|
|
||||||
|
type clientTableStandIn struct{}
|
||||||
|
|
||||||
|
func (clientTableStandIn) Len() int { return 3 }
|
||||||
|
|
||||||
|
type ruleFilesStandIn struct{}
|
||||||
|
|
||||||
|
func (ruleFilesStandIn) Len() int { return 4 }
|
||||||
|
|
||||||
|
type listsStandIn struct{}
|
||||||
|
|
||||||
|
func (listsStandIn) URLs() []string { return []string{listURL} }
|
||||||
|
|
||||||
|
func (listsStandIn) Failures(string) int { return 5 }
|
||||||
|
|
||||||
|
func (listsStandIn) Fetched(string) time.Time { return time.Time{} }
|
||||||
|
|
||||||
|
type dnsblStandIn struct{}
|
||||||
|
|
||||||
|
func (dnsblStandIn) Zones() []string { return []string{zone} }
|
||||||
|
|
||||||
|
func (dnsblStandIn) Queries(string) int { return 6 }
|
||||||
|
|
||||||
|
func (dnsblStandIn) Failures(string) int { return 0 }
|
||||||
|
|
||||||
|
type abuseIPDBStandIn struct{}
|
||||||
|
|
||||||
|
func (abuseIPDBStandIn) Checked() int { return 0 }
|
||||||
|
|
||||||
|
func (abuseIPDBStandIn) Failures() int { return 0 }
|
||||||
|
|
||||||
|
func (abuseIPDBStandIn) BudgetLeft() int { return 7 }
|
||||||
|
|
||||||
|
type alertQueueStandIn struct{}
|
||||||
|
|
||||||
|
func (alertQueueStandIn) DestinationsSet() []string {
|
||||||
|
return []string{destination}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (alertQueueStandIn) Counts(string) alerts.Counts {
|
||||||
|
return alerts.Counts{Sent: 8}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (alertQueueStandIn) Suppressed() int64 { return 0 }
|
||||||
|
|
||||||
|
type lookupDatabaseStandIn struct{}
|
||||||
|
|
||||||
|
func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} }
|
||||||
|
|
||||||
|
func (lookupDatabaseStandIn) ReadFailures() int { return 9 }
|
||||||
|
|
||||||
|
type remoteLogStandIn struct{}
|
||||||
|
|
||||||
|
func (remoteLogStandIn) Sent() int64 { return 10 }
|
||||||
|
|
||||||
|
func (remoteLogStandIn) Dropped() int64 { return 0 }
|
||||||
|
|
||||||
|
func (remoteLogStandIn) Depth() int { return 0 }
|
||||||
|
|
||||||
|
func TestMetricsAreReadFromEachPart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := metrics.New(metrics.Params{
|
||||||
|
BusiestCount: 1,
|
||||||
|
InstanceName: "app",
|
||||||
|
Now: func() time.Time { return time.Time{} },
|
||||||
|
Bans: ledgerStandIn{},
|
||||||
|
Clients: clientTableStandIn{},
|
||||||
|
Rules: ruleFilesStandIn{},
|
||||||
|
Lists: listsStandIn{},
|
||||||
|
DNSBL: dnsblStandIn{},
|
||||||
|
AbuseIPDB: abuseIPDBStandIn{},
|
||||||
|
Alerts: alertQueueStandIn{},
|
||||||
|
LookupFile: lookupDatabaseStandIn{},
|
||||||
|
RemoteLog: remoteLogStandIn{},
|
||||||
|
})
|
||||||
|
scraped := scrape(t, m)
|
||||||
|
|
||||||
|
for _, line := range []string{
|
||||||
|
`sww_bans_made_total{cause="limit",instance="app"} 1`,
|
||||||
|
`sww_active_bans{instance="app"} 2`,
|
||||||
|
`sww_tracked_clients{instance="app"} 3`,
|
||||||
|
`sww_rules_loaded{instance="app"} 4`,
|
||||||
|
`sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`,
|
||||||
|
`sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`,
|
||||||
|
`sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`,
|
||||||
|
`sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`,
|
||||||
|
`sww_lookup_database_read_failures_total{instance="app"} 9`,
|
||||||
|
`sww_remote_log_lines_sent_total{instance="app"} 10`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(scraped, "\n"+line+"\n") {
|
||||||
|
t.Errorf("metrics\n%s\nwant\n%s", scraped, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartThatIsOffAddsNoMetric(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
scraped := scrape(t, metrics.New(metrics.Params{
|
||||||
|
BusiestCount: 1,
|
||||||
|
InstanceName: "app",
|
||||||
|
}))
|
||||||
|
|
||||||
|
for _, name := range []string{
|
||||||
|
"sww_bans_made_total",
|
||||||
|
"sww_tracked_clients",
|
||||||
|
"sww_rules_loaded",
|
||||||
|
"sww_reputation_",
|
||||||
|
"sww_alert",
|
||||||
|
"sww_lookup_database_",
|
||||||
|
"sww_remote_log_",
|
||||||
|
} {
|
||||||
|
if strings.Contains(scraped, name) {
|
||||||
|
t.Errorf("metrics\n%s\nshow %s", scraped, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// scrape returns the metrics m serves.
|
||||||
|
func scrape(t *testing.T, m *metrics.Metrics) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
scraped := httptest.NewRecorder()
|
||||||
|
m.ServeHTTP(
|
||||||
|
scraped,
|
||||||
|
httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, "/", http.NoBody,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return scraped.Body.String()
|
||||||
|
}
|
||||||
@@ -445,12 +445,9 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
|||||||
scores: map[string]int64{suspect: 100},
|
scores: map[string]int64{suspect: 100},
|
||||||
}
|
}
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(metrics.Params{
|
||||||
m.AddReputation(
|
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
||||||
reputation.New(params()),
|
})
|
||||||
reputation.NewDNSBL(dnsblParams()),
|
|
||||||
)
|
|
||||||
m.AddAbuseIPDB(checker)
|
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one it refuses with an answer
|
// One check that AbuseIPDB answers, and one it refuses with an answer
|
||||||
// that names the key.
|
// that names the key.
|
||||||
@@ -503,12 +500,9 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
|
|||||||
p := abuseIPDBParams()
|
p := abuseIPDBParams()
|
||||||
p.DailyBudget = 5
|
p.DailyBudget = 5
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(metrics.Params{
|
||||||
m.AddReputation(
|
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
||||||
reputation.New(params()),
|
})
|
||||||
reputation.NewDNSBL(dnsblParams()),
|
|
||||||
)
|
|
||||||
m.AddAbuseIPDB(checker)
|
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one that fails.
|
// One check that AbuseIPDB answers, and one that fails.
|
||||||
hitFrom(t, checker, suspect, true)
|
hitFrom(t, checker, suspect, true)
|
||||||
@@ -811,6 +805,9 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// instanceName is the label instance of the metrics the tests build.
|
||||||
|
const instanceName = "app"
|
||||||
|
|
||||||
// scrapeMetrics returns the metrics m serves.
|
// scrapeMetrics returns the metrics m serves.
|
||||||
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -358,8 +358,9 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
|||||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||||
}}
|
}}
|
||||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(metrics.Params{
|
||||||
m.AddReputation(reputation.New(params()), dnsbl)
|
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
||||||
|
})
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
wantZones(t, dnsbl, listed)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
@@ -404,8 +405,9 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) {
|
|||||||
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
||||||
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
||||||
}}, p)
|
}}, p)
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(metrics.Params{
|
||||||
m.AddReputation(reputation.New(params()), dnsbl)
|
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
||||||
|
})
|
||||||
|
|
||||||
// Both clients are asked about before either answer comes, so that
|
// Both clients are asked about before either answer comes, so that
|
||||||
// the failure does not keep the zone from the other query.
|
// the failure does not keep the zone from the other query.
|
||||||
|
|||||||
@@ -2430,7 +2430,7 @@ func midnight() time.Time {
|
|||||||
// counts, with thresholds fill does not reach.
|
// counts, with thresholds fill does not reach.
|
||||||
func newParams(dir string) state.Params {
|
func newParams(dir string) state.Params {
|
||||||
discard := slog.New(slog.DiscardHandler)
|
discard := slog.New(slog.DiscardHandler)
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"})
|
||||||
queue := alerts.New(alerts.Params{
|
queue := alerts.New(alerts.Params{
|
||||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||||
Events: alerts.Events(),
|
Events: alerts.Events(),
|
||||||
|
|||||||
+18
-11
@@ -392,26 +392,33 @@ func newMetrics(
|
|||||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||||
remote *remotelog.Sender,
|
remote *remotelog.Sender,
|
||||||
) *metrics.Metrics {
|
) *metrics.Metrics {
|
||||||
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
params := metrics.Params{
|
||||||
m.AddBansAndClients(ledger, limiter, utcNow)
|
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
||||||
m.AddRules(ruleFiles)
|
InstanceName: cfg.InstanceName,
|
||||||
m.AddReputation(lists, dnsbl)
|
Now: utcNow,
|
||||||
|
Bans: ledger,
|
||||||
if cfg.AbuseIPDBKey != "" {
|
Clients: limiter,
|
||||||
m.AddAbuseIPDB(abuseIPDB)
|
Rules: ruleFiles,
|
||||||
|
Lists: lists,
|
||||||
|
DNSBL: dnsbl,
|
||||||
|
Alerts: alertQueue,
|
||||||
}
|
}
|
||||||
|
|
||||||
m.AddAlerts(alertQueue)
|
// A nil pointer in an interface field is not a nil interface, so a part
|
||||||
|
// that is off is left out rather than set to nil.
|
||||||
|
if cfg.AbuseIPDBKey != "" {
|
||||||
|
params.AbuseIPDB = abuseIPDB
|
||||||
|
}
|
||||||
|
|
||||||
if lookupFile != nil {
|
if lookupFile != nil {
|
||||||
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
params.LookupFile = lookupFile
|
||||||
}
|
}
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
m.AddRemoteLog(remote)
|
params.RemoteLog = remote
|
||||||
}
|
}
|
||||||
|
|
||||||
return m
|
return metrics.New(params)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
|
|||||||
+92
-147
@@ -2,7 +2,6 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/subtle"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -15,7 +14,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -41,80 +39,23 @@ var (
|
|||||||
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
||||||
)
|
)
|
||||||
|
|
||||||
// answerHealth answers a request for the health endpoint.
|
// handleHealth answers GET wafprotocol.HealthPath.
|
||||||
func (rq *request) answerHealth() {
|
func handleHealth() http.HandlerFunc {
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
return func(w http.ResponseWriter, _ *http.Request) {
|
||||||
// Set here rather than left to Go's server, which would set it only
|
// Set here rather than left to Go's server, which would set it only
|
||||||
// after the log line has taken the response's headers.
|
// after the log line has taken the response's headers.
|
||||||
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
_, _ = io.WriteString(rq.out, "ok\n")
|
_, _ = io.WriteString(w, "ok\n")
|
||||||
}
|
|
||||||
|
|
||||||
// answerAdmin answers a request for one of waf's own endpoints. Each
|
|
||||||
// needs a token, sent as Authorization: Bearer <token>: the metrics
|
|
||||||
// SWW_WAF_METRICS_TOKEN, the others SWW_WAF_ADMIN_TOKEN. A request without it is
|
|
||||||
// refused with 401. An endpoint whose token is unset answers 404, as any other
|
|
||||||
// path does.
|
|
||||||
func (rq *request) answerAdmin() {
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
|
|
||||||
token, answer := rq.endpoint()
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case token == "":
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusNotFound),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
case !hasToken(rq.in, token):
|
|
||||||
rq.out.Header().Set("WWW-Authenticate", "Bearer")
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusUnauthorized),
|
|
||||||
http.StatusUnauthorized,
|
|
||||||
)
|
|
||||||
default:
|
|
||||||
answer()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// endpoint returns the token the request's endpoint needs, and what
|
// handleListBans answers GET BansPath with every ban held.
|
||||||
// answers the request there; "" when there is no such endpoint.
|
func (h *handler) handleListBans() http.HandlerFunc {
|
||||||
func (rq *request) endpoint() (string, func()) {
|
return func(w http.ResponseWriter, _ *http.Request) {
|
||||||
cfg := rq.h.config
|
h.answerBans(w, h.ledger.Snapshot())
|
||||||
method, path := rq.in.Method, rq.in.URL.Path
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case method == http.MethodGet && path == MetricsPath:
|
|
||||||
return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) }
|
|
||||||
case method == http.MethodGet && path == BansPath:
|
|
||||||
return cfg.AdminToken, rq.listBans
|
|
||||||
case method == http.MethodPost && path == BansPath:
|
|
||||||
return cfg.AdminToken, rq.addBan
|
|
||||||
case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"):
|
|
||||||
return cfg.AdminToken, rq.liftBans
|
|
||||||
case method == http.MethodGet && strings.HasPrefix(path, ClientsPath):
|
|
||||||
return cfg.AdminToken, rq.showClient
|
|
||||||
default:
|
|
||||||
return "", nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// hasToken reports whether r carries token, as Authorization: Bearer
|
|
||||||
// <token>.
|
|
||||||
func hasToken(r *http.Request, token string) bool {
|
|
||||||
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
|
|
||||||
|
|
||||||
return strings.EqualFold(scheme, "Bearer") &&
|
|
||||||
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// listBans answers GET BansPath with every ban held.
|
|
||||||
func (rq *request) listBans() {
|
|
||||||
rq.answerBans(rq.h.ledger.Snapshot())
|
|
||||||
}
|
|
||||||
|
|
||||||
// banToAdd is the body of POST BansPath.
|
// banToAdd is the body of POST BansPath.
|
||||||
type banToAdd struct {
|
type banToAdd struct {
|
||||||
// Netblock is a netblock, or a client's address, which stands for the
|
// Netblock is a netblock, or a client's address, which stands for the
|
||||||
@@ -126,44 +67,44 @@ type banToAdd struct {
|
|||||||
Reason string `json:"reason"`
|
Reason string `json:"reason"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// addBan answers POST BansPath: it bans the netblock the body names, as
|
// handleAddBan answers POST BansPath: it bans the netblock the body names,
|
||||||
// an admin, from now for the duration the body gives, with its reason,
|
// as an admin, from now for the duration the body gives, with its reason,
|
||||||
// and answers with that ban.
|
// and answers with that ban.
|
||||||
func (rq *request) addBan() {
|
func (h *handler) handleAddBan() http.HandlerFunc {
|
||||||
toAdd, err := rq.readBanToAdd()
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
toAdd, err := readBanToAdd(r)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
netblock netip.Prefix
|
netblock netip.Prefix
|
||||||
expires time.Time
|
expires time.Time
|
||||||
now = rq.h.now()
|
now = h.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
|
netblock, err = h.parseBanNetblock(toAdd.Netblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
expires, err = expiry(toAdd.Duration, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ban := h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
|
||||||
|
h.answerBans(w, []bans.Ban{ban})
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
expires, err = expiry(toAdd.Duration, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
|
|
||||||
rq.answerBans([]bans.Ban{ban})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
||||||
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
||||||
func (rq *request) readBanToAdd() (banToAdd, error) {
|
func readBanToAdd(r *http.Request) (banToAdd, error) {
|
||||||
data, err := io.ReadAll(
|
data, err := io.ReadAll(
|
||||||
http.MaxBytesReader(nil, rq.in.Body, banBodyMaxBytes),
|
http.MaxBytesReader(nil, r.Body, banBodyMaxBytes),
|
||||||
)
|
)
|
||||||
rq.requestBytes = int64(len(data))
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
||||||
}
|
}
|
||||||
@@ -250,29 +191,31 @@ func expiry(duration string, now time.Time) (time.Time, error) {
|
|||||||
return now.Add(length), nil
|
return now.Add(length), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// liftBans answers DELETE BansPath/<client>: it lifts every ban active on
|
// handleLiftBans answers DELETE BansPath/<client>: it lifts every ban
|
||||||
// a netblock the client's address is in, and answers with those bans, or
|
// active on a netblock the client's address is in, and answers with those
|
||||||
// with 404 when none is active.
|
// bans, or with 404 when none is active.
|
||||||
func (rq *request) liftBans() {
|
func (h *handler) handleLiftBans() http.HandlerFunc {
|
||||||
client, err := pathAddress(rq.in.URL.Path, BansPath+"/")
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
if err != nil {
|
client, err := pathAddress(r.URL.Path, BansPath+"/")
|
||||||
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lifted := h.ledger.Lift(client, h.now())
|
||||||
|
if len(lifted) == 0 {
|
||||||
|
http.Error(
|
||||||
|
w,
|
||||||
|
"no ban is active on "+client.String(),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.answerBans(w, lifted)
|
||||||
}
|
}
|
||||||
|
|
||||||
lifted := rq.h.ledger.Lift(client, rq.h.now())
|
|
||||||
if len(lifted) == 0 {
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
"no ban is active on "+client.String(),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.answerBans(lifted)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
|
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
|
||||||
@@ -284,29 +227,31 @@ type clientAnswer struct {
|
|||||||
Bans []state.BanEntry `json:"bans"`
|
Bans []state.BanEntry `json:"bans"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// showClient answers GET ClientsPath<ip> with what waf knows of the client:
|
// handleShowClient answers GET ClientsPath<ip> with what waf knows of the
|
||||||
// its counters, its history, which holds its country as last looked up and its
|
// client: its counters, its history, which holds its country as last looked
|
||||||
// offences, and its bans with their notes.
|
// up and its offences, and its bans with their notes.
|
||||||
func (rq *request) showClient() {
|
func (h *handler) handleShowClient() http.HandlerFunc {
|
||||||
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
if err != nil {
|
addr, err := pathAddress(r.URL.Path, ClientsPath)
|
||||||
http.Error(rq.out, err.Error(), http.StatusBadRequest)
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
answer := clientAnswer{
|
||||||
|
Bans: state.BanEntries(h.ledger.Covering(addr)),
|
||||||
|
}
|
||||||
|
|
||||||
|
client, seen := h.limiter.Client(
|
||||||
|
clientGroup(h.config.IPv6GroupPrefix, addr),
|
||||||
|
)
|
||||||
|
if seen {
|
||||||
|
answer.Client = &client
|
||||||
|
}
|
||||||
|
|
||||||
|
h.answerJSON(w, answer)
|
||||||
}
|
}
|
||||||
|
|
||||||
answer := clientAnswer{
|
|
||||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
|
||||||
}
|
|
||||||
|
|
||||||
client, seen := rq.h.limiter.Client(
|
|
||||||
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
|
||||||
)
|
|
||||||
if seen {
|
|
||||||
answer.Client = &client
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.answerJSON(answer)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathAddress reads the client's address that follows prefix in path.
|
// pathAddress reads the client's address that follows prefix in path.
|
||||||
@@ -322,27 +267,27 @@ func pathAddress(path, prefix string) (netip.Addr, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// answerBans answers with held under bans, as bans.json lists them.
|
// answerBans answers with held under bans, as bans.json lists them.
|
||||||
func (rq *request) answerBans(held []bans.Ban) {
|
func (h *handler) answerBans(w http.ResponseWriter, held []bans.Ban) {
|
||||||
rq.answerJSON(struct {
|
h.answerJSON(w, struct {
|
||||||
Bans []state.BanEntry `json:"bans"`
|
Bans []state.BanEntry `json:"bans"`
|
||||||
}{state.BanEntries(held)})
|
}{state.BanEntries(held)})
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerJSON answers with value as indented JSON.
|
// answerJSON answers with value as indented JSON.
|
||||||
func (rq *request) answerJSON(value any) {
|
func (h *handler) answerJSON(w http.ResponseWriter, value any) {
|
||||||
body, err := json.MarshalIndent(value, "", " ")
|
body, err := json.MarshalIndent(value, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rq.h.processLog.Error(
|
h.processLog.Error(
|
||||||
"encoding an answer failed",
|
"encoding an answer failed",
|
||||||
"error",
|
"error",
|
||||||
err.Error(),
|
err.Error(),
|
||||||
)
|
)
|
||||||
http.Error(rq.out, http.StatusText(http.StatusInternalServerError),
|
http.Error(w, http.StatusText(http.StatusInternalServerError),
|
||||||
http.StatusInternalServerError)
|
http.StatusInternalServerError)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.out.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_, _ = rq.out.Write(append(body, '\n'))
|
_, _ = w.Write(append(body, '\n'))
|
||||||
}
|
}
|
||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -333,6 +334,39 @@ func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBanToAddIsLoggedWithTheBytesOfItsBody(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(
|
||||||
|
t,
|
||||||
|
"",
|
||||||
|
map[string]string{adminToken: adminSecret},
|
||||||
|
)
|
||||||
|
|
||||||
|
s.admin(http.MethodPost, waf.BansPath, banOtherClient, http.StatusOK)
|
||||||
|
|
||||||
|
line := s.out.endedLines(t, s.sent)[s.sent-1]
|
||||||
|
if line.RequestBytes != int64(len(banOtherClient)) {
|
||||||
|
t.Errorf(
|
||||||
|
"the ban to add was logged with request_bytes %d, want %d",
|
||||||
|
line.RequestBytes,
|
||||||
|
len(banOtherClient),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The health endpoint reads none of a body sent to it.
|
||||||
|
s.ownEndpointRequest("", http.MethodGet, wafprotocol.HealthPath, "ok",
|
||||||
|
http.StatusOK)
|
||||||
|
|
||||||
|
line = s.out.endedLines(t, s.sent)[s.sent-1]
|
||||||
|
if line.RequestBytes != 0 {
|
||||||
|
t.Errorf(
|
||||||
|
"the health request was logged with request_bytes %d, want 0",
|
||||||
|
line.RequestBytes,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+12
-8
@@ -45,8 +45,11 @@ func (a activeBan) check(
|
|||||||
}
|
}
|
||||||
|
|
||||||
ban, banned, madePermanent := check(rq.client, now)
|
ban, banned, madePermanent := check(rq.client, now)
|
||||||
|
|
||||||
|
var expires string
|
||||||
if banned {
|
if banned {
|
||||||
rq.line.BanExpires = banExpires(ban)
|
expires = banExpires(ban)
|
||||||
|
rq.line.BanExpires = expires
|
||||||
}
|
}
|
||||||
|
|
||||||
if madePermanent {
|
if madePermanent {
|
||||||
@@ -61,7 +64,7 @@ func (a activeBan) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: banAction(ban.Permanent()),
|
action: banAction(ban.Permanent()),
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires,
|
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -148,6 +151,7 @@ func (b banMaker) banForLimit(
|
|||||||
rq.line.LimitHit = hit.Window
|
rq.line.LimitHit = hit.Window
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rq.brokeLimit = true
|
||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
|
|
||||||
b.metrics.LimitHit(hit)
|
b.metrics.LimitHit(hit)
|
||||||
@@ -259,9 +263,9 @@ func (b banMaker) withClientNotes(
|
|||||||
status int,
|
status int,
|
||||||
netblock netip.Prefix,
|
netblock netip.Prefix,
|
||||||
) bans.Notes {
|
) bans.Notes {
|
||||||
notes.ASN = rq.line.ASN
|
notes.ASN = rq.lookupAnswer.ASN
|
||||||
notes.ASName = rq.line.ASName
|
notes.ASName = rq.lookupAnswer.ASName
|
||||||
notes.Country = rq.line.Country
|
notes.Country = rq.lookupAnswer.Country
|
||||||
notes.Reputation = rq.reputation
|
notes.Reputation = rq.reputation
|
||||||
notes.Request = rq.noted(now, status)
|
notes.Request = rq.noted(now, status)
|
||||||
notes.Requests = b.netblockRequests(netblock)
|
notes.Requests = b.netblockRequests(netblock)
|
||||||
@@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
|
|||||||
func (rq *request) noted(now time.Time, status int) bans.Request {
|
func (rq *request) noted(now time.Time, status int) bans.Request {
|
||||||
return bans.Request{
|
return bans.Request{
|
||||||
Time: now,
|
Time: now,
|
||||||
Method: rq.line.Method,
|
Method: rq.method,
|
||||||
Host: rq.line.Host,
|
Host: rq.site,
|
||||||
Path: rq.requestURI,
|
Path: rq.requestURI,
|
||||||
Status: status,
|
Status: status,
|
||||||
UserAgent: rq.line.UserAgent,
|
UserAgent: rq.userAgent,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ type biasedThresholds struct {
|
|||||||
func (b biasedThresholds) limitPercentages(
|
func (b biasedThresholds) limitPercentages(
|
||||||
rq *request,
|
rq *request,
|
||||||
) (percentage, percentage) {
|
) (percentage, percentage) {
|
||||||
asn, country := rq.line.ASN, rq.line.Country
|
asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country
|
||||||
|
|
||||||
unknown := percentage{percent: whole}
|
unknown := percentage{percent: whole}
|
||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
@@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages(
|
|||||||
|
|
||||||
otherTags := percentage{percent: whole}
|
otherTags := percentage{percent: whole}
|
||||||
|
|
||||||
for _, tag := range rq.line.Tags {
|
for _, tag := range rq.tags {
|
||||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||||
otherTags = lowest(
|
otherTags = lowest(
|
||||||
otherTags,
|
otherTags,
|
||||||
|
|||||||
@@ -43,6 +43,15 @@ func (b blocklists) check(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusesWhateverTheLookupAnswers reports whether blocklists refuses rq, as
|
||||||
|
// refusedByBlocklists says. The lookup's answer plays no part in it.
|
||||||
|
func (b blocklists) refusesWhateverTheLookupAnswers(
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) bool {
|
||||||
|
return b.refusedByBlocklists(b.lists.ListedBy(rq.client))
|
||||||
|
}
|
||||||
|
|
||||||
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ func (b byteLimits) check(
|
|||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if !rq.counted || rq.line.WouldAction != "" {
|
if !rq.counted || rq.wouldAction != "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
|
|||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.lookupAnswer.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.lookupAnswer.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.lookupAnswer.Country,
|
||||||
Reason: "scored by the Core Rule Set at or over " +
|
Reason: "scored by the Core Rule Set at or over " +
|
||||||
"SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD",
|
"SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD",
|
||||||
Detail: detail,
|
Detail: detail,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func (c countries) check(
|
|||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
allowed := c.allowed
|
allowed := c.allowed
|
||||||
country := rq.line.Country
|
country := rq.lookupAnswer.Country
|
||||||
|
|
||||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -58,3 +58,14 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHealthEndpointIsFoundAtItsPathPercentEncoded(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// %68 is h. An endpoint is found by the decoded path.
|
||||||
|
s, _, _ := startWithClock(t, "", nil)
|
||||||
|
|
||||||
|
got := s.ownEndpointRequest("", http.MethodGet, "/_sww/%68ealthz", "",
|
||||||
|
http.StatusOK)
|
||||||
|
wantBody(t, got, "ok\n")
|
||||||
|
}
|
||||||
@@ -9,6 +9,16 @@ import (
|
|||||||
// and holds the settings and parts it reads. Add one line for it to one of the
|
// and holds the settings and parts it reads. Add one line for it to one of the
|
||||||
// lists that newHandler builds, which hands it those settings and parts. Its
|
// lists that newHandler builds, which hands it those settings and parts. Its
|
||||||
// settings go in config.WAF, and the comment at the top of its file names them.
|
// settings go in config.WAF, and the comment at the top of its file names them.
|
||||||
|
// A heuristic of afterLookup that can refuse a request whatever the lookup
|
||||||
|
// answers also implements refusesWithoutLookup.
|
||||||
|
//
|
||||||
|
// Three places outside its file may change with it:
|
||||||
|
// - waitForGeoJS in newHandler lists the settings that need the lookup's
|
||||||
|
// answer;
|
||||||
|
// - requestCanGetTag in internal/config/waf.go holds which setting can give a
|
||||||
|
// request which tag;
|
||||||
|
// - findBesideLookup does the work of each heuristic of besideLookup, by
|
||||||
|
// name.
|
||||||
|
|
||||||
// heuristic is one of the checks that decide what waf does with a
|
// heuristic is one of the checks that decide what waf does with a
|
||||||
// request.
|
// request.
|
||||||
@@ -20,6 +30,15 @@ type heuristic interface {
|
|||||||
check(ctx context.Context, rq *request, now time.Time) *verdict
|
check(ctx context.Context, rq *request, now time.Time) *verdict
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusesWithoutLookup is a heuristic of afterLookup that can tell, before the
|
||||||
|
// lookup answers, that it will refuse a request whatever the answer.
|
||||||
|
type refusesWithoutLookup interface {
|
||||||
|
// refusesWhateverTheLookupAnswers reports whether the heuristic will refuse
|
||||||
|
// rq at now, whatever the lookup answers about its client. It changes
|
||||||
|
// nothing.
|
||||||
|
refusesWhateverTheLookupAnswers(rq *request, now time.Time) bool
|
||||||
|
}
|
||||||
|
|
||||||
// verdict is a heuristic refusing a request, or banning its client: the action
|
// verdict is a heuristic refusing a request, or banning its client: the action
|
||||||
// the log line names, one of requestlog's Action constants, and the figures the
|
// the log line names, one of requestlog's Action constants, and the figures the
|
||||||
// heuristic acted on, by name, such as the count and the limit.
|
// heuristic acted on, by name, such as the count and the limit.
|
||||||
|
|||||||
@@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) {
|
|||||||
rq.line.ASName = rq.lookupAnswer.ASName
|
rq.line.ASName = rq.lookupAnswer.ASName
|
||||||
rq.line.Country = rq.lookupAnswer.Country
|
rq.line.Country = rq.lookupAnswer.Country
|
||||||
|
|
||||||
if tag := asnTag(rq.line.ASN); tag != "" {
|
if tag := asnTag(rq.lookupAnswer.ASN); tag != "" {
|
||||||
rq.addTag(tag)
|
rq.addTag(tag)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tag := countryTag(rq.line.Country); tag != "" {
|
if tag := countryTag(rq.lookupAnswer.Country); tag != "" {
|
||||||
rq.addTag(tag)
|
rq.addTag(tag)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-31
@@ -177,30 +177,6 @@ func newHandler(params Params) *handler {
|
|||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
}
|
}
|
||||||
|
|
||||||
h.blocklists = blocklists{
|
|
||||||
lists: params.Lists,
|
|
||||||
action: cfg.BlocklistAction,
|
|
||||||
hitNoter: hitNoter,
|
|
||||||
}
|
|
||||||
h.rateLimits = rateLimits{
|
|
||||||
limiter: params.Limiter,
|
|
||||||
exemptNets: cfg.RateLimitExemptNets,
|
|
||||||
exemptPaths: cfg.RateLimitExemptPaths,
|
|
||||||
banResponse: cfg.BanResponse,
|
|
||||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
||||||
biasedThresholds: biasedThresholds{
|
|
||||||
lists: params.Lists,
|
|
||||||
tagLimitPercent: cfg.TagLimitPercent,
|
|
||||||
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
|
||||||
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
|
||||||
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
|
||||||
blocklistAction: cfg.BlocklistAction,
|
|
||||||
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
|
||||||
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
|
||||||
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
|
||||||
},
|
|
||||||
banMaker: sharedBanMaker,
|
|
||||||
}
|
|
||||||
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
||||||
h.ruleFiles = ruleFiles{
|
h.ruleFiles = ruleFiles{
|
||||||
rules: params.Rules,
|
rules: params.Rules,
|
||||||
@@ -230,7 +206,11 @@ func newHandler(params Params) *handler {
|
|||||||
}
|
}
|
||||||
h.afterLookup = []heuristic{
|
h.afterLookup = []heuristic{
|
||||||
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
||||||
h.blocklists,
|
blocklists{
|
||||||
|
lists: params.Lists,
|
||||||
|
action: cfg.BlocklistAction,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
dnsbl{
|
dnsbl{
|
||||||
dnsbl: params.DNSBL,
|
dnsbl: params.DNSBL,
|
||||||
action: cfg.DNSBLAbuseIPDBAction,
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
@@ -246,7 +226,25 @@ func newHandler(params Params) *handler {
|
|||||||
hitNoter: hitNoter,
|
hitNoter: hitNoter,
|
||||||
},
|
},
|
||||||
tagDeny{tags: cfg.TagDeny},
|
tagDeny{tags: cfg.TagDeny},
|
||||||
h.rateLimits,
|
rateLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
exemptNets: cfg.RateLimitExemptNets,
|
||||||
|
exemptPaths: cfg.RateLimitExemptPaths,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
biasedThresholds: biasedThresholds{
|
||||||
|
lists: params.Lists,
|
||||||
|
tagLimitPercent: cfg.TagLimitPercent,
|
||||||
|
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
||||||
|
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
||||||
|
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
||||||
|
blocklistAction: cfg.BlocklistAction,
|
||||||
|
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
||||||
|
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
|
},
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||||
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
||||||
@@ -268,6 +266,8 @@ func newHandler(params Params) *handler {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
h.ownEndpoints = h.routes()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,6 +293,9 @@ type handler struct {
|
|||||||
// awaitingReport holds the decision on each answered question by its id
|
// awaitingReport holds the decision on each answered question by its id
|
||||||
// until its report comes, and nil for a question still being handled.
|
// until its report comes, and nil for a question still being handled.
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
awaitingReport *lru.Cache[string, *decision]
|
||||||
|
// ownEndpoints answers every request that is not a question, as routes
|
||||||
|
// lists the endpoints.
|
||||||
|
ownEndpoints http.Handler
|
||||||
|
|
||||||
// The lists of heuristics, each in the order its heuristics run. A client
|
// The lists of heuristics, each in the order its heuristics run. A client
|
||||||
// in SWW_WAF_ALLOW_NETS skips them all.
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||||
@@ -323,11 +326,7 @@ type handler struct {
|
|||||||
// afterAnswer, they only ban.
|
// afterAnswer, they only ban.
|
||||||
onReport []heuristic
|
onReport []heuristic
|
||||||
|
|
||||||
// The heuristics of the lists that other steps of a request use too:
|
// The heuristics of besideLookup, whose work findBesideLookup does.
|
||||||
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
|
|
||||||
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
|
|
||||||
blocklists blocklists
|
|
||||||
rateLimits rateLimits
|
|
||||||
trapPaths trapPaths
|
trapPaths trapPaths
|
||||||
ruleFiles ruleFiles
|
ruleFiles ruleFiles
|
||||||
coreRuleSet coreRuleSet
|
coreRuleSet coreRuleSet
|
||||||
|
|||||||
+27
-20
@@ -548,16 +548,11 @@ func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
|||||||
params.CoreRuleSet = coreRuleSet
|
params.CoreRuleSet = coreRuleSet
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCounters adds to params the metrics, the table of clients, the ledger
|
// addCounters adds to params the table of clients, the ledger and the
|
||||||
// and the anomaly counters, as sww builds them from params' settings. The
|
// anomaly counters, as sww builds them from params' settings.
|
||||||
// metrics read the ledger, the table of clients and the rules.
|
|
||||||
func addCounters(params *waf.Params) {
|
func addCounters(params *waf.Params) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
|
|
||||||
params.Metrics = metrics.New(
|
|
||||||
cfg.MetricsBusiestCountriesAndASNs,
|
|
||||||
cfg.InstanceName,
|
|
||||||
)
|
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
Limits: ratelimit.Limits{
|
Limits: ratelimit.Limits{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
@@ -587,26 +582,18 @@ func addCounters(params *waf.Params) {
|
|||||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
Alerts: params.Alerts,
|
Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|
||||||
params.Metrics.AddBansAndClients(
|
|
||||||
params.Ledger,
|
|
||||||
params.Limiter,
|
|
||||||
params.Now,
|
|
||||||
)
|
|
||||||
params.Metrics.AddRules(params.Rules)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||||
// setting is set, and GeoJS at geojsURL, as sww builds
|
// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as
|
||||||
// them from params' settings and the parts addCounters added. The metrics
|
// sww builds them from params' settings and the parts addCounters added.
|
||||||
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
|
||||||
func addOutsideSources(
|
func addOutsideSources(
|
||||||
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||||
) {
|
) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
limiter, ledger := params.Limiter, params.Ledger
|
||||||
processLog, alertQueue := params.ProcessLog, params.Alerts
|
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||||
|
|
||||||
params.Lists = reputation.New(reputation.Params{
|
params.Lists = reputation.New(reputation.Params{
|
||||||
@@ -655,6 +642,9 @@ func addOutsideSources(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
m := newMetrics(params)
|
||||||
|
params.Metrics = m
|
||||||
|
|
||||||
params.GeoJS = geojs.New(geojs.Params{
|
params.GeoJS = geojs.New(geojs.Params{
|
||||||
URL: geojsURL,
|
URL: geojsURL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
@@ -667,12 +657,29 @@ func addOutsideSources(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
m.AddReputation(params.Lists, params.DNSBL)
|
// newMetrics returns the metrics, as sww builds them from params' settings,
|
||||||
|
// read from the ledger, the table of clients, the rules, the lists, the
|
||||||
|
// zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
|
func newMetrics(params *waf.Params) *metrics.Metrics {
|
||||||
|
cfg := params.Config
|
||||||
|
metricsParams := metrics.Params{
|
||||||
|
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
||||||
|
InstanceName: cfg.InstanceName,
|
||||||
|
Now: params.Now,
|
||||||
|
Bans: params.Ledger,
|
||||||
|
Clients: params.Limiter,
|
||||||
|
Rules: params.Rules,
|
||||||
|
Lists: params.Lists,
|
||||||
|
DNSBL: params.DNSBL,
|
||||||
|
}
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
if cfg.AbuseIPDBKey != "" {
|
||||||
m.AddAbuseIPDB(params.AbuseIPDB)
|
metricsParams.AbuseIPDB = params.AbuseIPDB
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return metrics.New(metricsParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newClient returns an HTTP client that sends requests as they are made,
|
// newClient returns an HTTP client that sends requests as they are made,
|
||||||
|
|||||||
@@ -67,11 +67,14 @@ func (r rateLimits) check(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
|
// refusesWhateverTheLookupAnswers reports whether the heuristic rate_limits
|
||||||
// at now, whatever its client's limit percentages: the rate limits count it,
|
// will refuse rq at now, whatever its client's limit percentages, which the
|
||||||
// and one more request from its client takes it over a whole rate limit. It
|
// lookup's answer can lower: the rate limits count it, and one more request
|
||||||
// counts nothing.
|
// from its client takes it over a whole rate limit. It counts nothing.
|
||||||
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
|
func (r rateLimits) refusesWhateverTheLookupAnswers(
|
||||||
|
rq *request,
|
||||||
|
now time.Time,
|
||||||
|
) bool {
|
||||||
return r.countedByRateLimits(rq) &&
|
return r.countedByRateLimits(rq) &&
|
||||||
r.limiter.OverARateLimit(
|
r.limiter.OverARateLimit(
|
||||||
clientGroup(r.ipv6GroupPrefix, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (r refusalLimit) check(
|
|||||||
// What the client was sent, or in observe mode would have been.
|
// What the client was sent, or in observe mode would have been.
|
||||||
status := rq.out.status
|
status := rq.out.status
|
||||||
|
|
||||||
switch rq.line.WouldAction {
|
switch rq.wouldAction {
|
||||||
case requestlog.ActionBlockedByBlockRule,
|
case requestlog.ActionBlockedByBlockRule,
|
||||||
requestlog.ActionBlockedByCoreRuleSet:
|
requestlog.ActionBlockedByCoreRuleSet:
|
||||||
status = http.StatusForbidden
|
status = http.StatusForbidden
|
||||||
|
|||||||
@@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit(
|
|||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.lookupAnswer.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.lookupAnswer.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.lookupAnswer.Country,
|
||||||
Reason: reason,
|
Reason: reason,
|
||||||
Detail: detail,
|
Detail: detail,
|
||||||
})
|
})
|
||||||
|
|||||||
+42
-20
@@ -24,42 +24,64 @@ type refusal struct {
|
|||||||
action string
|
action string
|
||||||
}
|
}
|
||||||
|
|
||||||
// decision is what the checks note of a request: its log line, and what
|
// decision is what the checks note of a request: its log line, what the
|
||||||
// counting the request needs once it has ended. waf keeps a copy of it
|
// heuristics tell each other, and what counting the request needs once it has
|
||||||
// until the request's report comes. It holds no credentials, since the log line
|
// ended. Nothing reads the log line back to decide. waf keeps a copy of it
|
||||||
// never does, and neither the request's objects nor their bodies.
|
// until the request's report comes. It holds no credentials, since the log
|
||||||
|
// line never does, and neither the request's objects nor their bodies.
|
||||||
type decision struct {
|
type decision struct {
|
||||||
line requestlog.Line
|
line requestlog.Line
|
||||||
client netip.Addr
|
client netip.Addr
|
||||||
// requestURI is the path with the query, as a ban's notes keep it.
|
// method and userAgent are the request's method and User-Agent, and
|
||||||
requestURI string
|
// requestURI its path with the query, as a ban's notes keep them.
|
||||||
|
// startRequest sets them.
|
||||||
|
method, userAgent, requestURI string
|
||||||
|
// tags are the request's tags, sorted, each once. addTag adds them, for
|
||||||
|
// tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and
|
||||||
|
// coreRuleSet.
|
||||||
|
tags []string
|
||||||
// site is, for a question, the host the visitor asked for, which the
|
// site is, for a question, the host the visitor asked for, which the
|
||||||
// alerts name, since one waf serves many sites.
|
// alerts and a ban's notes name, since one waf serves many sites.
|
||||||
|
// answerQuestion sets it.
|
||||||
site string
|
site string
|
||||||
// allowed is true for a question answered with DispositionAllow.
|
// allowed is true for a question answered with DispositionAllow.
|
||||||
allowed bool
|
allowed bool
|
||||||
// lookedUp is true once the client's AS number and country have been
|
// lookedUp is true once the client's AS number and country have been
|
||||||
// looked up, whether or not an answer was there, and lookupAnswer is
|
// looked up, whether or not an answer was there, and lookupAnswer is
|
||||||
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
||||||
|
// noteLookup sets them.
|
||||||
lookedUp bool
|
lookedUp bool
|
||||||
lookupAnswer geojs.Answer
|
lookupAnswer geojs.Answer
|
||||||
// skippedLookupWait is true for a request that did not wait for GeoJS and
|
// skippedLookupWait is true for a request that did not wait for GeoJS and
|
||||||
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
|
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
|
||||||
// so that not waiting never lowers its client's limits.
|
// so that not waiting never lowers its client's limits. noteLookup sets
|
||||||
|
// it.
|
||||||
skippedLookupWait bool
|
skippedLookupWait bool
|
||||||
// counted is true for a request the rate limits counted, whose bytes
|
// counted is true for a request the rate limits counted, whose bytes
|
||||||
// the byte limits count once it has ended. rateLimitPercent and
|
// the byte limits count once it has ended. rateLimitPercent and
|
||||||
// bytesLimitPercent are then its client's limit percentages for the rate
|
// bytesLimitPercent are then its client's limit percentages for the rate
|
||||||
// limits and for the byte limits.
|
// limits and for the byte limits. rateLimits sets them.
|
||||||
counted bool
|
counted bool
|
||||||
rateLimitPercent, bytesLimitPercent percentage
|
rateLimitPercent, bytesLimitPercent percentage
|
||||||
// attack is true for a request that matched a ban rule or asked for a
|
// attack is true for a request that matched a ban rule or asked for a
|
||||||
// trap path, blockedByBlockRule for one a block rule refused, and
|
// trap path, blockedByBlockRule for one a block rule refused, and
|
||||||
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
|
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
|
||||||
// its client's history counts.
|
// its client's history counts. trapPaths and ruleFiles set attack,
|
||||||
|
// ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet.
|
||||||
attack, blockedByBlockRule, blockedByCoreRuleSet bool
|
attack, blockedByBlockRule, blockedByCoreRuleSet bool
|
||||||
|
// wouldAction is, in observe mode, the action enforce mode would have
|
||||||
|
// refused the request with, one of requestlog's Action constants, and ""
|
||||||
|
// otherwise. check sets it.
|
||||||
|
wouldAction string
|
||||||
|
// brokeLimit is true for a request that broke a limit, an offence its
|
||||||
|
// client's history counts. banForLimit sets it.
|
||||||
|
brokeLimit bool
|
||||||
|
// blocklisted is true once blocklists finds a blocklist that lists the
|
||||||
|
// client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does,
|
||||||
|
// and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit.
|
||||||
|
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||||
// reputation is the reputation sources that list the client, for the
|
// reputation is the reputation sources that list the client, for the
|
||||||
// notes of a ban the request makes.
|
// notes of a ban the request makes. noteHit adds them.
|
||||||
reputation []bans.ReputationHit
|
reputation []bans.ReputationHit
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,11 +94,7 @@ type request struct {
|
|||||||
in *http.Request
|
in *http.Request
|
||||||
out *responseWriter
|
out *responseWriter
|
||||||
|
|
||||||
// blocklisted is true once a blocklist is found to list the client,
|
start time.Time
|
||||||
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
|
||||||
// AbuseIPDB's score of it is a hit.
|
|
||||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
|
||||||
start time.Time
|
|
||||||
// checked is when the checks were done.
|
// checked is when the checks were done.
|
||||||
checked time.Time
|
checked time.Time
|
||||||
// refused is what the request was refused with, nil if it was not.
|
// refused is what the request was refused with, nil if it was not.
|
||||||
@@ -106,6 +124,8 @@ func (h *handler) startRequest(
|
|||||||
rq := &request{
|
rq := &request{
|
||||||
decision: decision{
|
decision: decision{
|
||||||
client: client,
|
client: client,
|
||||||
|
method: r.Method,
|
||||||
|
userAgent: r.UserAgent(),
|
||||||
requestURI: r.URL.RequestURI(),
|
requestURI: r.URL.RequestURI(),
|
||||||
line: requestlog.Line{
|
line: requestlog.Line{
|
||||||
Time: requestlog.FormatTime(start),
|
Time: requestlog.FormatTime(start),
|
||||||
@@ -151,11 +171,12 @@ func (h *handler) startRequest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// addTag adds tag to the request's tags, which stay sorted and hold each tag
|
// addTag adds tag to the request's tags, which stay sorted and hold each tag
|
||||||
// once.
|
// once, and to its log line.
|
||||||
func (rq *request) addTag(tag string) {
|
func (rq *request) addTag(tag string) {
|
||||||
at, found := slices.BinarySearch(rq.line.Tags, tag)
|
at, found := slices.BinarySearch(rq.tags, tag)
|
||||||
if !found {
|
if !found {
|
||||||
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag)
|
rq.tags = slices.Insert(rq.tags, at, tag)
|
||||||
|
rq.line.Tags = rq.tags
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -196,6 +217,7 @@ func (rq *request) check(ctx context.Context) *refusal {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
|
rq.wouldAction = action
|
||||||
rq.line.WouldAction = action
|
rq.line.WouldAction = action
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -371,7 +393,7 @@ func (rq *request) addToHistory() {
|
|||||||
Status: rq.out.status,
|
Status: rq.out.status,
|
||||||
RequestBytes: rq.requestBytes,
|
RequestBytes: rq.requestBytes,
|
||||||
ResponseBytes: rq.out.bytes,
|
ResponseBytes: rq.out.bytes,
|
||||||
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
BrokeLimit: rq.brokeLimit,
|
||||||
Attack: rq.attack,
|
Attack: rq.attack,
|
||||||
BlockedByBlockRule: rq.blockedByBlockRule,
|
BlockedByBlockRule: rq.blockedByBlockRule,
|
||||||
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
|
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/subtle"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
|
)
|
||||||
|
|
||||||
|
// routes returns the router of waf's own endpoints. A question never reaches
|
||||||
|
// it, since ServeHTTP answers a question first. No check runs on a request
|
||||||
|
// here: waf's address is on the private network, where the peers are rproxy
|
||||||
|
// and the admin's tools, not visitors.
|
||||||
|
func (h *handler) routes() http.Handler {
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Use(matchDecodedPath)
|
||||||
|
|
||||||
|
// The report writes no log line of its own: it finishes the line of the
|
||||||
|
// request it reports.
|
||||||
|
r.Post(wafprotocol.ReportPath, h.handleReport())
|
||||||
|
|
||||||
|
r.Group(func(r chi.Router) {
|
||||||
|
r.Use(h.logOwnEndpoint)
|
||||||
|
// The answer to a path with no endpoint, and to a method the path has
|
||||||
|
// no endpoint for, is set in this group, so that it is logged too.
|
||||||
|
r.NotFound(answerNotFound)
|
||||||
|
r.MethodNotAllowed(answerNotFound)
|
||||||
|
|
||||||
|
r.Get(wafprotocol.HealthPath, handleHealth())
|
||||||
|
|
||||||
|
r.Group(func(r chi.Router) {
|
||||||
|
r.Use(requireToken(h.config.MetricsToken))
|
||||||
|
r.Get(MetricsPath, h.metrics.ServeHTTP)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The handlers under BansPath/ and ClientsPath read the address from
|
||||||
|
// the path themselves, so that one that is not an address gets 400
|
||||||
|
// after the token check.
|
||||||
|
r.Group(func(r chi.Router) {
|
||||||
|
r.Use(requireToken(h.config.AdminToken))
|
||||||
|
r.Get(BansPath, h.handleListBans())
|
||||||
|
r.Post(BansPath, h.handleAddBan())
|
||||||
|
r.Delete(BansPath+"/*", h.handleLiftBans())
|
||||||
|
r.Get(ClientsPath+"*", h.handleShowClient())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchDecodedPath has the router match the decoded path, which the handlers
|
||||||
|
// read. chi matches the path as sent when it is encoded in an unusual way,
|
||||||
|
// which would make /_sww/%68ealthz a path with no endpoint.
|
||||||
|
func matchDecodedPath(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// logOwnEndpoint writes one log line for each request, with the action
|
||||||
|
// own_endpoint. The line's request_bytes is how much of the body the handler
|
||||||
|
// read.
|
||||||
|
func (h *handler) logOwnEndpoint(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
peer := peerAddress(r)
|
||||||
|
|
||||||
|
rq := h.startRequest(w, r, peer, peer, false)
|
||||||
|
defer rq.finish()
|
||||||
|
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
|
||||||
|
counted := r.Clone(r.Context())
|
||||||
|
counted.Body = &countedBody{
|
||||||
|
ReadCloser: r.Body,
|
||||||
|
read: &rq.requestBytes,
|
||||||
|
}
|
||||||
|
|
||||||
|
next.ServeHTTP(rq.out, counted)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// countedBody is a request's body that adds the bytes read of it to read.
|
||||||
|
type countedBody struct {
|
||||||
|
io.ReadCloser
|
||||||
|
|
||||||
|
read *int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads part of the body.
|
||||||
|
func (b *countedBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.ReadCloser.Read(p)
|
||||||
|
*b.read += int64(n)
|
||||||
|
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireToken returns middleware that lets a request through only when it
|
||||||
|
// carries token, as Authorization: Bearer <token>. It answers 401 to a request
|
||||||
|
// without it. While token is unset, it answers 404, as a path with no endpoint
|
||||||
|
// does.
|
||||||
|
func requireToken(token string) func(http.Handler) http.Handler {
|
||||||
|
return func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case token == "":
|
||||||
|
answerNotFound(w, r)
|
||||||
|
case !hasToken(r, token):
|
||||||
|
w.Header().Set("WWW-Authenticate", "Bearer")
|
||||||
|
http.Error(
|
||||||
|
w,
|
||||||
|
http.StatusText(http.StatusUnauthorized),
|
||||||
|
http.StatusUnauthorized,
|
||||||
|
)
|
||||||
|
default:
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasToken reports whether r carries token, as Authorization: Bearer
|
||||||
|
// <token>.
|
||||||
|
func hasToken(r *http.Request, token string) bool {
|
||||||
|
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||||
|
|
||||||
|
return strings.EqualFold(scheme, "Bearer") &&
|
||||||
|
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerNotFound answers 404, for a path with no endpoint, and for a method
|
||||||
|
// the path has no endpoint for.
|
||||||
|
func answerNotFound(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||||
|
}
|
||||||
@@ -23,7 +23,7 @@ func (t tagDeny) check(
|
|||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
for _, tag := range rq.line.Tags {
|
for _, tag := range rq.tags {
|
||||||
if slices.Contains(t.tags, tag) {
|
if slices.Contains(t.tags, tag) {
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
|
|||||||
+36
-53
@@ -58,8 +58,8 @@ func New(params Params) *Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP handles one request to waf: a request carrying
|
// ServeHTTP handles one request to waf: a request carrying
|
||||||
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
|
// wafprotocol.ClientHeader is rproxy's question, and any other is for one of
|
||||||
// itself.
|
// waf's own endpoints.
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
||||||
h.answerQuestion(w, r)
|
h.answerQuestion(w, r)
|
||||||
@@ -67,23 +67,27 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.answerOwnEndpoint(w, r)
|
h.ownEndpoints.ServeHTTP(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
||||||
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
||||||
// its client: the heuristic blocklists or rate_limits will refuse it, as a
|
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
|
||||||
// function in that heuristic's file says. In observe mode, which refuses
|
// says it will refuse it. In observe mode, which refuses nothing, none is.
|
||||||
// nothing, none is. rproxy lets a request through when waf answers too late.
|
// rproxy lets a request through when waf answers too late.
|
||||||
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
||||||
if rq.h.config.Observe {
|
if rq.h.config.Observe {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
|
for _, each := range rq.h.afterLookup {
|
||||||
|
knows, ok := each.(refusesWithoutLookup)
|
||||||
|
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
|
return false
|
||||||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerQuestion runs the checks on the visitor's request a question
|
// answerQuestion runs the checks on the visitor's request a question
|
||||||
@@ -202,8 +206,8 @@ func (rq *request) allow() {
|
|||||||
|
|
||||||
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
||||||
if rq.h.config.AddLookupHeaders {
|
if rq.h.config.AddLookupHeaders {
|
||||||
answer.ASN = rq.line.ASN
|
answer.ASN = rq.lookupAnswer.ASN
|
||||||
answer.Country = rq.line.Country
|
answer.Country = rq.lookupAnswer.Country
|
||||||
}
|
}
|
||||||
|
|
||||||
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
||||||
@@ -240,50 +244,29 @@ func (rq *request) deny(r refusal) {
|
|||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerOwnEndpoint answers a request to waf's own address. No check
|
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
|
||||||
// runs on it: the address is on the private network, where the peers are
|
// the report names, if it holds that request's decision, and ignores the
|
||||||
// rproxy and the admin's tools, not visitors.
|
// report otherwise.
|
||||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) handleReport() http.HandlerFunc {
|
||||||
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
h.takeReport(w, r)
|
report, err := wafprotocol.ReadReport(w, r)
|
||||||
|
if err != nil {
|
||||||
|
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||||
|
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
held, found := h.awaitingReport.Peek(report.RequestID)
|
||||||
|
// Remove reports true to one report only, of several that name the
|
||||||
|
// id. A question still being handled is held as nil.
|
||||||
|
if found && held != nil &&
|
||||||
|
h.awaitingReport.Remove(report.RequestID) {
|
||||||
|
h.finishWith(r.Context(), held, report)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
peer := peerAddress(r)
|
|
||||||
|
|
||||||
rq := h.startRequest(w, r, peer, peer, false)
|
|
||||||
defer rq.finish()
|
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
|
||||||
rq.answerHealth()
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.answerAdmin()
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
|
|
||||||
// report names, if it holds that request's decision, and ignores the report
|
|
||||||
// otherwise.
|
|
||||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
|
||||||
report, err := wafprotocol.ReadReport(w, r)
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
|
||||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
held, found := h.awaitingReport.Peek(report.RequestID)
|
|
||||||
// Remove reports true to one report only, of several that name the id. A
|
|
||||||
// question still being handled is held as nil.
|
|
||||||
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
|
|
||||||
h.finishWith(r.Context(), held, report)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
// finishWith finishes a request with what its report says, and writes the
|
||||||
|
|||||||
Reference in new issue
Block a user