diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index a1c4aa8..21613f4 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -15,12 +15,33 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" - "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/rules" ) +// Params are what New builds the metrics from. Each part is nil while it is +// off, and a part that is nil adds no metric of its own. +type Params struct { + // BusiestCount is how many countries and how many AS numbers get series + // of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). + BusiestCount int + // InstanceName is the label instance every metric carries + // (SWW_WAF_INSTANCE_NAME). + InstanceName string + // Now returns the time at which Bans counts the bans active. + Now func() time.Time + + Bans banLedger + Clients sized + Rules sized + Lists fetchedLists + DNSBL dnsblZones + AbuseIPDB abuseIPDBChecker + Alerts alertQueue + LookupFile lookupDatabase + RemoteLog remoteLogSender +} + // Metrics are sww's metrics. They are safe for concurrent use. type Metrics struct { // registry gives every metric registered with it the label instance. @@ -36,12 +57,10 @@ type Metrics struct { rateLimitHits *prometheus.CounterVec offences *prometheus.CounterVec coreRuleSetMatches *prometheus.CounterVec - // ruleMatches are made by AddRules, and reputationHits by - // AddReputation. - ruleMatches *prometheus.CounterVec - reputationHits *prometheus.CounterVec - countries *busiest - asns *busiest + ruleMatches *prometheus.CounterVec + reputationHits *prometheus.CounterVec + countries *busiest + asns *busiest // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those // that failed. RequestsWithoutGeoJSAnswer are the requests that needed @@ -59,20 +78,18 @@ type Metrics struct { stateFileEditsSetAside *prometheus.CounterVec } -// New returns the metrics, with the Go runtime's and the process's own. -// busiestCount is how many countries and how many AS numbers get series of -// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric -// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance. +// New returns the metrics, with the Go runtime's and the process's own, and +// those read from each part in params as the metrics are asked for. // //nolint:funlen // a few lines for each metric, a list that grows with them -func New(busiestCount int, instanceName string) *Metrics { +func New(params Params) *Metrics { byStatus := []string{"status_class", "action"} byFile := []string{"file"} registry := prometheus.NewRegistry() m := &Metrics{ registry: prometheus.WrapRegistererWith( - prometheus.Labels{"instance": instanceName}, registry), + prometheus.Labels{"instance": params.InstanceName}, registry), handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "sww_requests_in_flight", @@ -113,8 +130,19 @@ func New(busiestCount int, instanceName string) *Metrics { "and the rule's id.", []string{"mode", "rule_id"}, ), - countries: newCountries(busiestCount), - asns: newASNs(busiestCount), + ruleMatches: counterVec( + "sww_rule_matches_total", + "Requests that matched a rule of the rule files, by its id and action.", + []string{"rule_id", "action"}, + ), + reputationHits: counterVec( + "sww_reputation_hits_total", + "Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+ + "by the list's URL, the zone, or abuseipdb.", + []string{sourceLabel}, + ), + countries: newCountries(params.BusiestCount), + asns: newASNs(params.BusiestCount), GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ Name: "sww_geojs_requests_total", Help: "Requests to GeoJS.", @@ -166,6 +194,7 @@ func New(busiestCount int, instanceName string) *Metrics { m.inFlight, m.requests, m.requestBytes, m.responseBytes, m.requestDuration, m.upstreamDuration, m.rateLimitHits, m.offences, m.coreRuleSetMatches, + m.ruleMatches, m.reputationHits, m.countries, m.asns, m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer, m.stateFileWrites, m.stateFileWriteFailures, @@ -173,261 +202,45 @@ func New(busiestCount int, instanceName string) *Metrics { m.stateFileEditsTakenIn, m.stateFileEditsSetAside, ) + if params.Bans != nil { + m.addBans(params.Bans, params.Now) + } + + if params.Clients != nil { + m.addClients(params.Clients) + } + + if params.Rules != nil { + m.addRules(params.Rules) + } + + if params.Lists != nil { + m.addLists(params.Lists) + } + + if params.DNSBL != nil { + m.addDNSBL(params.DNSBL) + } + + if params.AbuseIPDB != nil { + m.addAbuseIPDB(params.AbuseIPDB) + } + + if params.Alerts != nil { + m.addAlerts(params.Alerts) + } + + if params.LookupFile != nil { + m.addLookupFile(params.LookupFile) + } + + if params.RemoteLog != nil { + m.addRemoteLog(params.RemoteLog) + } + return m } -// AddBansAndClients adds the metrics read from the ledger and the table -// of clients as the metrics are asked for: the bans made since the start, -// by cause, the bans active and permanent at now, and the clients in the -// table. -func (m *Metrics) AddBansAndClients( - ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, -) { - for _, cause := range []string{ - bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB, - } { - m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_bans_made_total", - Help: "Bans made, by cause.", - ConstLabels: prometheus.Labels{"cause": cause}, - }, func() float64 { - return float64(ledger.Made(cause)) - }), - ) - } - - m.registry.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_active_bans", - Help: "Bans active now, the permanent ones included.", - }, func() float64 { - active, _ := ledger.Count(now()) - - return float64(active) - }), - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_permanent_bans", - Help: "Permanent bans not lifted.", - }, func() float64 { - _, permanent := ledger.Count(now()) - - return float64(permanent) - }), - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_tracked_clients", - Help: "Clients in the table of clients.", - }, func() float64 { - return float64(limiter.Len()) - }), - ) -} - -// AddRules adds the metrics of the rule files: the requests that matched -// each rule, which RuleMatched counts, and the rules loaded from -// ruleFiles, read as the metrics are asked for. It is called once, before -// RuleMatched. -func (m *Metrics) AddRules(ruleFiles *rules.Files) { - m.ruleMatches = counterVec( - "sww_rule_matches_total", - "Requests that matched a rule of the rule files, by its id and action.", - []string{"rule_id", "action"}, - ) - - m.registry.MustRegister(m.ruleMatches, - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_rules_loaded", - Help: "Rules loaded from the rule files.", - }, func() float64 { - return float64(ruleFiles.Len()) - })) -} - -// AddRemoteLog adds the metrics of sending the log lines to -// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the -// lines sent, those dropped, and those waiting in the buffer. -func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { - m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_remote_log_lines_sent_total", - Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.", - }, func() float64 { - return float64(remote.Sent()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_remote_log_lines_dropped_total", - Help: "Log lines dropped: the oldest in a full buffer, and those " + - "whose sending failed.", - }, func() float64 { - return float64(remote.Dropped()) - }), - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_remote_log_lines_waiting", - Help: "Log lines in the buffer, waiting to be sent.", - }, func() float64 { - return float64(remote.Depth()) - }), - ) -} - -// AddLookupFile adds the metrics of the lookup database, read as the -// metrics are asked for: when the file in use was read, which lastRead -// returns, and the replacements of it that could not be read, which -// readFailures returns. -func (m *Metrics) AddLookupFile( - lastRead func() time.Time, - readFailures func() int, -) { - m.registry.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_lookup_database_last_read_timestamp_seconds", - Help: "When the lookup database in use was read, in seconds since 1970.", - }, func() float64 { - return float64(lastRead().Unix()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_lookup_database_read_failures_total", - Help: "Replacements of the lookup database that could not be read.", - }, func() float64 { - return float64(readFailures()) - }), - ) -} - -// sourceLabel is the label of the reputation metrics: a list's URL, a -// DNSBL zone, its key masked, or abuseipdb. -const sourceLabel = "source" - -// AddReputation adds the metrics of the lists fetched from URLs and of the -// DNSBL zones, by source, each list's URL or each zone, its key masked as -// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a -// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, -// and, read from lists and dnsbl as the metrics are asked for, for a list, -// the fetches that failed and when the copy in use was fetched, and for a -// zone, the queries made and those that failed. It is called once, before -// ReputationHit. -func (m *Metrics) AddReputation( - lists *reputation.Lists, - dnsbl *reputation.DNSBL, -) { - m.reputationHits = counterVec( - "sww_reputation_hits_total", - "Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+ - "by the list's URL, the zone, or abuseipdb.", - []string{sourceLabel}, - ) - m.registry.MustRegister(m.reputationHits) - - for _, zone := range dnsbl.Zones() { - source := prometheus.Labels{ - sourceLabel: reputation.MaskZoneKey(zone), - } - - m.addReputationQueries( - source, - func() int { return dnsbl.Queries(zone) }, - ) - m.addReputationFailures( - source, - func() int { return dnsbl.Failures(zone) }, - ) - } - - for _, listURL := range lists.URLs() { - source := prometheus.Labels{sourceLabel: listURL} - - m.addReputationFailures( - source, - func() int { return lists.Failures(listURL) }, - ) - m.registry.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_reputation_last_fetch_timestamp_seconds", - Help: "When the copy of the list in use was fetched, in seconds since " + - "1970, or 0 while there is none.", - ConstLabels: source, - }, func() float64 { - fetched := lists.Fetched(listURL) - if fetched.IsZero() { - return 0 - } - - return float64(fetched.Unix()) - }), - ) - } -} - -// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb, -// read from abuseIPDB as the metrics are asked for: the checks made, those -// that failed, and how many checks the day's budget has left. It is -// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set. -func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) { - source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource} - - m.addReputationQueries(source, abuseIPDB.Checked) - m.addReputationFailures(source, abuseIPDB.Failures) - m.registry.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "sww_reputation_daily_budget_remaining", - Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.", - ConstLabels: source, - }, func() float64 { - return float64(abuseIPDB.BudgetLeft()) - }), - ) -} - -// ReputationHit counts a request whose client source lists: a blocklist, -// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb. -func (m *Metrics) ReputationHit(source string) { - m.reputationHits.WithLabelValues(source).Inc() -} - -// AddAlerts adds the metrics of the alerts sent to each destination set, -// read from queue as the metrics are asked for, by destination: the -// alerts sent, the requests to the destination that failed, the alerts -// held back, which are the same for every destination, and those -// dropped. With no destination set, it adds none. -func (m *Metrics) AddAlerts(queue *alerts.Queue) { - for _, name := range queue.DestinationsSet() { - destination := prometheus.Labels{"destination": name} - - m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_alerts_sent_total", - Help: "Alerts the destination took.", - ConstLabels: destination, - }, func() float64 { - return float64(queue.Counts(name).Sent) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_alert_send_failures_total", - Help: "Requests to the destination that failed.", - ConstLabels: destination, - }, func() float64 { - return float64(queue.Counts(name).SendFailures) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_alerts_suppressed_total", - Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " + - "alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.", - ConstLabels: destination, - }, func() float64 { - return float64(queue.Suppressed()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "sww_alerts_dropped_total", - Help: "Alerts dropped, the oldest first, from a full queue, and " + - "alerts given up as the destination refused them.", - ConstLabels: destination, - }, func() float64 { - return float64(queue.Counts(name).Dropped) - }), - ) - } -} - // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) @@ -497,6 +310,12 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) { m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc() } +// ReputationHit counts a request whose client source lists: a blocklist, +// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb. +func (m *Metrics) ReputationHit(source string) { + m.reputationHits.WithLabelValues(source).Inc() +} + // StateFileWritten counts a write of the state or cache file name, of size // bytes, that ended with err. func (m *Metrics) StateFileWritten(name string, size int, err error) { @@ -528,6 +347,312 @@ func (m *Metrics) StateFileEditSetAside(name string) { m.stateFileEditsSetAside.WithLabelValues(name).Inc() } +// banLedger is what the metrics read from the ledger of bans. +type banLedger interface { + // Made returns the bans made since the start with cause. + Made(cause string) int + // Count returns the bans active at now, the permanent ones included, + // and the permanent ones. + Count(now time.Time) (active, permanent int) +} + +// addBans adds the metrics read from ledger as the metrics are asked for: +// the bans made since the start, by cause, and the bans active and +// permanent at now. +func (m *Metrics) addBans(ledger banLedger, now func() time.Time) { + for _, cause := range []string{ + bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB, + } { + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_bans_made_total", + Help: "Bans made, by cause.", + ConstLabels: prometheus.Labels{"cause": cause}, + }, func() float64 { + return float64(ledger.Made(cause)) + }), + ) + } + + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_active_bans", + Help: "Bans active now, the permanent ones included.", + }, func() float64 { + active, _ := ledger.Count(now()) + + return float64(active) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_permanent_bans", + Help: "Permanent bans not lifted.", + }, func() float64 { + _, permanent := ledger.Count(now()) + + return float64(permanent) + }), + ) +} + +// sized is what the metrics read from the table of clients and from the +// rule files. +type sized interface { + // Len returns how many clients the table holds, or how many rules were + // loaded from the rule files. + Len() int +} + +// addClients adds the metric of the clients in table, read as the metrics +// are asked for. +func (m *Metrics) addClients(table sized) { + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_tracked_clients", + Help: "Clients in the table of clients.", + }, func() float64 { + return float64(table.Len()) + }), + ) +} + +// addRules adds the metric of the rules loaded from files, read as the +// metrics are asked for. +func (m *Metrics) addRules(files sized) { + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_rules_loaded", + Help: "Rules loaded from the rule files.", + }, func() float64 { + return float64(files.Len()) + })) +} + +// remoteLogSender is what the metrics read from what sends the log lines to +// SWW_WAF_LOG_REMOTE_URL. +type remoteLogSender interface { + // Sent returns the lines sent. + Sent() int64 + // Dropped returns the lines dropped. + Dropped() int64 + // Depth returns the lines waiting in the buffer. + Depth() int +} + +// addRemoteLog adds the metrics of sending the log lines to +// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the +// lines sent, those dropped, and those waiting in the buffer. +func (m *Metrics) addRemoteLog(remote remoteLogSender) { + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_remote_log_lines_sent_total", + Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.", + }, func() float64 { + return float64(remote.Sent()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_remote_log_lines_dropped_total", + Help: "Log lines dropped: the oldest in a full buffer, and those " + + "whose sending failed.", + }, func() float64 { + return float64(remote.Dropped()) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_remote_log_lines_waiting", + Help: "Log lines in the buffer, waiting to be sent.", + }, func() float64 { + return float64(remote.Depth()) + }), + ) +} + +// lookupDatabase is what the metrics read from the lookup database. +type lookupDatabase interface { + // LastRead returns when the file in use was read. + LastRead() time.Time + // ReadFailures returns the replacements of the file that could not be + // read. + ReadFailures() int +} + +// addLookupFile adds the metrics of the lookup database, read from file as +// the metrics are asked for: when the file in use was read, and the +// replacements of it that could not be read. +func (m *Metrics) addLookupFile(file lookupDatabase) { + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_lookup_database_last_read_timestamp_seconds", + Help: "When the lookup database in use was read, in seconds since 1970.", + }, func() float64 { + return float64(file.LastRead().Unix()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_lookup_database_read_failures_total", + Help: "Replacements of the lookup database that could not be read.", + }, func() float64 { + return float64(file.ReadFailures()) + }), + ) +} + +// sourceLabel is the label of the reputation metrics: a list's URL, a +// DNSBL zone, its key masked, or abuseipdb. +const sourceLabel = "source" + +// dnsblZones is what the metrics read from what asks the DNSBL zones. +type dnsblZones interface { + // Zones returns the zones asked, their keys unmasked. + Zones() []string + // Queries returns the queries made to zone. + Queries(zone string) int + // Failures returns the queries to zone that failed. + Failures(zone string) int +} + +// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone, +// its key masked as reputation.MaskZoneKey masks it, read as the metrics are +// asked for: the queries made and those that failed. +func (m *Metrics) addDNSBL(dnsbl dnsblZones) { + for _, zone := range dnsbl.Zones() { + source := prometheus.Labels{ + sourceLabel: reputation.MaskZoneKey(zone), + } + + m.addReputationQueries( + source, + func() int { return dnsbl.Queries(zone) }, + ) + m.addReputationFailures( + source, + func() int { return dnsbl.Failures(zone) }, + ) + } +} + +// fetchedLists is what the metrics read from the lists fetched from URLs. +type fetchedLists interface { + // URLs returns the lists' URLs. + URLs() []string + // Failures returns the fetches of the list at listURL that failed. + Failures(listURL string) int + // Fetched returns when the copy in use of the list at listURL was + // fetched, or the zero time while there is none. + Fetched(listURL string) time.Time +} + +// addLists adds the metrics of each list fetched from a URL, by source, the +// list's URL, read from lists as the metrics are asked for: the fetches +// that failed and when the copy in use was fetched. +func (m *Metrics) addLists(lists fetchedLists) { + for _, listURL := range lists.URLs() { + source := prometheus.Labels{sourceLabel: listURL} + + m.addReputationFailures( + source, + func() int { return lists.Failures(listURL) }, + ) + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_reputation_last_fetch_timestamp_seconds", + Help: "When the copy of the list in use was fetched, in seconds since " + + "1970, or 0 while there is none.", + ConstLabels: source, + }, func() float64 { + fetched := lists.Fetched(listURL) + if fetched.IsZero() { + return 0 + } + + return float64(fetched.Unix()) + }), + ) + } +} + +// abuseIPDBChecker is what the metrics read from what checks clients with +// AbuseIPDB. +type abuseIPDBChecker interface { + // Checked returns the checks made. + Checked() int + // Failures returns the checks that failed. + Failures() int + // BudgetLeft returns how many checks the day's budget has left. + BudgetLeft() int +} + +// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb, +// read from abuseIPDB as the metrics are asked for: the checks made, those +// that failed, and how many checks the day's budget has left. +func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) { + source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource} + + m.addReputationQueries(source, abuseIPDB.Checked) + m.addReputationFailures(source, abuseIPDB.Failures) + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "sww_reputation_daily_budget_remaining", + Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.", + ConstLabels: source, + }, func() float64 { + return float64(abuseIPDB.BudgetLeft()) + }), + ) +} + +// alertQueue is what the metrics read from the queue of alerts. +type alertQueue interface { + // DestinationsSet returns the names of the destinations set. + DestinationsSet() []string + // Counts returns the alerts sent to the destination name, the requests + // to it that failed, and the alerts dropped. + Counts(name string) alerts.Counts + // Suppressed returns the alerts held back. + Suppressed() int64 +} + +// addAlerts adds the metrics of the alerts sent to each destination set, +// read from queue as the metrics are asked for, by destination: the +// alerts sent, the requests to the destination that failed, the alerts +// held back, which are the same for every destination, and those +// dropped. With no destination set, it adds none. +func (m *Metrics) addAlerts(queue alertQueue) { + for _, name := range queue.DestinationsSet() { + destination := prometheus.Labels{"destination": name} + + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_alerts_sent_total", + Help: "Alerts the destination took.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).Sent) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_alert_send_failures_total", + Help: "Requests to the destination that failed.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).SendFailures) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_alerts_suppressed_total", + Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " + + "alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Suppressed()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "sww_alerts_dropped_total", + Help: "Alerts dropped, the oldest first, from a full queue, and " + + "alerts given up as the destination refused them.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).Dropped) + }), + ) + } +} + // count counts a request that has ended, as RequestEnded and RequestReported // describe. func (m *Metrics) count( diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..d1dd7ec --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,160 @@ +package metrics_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/metrics" +) + +const ( + listURL = "https://lists.example/block.txt" + zone = "dnsbl.example" + destination = "webhook" +) + +// Each stand-in answers with a number of its own, so that each metric +// shows which part it was read from. + +type ledgerStandIn struct{} + +func (ledgerStandIn) Made(string) int { return 1 } + +func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 } + +type clientTableStandIn struct{} + +func (clientTableStandIn) Len() int { return 3 } + +type ruleFilesStandIn struct{} + +func (ruleFilesStandIn) Len() int { return 4 } + +type listsStandIn struct{} + +func (listsStandIn) URLs() []string { return []string{listURL} } + +func (listsStandIn) Failures(string) int { return 5 } + +func (listsStandIn) Fetched(string) time.Time { return time.Time{} } + +type dnsblStandIn struct{} + +func (dnsblStandIn) Zones() []string { return []string{zone} } + +func (dnsblStandIn) Queries(string) int { return 6 } + +func (dnsblStandIn) Failures(string) int { return 0 } + +type abuseIPDBStandIn struct{} + +func (abuseIPDBStandIn) Checked() int { return 0 } + +func (abuseIPDBStandIn) Failures() int { return 0 } + +func (abuseIPDBStandIn) BudgetLeft() int { return 7 } + +type alertQueueStandIn struct{} + +func (alertQueueStandIn) DestinationsSet() []string { + return []string{destination} +} + +func (alertQueueStandIn) Counts(string) alerts.Counts { + return alerts.Counts{Sent: 8} +} + +func (alertQueueStandIn) Suppressed() int64 { return 0 } + +type lookupDatabaseStandIn struct{} + +func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} } + +func (lookupDatabaseStandIn) ReadFailures() int { return 9 } + +type remoteLogStandIn struct{} + +func (remoteLogStandIn) Sent() int64 { return 10 } + +func (remoteLogStandIn) Dropped() int64 { return 0 } + +func (remoteLogStandIn) Depth() int { return 0 } + +func TestMetricsAreReadFromEachPart(t *testing.T) { + t.Parallel() + + m := metrics.New(metrics.Params{ + BusiestCount: 1, + InstanceName: "app", + Now: func() time.Time { return time.Time{} }, + Bans: ledgerStandIn{}, + Clients: clientTableStandIn{}, + Rules: ruleFilesStandIn{}, + Lists: listsStandIn{}, + DNSBL: dnsblStandIn{}, + AbuseIPDB: abuseIPDBStandIn{}, + Alerts: alertQueueStandIn{}, + LookupFile: lookupDatabaseStandIn{}, + RemoteLog: remoteLogStandIn{}, + }) + scraped := scrape(t, m) + + for _, line := range []string{ + `sww_bans_made_total{cause="limit",instance="app"} 1`, + `sww_active_bans{instance="app"} 2`, + `sww_tracked_clients{instance="app"} 3`, + `sww_rules_loaded{instance="app"} 4`, + `sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`, + `sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`, + `sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`, + `sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`, + `sww_lookup_database_read_failures_total{instance="app"} 9`, + `sww_remote_log_lines_sent_total{instance="app"} 10`, + } { + if !strings.Contains(scraped, "\n"+line+"\n") { + t.Errorf("metrics\n%s\nwant\n%s", scraped, line) + } + } +} + +func TestPartThatIsOffAddsNoMetric(t *testing.T) { + t.Parallel() + + scraped := scrape(t, metrics.New(metrics.Params{ + BusiestCount: 1, + InstanceName: "app", + })) + + for _, name := range []string{ + "sww_bans_made_total", + "sww_tracked_clients", + "sww_rules_loaded", + "sww_reputation_", + "sww_alert", + "sww_lookup_database_", + "sww_remote_log_", + } { + if strings.Contains(scraped, name) { + t.Errorf("metrics\n%s\nshow %s", scraped, name) + } + } +} + +// scrape returns the metrics m serves. +func scrape(t *testing.T, m *metrics.Metrics) string { + t.Helper() + + scraped := httptest.NewRecorder() + m.ServeHTTP( + scraped, + httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/", http.NoBody, + ), + ) + + return scraped.Body.String() +} diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go index 68257a2..0da80dc 100644 --- a/internal/reputation/abuseipdb_test.go +++ b/internal/reputation/abuseipdb_test.go @@ -445,12 +445,9 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) { scores: map[string]int64{suspect: 100}, } checker := newAbuseIPDB(abuseIPDB, p) - m := metrics.New(1, "app") - m.AddReputation( - reputation.New(params()), - reputation.NewDNSBL(dnsblParams()), - ) - m.AddAbuseIPDB(checker) + m := metrics.New(metrics.Params{ + BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker, + }) // One check that AbuseIPDB answers, and one it refuses with an answer // that names the key. @@ -503,12 +500,9 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) { p := abuseIPDBParams() p.DailyBudget = 5 checker := newAbuseIPDB(abuseIPDB, p) - m := metrics.New(1, "app") - m.AddReputation( - reputation.New(params()), - reputation.NewDNSBL(dnsblParams()), - ) - m.AddAbuseIPDB(checker) + m := metrics.New(metrics.Params{ + BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker, + }) // One check that AbuseIPDB answers, and one that fails. hitFrom(t, checker, suspect, true) @@ -811,6 +805,9 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) { } } +// instanceName is the label instance of the metrics the tests build. +const instanceName = "app" + // scrapeMetrics returns the metrics m serves. func scrapeMetrics(t *testing.T, m *metrics.Metrics) string { t.Helper() diff --git a/internal/reputation/dnsbl_test.go b/internal/reputation/dnsbl_test.go index e86c5d5..dba9d0c 100644 --- a/internal/reputation/dnsbl_test.go +++ b/internal/reputation/dnsbl_test.go @@ -358,8 +358,9 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) { "99.2.0.192." + otherZone + ".": {rcode: serverFailure}, }} dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone)) - m := metrics.New(1, "app") - m.AddReputation(reputation.New(params()), dnsbl) + m := metrics.New(metrics.Params{ + BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl, + }) wantZones(t, dnsbl, listed) synctest.Wait() @@ -404,8 +405,9 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) { "99.2.0.192." + keyed + ".": {addrs: []string{listing}}, "100.2.0.192." + keyed + ".": {rcode: serverFailure}, }}, p) - m := metrics.New(1, "app") - m.AddReputation(reputation.New(params()), dnsbl) + m := metrics.New(metrics.Params{ + BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl, + }) // Both clients are asked about before either answer comes, so that // the failure does not keep the zone from the other query. diff --git a/internal/state/state_test.go b/internal/state/state_test.go index ad3ab53..aa7cdb3 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -2430,7 +2430,7 @@ func midnight() time.Time { // counts, with thresholds fill does not reach. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) - m := metrics.New(1, "app") + m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"}) queue := alerts.New(alerts.Params{ WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, Events: alerts.Events(), diff --git a/internal/sww/waf.go b/internal/sww/waf.go index bf62f9c..cd64dd5 100644 --- a/internal/sww/waf.go +++ b/internal/sww/waf.go @@ -392,26 +392,33 @@ func newMetrics( alertQueue *alerts.Queue, lookupFile *lookup.File, remote *remotelog.Sender, ) *metrics.Metrics { - m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName) - m.AddBansAndClients(ledger, limiter, utcNow) - m.AddRules(ruleFiles) - m.AddReputation(lists, dnsbl) - - if cfg.AbuseIPDBKey != "" { - m.AddAbuseIPDB(abuseIPDB) + params := metrics.Params{ + BusiestCount: cfg.MetricsBusiestCountriesAndASNs, + InstanceName: cfg.InstanceName, + Now: utcNow, + Bans: ledger, + Clients: limiter, + Rules: ruleFiles, + Lists: lists, + DNSBL: dnsbl, + Alerts: alertQueue, } - m.AddAlerts(alertQueue) + // A nil pointer in an interface field is not a nil interface, so a part + // that is off is left out rather than set to nil. + if cfg.AbuseIPDBKey != "" { + params.AbuseIPDB = abuseIPDB + } if lookupFile != nil { - m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures) + params.LookupFile = lookupFile } if remote != nil { - m.AddRemoteLog(remote) + params.RemoteLog = remote } - return m + return metrics.New(params) } // newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each diff --git a/internal/waf/proxy_test.go b/internal/waf/proxy_test.go index d7576d7..5a14f92 100644 --- a/internal/waf/proxy_test.go +++ b/internal/waf/proxy_test.go @@ -548,16 +548,11 @@ func addCoreRuleSet(t *testing.T, params *waf.Params) { params.CoreRuleSet = coreRuleSet } -// addCounters adds to params the metrics, the table of clients, the ledger -// and the anomaly counters, as sww builds them from params' settings. The -// metrics read the ledger, the table of clients and the rules. +// addCounters adds to params the table of clients, the ledger and the +// anomaly counters, as sww builds them from params' settings. func addCounters(params *waf.Params) { cfg := params.Config - params.Metrics = metrics.New( - cfg.MetricsBusiestCountriesAndASNs, - cfg.InstanceName, - ) params.Limiter = ratelimit.New(ratelimit.Params{ Limits: ratelimit.Limits{ PerMinute: cfg.RateLimitPerMinute, @@ -587,26 +582,18 @@ func addCounters(params *waf.Params) { NamedNetblocks: cfg.AnomalyWatchedNets, Alerts: params.Alerts, }) - - params.Metrics.AddBansAndClients( - params.Ledger, - params.Limiter, - params.Now, - ) - params.Metrics.AddRules(params.Rules) } // addOutsideSources adds to params the lists fetched from URLs, the DNSBL // zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its // blacklist from abuseIPDBBlacklistURL, routewatch's answers while its -// setting is set, and GeoJS at geojsURL, as sww builds -// them from params' settings and the parts addCounters added. The metrics -// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set. +// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as +// sww builds them from params' settings and the parts addCounters added. func addOutsideSources( params *waf.Params, geojsURL, abuseIPDBCheckURL string, ) { cfg := params.Config - m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger + limiter, ledger := params.Limiter, params.Ledger processLog, alertQueue := params.ProcessLog, params.Alerts params.Lists = reputation.New(reputation.Params{ @@ -655,6 +642,9 @@ func addOutsideSources( }) } + m := newMetrics(params) + params.Metrics = m + params.GeoJS = geojs.New(geojs.Params{ URL: geojsURL, Answered: func(answer geojs.Answer) { @@ -667,12 +657,29 @@ func addOutsideSources( }, WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc, }) +} - m.AddReputation(params.Lists, params.DNSBL) +// newMetrics returns the metrics, as sww builds them from params' settings, +// read from the ledger, the table of clients, the rules, the lists, the +// zones' verdicts, and AbuseIPDB's while its key is set. +func newMetrics(params *waf.Params) *metrics.Metrics { + cfg := params.Config + metricsParams := metrics.Params{ + BusiestCount: cfg.MetricsBusiestCountriesAndASNs, + InstanceName: cfg.InstanceName, + Now: params.Now, + Bans: params.Ledger, + Clients: params.Limiter, + Rules: params.Rules, + Lists: params.Lists, + DNSBL: params.DNSBL, + } if cfg.AbuseIPDBKey != "" { - m.AddAbuseIPDB(params.AbuseIPDB) + metricsParams.AbuseIPDB = params.AbuseIPDB } + + return metrics.New(metricsParams) } // newClient returns an HTTP client that sends requests as they are made,