// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics // endpoint" section of SPEC.md lists them, and serves them in the // Prometheus text format. No metric carries a client's address. package metrics import ( "net/http" "strconv" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) // Metrics are smallwebwaf's metrics. They are safe for concurrent use. type Metrics struct { // registry gives every metric registered with it the label instance. registry prometheus.Registerer handler http.Handler inFlight prometheus.Gauge requests *prometheus.CounterVec requestBytes *prometheus.CounterVec responseBytes *prometheus.CounterVec requestDuration prometheus.Histogram upstreamDuration prometheus.Histogram rateLimitHits *prometheus.CounterVec sizeAndTimeLimitHits *prometheus.CounterVec offences *prometheus.CounterVec // ruleMatches are made by AddRules. ruleMatches *prometheus.CounterVec countries *busiest asns *busiest // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those // that failed. GeoJSUnanswered are the requests that needed their // client's answer, for a setting that acts on it, and went on without // it because GeoJS had not given it in time. GeoJSRequests prometheus.Counter GeoJSFailures prometheus.Counter GeoJSUnanswered prometheus.Counter stateFileWrites *prometheus.CounterVec stateFileWriteFailures *prometheus.CounterVec stateFileLastWrite *prometheus.GaugeVec stateFileSize *prometheus.GaugeVec stateFileEditsTakenIn *prometheus.CounterVec stateFileEditsSetAside *prometheus.CounterVec } // New returns the metrics, with the Go runtime's and the process's own. // topN is how many countries and how many AS numbers get series of their // own (SWWAF_METRICS_TOP_N). Every metric carries instanceName // (SWWAF_INSTANCE_NAME) as its label instance. func New(topN int, instanceName string) *Metrics { byStatus := []string{"status_class", "action"} byFile := []string{"file"} registry := prometheus.NewRegistry() m := &Metrics{ registry: prometheus.WrapRegistererWith( prometheus.Labels{"instance": instanceName}, registry), handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "smallwebwaf_requests_in_flight", Help: "Requests under way.", }), requests: counterVec("smallwebwaf_requests_total", "Requests, by the class of their status and their action.", byStatus), requestBytes: counterVec("smallwebwaf_request_bytes_total", "Request body bytes, by the class of the status and the action.", byStatus), responseBytes: counterVec("smallwebwaf_response_bytes_total", "Response body bytes, by the class of the status and the action.", byStatus), requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "smallwebwaf_request_duration_seconds", Help: "How long requests took, from their arrival to their end.", }), upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "smallwebwaf_upstream_duration_seconds", Help: "How long requests passed to the app took, from then to their end.", }), rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total", "Requests that broke a rate limit, by its window.", []string{"window"}), sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total", "Requests that passed a size or time limit, by its setting.", []string{"limit"}), offences: counterVec("smallwebwaf_offences_total", "Offences, by kind.", []string{"kind"}), countries: newCountries(topN), asns: newASNs(topN), GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_requests_total", Help: "Requests to GeoJS.", }), GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_failures_total", Help: "Requests to GeoJS that failed.", }), GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_unanswered_total", Help: "Requests that needed their client's answer from GeoJS and " + "went on without it, because GeoJS had not given it in time.", }), stateFileWrites: counterVec("smallwebwaf_state_file_writes_total", "Writes of each state file.", byFile), stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total", "Writes of each state file that failed.", byFile), stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds", "When each state file was last written, in seconds since 1970.", byFile), stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes", "The size of each state file, as it was last written.", byFile), stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total", "Edits of each state file taken in while running.", byFile), stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total", "Edits of each state file renamed to .bad because they did not parse.", byFile), } m.registry.MustRegister( collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), m.inFlight, m.requests, m.requestBytes, m.responseBytes, m.requestDuration, m.upstreamDuration, m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns, m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, m.stateFileWrites, m.stateFileWriteFailures, m.stateFileLastWrite, m.stateFileSize, m.stateFileEditsTakenIn, m.stateFileEditsSetAside, ) return m } // AddBansAndClients adds the metrics read from the ledger and the table // of clients as the metrics are asked for: the bans made since the start, // by cause, the bans active and permanent at now, and the clients in the // table. func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} { m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_bans_made_total", Help: "Bans made, by cause.", ConstLabels: prometheus.Labels{"cause": cause}, }, func() float64 { return float64(ledger.Made(cause)) })) } m.registry.MustRegister( prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_active_bans", Help: "Bans active now, the permanent ones included.", }, func() float64 { active, _ := ledger.Count(now()) return float64(active) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_permanent_bans", Help: "Permanent bans not lifted.", }, func() float64 { _, permanent := ledger.Count(now()) return float64(permanent) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_tracked_clients", Help: "Clients in the table of clients.", }, func() float64 { return float64(limiter.Len()) }), ) } // AddRules adds the metrics of the rule files: the requests that matched // each rule, which RuleMatched counts, and the rules loaded from // ruleFiles, read as the metrics are asked for. It is called once, before // RuleMatched. func (m *Metrics) AddRules(ruleFiles *rules.Files) { m.ruleMatches = counterVec("smallwebwaf_rule_matches_total", "Requests that matched a rule of the rule files, by its id and action.", []string{"rule_id", "action"}) m.registry.MustRegister(m.ruleMatches, prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_rules_loaded", Help: "Rules loaded from the rule files.", }, func() float64 { return float64(ruleFiles.Len()) })) } // AddRemoteLog adds the metrics of sending the log lines to // SWWAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the // lines sent, those dropped, and those waiting in the buffer. func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { m.registry.MustRegister( prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_remote_log_lines_sent_total", Help: "Log lines sent to SWWAF_LOG_REMOTE_URL.", }, func() float64 { return float64(remote.Sent()) }), prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_remote_log_lines_dropped_total", Help: "Log lines dropped: the oldest in a full buffer, and those " + "whose sending failed.", }, func() float64 { return float64(remote.Dropped()) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_remote_log_buffer_depth", Help: "Log lines in the buffer, waiting to be sent.", }, func() float64 { return float64(remote.Depth()) }), ) } // AddLookupFile adds the metrics of the lookup database, read as the // metrics are asked for: when the file in use was read, which lastRead // returns, and the replacements of it that could not be read, which // readFailures returns. The lookup package's File, which has both, cannot // be named here: that package counts GeoJS's requests in these metrics. func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() int) { m.registry.MustRegister( prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_lookup_database_last_read_timestamp_seconds", Help: "When the lookup database in use was read, in seconds since 1970.", }, func() float64 { return float64(lastRead().Unix()) }), prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_lookup_database_read_failures_total", Help: "Replacements of the lookup database that could not be read.", }, func() float64 { return float64(readFailures()) }), ) } // AddAlerts adds the metrics of the alerts sent to each destination set, // read from queue as the metrics are asked for, by destination: the // alerts sent, the requests to the destination that failed, the alerts // held back, which are the same for every destination, and those // dropped. With no destination set, it adds none. func (m *Metrics) AddAlerts(queue *alerts.Queue) { for _, name := range queue.DestinationsSet() { destination := prometheus.Labels{"destination": name} m.registry.MustRegister( prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_alerts_sent_total", Help: "Alerts the destination took.", ConstLabels: destination, }, func() float64 { return float64(queue.Counts(name).Sent) }), prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_alerts_failed_total", Help: "Requests to the destination that failed.", ConstLabels: destination, }, func() float64 { return float64(queue.Counts(name).Failed) }), prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_alerts_suppressed_total", Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " + "alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.", ConstLabels: destination, }, func() float64 { return float64(queue.Suppressed()) }), prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_alerts_dropped_total", Help: "Alerts dropped, the oldest first, from a full queue, and " + "alerts given up as the destination refused them.", ConstLabels: destination, }, func() float64 { return float64(queue.Counts(name).Dropped) }), ) } } // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) } // RequestStarted counts a request as under way. func (m *Metrics) RequestStarted() { m.inFlight.Inc() } // RequestEnded counts a request that has ended, from its log line. limit // is the setting whose size or time limit the request passed, "" if none. // duration is how long the request took, and upstreamDuration how long it // took from when it was passed to the app, zero if it was not. func (m *Metrics) RequestEnded( line *requestlog.Line, limit string, duration, upstreamDuration time.Duration, ) { m.inFlight.Dec() class := statusClass(line.Status) m.requests.WithLabelValues(class, line.Action).Inc() m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes)) m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes)) m.requestDuration.Observe(duration.Seconds()) if upstreamDuration > 0 { m.upstreamDuration.Observe(upstreamDuration.Seconds()) } if line.LimitHit != "" { m.rateLimitHits.WithLabelValues(line.LimitHit).Inc() } if limit != "" { m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc() } if line.Offence != "" { m.offences.WithLabelValues(line.Offence).Inc() } if line.Country != "" { m.countries.add(line.Country, line) } if line.ASN != "" { m.asns.add(line.ASN, line) } } // RuleMatched counts a request that matched the rule id, whose action is // action. func (m *Metrics) RuleMatched(id, action string) { m.ruleMatches.WithLabelValues(id, action).Inc() } // StateFileWritten counts a write of the state file name, of size bytes, // that ended with err. func (m *Metrics) StateFileWritten(name string, size int, err error) { m.stateFileWrites.WithLabelValues(name).Inc() // The series of failures is there from the first write, at zero until // one fails. failures := m.stateFileWriteFailures.WithLabelValues(name) if err != nil { failures.Inc() return } m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime() m.stateFileSize.WithLabelValues(name).Set(float64(size)) } // StateFileEditTakenIn counts an admin's edit of the state file name // taken in while smallwebwaf runs. func (m *Metrics) StateFileEditTakenIn(name string) { m.stateFileEditsTakenIn.WithLabelValues(name).Inc() } // StateFileEditSetAside counts an admin's edit of the state file name // renamed to name.bad because it did not parse. func (m *Metrics) StateFileEditSetAside(name string) { m.stateFileEditsSetAside.WithLabelValues(name).Inc() } // statusClass returns the class of status, such as 2xx, or none when no // status was sent. func statusClass(status int) string { if status == 0 { return "none" } // A status's class is its hundreds: 404 is in 4xx. const hundred = 100 return strconv.Itoa(status/hundred) + "xx" } // counterVec returns a counter named name, described by help, with a // series for each set of values of labels. func counterVec(name, help string, labels []string) *prometheus.CounterVec { return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, labels) } // gaugeVec returns a gauge named name, described by help, with a series // for each set of values of labels. func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels) }