// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics // endpoint" section of SPEC.md lists them, and serves them in the // Prometheus text format. No metric carries a client's address. package metrics import ( "net/http" "strconv" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) // Metrics are smallwebwaf's metrics. They are safe for concurrent use. type Metrics struct { registry *prometheus.Registry handler http.Handler inFlight prometheus.Gauge requests *prometheus.CounterVec requestBytes *prometheus.CounterVec responseBytes *prometheus.CounterVec requestDuration prometheus.Histogram upstreamDuration prometheus.Histogram rateLimitHits *prometheus.CounterVec sizeAndTimeLimitHits *prometheus.CounterVec offences *prometheus.CounterVec countries *countries // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those // that failed. GeoJSUnanswered are the requests whose client counted // as coming from an unknown country because GeoJS had not answered // about it in time. GeoJSRequests prometheus.Counter GeoJSFailures prometheus.Counter GeoJSUnanswered prometheus.Counter stateFileWrites *prometheus.CounterVec stateFileWriteFailures *prometheus.CounterVec stateFileLastWrite *prometheus.GaugeVec stateFileSize *prometheus.GaugeVec stateFileEditsTakenIn *prometheus.CounterVec stateFileEditsSetAside *prometheus.CounterVec } // New returns the metrics, with the Go runtime's and the process's own. // topN is how many countries get series of their own // (SWWAF_METRICS_TOP_N). func New(topN int) *Metrics { byStatus := []string{"status_class", "action"} byFile := []string{"file"} m := &Metrics{ registry: prometheus.NewRegistry(), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "smallwebwaf_requests_in_flight", Help: "Requests under way.", }), requests: counterVec("smallwebwaf_requests_total", "Requests, by the class of their status and their action.", byStatus), requestBytes: counterVec("smallwebwaf_request_bytes_total", "Request body bytes, by the class of the status and the action.", byStatus), responseBytes: counterVec("smallwebwaf_response_bytes_total", "Response body bytes, by the class of the status and the action.", byStatus), requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "smallwebwaf_request_duration_seconds", Help: "How long requests took, from their arrival to their end.", }), upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "smallwebwaf_upstream_duration_seconds", Help: "How long requests passed to the app took, from then to their end.", }), rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total", "Requests that broke a rate limit, by its window.", []string{"window"}), sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total", "Requests that passed a size or time limit, by its setting.", []string{"limit"}), offences: counterVec("smallwebwaf_offences_total", "Offences, by kind.", []string{"kind"}), countries: newCountries(topN), GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_requests_total", Help: "Requests to GeoJS.", }), GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_failures_total", Help: "Requests to GeoJS that failed.", }), GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_unanswered_total", Help: "Requests whose client counted as coming from an unknown " + "country because GeoJS had not answered about it in time.", }), stateFileWrites: counterVec("smallwebwaf_state_file_writes_total", "Writes of each state file.", byFile), stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total", "Writes of each state file that failed.", byFile), stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds", "When each state file was last written, in seconds since 1970.", byFile), stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes", "The size of each state file, as it was last written.", byFile), stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total", "Edits of each state file taken in while running.", byFile), stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total", "Edits of each state file renamed to .bad because they did not parse.", byFile), } m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}) m.registry.MustRegister( collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), m.inFlight, m.requests, m.requestBytes, m.responseBytes, m.requestDuration, m.upstreamDuration, m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries.requests, m.countries.requestBytes, m.countries.responseBytes, m.countries.refused, m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, m.stateFileWrites, m.stateFileWriteFailures, m.stateFileLastWrite, m.stateFileSize, m.stateFileEditsTakenIn, m.stateFileEditsSetAside, ) return m } // AddBansAndClients adds the metrics read from the ledger and the table // of clients as the metrics are asked for: the bans made since the start, // the bans active and permanent at now, and the clients in the table. func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { m.registry.MustRegister( // Every ban smallwebwaf makes so far is for a broken limit. prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_bans_made_total", Help: "Bans made, by cause.", ConstLabels: prometheus.Labels{"cause": "limit"}, }, func() float64 { return float64(ledger.Made()) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_active_bans", Help: "Bans active now, the permanent ones included.", }, func() float64 { active, _ := ledger.Count(now()) return float64(active) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_permanent_bans", Help: "Permanent bans.", }, func() float64 { _, permanent := ledger.Count(now()) return float64(permanent) }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_tracked_clients", Help: "Clients in the table of clients.", }, func() float64 { return float64(limiter.Len()) }), ) } // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) } // RequestStarted counts a request as under way. func (m *Metrics) RequestStarted() { m.inFlight.Inc() } // RequestEnded counts a request that has ended, from its log line. limit // is the setting whose size or time limit the request passed, "" if none. // duration is how long the request took, and upstreamDuration how long it // took from when it was passed to the app, zero if it was not. func (m *Metrics) RequestEnded( line *requestlog.Line, limit string, duration, upstreamDuration time.Duration, ) { m.inFlight.Dec() class := statusClass(line.Status) m.requests.WithLabelValues(class, line.Action).Inc() m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes)) m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes)) m.requestDuration.Observe(duration.Seconds()) if upstreamDuration > 0 { m.upstreamDuration.Observe(upstreamDuration.Seconds()) } if line.LimitHit != "" { m.rateLimitHits.WithLabelValues(line.LimitHit).Inc() } if limit != "" { m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc() } if line.Offence != "" { m.offences.WithLabelValues(line.Offence).Inc() } if line.Country != "" { m.countries.add(line) } } // StateFileWritten counts a write of the state file name, of size bytes, // that ended with err. func (m *Metrics) StateFileWritten(name string, size int, err error) { m.stateFileWrites.WithLabelValues(name).Inc() // The series of failures is there from the first write, at zero until // one fails. failures := m.stateFileWriteFailures.WithLabelValues(name) if err != nil { failures.Inc() return } m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime() m.stateFileSize.WithLabelValues(name).Set(float64(size)) } // StateFileEditTakenIn counts an admin's edit of the state file name // taken in while smallwebwaf runs. func (m *Metrics) StateFileEditTakenIn(name string) { m.stateFileEditsTakenIn.WithLabelValues(name).Inc() } // StateFileEditSetAside counts an admin's edit of the state file name // renamed to name.bad because it did not parse. func (m *Metrics) StateFileEditSetAside(name string) { m.stateFileEditsSetAside.WithLabelValues(name).Inc() } // statusClass returns the class of status, such as 2xx, or none when no // status was sent. func statusClass(status int) string { if status == 0 { return "none" } // A status's class is its hundreds: 404 is in 4xx. const hundred = 100 return strconv.Itoa(status/hundred) + "xx" } // counterVec returns a counter named name, described by help, with a // series for each set of values of labels. func counterVec(name, help string, labels []string) *prometheus.CounterVec { return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, labels) } // gaugeVec returns a gauge named name, described by help, with a series // for each set of values of labels. func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels) }