Serve Prometheus metrics behind SWWAF_METRICS_TOKEN (closes #23)

GET /_smallwebwaf/metrics answers in the Prometheus text format for a request carrying SWWAF_METRICS_TOKEN, 401 without it and 404 while it is unset. Every request under /_smallwebwaf/ but the health check now goes through the checks and is answered where it would be forwarded, 404 for any path but the metrics, so none reaches the app. In the client's history a 401 counts as refused, the metrics and the 404s as neither. SWWAF_METRICS_TOP_N bounds the series by country, the rest counted as other. Deviation: go.mod and go.sum written by hand, as go runs only through make. Deviation: no metrics yet for state files read again after an edit or edits set aside; that work is not merged. Model: opus-5-5
2026-10-06 11:40:27 +02:00
parent 68f687cb0c
commit 234c5eac60
25 changed files with 1548 additions and 83 deletions
@@ -0,0 +1,116 @@
+package metrics
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// other is the label under which the countries outside the busiest are
+// counted.
+const other = "other"
+
+// countries are the metrics by the client's country, for requests whose
+// client's country is known. The topN busiest countries, by their requests
+// since the start, have series of their own, and the others are counted
+// under other, so that there are never more than topN + 1 series. A
+// country that drops out of the busiest loses its series, and its next
+// requests are counted under other; one that becomes one of them gets a
+// series that counts from then on. Each series therefore only ever goes
+// up.
+type countries struct {
+	topN int
+
+	requests      *prometheus.CounterVec
+	requestBytes  *prometheus.CounterVec
+	responseBytes *prometheus.CounterVec
+	// refused are the requests the country lists refused.
+	refused *prometheus.CounterVec
+
+	mu sync.Mutex
+	// seen is each country's requests since the start, by which the
+	// countries are ranked. GeoJS gives two-letter codes, so it holds at
+	// most a few hundred.
+	seen map[string]int64
+	// top are the countries with series of their own.
+	top map[string]bool
+}
+
+// newCountries returns the metrics by country, with series of their own
+// for the topN busiest countries.
+func newCountries(topN int) *countries {
+	byCountry := []string{"country"}
+
+	return &countries{
+		topN: topN,
+		requests: counterVec("smallwebwaf_country_requests_total",
+			"Requests, by the client's country.", byCountry),
+		requestBytes: counterVec("smallwebwaf_country_request_bytes_total",
+			"Request body bytes, by the client's country.", byCountry),
+		responseBytes: counterVec("smallwebwaf_country_response_bytes_total",
+			"Response body bytes, by the client's country.", byCountry),
+		refused: counterVec("smallwebwaf_country_list_refusals_total",
+			"Requests the country lists refused, by the client's country.",
+			byCountry),
+		seen: map[string]int64{},
+		top:  map[string]bool{},
+	}
+}
+
+// add counts a request from its log line, whose country is known.
+func (c *countries) add(line *requestlog.Line) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.seen[line.Country]++
+
+	label := c.label(line.Country)
+	c.requests.WithLabelValues(label).Inc()
+	c.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes))
+	c.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes))
+
+	if line.Action == requestlog.ActionCountryDenied {
+		c.refused.WithLabelValues(label).Inc()
+	}
+}
+
+// label returns the label a request from country is counted under: the
+// country while it is one of the busiest, other while it is not. A
+// country busier than the least busy of them takes its place, and that
+// country's series are dropped.
+func (c *countries) label(country string) string {
+	if c.top[country] {
+		return country
+	}
+
+	if len(c.top) < c.topN {
+		c.top[country] = true
+
+		return country
+	}
+
+	least := ""
+
+	for top := range c.top {
+		if least == "" || c.seen[top] < c.seen[least] {
+			least = top
+		}
+	}
+
+	if c.seen[country] <= c.seen[least] {
+		return other
+	}
+
+	delete(c.top, least)
+
+	for _, vec := range []*prometheus.CounterVec{
+		c.requests, c.requestBytes, c.responseBytes, c.refused,
+	} {
+		vec.DeleteLabelValues(least)
+	}
+
+	c.top[country] = true
+
+	return country
+}
@@ -0,0 +1,257 @@
+// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics
+// endpoint" section of SPEC.md lists them, and serves them in the
+// Prometheus text format. No metric carries a client's address.
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// Metrics are smallwebwaf's metrics. They are safe for concurrent use.
+type Metrics struct {
+	registry *prometheus.Registry
+	handler  http.Handler
+
+	inFlight             prometheus.Gauge
+	requests             *prometheus.CounterVec
+	requestBytes         *prometheus.CounterVec
+	responseBytes        *prometheus.CounterVec
+	requestDuration      prometheus.Histogram
+	upstreamDuration     prometheus.Histogram
+	rateLimitHits        *prometheus.CounterVec
+	sizeAndTimeLimitHits *prometheus.CounterVec
+	offences             *prometheus.CounterVec
+	countries            *countries
+
+	// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
+	// that failed. GeoJSUnanswered are the requests whose client counted
+	// as coming from an unknown country because GeoJS had not answered
+	// about it in time.
+	GeoJSRequests   prometheus.Counter
+	GeoJSFailures   prometheus.Counter
+	GeoJSUnanswered prometheus.Counter
+
+	stateFileWrites        *prometheus.CounterVec
+	stateFileWriteFailures *prometheus.CounterVec
+	stateFileLastWrite     *prometheus.GaugeVec
+	stateFileSize          *prometheus.GaugeVec
+}
+
+// New returns the metrics, with the Go runtime's and the process's own.
+// topN is how many countries get series of their own
+// (SWWAF_METRICS_TOP_N).
+func New(topN int) *Metrics {
+	byStatus := []string{"status_class", "action"}
+	byFile := []string{"file"}
+
+	m := &Metrics{
+		registry: prometheus.NewRegistry(),
+		inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "smallwebwaf_requests_in_flight",
+			Help: "Requests under way.",
+		}),
+		requests: counterVec("smallwebwaf_requests_total",
+			"Requests, by the class of their status and their action.", byStatus),
+		requestBytes: counterVec("smallwebwaf_request_bytes_total",
+			"Request body bytes, by the class of the status and the action.",
+			byStatus),
+		responseBytes: counterVec("smallwebwaf_response_bytes_total",
+			"Response body bytes, by the class of the status and the action.",
+			byStatus),
+		requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name: "smallwebwaf_request_duration_seconds",
+			Help: "How long requests took, from their arrival to their end.",
+		}),
+		upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name: "smallwebwaf_upstream_duration_seconds",
+			Help: "How long requests passed to the app took, from then to their end.",
+		}),
+		rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total",
+			"Requests that broke a rate limit, by its window.",
+			[]string{"window"}),
+		sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total",
+			"Requests that passed a size or time limit, by its setting.",
+			[]string{"limit"}),
+		offences: counterVec("smallwebwaf_offences_total",
+			"Offences, by kind.", []string{"kind"}),
+		countries: newCountries(topN),
+		GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "smallwebwaf_geojs_requests_total",
+			Help: "Requests to GeoJS.",
+		}),
+		GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "smallwebwaf_geojs_failures_total",
+			Help: "Requests to GeoJS that failed.",
+		}),
+		GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "smallwebwaf_geojs_unanswered_total",
+			Help: "Requests whose client counted as coming from an unknown " +
+				"country because GeoJS had not answered about it in time.",
+		}),
+		stateFileWrites: counterVec("smallwebwaf_state_file_writes_total",
+			"Writes of each state file.", byFile),
+		stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total",
+			"Writes of each state file that failed.", byFile),
+		stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds",
+			"When each state file was last written, in seconds since 1970.", byFile),
+		stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes",
+			"The size of each state file, as it was last written.", byFile),
+	}
+
+	m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})
+
+	m.registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		m.inFlight, m.requests, m.requestBytes, m.responseBytes,
+		m.requestDuration, m.upstreamDuration,
+		m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences,
+		m.countries.requests, m.countries.requestBytes, m.countries.responseBytes,
+		m.countries.refused,
+		m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
+		m.stateFileWrites, m.stateFileWriteFailures,
+		m.stateFileLastWrite, m.stateFileSize,
+	)
+
+	return m
+}
+
+// AddBansAndClients adds the metrics read from the ledger and the table
+// of clients as the metrics are asked for: the bans made since the start,
+// the bans active and permanent at now, and the clients in the table.
+func (m *Metrics) AddBansAndClients(
+	ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
+) {
+	m.registry.MustRegister(
+		// Every ban smallwebwaf makes so far is for a broken limit.
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name:        "smallwebwaf_bans_made_total",
+			Help:        "Bans made, by cause.",
+			ConstLabels: prometheus.Labels{"cause": "limit"},
+		}, func() float64 {
+			return float64(ledger.Made())
+		}),
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_active_bans",
+			Help: "Bans active now, the permanent ones included.",
+		}, func() float64 {
+			active, _ := ledger.Count(now())
+
+			return float64(active)
+		}),
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_permanent_bans",
+			Help: "Permanent bans.",
+		}, func() float64 {
+			_, permanent := ledger.Count(now())
+
+			return float64(permanent)
+		}),
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_tracked_clients",
+			Help: "Clients in the table of clients.",
+		}, func() float64 {
+			return float64(limiter.Len())
+		}),
+	)
+}
+
+// ServeHTTP answers with the metrics in the Prometheus text format.
+func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	m.handler.ServeHTTP(w, r)
+}
+
+// RequestStarted counts a request as under way.
+func (m *Metrics) RequestStarted() {
+	m.inFlight.Inc()
+}
+
+// RequestEnded counts a request that has ended, from its log line. limit
+// is the setting whose size or time limit the request passed, "" if none.
+// duration is how long the request took, and upstreamDuration how long it
+// took from when it was passed to the app, zero if it was not.
+func (m *Metrics) RequestEnded(
+	line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
+) {
+	m.inFlight.Dec()
+
+	class := statusClass(line.Status)
+	m.requests.WithLabelValues(class, line.Action).Inc()
+	m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
+	m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
+	m.requestDuration.Observe(duration.Seconds())
+
+	if upstreamDuration > 0 {
+		m.upstreamDuration.Observe(upstreamDuration.Seconds())
+	}
+
+	if line.LimitHit != "" {
+		m.rateLimitHits.WithLabelValues(line.LimitHit).Inc()
+	}
+
+	if limit != "" {
+		m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
+	}
+
+	if line.Offence != "" {
+		m.offences.WithLabelValues(line.Offence).Inc()
+	}
+
+	if line.Country != "" {
+		m.countries.add(line)
+	}
+}
+
+// StateFileWritten counts a write of the state file name, of size bytes,
+// that ended with err.
+func (m *Metrics) StateFileWritten(name string, size int, err error) {
+	m.stateFileWrites.WithLabelValues(name).Inc()
+
+	// The series of failures is there from the first write, at zero until
+	// one fails.
+	failures := m.stateFileWriteFailures.WithLabelValues(name)
+
+	if err != nil {
+		failures.Inc()
+
+		return
+	}
+
+	m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime()
+	m.stateFileSize.WithLabelValues(name).Set(float64(size))
+}
+
+// statusClass returns the class of status, such as 2xx, or none when no
+// status was sent.
+func statusClass(status int) string {
+	if status == 0 {
+		return "none"
+	}
+
+	// A status's class is its hundreds: 404 is in 4xx.
+	const hundred = 100
+
+	return strconv.Itoa(status/hundred) + "xx"
+}
+
+// counterVec returns a counter named name, described by help, with a
+// series for each set of values of labels.
+func counterVec(name, help string, labels []string) *prometheus.CounterVec {
+	return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help},
+		labels)
+}
+
+// gaugeVec returns a gauge named name, described by help, with a series
+// for each set of values of labels.
+func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec {
+	return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels)
+}