smallwebwaf/internal/metrics/metrics.go

// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics
// endpoint" section of SPEC.md lists them, and serves them in the
// Prometheus text format. No metric carries a client's address.
package metrics

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/collectors"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
	"sneak.berlin/go/smallwebwaf/internal/remotelog"
	"sneak.berlin/go/smallwebwaf/internal/reputation"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
	"sneak.berlin/go/smallwebwaf/internal/rules"
)

// Metrics are smallwebwaf's metrics. They are safe for concurrent use.
type Metrics struct {
	// registry gives every metric registered with it the label instance.
	registry prometheus.Registerer
	handler  http.Handler

	inFlight             prometheus.Gauge
	requests             *prometheus.CounterVec
	requestBytes         *prometheus.CounterVec
	responseBytes        *prometheus.CounterVec
	requestDuration      prometheus.Histogram
	upstreamDuration     prometheus.Histogram
	rateLimitHits        *prometheus.CounterVec
	sizeAndTimeLimitHits *prometheus.CounterVec
	offences             *prometheus.CounterVec
	// ruleMatches are made by AddRules, and reputationHits by
	// AddReputation.
	ruleMatches    *prometheus.CounterVec
	reputationHits *prometheus.CounterVec
	countries      *busiest
	asns           *busiest

	// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
	// that failed. GeoJSUnanswered are the requests that needed their
	// client's answer, for a setting that acts on it, and went on without
	// it because GeoJS had not given it in time.
	GeoJSRequests   prometheus.Counter
	GeoJSFailures   prometheus.Counter
	GeoJSUnanswered prometheus.Counter

	stateFileWrites        *prometheus.CounterVec
	stateFileWriteFailures *prometheus.CounterVec
	stateFileLastWrite     *prometheus.GaugeVec
	stateFileSize          *prometheus.GaugeVec
	stateFileEditsTakenIn  *prometheus.CounterVec
	stateFileEditsSetAside *prometheus.CounterVec
}

// New returns the metrics, with the Go runtime's and the process's own.
// topN is how many countries and how many AS numbers get series of their
// own (SWWAF_METRICS_TOP_N). Every metric carries instanceName
// (SWWAF_INSTANCE_NAME) as its label instance.
func New(topN int, instanceName string) *Metrics {
	byStatus := []string{"status_class", "action"}
	byFile := []string{"file"}
	registry := prometheus.NewRegistry()

	m := &Metrics{
		registry: prometheus.WrapRegistererWith(
			prometheus.Labels{"instance": instanceName}, registry),
		handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
		inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "smallwebwaf_requests_in_flight",
			Help: "Requests under way.",
		}),
		requests: counterVec("smallwebwaf_requests_total",
			"Requests, by the class of their status and their action.", byStatus),
		requestBytes: counterVec("smallwebwaf_request_bytes_total",
			"Request body bytes, by the class of the status and the action.",
			byStatus),
		responseBytes: counterVec("smallwebwaf_response_bytes_total",
			"Response body bytes, by the class of the status and the action.",
			byStatus),
		requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
			Name: "smallwebwaf_request_duration_seconds",
			Help: "How long requests took, from their arrival to their end.",
		}),
		upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
			Name: "smallwebwaf_upstream_duration_seconds",
			Help: "How long requests passed to the app took, from then to their end.",
		}),
		rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total",
			"Requests that broke a rate limit or a byte limit, by its window and "+
				"its kind, requests or bytes.",
			[]string{"window", "kind"}),
		sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total",
			"Requests that passed a size or time limit, by its setting.",
			[]string{"limit"}),
		offences: counterVec("smallwebwaf_offences_total",
			"Offences, by kind.", []string{"kind"}),
		countries: newCountries(topN),
		asns:      newASNs(topN),
		GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_requests_total",
			Help: "Requests to GeoJS.",
		}),
		GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_failures_total",
			Help: "Requests to GeoJS that failed.",
		}),
		GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_unanswered_total",
			Help: "Requests that needed their client's answer from GeoJS and " +
				"went on without it, because GeoJS had not given it in time.",
		}),
		stateFileWrites: counterVec("smallwebwaf_state_file_writes_total",
			"Writes of each state file.", byFile),
		stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total",
			"Writes of each state file that failed.", byFile),
		stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds",
			"When each state file was last written, in seconds since 1970.", byFile),
		stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes",
			"The size of each state file, as it was last written.", byFile),
		stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total",
			"Edits of each state file taken in while running.", byFile),
		stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total",
			"Edits of each state file renamed to <name>.bad because they did not parse.",
			byFile),
	}

	m.registry.MustRegister(
		collectors.NewGoCollector(),
		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
		m.inFlight, m.requests, m.requestBytes, m.responseBytes,
		m.requestDuration, m.upstreamDuration,
		m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns,
		m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
		m.stateFileWrites, m.stateFileWriteFailures,
		m.stateFileLastWrite, m.stateFileSize,
		m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
	)

	return m
}

// AddBansAndClients adds the metrics read from the ledger and the table
// of clients as the metrics are asked for: the bans made since the start,
// by cause, the bans active and permanent at now, and the clients in the
// table.
func (m *Metrics) AddBansAndClients(
	ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
) {
	for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} {
		m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name:        "smallwebwaf_bans_made_total",
			Help:        "Bans made, by cause.",
			ConstLabels: prometheus.Labels{"cause": cause},
		}, func() float64 {
			return float64(ledger.Made(cause))
		}))
	}

	m.registry.MustRegister(
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_active_bans",
			Help: "Bans active now, the permanent ones included.",
		}, func() float64 {
			active, _ := ledger.Count(now())

			return float64(active)
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_permanent_bans",
			Help: "Permanent bans not lifted.",
		}, func() float64 {
			_, permanent := ledger.Count(now())

			return float64(permanent)
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_tracked_clients",
			Help: "Clients in the table of clients.",
		}, func() float64 {
			return float64(limiter.Len())
		}),
	)
}

// AddRules adds the metrics of the rule files: the requests that matched
// each rule, which RuleMatched counts, and the rules loaded from
// ruleFiles, read as the metrics are asked for. It is called once, before
// RuleMatched.
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
	m.ruleMatches = counterVec("smallwebwaf_rule_matches_total",
		"Requests that matched a rule of the rule files, by its id and action.",
		[]string{"rule_id", "action"})

	m.registry.MustRegister(m.ruleMatches,
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_rules_loaded",
			Help: "Rules loaded from the rule files.",
		}, func() float64 {
			return float64(ruleFiles.Len())
		}))
}

// AddRemoteLog adds the metrics of sending the log lines to
// SWWAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
// lines sent, those dropped, and those waiting in the buffer.
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
	m.registry.MustRegister(
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_remote_log_lines_sent_total",
			Help: "Log lines sent to SWWAF_LOG_REMOTE_URL.",
		}, func() float64 {
			return float64(remote.Sent())
		}),
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_remote_log_lines_dropped_total",
			Help: "Log lines dropped: the oldest in a full buffer, and those " +
				"whose sending failed.",
		}, func() float64 {
			return float64(remote.Dropped())
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_remote_log_buffer_depth",
			Help: "Log lines in the buffer, waiting to be sent.",
		}, func() float64 {
			return float64(remote.Depth())
		}),
	)
}

// AddLookupFile adds the metrics of the lookup database, read as the
// metrics are asked for: when the file in use was read, which lastRead
// returns, and the replacements of it that could not be read, which
// readFailures returns. The lookup package's File, which has both, cannot
// be named here: that package counts GeoJS's requests in these metrics.
func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() int) {
	m.registry.MustRegister(
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_lookup_database_last_read_timestamp_seconds",
			Help: "When the lookup database in use was read, in seconds since 1970.",
		}, func() float64 {
			return float64(lastRead().Unix())
		}),
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_lookup_database_read_failures_total",
			Help: "Replacements of the lookup database that could not be read.",
		}, func() float64 {
			return float64(readFailures())
		}),
	)
}

// AddReputation adds the metrics of the lists fetched from URLs, by
// source, each list's URL: the requests whose client a blocklist lists,
// which ReputationHit counts, and, read from lists as the metrics are
// asked for, the fetches that failed and when the copy in use was fetched.
// It is called once, before ReputationHit.
func (m *Metrics) AddReputation(lists *reputation.Lists) {
	m.reputationHits = counterVec("smallwebwaf_reputation_hits_total",
		"Requests whose client a blocklist lists, by the blocklist's URL.",
		[]string{"source"})
	m.registry.MustRegister(m.reputationHits)

	for _, listURL := range lists.URLs() {
		source := prometheus.Labels{"source": listURL}

		m.registry.MustRegister(
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name:        "smallwebwaf_reputation_failures_total",
				Help:        "Fetches of the list that failed.",
				ConstLabels: source,
			}, func() float64 {
				return float64(lists.Failures(listURL))
			}),
			prometheus.NewGaugeFunc(prometheus.GaugeOpts{
				Name: "smallwebwaf_reputation_last_fetch_timestamp_seconds",
				Help: "When the copy of the list in use was fetched, in seconds since " +
					"1970, or 0 while there is none.",
				ConstLabels: source,
			}, func() float64 {
				fetched := lists.Fetched(listURL)
				if fetched.IsZero() {
					return 0
				}

				return float64(fetched.Unix())
			}),
		)
	}
}

// ReputationHit counts a request whose client the blocklist at source, its
// URL, lists.
func (m *Metrics) ReputationHit(source string) {
	m.reputationHits.WithLabelValues(source).Inc()
}

// AddAlerts adds the metrics of the alerts sent to each destination set,
// read from queue as the metrics are asked for, by destination: the
// alerts sent, the requests to the destination that failed, the alerts
// held back, which are the same for every destination, and those
// dropped. With no destination set, it adds none.
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
	for _, name := range queue.DestinationsSet() {
		destination := prometheus.Labels{"destination": name}

		m.registry.MustRegister(
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name:        "smallwebwaf_alerts_sent_total",
				Help:        "Alerts the destination took.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Sent)
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name:        "smallwebwaf_alerts_failed_total",
				Help:        "Requests to the destination that failed.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Failed)
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name: "smallwebwaf_alerts_suppressed_total",
				Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
					"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Suppressed())
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name: "smallwebwaf_alerts_dropped_total",
				Help: "Alerts dropped, the oldest first, from a full queue, and " +
					"alerts given up as the destination refused them.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Dropped)
			}),
		)
	}
}

// ServeHTTP answers with the metrics in the Prometheus text format.
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.handler.ServeHTTP(w, r)
}

// RequestStarted counts a request as under way.
func (m *Metrics) RequestStarted() {
	m.inFlight.Inc()
}

// RequestEnded counts a request that has ended, from its log line. limit
// is the setting whose size or time limit the request passed, "" if none.
// duration is how long the request took, and upstreamDuration how long it
// took from when it was passed to the app, zero if it was not.
func (m *Metrics) RequestEnded(
	line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
) {
	m.inFlight.Dec()

	class := statusClass(line.Status)
	m.requests.WithLabelValues(class, line.Action).Inc()
	m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
	m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
	m.requestDuration.Observe(duration.Seconds())

	if upstreamDuration > 0 {
		m.upstreamDuration.Observe(upstreamDuration.Seconds())
	}

	if line.LimitHit != "" {
		// The log line names a byte limit's window with _bytes after it.
		window, isBytes := strings.CutSuffix(line.LimitHit, "_bytes")

		kind := ratelimit.KindRequests
		if isBytes {
			kind = ratelimit.KindBytes
		}

		m.rateLimitHits.WithLabelValues(window, kind).Inc()
	}

	if limit != "" {
		m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
	}

	if line.Offence != "" {
		m.offences.WithLabelValues(line.Offence).Inc()
	}

	if line.Country != "" {
		m.countries.add(line.Country, line)
	}

	if line.ASN != "" {
		m.asns.add(line.ASN, line)
	}
}

// RuleMatched counts a request that matched the rule id, whose action is
// action.
func (m *Metrics) RuleMatched(id, action string) {
	m.ruleMatches.WithLabelValues(id, action).Inc()
}

// StateFileWritten counts a write of the state file name, of size bytes,
// that ended with err.
func (m *Metrics) StateFileWritten(name string, size int, err error) {
	m.stateFileWrites.WithLabelValues(name).Inc()

	// The series of failures is there from the first write, at zero until
	// one fails.
	failures := m.stateFileWriteFailures.WithLabelValues(name)

	if err != nil {
		failures.Inc()

		return
	}

	m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime()
	m.stateFileSize.WithLabelValues(name).Set(float64(size))
}

// StateFileEditTakenIn counts an admin's edit of the state file name
// taken in while smallwebwaf runs.
func (m *Metrics) StateFileEditTakenIn(name string) {
	m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
}

// StateFileEditSetAside counts an admin's edit of the state file name
// renamed to name.bad because it did not parse.
func (m *Metrics) StateFileEditSetAside(name string) {
	m.stateFileEditsSetAside.WithLabelValues(name).Inc()
}

// statusClass returns the class of status, such as 2xx, or none when no
// status was sent.
func statusClass(status int) string {
	if status == 0 {
		return "none"
	}

	// A status's class is its hundreds: 404 is in 4xx.
	const hundred = 100

	return strconv.Itoa(status/hundred) + "xx"
}

// counterVec returns a counter named name, described by help, with a
// series for each set of values of labels.
func counterVec(name, help string, labels []string) *prometheus.CounterVec {
	return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help},
		labels)
}

// gaugeVec returns a gauge named name, described by help, with a series
// for each set of values of labels.
func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec {
	return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels)
}