smallwebwaf/internal/metrics/metrics.go

// Package metrics keeps smallwebwaf's Prometheus metrics, as
// docs/metrics.md lists them, and serves them in the Prometheus text
// format, as the "Metrics endpoint" section of SPEC.md requires. No
// metric carries a client's address.
package metrics

import (
	"net/http"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/collectors"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/config"
	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
	"sneak.berlin/go/smallwebwaf/internal/remotelog"
	"sneak.berlin/go/smallwebwaf/internal/reputation"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
	"sneak.berlin/go/smallwebwaf/internal/rules"
)

// Metrics are smallwebwaf's metrics. They are safe for concurrent use.
type Metrics struct {
	// registry gives every metric registered with it the label instance.
	registry prometheus.Registerer
	handler  http.Handler

	inFlight             prometheus.Gauge
	requests             *prometheus.CounterVec
	requestBytes         *prometheus.CounterVec
	responseBytes        *prometheus.CounterVec
	requestDuration      prometheus.Histogram
	upstreamDuration     prometheus.Histogram
	rateLimitHits        *prometheus.CounterVec
	sizeAndTimeLimitHits *prometheus.CounterVec
	offences             *prometheus.CounterVec
	wafMatches           *prometheus.CounterVec
	// ruleMatches are made by AddRules, and reputationHits by
	// AddReputation.
	ruleMatches    *prometheus.CounterVec
	reputationHits *prometheus.CounterVec
	countries      *busiest
	asns           *busiest

	// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
	// that failed. GeoJSUnanswered are the requests that needed their
	// client's answer, for a setting that acts on it, and went on without
	// it because GeoJS had not given it in time.
	GeoJSRequests   prometheus.Counter
	GeoJSFailures   prometheus.Counter
	GeoJSUnanswered prometheus.Counter

	stateFileWrites        *prometheus.CounterVec
	stateFileWriteFailures *prometheus.CounterVec
	stateFileLastWrite     *prometheus.GaugeVec
	stateFileSize          *prometheus.GaugeVec
	stateFileEditsTakenIn  *prometheus.CounterVec
	stateFileEditsSetAside *prometheus.CounterVec
}

// New returns the metrics, with the Go runtime's and the process's own.
// topN is how many countries and how many AS numbers get series of their
// own (SWWAF_METRICS_TOP_N). Every metric carries instanceName
// (SWWAF_INSTANCE_NAME) as its label instance.
//
//nolint:funlen // a few lines for each metric, a list that grows with them
func New(topN int, instanceName string) *Metrics {
	byStatus := []string{"status_class", "action"}
	byFile := []string{"file"}
	registry := prometheus.NewRegistry()

	m := &Metrics{
		registry: prometheus.WrapRegistererWith(
			prometheus.Labels{"instance": instanceName}, registry),
		handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
		inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "smallwebwaf_requests_in_flight",
			Help: "Requests under way.",
		}),
		requests: counterVec("smallwebwaf_requests_total",
			"Requests, by the class of their status and their action.", byStatus),
		requestBytes: counterVec("smallwebwaf_request_bytes_total",
			"Request body bytes, by the class of the status and the action.",
			byStatus),
		responseBytes: counterVec("smallwebwaf_response_bytes_total",
			"Response body bytes, by the class of the status and the action.",
			byStatus),
		requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
			Name: "smallwebwaf_request_duration_seconds",
			Help: "How long requests took, from their arrival to their end.",
		}),
		upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
			Name: "smallwebwaf_upstream_duration_seconds",
			Help: "How long requests passed to the app took, from then to their end.",
		}),
		rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total",
			"Requests that broke a rate limit, a byte limit or the error burst, by "+
				"its window and its kind, requests, bytes or refusals.",
			[]string{"window", "kind"}),
		sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total",
			"Requests that passed a size or time limit, by its setting.",
			[]string{"limit"}),
		offences: counterVec("smallwebwaf_offences_total",
			"Offences, by kind.", []string{"kind"}),
		wafMatches: counterVec("smallwebwaf_waf_matches_total",
			"Requests that matched a rule of the Core Rule Set, by SWWAF_WAF_MODE "+
				"and the rule's id.", []string{"mode", "rule_id"}),
		countries: newCountries(topN),
		asns:      newASNs(topN),
		GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_requests_total",
			Help: "Requests to GeoJS.",
		}),
		GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_failures_total",
			Help: "Requests to GeoJS that failed.",
		}),
		GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{
			Name: "smallwebwaf_geojs_unanswered_total",
			Help: "Requests that needed their client's answer from GeoJS and " +
				"went on without it, because GeoJS had not given it in time.",
		}),
		stateFileWrites: counterVec("smallwebwaf_state_file_writes_total",
			"Writes of each state file.", byFile),
		stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total",
			"Writes of each state file that failed.", byFile),
		stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds",
			"When each state file was last written, in seconds since 1970.", byFile),
		stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes",
			"The size of each state file, as it was last written.", byFile),
		stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total",
			"Edits of each state file taken in while running.", byFile),
		stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total",
			"Edits of each state file renamed to <name>.bad because they did not parse.",
			byFile),
	}

	m.registry.MustRegister(
		collectors.NewGoCollector(),
		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
		m.inFlight, m.requests, m.requestBytes, m.responseBytes,
		m.requestDuration, m.upstreamDuration,
		m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.wafMatches,
		m.countries, m.asns,
		m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
		m.stateFileWrites, m.stateFileWriteFailures,
		m.stateFileLastWrite, m.stateFileSize,
		m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
	)

	return m
}

// AddBansAndClients adds the metrics read from the ledger and the table
// of clients as the metrics are asked for: the bans made since the start,
// by cause, the bans active and permanent at now, and the clients in the
// table.
func (m *Metrics) AddBansAndClients(
	ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
) {
	for _, cause := range []string{
		bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseCrowdSec,
	} {
		m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name:        "smallwebwaf_bans_made_total",
			Help:        "Bans made, by cause.",
			ConstLabels: prometheus.Labels{"cause": cause},
		}, func() float64 {
			return float64(ledger.Made(cause))
		}))
	}

	m.registry.MustRegister(
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_active_bans",
			Help: "Bans active now, the permanent ones included.",
		}, func() float64 {
			active, _ := ledger.Count(now())

			return float64(active)
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_permanent_bans",
			Help: "Permanent bans not lifted.",
		}, func() float64 {
			_, permanent := ledger.Count(now())

			return float64(permanent)
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_tracked_clients",
			Help: "Clients in the table of clients.",
		}, func() float64 {
			return float64(limiter.Len())
		}),
	)
}

// AddRules adds the metrics of the rule files: the requests that matched
// each rule, which RuleMatched counts, and the rules loaded from
// ruleFiles, read as the metrics are asked for. It is called once, before
// RuleMatched.
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
	m.ruleMatches = counterVec("smallwebwaf_rule_matches_total",
		"Requests that matched a rule of the rule files, by its id and action.",
		[]string{"rule_id", "action"})

	m.registry.MustRegister(m.ruleMatches,
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_rules_loaded",
			Help: "Rules loaded from the rule files.",
		}, func() float64 {
			return float64(ruleFiles.Len())
		}))
}

// AddRemoteLog adds the metrics of sending the log lines to
// SWWAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
// lines sent, those dropped, and those waiting in the buffer.
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
	m.registry.MustRegister(
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_remote_log_lines_sent_total",
			Help: "Log lines sent to SWWAF_LOG_REMOTE_URL.",
		}, func() float64 {
			return float64(remote.Sent())
		}),
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_remote_log_lines_dropped_total",
			Help: "Log lines dropped: the oldest in a full buffer, and those " +
				"whose sending failed.",
		}, func() float64 {
			return float64(remote.Dropped())
		}),
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_remote_log_buffer_depth",
			Help: "Log lines in the buffer, waiting to be sent.",
		}, func() float64 {
			return float64(remote.Depth())
		}),
	)
}

// AddLookupFile adds the metrics of the lookup database, read as the
// metrics are asked for: when the file in use was read, which lastRead
// returns, and the replacements of it that could not be read, which
// readFailures returns. The lookup package's File, which has both, cannot
// be named here: that package counts GeoJS's requests in these metrics.
func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() int) {
	m.registry.MustRegister(
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name: "smallwebwaf_lookup_database_last_read_timestamp_seconds",
			Help: "When the lookup database in use was read, in seconds since 1970.",
		}, func() float64 {
			return float64(lastRead().Unix())
		}),
		prometheus.NewCounterFunc(prometheus.CounterOpts{
			Name: "smallwebwaf_lookup_database_read_failures_total",
			Help: "Replacements of the lookup database that could not be read.",
		}, func() float64 {
			return float64(readFailures())
		}),
	)
}

// sourceLabel is the label of the reputation metrics: a list's URL, a
// DNSBL zone, its key masked, or abuseipdb.
const sourceLabel = "source"

// AddReputation adds the metrics of the lists fetched from URLs and of the
// DNSBL zones, by source, each list's URL or each zone, its key masked as
// config.MaskZoneKey masks it: the requests whose client a blocklist, the
// CrowdSec decision list, a zone's verdict or AbuseIPDB's score lists,
// which ReputationHit counts, and, read from lists and dnsbl as the
// metrics are asked for, for a list, the fetches that failed and when the
// copy in use was fetched, and for a zone, the queries made and those that
// failed. It is called once, before ReputationHit.
func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) {
	m.reputationHits = counterVec("smallwebwaf_reputation_hits_total",
		"Requests whose client a blocklist, the CrowdSec decision list, a DNSBL "+
			"zone or AbuseIPDB lists, by the list's URL, the zone, or abuseipdb.",
		[]string{sourceLabel})
	m.registry.MustRegister(m.reputationHits)

	for _, zone := range dnsbl.Zones() {
		source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}

		m.addReputationQueries(source, func() int { return dnsbl.Queries(zone) })
		m.addReputationFailures(source, func() int { return dnsbl.Failures(zone) })
	}

	for _, listURL := range lists.URLs() {
		source := prometheus.Labels{sourceLabel: listURL}

		m.addReputationFailures(source, func() int { return lists.Failures(listURL) })
		m.registry.MustRegister(
			prometheus.NewGaugeFunc(prometheus.GaugeOpts{
				Name: "smallwebwaf_reputation_last_fetch_timestamp_seconds",
				Help: "When the copy of the list in use was fetched, in seconds since " +
					"1970, or 0 while there is none.",
				ConstLabels: source,
			}, func() float64 {
				fetched := lists.Fetched(listURL)
				if fetched.IsZero() {
					return 0
				}

				return float64(fetched.Unix())
			}),
		)
	}
}

// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
// read from abuseIPDB as the metrics are asked for: the checks made, those
// that failed, and how many checks the day's budget has left. It is
// called once, after AddReputation, while SWWAF_ABUSEIPDB_KEY is set.
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
	source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}

	m.addReputationQueries(source, abuseIPDB.Checked)
	m.addReputationFailures(source, abuseIPDB.Failures)
	m.registry.MustRegister(
		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
			Name:        "smallwebwaf_reputation_daily_budget_remaining",
			Help:        "Checks of the day's SWWAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
			ConstLabels: source,
		}, func() float64 {
			return float64(abuseIPDB.BudgetLeft())
		}),
	)
}

// ReputationHit counts a request whose client source lists: a blocklist
// or the CrowdSec decision list, by its URL, a DNSBL zone, its key masked,
// or AbuseIPDB, abuseipdb.
func (m *Metrics) ReputationHit(source string) {
	m.reputationHits.WithLabelValues(source).Inc()
}

// AddAlerts adds the metrics of the alerts sent to each destination set,
// read from queue as the metrics are asked for, by destination: the
// alerts sent, the requests to the destination that failed, the alerts
// held back, which are the same for every destination, and those
// dropped. With no destination set, it adds none.
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
	for _, name := range queue.DestinationsSet() {
		destination := prometheus.Labels{"destination": name}

		m.registry.MustRegister(
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name:        "smallwebwaf_alerts_sent_total",
				Help:        "Alerts the destination took.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Sent)
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name:        "smallwebwaf_alerts_failed_total",
				Help:        "Requests to the destination that failed.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Failed)
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name: "smallwebwaf_alerts_suppressed_total",
				Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
					"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Suppressed())
			}),
			prometheus.NewCounterFunc(prometheus.CounterOpts{
				Name: "smallwebwaf_alerts_dropped_total",
				Help: "Alerts dropped, the oldest first, from a full queue, and " +
					"alerts given up as the destination refused them.",
				ConstLabels: destination,
			}, func() float64 {
				return float64(queue.Counts(name).Dropped)
			}),
		)
	}
}

// ServeHTTP answers with the metrics in the Prometheus text format.
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.handler.ServeHTTP(w, r)
}

// RequestStarted counts a request as under way.
func (m *Metrics) RequestStarted() {
	m.inFlight.Inc()
}

// RequestEnded counts a request that has ended, from its log line. limit
// is the setting whose size or time limit the request passed, "" if none.
// duration is how long the request took, and upstreamDuration how long it
// took from when it was passed to the app, zero if it was not.
func (m *Metrics) RequestEnded(
	line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
) {
	m.inFlight.Dec()
	m.count(line, limit, duration, upstreamDuration)
}

// RequestAllowed counts a question of rproxy's, answered allow, as no longer
// under way. RequestReported counts the request it was about.
func (m *Metrics) RequestAllowed() {
	m.inFlight.Dec()
}

// RequestReported counts a request rproxy passed to the app, as RequestEnded
// does, from what its report says, which line holds.
func (m *Metrics) RequestReported(
	line *requestlog.Line, duration, upstreamDuration time.Duration,
) {
	m.count(line, "", duration, upstreamDuration)
}

// LimitHit counts a request that broke a rate limit, a byte limit or the
// error burst, by the window and the kind of hit.
func (m *Metrics) LimitHit(hit ratelimit.Hit) {
	m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc()
}

// Offences counts the offences of r, a request that has ended, as its
// client's history counts them, by kind, named as clients.json names
// them.
func (m *Metrics) Offences(r ratelimit.Request) {
	for kind, committed := range map[string]bool{
		"limit":         r.BrokeLimit,
		"attack":        r.Attack,
		"rule_blocked":  r.RuleBlocked,
		"waf_blocked":   r.WAFBlocked,
		"token_refused": r.TokenRefused,
	} {
		if committed {
			m.offences.WithLabelValues(kind).Inc()
		}
	}
}

// RuleMatched counts a request that matched the rule id, whose action is
// action.
func (m *Metrics) RuleMatched(id, action string) {
	m.ruleMatches.WithLabelValues(id, action).Inc()
}

// WAFMatched counts a request that matched the Core Rule Set's rule id,
// with SWWAF_WAF_MODE at mode.
func (m *Metrics) WAFMatched(mode string, id int) {
	m.wafMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
}

// StateFileWritten counts a write of the state file name, of size bytes,
// that ended with err.
func (m *Metrics) StateFileWritten(name string, size int, err error) {
	m.stateFileWrites.WithLabelValues(name).Inc()

	// The series of failures is there from the first write, at zero until
	// one fails.
	failures := m.stateFileWriteFailures.WithLabelValues(name)

	if err != nil {
		failures.Inc()

		return
	}

	m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime()
	m.stateFileSize.WithLabelValues(name).Set(float64(size))
}

// StateFileEditTakenIn counts an admin's edit of the state file name
// taken in while smallwebwaf runs.
func (m *Metrics) StateFileEditTakenIn(name string) {
	m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
}

// StateFileEditSetAside counts an admin's edit of the state file name
// renamed to name.bad because it did not parse.
func (m *Metrics) StateFileEditSetAside(name string) {
	m.stateFileEditsSetAside.WithLabelValues(name).Inc()
}

// count counts a request that has ended, as RequestEnded describes.
func (m *Metrics) count(
	line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
) {
	class := statusClass(line.Status)
	m.requests.WithLabelValues(class, line.Action).Inc()
	m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
	m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
	m.requestDuration.Observe(duration.Seconds())

	if upstreamDuration > 0 {
		m.upstreamDuration.Observe(upstreamDuration.Seconds())
	}

	if limit != "" {
		m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
	}

	if line.Country != "" {
		m.countries.add(line.Country, line)
	}

	if line.ASN != "" {
		m.asns.add(line.ASN, line)
	}
}

// addReputationQueries adds the counter of the queries to source, a DNSBL
// zone, or of the checks of clients with AbuseIPDB, which count tells.
func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) {
	m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
		Name:        "smallwebwaf_reputation_queries_total",
		Help:        "Queries to the DNSBL zone, or checks of clients with AbuseIPDB.",
		ConstLabels: source,
	}, func() float64 {
		return float64(count())
	}))
}

// addReputationFailures adds the counter of the fetches of source, a
// list, the queries to it, a DNSBL zone, or the checks with it, AbuseIPDB,
// that failed, which count tells.
func (m *Metrics) addReputationFailures(source prometheus.Labels, count func() int) {
	m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
		Name: "smallwebwaf_reputation_failures_total",
		Help: "Fetches of the list, queries to the DNSBL zone, or checks with " +
			"AbuseIPDB, that failed.",
		ConstLabels: source,
	}, func() float64 {
		return float64(count())
	}))
}

// statusClass returns the class of status, such as 2xx, or none when no
// status was sent.
func statusClass(status int) string {
	if status == 0 {
		return "none"
	}

	// A status's class is its hundreds: 404 is in 4xx.
	const hundred = 100

	return strconv.Itoa(status/hundred) + "xx"
}

// counterVec returns a counter named name, described by help, with a
// series for each set of values of labels.
func counterVec(name, help string, labels []string) *prometheus.CounterVec {
	return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help},
		labels)
}

// gaugeVec returns a gauge named name, described by help, with a series
// for each set of values of labels.
func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec {
	return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels)
}