check / check (push) Waiting to run
metrics.New takes a Params holding every part at once, each behind a small interface declared in metrics beside the metrics it reads. A part that is off is left nil and adds no metric. The rule-match and reputation-hit counters are made in New, so no metric field is nil and no call order is needed. metrics no longer imports remotelog or rules. Model: opus-5-5
744 lines
24 KiB
Go
744 lines
24 KiB
Go
// Package metrics keeps sww's Prometheus metrics, as
|
|
// docs/metrics.md lists them, and serves them in the Prometheus text
|
|
// format, as the "Metrics endpoint" section of SPEC.md requires. No
|
|
// metric carries a client's address.
|
|
package metrics
|
|
|
|
import (
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/collectors"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
)
|
|
|
|
// Params are what New builds the metrics from. Each part is nil while it is
|
|
// off, and a part that is nil adds no metric of its own.
|
|
type Params struct {
|
|
// BusiestCount is how many countries and how many AS numbers get series
|
|
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
|
BusiestCount int
|
|
// InstanceName is the label instance every metric carries
|
|
// (SWW_WAF_INSTANCE_NAME).
|
|
InstanceName string
|
|
// Now returns the time at which Bans counts the bans active.
|
|
Now func() time.Time
|
|
|
|
Bans banLedger
|
|
Clients sized
|
|
Rules sized
|
|
Lists fetchedLists
|
|
DNSBL dnsblZones
|
|
AbuseIPDB abuseIPDBChecker
|
|
Alerts alertQueue
|
|
LookupFile lookupDatabase
|
|
RemoteLog remoteLogSender
|
|
}
|
|
|
|
// Metrics are sww's metrics. They are safe for concurrent use.
|
|
type Metrics struct {
|
|
// registry gives every metric registered with it the label instance.
|
|
registry prometheus.Registerer
|
|
handler http.Handler
|
|
|
|
inFlight prometheus.Gauge
|
|
requests *prometheus.CounterVec
|
|
requestBytes *prometheus.CounterVec
|
|
responseBytes *prometheus.CounterVec
|
|
requestDuration prometheus.Histogram
|
|
upstreamDuration prometheus.Histogram
|
|
rateLimitHits *prometheus.CounterVec
|
|
offences *prometheus.CounterVec
|
|
coreRuleSetMatches *prometheus.CounterVec
|
|
ruleMatches *prometheus.CounterVec
|
|
reputationHits *prometheus.CounterVec
|
|
countries *busiest
|
|
asns *busiest
|
|
|
|
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
|
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
|
// their client's answer, for a setting that acts on it, and went on
|
|
// without it because GeoJS had not given it in time.
|
|
GeoJSRequests prometheus.Counter
|
|
GeoJSFailures prometheus.Counter
|
|
RequestsWithoutGeoJSAnswer prometheus.Counter
|
|
|
|
stateFileWrites *prometheus.CounterVec
|
|
stateFileWriteFailures *prometheus.CounterVec
|
|
stateFileLastWrite *prometheus.GaugeVec
|
|
stateFileSize *prometheus.GaugeVec
|
|
stateFileEditsTakenIn *prometheus.CounterVec
|
|
stateFileEditsSetAside *prometheus.CounterVec
|
|
}
|
|
|
|
// New returns the metrics, with the Go runtime's and the process's own, and
|
|
// those read from each part in params as the metrics are asked for.
|
|
//
|
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
|
func New(params Params) *Metrics {
|
|
byStatus := []string{"status_class", "action"}
|
|
byFile := []string{"file"}
|
|
registry := prometheus.NewRegistry()
|
|
|
|
m := &Metrics{
|
|
registry: prometheus.WrapRegistererWith(
|
|
prometheus.Labels{"instance": params.InstanceName}, registry),
|
|
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
|
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "sww_requests_in_flight",
|
|
Help: "Requests under way.",
|
|
}),
|
|
requests: counterVec(
|
|
"sww_requests_total",
|
|
"Requests, by the class of their status and their action.",
|
|
byStatus,
|
|
),
|
|
requestBytes: counterVec("sww_request_bytes_total",
|
|
"Request body bytes, by the class of the status and the action.",
|
|
byStatus),
|
|
responseBytes: counterVec(
|
|
"sww_response_bytes_total",
|
|
"Response body bytes, by the class of the status and the action.",
|
|
byStatus,
|
|
),
|
|
requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "sww_request_duration_seconds",
|
|
Help: "How long requests took, from their arrival to their end.",
|
|
}),
|
|
upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "sww_upstream_duration_seconds",
|
|
Help: "How long requests passed to the app took, from then to their end.",
|
|
}),
|
|
rateLimitHits: counterVec(
|
|
"sww_rate_limit_hits_total",
|
|
"Requests that broke a rate limit, a byte limit or the refusal limit, by "+
|
|
"its window and its kind, requests, bytes or refusals.",
|
|
[]string{"window", "kind"},
|
|
),
|
|
offences: counterVec("sww_offences_total",
|
|
"Offences, by kind.", []string{"kind"}),
|
|
coreRuleSetMatches: counterVec(
|
|
"sww_core_rule_set_matches_total",
|
|
"Requests that matched a rule of the Core Rule Set, by SWW_WAF_CORE_RULE_SET_MODE "+
|
|
"and the rule's id.",
|
|
[]string{"mode", "rule_id"},
|
|
),
|
|
ruleMatches: counterVec(
|
|
"sww_rule_matches_total",
|
|
"Requests that matched a rule of the rule files, by its id and action.",
|
|
[]string{"rule_id", "action"},
|
|
),
|
|
reputationHits: counterVec(
|
|
"sww_reputation_hits_total",
|
|
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
|
"by the list's URL, the zone, or abuseipdb.",
|
|
[]string{sourceLabel},
|
|
),
|
|
countries: newCountries(params.BusiestCount),
|
|
asns: newASNs(params.BusiestCount),
|
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "sww_geojs_requests_total",
|
|
Help: "Requests to GeoJS.",
|
|
}),
|
|
GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "sww_geojs_failures_total",
|
|
Help: "Requests to GeoJS that failed.",
|
|
}),
|
|
RequestsWithoutGeoJSAnswer: prometheus.NewCounter(
|
|
prometheus.CounterOpts{
|
|
Name: "sww_requests_without_geojs_answer_total",
|
|
Help: "Requests that needed their client's answer from GeoJS and " +
|
|
"went on without it, because GeoJS had not given it in time.",
|
|
},
|
|
),
|
|
stateFileWrites: counterVec("sww_state_file_writes_total",
|
|
"Writes of each state or cache file.", byFile),
|
|
stateFileWriteFailures: counterVec(
|
|
"sww_state_file_write_failures_total",
|
|
"Writes of each state or cache file that failed.",
|
|
byFile,
|
|
),
|
|
stateFileLastWrite: gaugeVec(
|
|
"sww_state_file_last_write_timestamp_seconds",
|
|
"When each state or cache file was last written, in seconds since 1970.",
|
|
byFile,
|
|
),
|
|
stateFileSize: gaugeVec(
|
|
"sww_state_file_size_bytes",
|
|
"The size of each state or cache file, as it was last written.",
|
|
byFile,
|
|
),
|
|
stateFileEditsTakenIn: counterVec(
|
|
"sww_state_file_edits_taken_in_total",
|
|
"Edits of each state or cache file taken in while running.",
|
|
byFile,
|
|
),
|
|
stateFileEditsSetAside: counterVec(
|
|
"sww_state_file_edits_set_aside_total",
|
|
"Edits of each state or cache file renamed to <name>.bad because they "+
|
|
"did not parse.",
|
|
byFile,
|
|
),
|
|
}
|
|
|
|
m.registry.MustRegister(
|
|
collectors.NewGoCollector(),
|
|
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
|
m.requestDuration, m.upstreamDuration,
|
|
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
|
m.ruleMatches, m.reputationHits,
|
|
m.countries, m.asns,
|
|
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
|
m.stateFileWrites, m.stateFileWriteFailures,
|
|
m.stateFileLastWrite, m.stateFileSize,
|
|
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
|
)
|
|
|
|
if params.Bans != nil {
|
|
m.addBans(params.Bans, params.Now)
|
|
}
|
|
|
|
if params.Clients != nil {
|
|
m.addClients(params.Clients)
|
|
}
|
|
|
|
if params.Rules != nil {
|
|
m.addRules(params.Rules)
|
|
}
|
|
|
|
if params.Lists != nil {
|
|
m.addLists(params.Lists)
|
|
}
|
|
|
|
if params.DNSBL != nil {
|
|
m.addDNSBL(params.DNSBL)
|
|
}
|
|
|
|
if params.AbuseIPDB != nil {
|
|
m.addAbuseIPDB(params.AbuseIPDB)
|
|
}
|
|
|
|
if params.Alerts != nil {
|
|
m.addAlerts(params.Alerts)
|
|
}
|
|
|
|
if params.LookupFile != nil {
|
|
m.addLookupFile(params.LookupFile)
|
|
}
|
|
|
|
if params.RemoteLog != nil {
|
|
m.addRemoteLog(params.RemoteLog)
|
|
}
|
|
|
|
return m
|
|
}
|
|
|
|
// ServeHTTP answers with the metrics in the Prometheus text format.
|
|
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
m.handler.ServeHTTP(w, r)
|
|
}
|
|
|
|
// RequestStarted counts a request as under way.
|
|
func (m *Metrics) RequestStarted() {
|
|
m.inFlight.Inc()
|
|
}
|
|
|
|
// RequestEnded counts a request that has ended without being passed to the app,
|
|
// from its log line. duration is how long the request took.
|
|
func (m *Metrics) RequestEnded(
|
|
line *requestlog.Line,
|
|
duration time.Duration,
|
|
) {
|
|
m.inFlight.Dec()
|
|
m.count(line, duration, 0)
|
|
}
|
|
|
|
// RequestAllowed counts a question of rproxy's, answered allow, as no longer
|
|
// under way. RequestReported counts the request it was about.
|
|
func (m *Metrics) RequestAllowed() {
|
|
m.inFlight.Dec()
|
|
}
|
|
|
|
// RequestReported counts a request rproxy passed to the app, from what its
|
|
// report says, which line holds: duration is how long the request took, and
|
|
// upstreamDuration how long it took from when it was passed to the app.
|
|
func (m *Metrics) RequestReported(
|
|
line *requestlog.Line, duration, upstreamDuration time.Duration,
|
|
) {
|
|
m.count(line, duration, upstreamDuration)
|
|
}
|
|
|
|
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
|
// refusal limit, by the window and the kind of hit.
|
|
func (m *Metrics) LimitHit(hit ratelimit.Hit) {
|
|
m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc()
|
|
}
|
|
|
|
// Offences counts the offences of r, a request that has ended, as its
|
|
// client's history counts them, by kind, named as clients.json names
|
|
// them.
|
|
func (m *Metrics) Offences(r ratelimit.Request) {
|
|
for kind, committed := range map[string]bool{
|
|
"limit": r.BrokeLimit,
|
|
"attack": r.Attack,
|
|
"blocked_by_block_rule": r.BlockedByBlockRule,
|
|
"blocked_by_core_rule_set": r.BlockedByCoreRuleSet,
|
|
} {
|
|
if committed {
|
|
m.offences.WithLabelValues(kind).Inc()
|
|
}
|
|
}
|
|
}
|
|
|
|
// RuleMatched counts a request that matched the rule id, whose action is
|
|
// action.
|
|
func (m *Metrics) RuleMatched(id, action string) {
|
|
m.ruleMatches.WithLabelValues(id, action).Inc()
|
|
}
|
|
|
|
// CoreRuleSetMatched counts a request that matched the Core Rule Set's rule id,
|
|
// with SWW_WAF_CORE_RULE_SET_MODE at mode.
|
|
func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
|
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
|
}
|
|
|
|
// ReputationHit counts a request whose client source lists: a blocklist,
|
|
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
|
func (m *Metrics) ReputationHit(source string) {
|
|
m.reputationHits.WithLabelValues(source).Inc()
|
|
}
|
|
|
|
// StateFileWritten counts a write of the state or cache file name, of size
|
|
// bytes, that ended with err.
|
|
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
|
m.stateFileWrites.WithLabelValues(name).Inc()
|
|
|
|
// The series of failures is there from the first write, at zero until
|
|
// one fails.
|
|
failures := m.stateFileWriteFailures.WithLabelValues(name)
|
|
|
|
if err != nil {
|
|
failures.Inc()
|
|
|
|
return
|
|
}
|
|
|
|
m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime()
|
|
m.stateFileSize.WithLabelValues(name).Set(float64(size))
|
|
}
|
|
|
|
// StateFileEditTakenIn counts an admin's edit of the state or cache file name
|
|
// taken in while sww runs.
|
|
func (m *Metrics) StateFileEditTakenIn(name string) {
|
|
m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
|
|
}
|
|
|
|
// StateFileEditSetAside counts an admin's edit of the state or cache file name
|
|
// renamed to name.bad because it did not parse.
|
|
func (m *Metrics) StateFileEditSetAside(name string) {
|
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
|
}
|
|
|
|
// banLedger is what the metrics read from the ledger of bans.
|
|
type banLedger interface {
|
|
// Made returns the bans made since the start with cause.
|
|
Made(cause string) int
|
|
// Count returns the bans active at now, the permanent ones included,
|
|
// and the permanent ones.
|
|
Count(now time.Time) (active, permanent int)
|
|
}
|
|
|
|
// addBans adds the metrics read from ledger as the metrics are asked for:
|
|
// the bans made since the start, by cause, and the bans active and
|
|
// permanent at now.
|
|
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
|
for _, cause := range []string{
|
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
|
} {
|
|
m.registry.MustRegister(
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_bans_made_total",
|
|
Help: "Bans made, by cause.",
|
|
ConstLabels: prometheus.Labels{"cause": cause},
|
|
}, func() float64 {
|
|
return float64(ledger.Made(cause))
|
|
}),
|
|
)
|
|
}
|
|
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_active_bans",
|
|
Help: "Bans active now, the permanent ones included.",
|
|
}, func() float64 {
|
|
active, _ := ledger.Count(now())
|
|
|
|
return float64(active)
|
|
}),
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_permanent_bans",
|
|
Help: "Permanent bans not lifted.",
|
|
}, func() float64 {
|
|
_, permanent := ledger.Count(now())
|
|
|
|
return float64(permanent)
|
|
}),
|
|
)
|
|
}
|
|
|
|
// sized is what the metrics read from the table of clients and from the
|
|
// rule files.
|
|
type sized interface {
|
|
// Len returns how many clients the table holds, or how many rules were
|
|
// loaded from the rule files.
|
|
Len() int
|
|
}
|
|
|
|
// addClients adds the metric of the clients in table, read as the metrics
|
|
// are asked for.
|
|
func (m *Metrics) addClients(table sized) {
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_tracked_clients",
|
|
Help: "Clients in the table of clients.",
|
|
}, func() float64 {
|
|
return float64(table.Len())
|
|
}),
|
|
)
|
|
}
|
|
|
|
// addRules adds the metric of the rules loaded from files, read as the
|
|
// metrics are asked for.
|
|
func (m *Metrics) addRules(files sized) {
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_rules_loaded",
|
|
Help: "Rules loaded from the rule files.",
|
|
}, func() float64 {
|
|
return float64(files.Len())
|
|
}))
|
|
}
|
|
|
|
// remoteLogSender is what the metrics read from what sends the log lines to
|
|
// SWW_WAF_LOG_REMOTE_URL.
|
|
type remoteLogSender interface {
|
|
// Sent returns the lines sent.
|
|
Sent() int64
|
|
// Dropped returns the lines dropped.
|
|
Dropped() int64
|
|
// Depth returns the lines waiting in the buffer.
|
|
Depth() int
|
|
}
|
|
|
|
// addRemoteLog adds the metrics of sending the log lines to
|
|
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
|
// lines sent, those dropped, and those waiting in the buffer.
|
|
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
|
m.registry.MustRegister(
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_remote_log_lines_sent_total",
|
|
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
|
}, func() float64 {
|
|
return float64(remote.Sent())
|
|
}),
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_remote_log_lines_dropped_total",
|
|
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
|
"whose sending failed.",
|
|
}, func() float64 {
|
|
return float64(remote.Dropped())
|
|
}),
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_remote_log_lines_waiting",
|
|
Help: "Log lines in the buffer, waiting to be sent.",
|
|
}, func() float64 {
|
|
return float64(remote.Depth())
|
|
}),
|
|
)
|
|
}
|
|
|
|
// lookupDatabase is what the metrics read from the lookup database.
|
|
type lookupDatabase interface {
|
|
// LastRead returns when the file in use was read.
|
|
LastRead() time.Time
|
|
// ReadFailures returns the replacements of the file that could not be
|
|
// read.
|
|
ReadFailures() int
|
|
}
|
|
|
|
// addLookupFile adds the metrics of the lookup database, read from file as
|
|
// the metrics are asked for: when the file in use was read, and the
|
|
// replacements of it that could not be read.
|
|
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
|
Help: "When the lookup database in use was read, in seconds since 1970.",
|
|
}, func() float64 {
|
|
return float64(file.LastRead().Unix())
|
|
}),
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_lookup_database_read_failures_total",
|
|
Help: "Replacements of the lookup database that could not be read.",
|
|
}, func() float64 {
|
|
return float64(file.ReadFailures())
|
|
}),
|
|
)
|
|
}
|
|
|
|
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
|
// DNSBL zone, its key masked, or abuseipdb.
|
|
const sourceLabel = "source"
|
|
|
|
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
|
type dnsblZones interface {
|
|
// Zones returns the zones asked, their keys unmasked.
|
|
Zones() []string
|
|
// Queries returns the queries made to zone.
|
|
Queries(zone string) int
|
|
// Failures returns the queries to zone that failed.
|
|
Failures(zone string) int
|
|
}
|
|
|
|
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
|
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
|
// asked for: the queries made and those that failed.
|
|
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
|
for _, zone := range dnsbl.Zones() {
|
|
source := prometheus.Labels{
|
|
sourceLabel: reputation.MaskZoneKey(zone),
|
|
}
|
|
|
|
m.addReputationQueries(
|
|
source,
|
|
func() int { return dnsbl.Queries(zone) },
|
|
)
|
|
m.addReputationFailures(
|
|
source,
|
|
func() int { return dnsbl.Failures(zone) },
|
|
)
|
|
}
|
|
}
|
|
|
|
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
|
type fetchedLists interface {
|
|
// URLs returns the lists' URLs.
|
|
URLs() []string
|
|
// Failures returns the fetches of the list at listURL that failed.
|
|
Failures(listURL string) int
|
|
// Fetched returns when the copy in use of the list at listURL was
|
|
// fetched, or the zero time while there is none.
|
|
Fetched(listURL string) time.Time
|
|
}
|
|
|
|
// addLists adds the metrics of each list fetched from a URL, by source, the
|
|
// list's URL, read from lists as the metrics are asked for: the fetches
|
|
// that failed and when the copy in use was fetched.
|
|
func (m *Metrics) addLists(lists fetchedLists) {
|
|
for _, listURL := range lists.URLs() {
|
|
source := prometheus.Labels{sourceLabel: listURL}
|
|
|
|
m.addReputationFailures(
|
|
source,
|
|
func() int { return lists.Failures(listURL) },
|
|
)
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
|
Help: "When the copy of the list in use was fetched, in seconds since " +
|
|
"1970, or 0 while there is none.",
|
|
ConstLabels: source,
|
|
}, func() float64 {
|
|
fetched := lists.Fetched(listURL)
|
|
if fetched.IsZero() {
|
|
return 0
|
|
}
|
|
|
|
return float64(fetched.Unix())
|
|
}),
|
|
)
|
|
}
|
|
}
|
|
|
|
// abuseIPDBChecker is what the metrics read from what checks clients with
|
|
// AbuseIPDB.
|
|
type abuseIPDBChecker interface {
|
|
// Checked returns the checks made.
|
|
Checked() int
|
|
// Failures returns the checks that failed.
|
|
Failures() int
|
|
// BudgetLeft returns how many checks the day's budget has left.
|
|
BudgetLeft() int
|
|
}
|
|
|
|
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
|
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
|
// that failed, and how many checks the day's budget has left.
|
|
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
|
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
|
|
|
m.addReputationQueries(source, abuseIPDB.Checked)
|
|
m.addReputationFailures(source, abuseIPDB.Failures)
|
|
m.registry.MustRegister(
|
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
Name: "sww_reputation_daily_budget_remaining",
|
|
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
|
ConstLabels: source,
|
|
}, func() float64 {
|
|
return float64(abuseIPDB.BudgetLeft())
|
|
}),
|
|
)
|
|
}
|
|
|
|
// alertQueue is what the metrics read from the queue of alerts.
|
|
type alertQueue interface {
|
|
// DestinationsSet returns the names of the destinations set.
|
|
DestinationsSet() []string
|
|
// Counts returns the alerts sent to the destination name, the requests
|
|
// to it that failed, and the alerts dropped.
|
|
Counts(name string) alerts.Counts
|
|
// Suppressed returns the alerts held back.
|
|
Suppressed() int64
|
|
}
|
|
|
|
// addAlerts adds the metrics of the alerts sent to each destination set,
|
|
// read from queue as the metrics are asked for, by destination: the
|
|
// alerts sent, the requests to the destination that failed, the alerts
|
|
// held back, which are the same for every destination, and those
|
|
// dropped. With no destination set, it adds none.
|
|
func (m *Metrics) addAlerts(queue alertQueue) {
|
|
for _, name := range queue.DestinationsSet() {
|
|
destination := prometheus.Labels{"destination": name}
|
|
|
|
m.registry.MustRegister(
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_alerts_sent_total",
|
|
Help: "Alerts the destination took.",
|
|
ConstLabels: destination,
|
|
}, func() float64 {
|
|
return float64(queue.Counts(name).Sent)
|
|
}),
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_alert_send_failures_total",
|
|
Help: "Requests to the destination that failed.",
|
|
ConstLabels: destination,
|
|
}, func() float64 {
|
|
return float64(queue.Counts(name).SendFailures)
|
|
}),
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_alerts_suppressed_total",
|
|
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
|
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
|
ConstLabels: destination,
|
|
}, func() float64 {
|
|
return float64(queue.Suppressed())
|
|
}),
|
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_alerts_dropped_total",
|
|
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
|
"alerts given up as the destination refused them.",
|
|
ConstLabels: destination,
|
|
}, func() float64 {
|
|
return float64(queue.Counts(name).Dropped)
|
|
}),
|
|
)
|
|
}
|
|
}
|
|
|
|
// count counts a request that has ended, as RequestEnded and RequestReported
|
|
// describe.
|
|
func (m *Metrics) count(
|
|
line *requestlog.Line, duration, upstreamDuration time.Duration,
|
|
) {
|
|
class := statusClass(line.Status)
|
|
m.requests.WithLabelValues(class, line.Action).Inc()
|
|
m.requestBytes.WithLabelValues(class, line.Action).
|
|
Add(float64(line.RequestBytes))
|
|
m.responseBytes.WithLabelValues(class, line.Action).
|
|
Add(float64(line.ResponseBytes))
|
|
m.requestDuration.Observe(duration.Seconds())
|
|
|
|
if upstreamDuration > 0 {
|
|
m.upstreamDuration.Observe(upstreamDuration.Seconds())
|
|
}
|
|
|
|
if line.Country != "" {
|
|
m.countries.add(line.Country, line)
|
|
}
|
|
|
|
if line.ASN != "" {
|
|
m.asns.add(line.ASN, line)
|
|
}
|
|
}
|
|
|
|
// addReputationQueries adds the counter of the queries to source, a DNSBL
|
|
// zone, or of the checks of clients with AbuseIPDB, which count tells.
|
|
func (m *Metrics) addReputationQueries(
|
|
source prometheus.Labels,
|
|
count func() int,
|
|
) {
|
|
m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_reputation_queries_total",
|
|
Help: "Queries to the DNSBL zone, or checks of clients with AbuseIPDB.",
|
|
ConstLabels: source,
|
|
}, func() float64 {
|
|
return float64(count())
|
|
}))
|
|
}
|
|
|
|
// addReputationFailures adds the counter of the fetches of source, a
|
|
// list, the queries to it, a DNSBL zone, or the checks with it, AbuseIPDB,
|
|
// that failed, which count tells.
|
|
func (m *Metrics) addReputationFailures(
|
|
source prometheus.Labels,
|
|
count func() int,
|
|
) {
|
|
m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
Name: "sww_reputation_failures_total",
|
|
Help: "Fetches of the list, queries to the DNSBL zone, or checks with " +
|
|
"AbuseIPDB, that failed.",
|
|
ConstLabels: source,
|
|
}, func() float64 {
|
|
return float64(count())
|
|
}))
|
|
}
|
|
|
|
// statusClass returns the class of status, such as 2xx, or none when no
|
|
// status was sent.
|
|
func statusClass(status int) string {
|
|
if status == 0 {
|
|
return "none"
|
|
}
|
|
|
|
// A status's class is its hundreds: 404 is in 4xx.
|
|
const hundred = 100
|
|
|
|
return strconv.Itoa(status/hundred) + "xx"
|
|
}
|
|
|
|
// counterVec returns a counter named name, described by help, with a
|
|
// series for each set of values of labels.
|
|
func counterVec(name, help string, labels []string) *prometheus.CounterVec {
|
|
return prometheus.NewCounterVec(
|
|
prometheus.CounterOpts{Name: name, Help: help},
|
|
labels,
|
|
)
|
|
}
|
|
|
|
// gaugeVec returns a gauge named name, described by help, with a series
|
|
// for each set of values of labels.
|
|
func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec {
|
|
return prometheus.NewGaugeVec(
|
|
prometheus.GaugeOpts{Name: name, Help: help},
|
|
labels,
|
|
)
|
|
}
|