Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68cfbf890f |
No files matched your search
+268
-393
@@ -15,33 +15,12 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Params are what New builds the metrics from. Each part is nil while it is
|
|
||||||
// off, and a part that is nil adds no metric of its own.
|
|
||||||
type Params struct {
|
|
||||||
// BusiestCount is how many countries and how many AS numbers get series
|
|
||||||
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
|
||||||
BusiestCount int
|
|
||||||
// InstanceName is the label instance every metric carries
|
|
||||||
// (SWW_WAF_INSTANCE_NAME).
|
|
||||||
InstanceName string
|
|
||||||
// Now returns the time at which Bans counts the bans active.
|
|
||||||
Now func() time.Time
|
|
||||||
|
|
||||||
Bans banLedger
|
|
||||||
Clients sized
|
|
||||||
Rules sized
|
|
||||||
Lists fetchedLists
|
|
||||||
DNSBL dnsblZones
|
|
||||||
AbuseIPDB abuseIPDBChecker
|
|
||||||
Alerts alertQueue
|
|
||||||
LookupFile lookupDatabase
|
|
||||||
RemoteLog remoteLogSender
|
|
||||||
}
|
|
||||||
|
|
||||||
// Metrics are sww's metrics. They are safe for concurrent use.
|
// Metrics are sww's metrics. They are safe for concurrent use.
|
||||||
type Metrics struct {
|
type Metrics struct {
|
||||||
// registry gives every metric registered with it the label instance.
|
// registry gives every metric registered with it the label instance.
|
||||||
@@ -57,10 +36,12 @@ type Metrics struct {
|
|||||||
rateLimitHits *prometheus.CounterVec
|
rateLimitHits *prometheus.CounterVec
|
||||||
offences *prometheus.CounterVec
|
offences *prometheus.CounterVec
|
||||||
coreRuleSetMatches *prometheus.CounterVec
|
coreRuleSetMatches *prometheus.CounterVec
|
||||||
ruleMatches *prometheus.CounterVec
|
// ruleMatches are made by AddRules, and reputationHits by
|
||||||
reputationHits *prometheus.CounterVec
|
// AddReputation.
|
||||||
countries *busiest
|
ruleMatches *prometheus.CounterVec
|
||||||
asns *busiest
|
reputationHits *prometheus.CounterVec
|
||||||
|
countries *busiest
|
||||||
|
asns *busiest
|
||||||
|
|
||||||
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
||||||
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
||||||
@@ -78,18 +59,20 @@ type Metrics struct {
|
|||||||
stateFileEditsSetAside *prometheus.CounterVec
|
stateFileEditsSetAside *prometheus.CounterVec
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the metrics, with the Go runtime's and the process's own, and
|
// New returns the metrics, with the Go runtime's and the process's own.
|
||||||
// those read from each part in params as the metrics are asked for.
|
// busiestCount is how many countries and how many AS numbers get series of
|
||||||
|
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
||||||
|
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
||||||
//
|
//
|
||||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||||
func New(params Params) *Metrics {
|
func New(busiestCount int, instanceName string) *Metrics {
|
||||||
byStatus := []string{"status_class", "action"}
|
byStatus := []string{"status_class", "action"}
|
||||||
byFile := []string{"file"}
|
byFile := []string{"file"}
|
||||||
registry := prometheus.NewRegistry()
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
m := &Metrics{
|
m := &Metrics{
|
||||||
registry: prometheus.WrapRegistererWith(
|
registry: prometheus.WrapRegistererWith(
|
||||||
prometheus.Labels{"instance": params.InstanceName}, registry),
|
prometheus.Labels{"instance": instanceName}, registry),
|
||||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
Name: "sww_requests_in_flight",
|
Name: "sww_requests_in_flight",
|
||||||
@@ -130,19 +113,8 @@ func New(params Params) *Metrics {
|
|||||||
"and the rule's id.",
|
"and the rule's id.",
|
||||||
[]string{"mode", "rule_id"},
|
[]string{"mode", "rule_id"},
|
||||||
),
|
),
|
||||||
ruleMatches: counterVec(
|
countries: newCountries(busiestCount),
|
||||||
"sww_rule_matches_total",
|
asns: newASNs(busiestCount),
|
||||||
"Requests that matched a rule of the rule files, by its id and action.",
|
|
||||||
[]string{"rule_id", "action"},
|
|
||||||
),
|
|
||||||
reputationHits: counterVec(
|
|
||||||
"sww_reputation_hits_total",
|
|
||||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
|
||||||
"by the list's URL, the zone, or abuseipdb.",
|
|
||||||
[]string{sourceLabel},
|
|
||||||
),
|
|
||||||
countries: newCountries(params.BusiestCount),
|
|
||||||
asns: newASNs(params.BusiestCount),
|
|
||||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "sww_geojs_requests_total",
|
Name: "sww_geojs_requests_total",
|
||||||
Help: "Requests to GeoJS.",
|
Help: "Requests to GeoJS.",
|
||||||
@@ -194,7 +166,6 @@ func New(params Params) *Metrics {
|
|||||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||||
m.requestDuration, m.upstreamDuration,
|
m.requestDuration, m.upstreamDuration,
|
||||||
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
||||||
m.ruleMatches, m.reputationHits,
|
|
||||||
m.countries, m.asns,
|
m.countries, m.asns,
|
||||||
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
||||||
m.stateFileWrites, m.stateFileWriteFailures,
|
m.stateFileWrites, m.stateFileWriteFailures,
|
||||||
@@ -202,45 +173,261 @@ func New(params Params) *Metrics {
|
|||||||
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
||||||
)
|
)
|
||||||
|
|
||||||
if params.Bans != nil {
|
|
||||||
m.addBans(params.Bans, params.Now)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Clients != nil {
|
|
||||||
m.addClients(params.Clients)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Rules != nil {
|
|
||||||
m.addRules(params.Rules)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Lists != nil {
|
|
||||||
m.addLists(params.Lists)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.DNSBL != nil {
|
|
||||||
m.addDNSBL(params.DNSBL)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.AbuseIPDB != nil {
|
|
||||||
m.addAbuseIPDB(params.AbuseIPDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Alerts != nil {
|
|
||||||
m.addAlerts(params.Alerts)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.LookupFile != nil {
|
|
||||||
m.addLookupFile(params.LookupFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.RemoteLog != nil {
|
|
||||||
m.addRemoteLog(params.RemoteLog)
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddBansAndClients adds the metrics read from the ledger and the table
|
||||||
|
// of clients as the metrics are asked for: the bans made since the start,
|
||||||
|
// by cause, the bans active and permanent at now, and the clients in the
|
||||||
|
// table.
|
||||||
|
func (m *Metrics) AddBansAndClients(
|
||||||
|
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||||
|
) {
|
||||||
|
for _, cause := range []string{
|
||||||
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||||
|
} {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_bans_made_total",
|
||||||
|
Help: "Bans made, by cause.",
|
||||||
|
ConstLabels: prometheus.Labels{"cause": cause},
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ledger.Made(cause))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_active_bans",
|
||||||
|
Help: "Bans active now, the permanent ones included.",
|
||||||
|
}, func() float64 {
|
||||||
|
active, _ := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(active)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_permanent_bans",
|
||||||
|
Help: "Permanent bans not lifted.",
|
||||||
|
}, func() float64 {
|
||||||
|
_, permanent := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(permanent)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_tracked_clients",
|
||||||
|
Help: "Clients in the table of clients.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(limiter.Len())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRules adds the metrics of the rule files: the requests that matched
|
||||||
|
// each rule, which RuleMatched counts, and the rules loaded from
|
||||||
|
// ruleFiles, read as the metrics are asked for. It is called once, before
|
||||||
|
// RuleMatched.
|
||||||
|
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
|
||||||
|
m.ruleMatches = counterVec(
|
||||||
|
"sww_rule_matches_total",
|
||||||
|
"Requests that matched a rule of the rule files, by its id and action.",
|
||||||
|
[]string{"rule_id", "action"},
|
||||||
|
)
|
||||||
|
|
||||||
|
m.registry.MustRegister(m.ruleMatches,
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_rules_loaded",
|
||||||
|
Help: "Rules loaded from the rule files.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ruleFiles.Len())
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRemoteLog adds the metrics of sending the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||||
|
// lines sent, those dropped, and those waiting in the buffer.
|
||||||
|
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_sent_total",
|
||||||
|
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Sent())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_dropped_total",
|
||||||
|
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||||
|
"whose sending failed.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Dropped())
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_remote_log_lines_waiting",
|
||||||
|
Help: "Log lines in the buffer, waiting to be sent.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Depth())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddLookupFile adds the metrics of the lookup database, read as the
|
||||||
|
// metrics are asked for: when the file in use was read, which lastRead
|
||||||
|
// returns, and the replacements of it that could not be read, which
|
||||||
|
// readFailures returns.
|
||||||
|
func (m *Metrics) AddLookupFile(
|
||||||
|
lastRead func() time.Time,
|
||||||
|
readFailures func() int,
|
||||||
|
) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||||
|
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(lastRead().Unix())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_lookup_database_read_failures_total",
|
||||||
|
Help: "Replacements of the lookup database that could not be read.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(readFailures())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||||
|
// DNSBL zone, its key masked, or abuseipdb.
|
||||||
|
const sourceLabel = "source"
|
||||||
|
|
||||||
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
|
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
|
// zone, the queries made and those that failed. It is called once, before
|
||||||
|
// ReputationHit.
|
||||||
|
func (m *Metrics) AddReputation(
|
||||||
|
lists *reputation.Lists,
|
||||||
|
dnsbl *reputation.DNSBL,
|
||||||
|
) {
|
||||||
|
m.reputationHits = counterVec(
|
||||||
|
"sww_reputation_hits_total",
|
||||||
|
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||||
|
"by the list's URL, the zone, or abuseipdb.",
|
||||||
|
[]string{sourceLabel},
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
|
for _, zone := range dnsbl.Zones() {
|
||||||
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
|
m.addReputationQueries(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Queries(zone) },
|
||||||
|
)
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Failures(zone) },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, listURL := range lists.URLs() {
|
||||||
|
source := prometheus.Labels{sourceLabel: listURL}
|
||||||
|
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return lists.Failures(listURL) },
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||||
|
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||||
|
"1970, or 0 while there is none.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
fetched := lists.Fetched(listURL)
|
||||||
|
if fetched.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return float64(fetched.Unix())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||||
|
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||||
|
// that failed, and how many checks the day's budget has left. It is
|
||||||
|
// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set.
|
||||||
|
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
||||||
|
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||||
|
|
||||||
|
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||||
|
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_daily_budget_remaining",
|
||||||
|
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(abuseIPDB.BudgetLeft())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||||
|
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||||
|
func (m *Metrics) ReputationHit(source string) {
|
||||||
|
m.reputationHits.WithLabelValues(source).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAlerts adds the metrics of the alerts sent to each destination set,
|
||||||
|
// read from queue as the metrics are asked for, by destination: the
|
||||||
|
// alerts sent, the requests to the destination that failed, the alerts
|
||||||
|
// held back, which are the same for every destination, and those
|
||||||
|
// dropped. With no destination set, it adds none.
|
||||||
|
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
|
||||||
|
for _, name := range queue.DestinationsSet() {
|
||||||
|
destination := prometheus.Labels{"destination": name}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_sent_total",
|
||||||
|
Help: "Alerts the destination took.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Sent)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alert_send_failures_total",
|
||||||
|
Help: "Requests to the destination that failed.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).SendFailures)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_suppressed_total",
|
||||||
|
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||||
|
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Suppressed())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_dropped_total",
|
||||||
|
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||||
|
"alerts given up as the destination refused them.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Dropped)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ServeHTTP answers with the metrics in the Prometheus text format.
|
// ServeHTTP answers with the metrics in the Prometheus text format.
|
||||||
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
m.handler.ServeHTTP(w, r)
|
m.handler.ServeHTTP(w, r)
|
||||||
@@ -310,12 +497,6 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
|||||||
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
|
||||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
|
||||||
func (m *Metrics) ReputationHit(source string) {
|
|
||||||
m.reputationHits.WithLabelValues(source).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// StateFileWritten counts a write of the state or cache file name, of size
|
// StateFileWritten counts a write of the state or cache file name, of size
|
||||||
// bytes, that ended with err.
|
// bytes, that ended with err.
|
||||||
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
||||||
@@ -347,312 +528,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
|||||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// banLedger is what the metrics read from the ledger of bans.
|
|
||||||
type banLedger interface {
|
|
||||||
// Made returns the bans made since the start with cause.
|
|
||||||
Made(cause string) int
|
|
||||||
// Count returns the bans active at now, the permanent ones included,
|
|
||||||
// and the permanent ones.
|
|
||||||
Count(now time.Time) (active, permanent int)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addBans adds the metrics read from ledger as the metrics are asked for:
|
|
||||||
// the bans made since the start, by cause, and the bans active and
|
|
||||||
// permanent at now.
|
|
||||||
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
|
||||||
for _, cause := range []string{
|
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
|
||||||
} {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_bans_made_total",
|
|
||||||
Help: "Bans made, by cause.",
|
|
||||||
ConstLabels: prometheus.Labels{"cause": cause},
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(ledger.Made(cause))
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_active_bans",
|
|
||||||
Help: "Bans active now, the permanent ones included.",
|
|
||||||
}, func() float64 {
|
|
||||||
active, _ := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(active)
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_permanent_bans",
|
|
||||||
Help: "Permanent bans not lifted.",
|
|
||||||
}, func() float64 {
|
|
||||||
_, permanent := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(permanent)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sized is what the metrics read from the table of clients and from the
|
|
||||||
// rule files.
|
|
||||||
type sized interface {
|
|
||||||
// Len returns how many clients the table holds, or how many rules were
|
|
||||||
// loaded from the rule files.
|
|
||||||
Len() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addClients adds the metric of the clients in table, read as the metrics
|
|
||||||
// are asked for.
|
|
||||||
func (m *Metrics) addClients(table sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_tracked_clients",
|
|
||||||
Help: "Clients in the table of clients.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(table.Len())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRules adds the metric of the rules loaded from files, read as the
|
|
||||||
// metrics are asked for.
|
|
||||||
func (m *Metrics) addRules(files sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_rules_loaded",
|
|
||||||
Help: "Rules loaded from the rule files.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(files.Len())
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// remoteLogSender is what the metrics read from what sends the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL.
|
|
||||||
type remoteLogSender interface {
|
|
||||||
// Sent returns the lines sent.
|
|
||||||
Sent() int64
|
|
||||||
// Dropped returns the lines dropped.
|
|
||||||
Dropped() int64
|
|
||||||
// Depth returns the lines waiting in the buffer.
|
|
||||||
Depth() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRemoteLog adds the metrics of sending the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
|
||||||
// lines sent, those dropped, and those waiting in the buffer.
|
|
||||||
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_sent_total",
|
|
||||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Sent())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_dropped_total",
|
|
||||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
|
||||||
"whose sending failed.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Dropped())
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_remote_log_lines_waiting",
|
|
||||||
Help: "Log lines in the buffer, waiting to be sent.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Depth())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookupDatabase is what the metrics read from the lookup database.
|
|
||||||
type lookupDatabase interface {
|
|
||||||
// LastRead returns when the file in use was read.
|
|
||||||
LastRead() time.Time
|
|
||||||
// ReadFailures returns the replacements of the file that could not be
|
|
||||||
// read.
|
|
||||||
ReadFailures() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLookupFile adds the metrics of the lookup database, read from file as
|
|
||||||
// the metrics are asked for: when the file in use was read, and the
|
|
||||||
// replacements of it that could not be read.
|
|
||||||
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
|
||||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.LastRead().Unix())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_lookup_database_read_failures_total",
|
|
||||||
Help: "Replacements of the lookup database that could not be read.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.ReadFailures())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
|
||||||
// DNSBL zone, its key masked, or abuseipdb.
|
|
||||||
const sourceLabel = "source"
|
|
||||||
|
|
||||||
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
|
||||||
type dnsblZones interface {
|
|
||||||
// Zones returns the zones asked, their keys unmasked.
|
|
||||||
Zones() []string
|
|
||||||
// Queries returns the queries made to zone.
|
|
||||||
Queries(zone string) int
|
|
||||||
// Failures returns the queries to zone that failed.
|
|
||||||
Failures(zone string) int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
|
||||||
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
|
||||||
// asked for: the queries made and those that failed.
|
|
||||||
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
|
||||||
source := prometheus.Labels{
|
|
||||||
sourceLabel: reputation.MaskZoneKey(zone),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.addReputationQueries(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Queries(zone) },
|
|
||||||
)
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Failures(zone) },
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
|
||||||
type fetchedLists interface {
|
|
||||||
// URLs returns the lists' URLs.
|
|
||||||
URLs() []string
|
|
||||||
// Failures returns the fetches of the list at listURL that failed.
|
|
||||||
Failures(listURL string) int
|
|
||||||
// Fetched returns when the copy in use of the list at listURL was
|
|
||||||
// fetched, or the zero time while there is none.
|
|
||||||
Fetched(listURL string) time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLists adds the metrics of each list fetched from a URL, by source, the
|
|
||||||
// list's URL, read from lists as the metrics are asked for: the fetches
|
|
||||||
// that failed and when the copy in use was fetched.
|
|
||||||
func (m *Metrics) addLists(lists fetchedLists) {
|
|
||||||
for _, listURL := range lists.URLs() {
|
|
||||||
source := prometheus.Labels{sourceLabel: listURL}
|
|
||||||
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return lists.Failures(listURL) },
|
|
||||||
)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
|
||||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
|
||||||
"1970, or 0 while there is none.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
fetched := lists.Fetched(listURL)
|
|
||||||
if fetched.IsZero() {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return float64(fetched.Unix())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// abuseIPDBChecker is what the metrics read from what checks clients with
|
|
||||||
// AbuseIPDB.
|
|
||||||
type abuseIPDBChecker interface {
|
|
||||||
// Checked returns the checks made.
|
|
||||||
Checked() int
|
|
||||||
// Failures returns the checks that failed.
|
|
||||||
Failures() int
|
|
||||||
// BudgetLeft returns how many checks the day's budget has left.
|
|
||||||
BudgetLeft() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
|
||||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
|
||||||
// that failed, and how many checks the day's budget has left.
|
|
||||||
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
|
||||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
|
||||||
|
|
||||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
|
||||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_daily_budget_remaining",
|
|
||||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(abuseIPDB.BudgetLeft())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// alertQueue is what the metrics read from the queue of alerts.
|
|
||||||
type alertQueue interface {
|
|
||||||
// DestinationsSet returns the names of the destinations set.
|
|
||||||
DestinationsSet() []string
|
|
||||||
// Counts returns the alerts sent to the destination name, the requests
|
|
||||||
// to it that failed, and the alerts dropped.
|
|
||||||
Counts(name string) alerts.Counts
|
|
||||||
// Suppressed returns the alerts held back.
|
|
||||||
Suppressed() int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAlerts adds the metrics of the alerts sent to each destination set,
|
|
||||||
// read from queue as the metrics are asked for, by destination: the
|
|
||||||
// alerts sent, the requests to the destination that failed, the alerts
|
|
||||||
// held back, which are the same for every destination, and those
|
|
||||||
// dropped. With no destination set, it adds none.
|
|
||||||
func (m *Metrics) addAlerts(queue alertQueue) {
|
|
||||||
for _, name := range queue.DestinationsSet() {
|
|
||||||
destination := prometheus.Labels{"destination": name}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_sent_total",
|
|
||||||
Help: "Alerts the destination took.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Sent)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alert_send_failures_total",
|
|
||||||
Help: "Requests to the destination that failed.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).SendFailures)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_suppressed_total",
|
|
||||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
|
||||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Suppressed())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_dropped_total",
|
|
||||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
|
||||||
"alerts given up as the destination refused them.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Dropped)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
// count counts a request that has ended, as RequestEnded and RequestReported
|
||||||
// describe.
|
// describe.
|
||||||
func (m *Metrics) count(
|
func (m *Metrics) count(
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
package metrics_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
listURL = "https://lists.example/block.txt"
|
|
||||||
zone = "dnsbl.example"
|
|
||||||
destination = "webhook"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Each stand-in answers with a number of its own, so that each metric
|
|
||||||
// shows which part it was read from.
|
|
||||||
|
|
||||||
type ledgerStandIn struct{}
|
|
||||||
|
|
||||||
func (ledgerStandIn) Made(string) int { return 1 }
|
|
||||||
|
|
||||||
func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 }
|
|
||||||
|
|
||||||
type clientTableStandIn struct{}
|
|
||||||
|
|
||||||
func (clientTableStandIn) Len() int { return 3 }
|
|
||||||
|
|
||||||
type ruleFilesStandIn struct{}
|
|
||||||
|
|
||||||
func (ruleFilesStandIn) Len() int { return 4 }
|
|
||||||
|
|
||||||
type listsStandIn struct{}
|
|
||||||
|
|
||||||
func (listsStandIn) URLs() []string { return []string{listURL} }
|
|
||||||
|
|
||||||
func (listsStandIn) Failures(string) int { return 5 }
|
|
||||||
|
|
||||||
func (listsStandIn) Fetched(string) time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
type dnsblStandIn struct{}
|
|
||||||
|
|
||||||
func (dnsblStandIn) Zones() []string { return []string{zone} }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Queries(string) int { return 6 }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Failures(string) int { return 0 }
|
|
||||||
|
|
||||||
type abuseIPDBStandIn struct{}
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Checked() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Failures() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) BudgetLeft() int { return 7 }
|
|
||||||
|
|
||||||
type alertQueueStandIn struct{}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) DestinationsSet() []string {
|
|
||||||
return []string{destination}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Counts(string) alerts.Counts {
|
|
||||||
return alerts.Counts{Sent: 8}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Suppressed() int64 { return 0 }
|
|
||||||
|
|
||||||
type lookupDatabaseStandIn struct{}
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) ReadFailures() int { return 9 }
|
|
||||||
|
|
||||||
type remoteLogStandIn struct{}
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Sent() int64 { return 10 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Dropped() int64 { return 0 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Depth() int { return 0 }
|
|
||||||
|
|
||||||
func TestMetricsAreReadFromEachPart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
Now: func() time.Time { return time.Time{} },
|
|
||||||
Bans: ledgerStandIn{},
|
|
||||||
Clients: clientTableStandIn{},
|
|
||||||
Rules: ruleFilesStandIn{},
|
|
||||||
Lists: listsStandIn{},
|
|
||||||
DNSBL: dnsblStandIn{},
|
|
||||||
AbuseIPDB: abuseIPDBStandIn{},
|
|
||||||
Alerts: alertQueueStandIn{},
|
|
||||||
LookupFile: lookupDatabaseStandIn{},
|
|
||||||
RemoteLog: remoteLogStandIn{},
|
|
||||||
})
|
|
||||||
scraped := scrape(t, m)
|
|
||||||
|
|
||||||
for _, line := range []string{
|
|
||||||
`sww_bans_made_total{cause="limit",instance="app"} 1`,
|
|
||||||
`sww_active_bans{instance="app"} 2`,
|
|
||||||
`sww_tracked_clients{instance="app"} 3`,
|
|
||||||
`sww_rules_loaded{instance="app"} 4`,
|
|
||||||
`sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`,
|
|
||||||
`sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`,
|
|
||||||
`sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`,
|
|
||||||
`sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`,
|
|
||||||
`sww_lookup_database_read_failures_total{instance="app"} 9`,
|
|
||||||
`sww_remote_log_lines_sent_total{instance="app"} 10`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(scraped, "\n"+line+"\n") {
|
|
||||||
t.Errorf("metrics\n%s\nwant\n%s", scraped, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPartThatIsOffAddsNoMetric(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
scraped := scrape(t, metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
}))
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"sww_bans_made_total",
|
|
||||||
"sww_tracked_clients",
|
|
||||||
"sww_rules_loaded",
|
|
||||||
"sww_reputation_",
|
|
||||||
"sww_alert",
|
|
||||||
"sww_lookup_database_",
|
|
||||||
"sww_remote_log_",
|
|
||||||
} {
|
|
||||||
if strings.Contains(scraped, name) {
|
|
||||||
t.Errorf("metrics\n%s\nshow %s", scraped, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// scrape returns the metrics m serves.
|
|
||||||
func scrape(t *testing.T, m *metrics.Metrics) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
scraped := httptest.NewRecorder()
|
|
||||||
m.ServeHTTP(
|
|
||||||
scraped,
|
|
||||||
httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, "/", http.NoBody,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
return scraped.Body.String()
|
|
||||||
}
|
|
||||||
@@ -445,9 +445,12 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
|||||||
scores: map[string]int64{suspect: 100},
|
scores: map[string]int64{suspect: 100},
|
||||||
}
|
}
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one it refuses with an answer
|
// One check that AbuseIPDB answers, and one it refuses with an answer
|
||||||
// that names the key.
|
// that names the key.
|
||||||
@@ -500,9 +503,12 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
|
|||||||
p := abuseIPDBParams()
|
p := abuseIPDBParams()
|
||||||
p.DailyBudget = 5
|
p.DailyBudget = 5
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one that fails.
|
// One check that AbuseIPDB answers, and one that fails.
|
||||||
hitFrom(t, checker, suspect, true)
|
hitFrom(t, checker, suspect, true)
|
||||||
@@ -805,9 +811,6 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// instanceName is the label instance of the metrics the tests build.
|
|
||||||
const instanceName = "app"
|
|
||||||
|
|
||||||
// scrapeMetrics returns the metrics m serves.
|
// scrapeMetrics returns the metrics m serves.
|
||||||
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -358,9 +358,8 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
|||||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||||
}}
|
}}
|
||||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
wantZones(t, dnsbl, listed)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
@@ -405,9 +404,8 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) {
|
|||||||
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
||||||
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
||||||
}}, p)
|
}}, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
// Both clients are asked about before either answer comes, so that
|
// Both clients are asked about before either answer comes, so that
|
||||||
// the failure does not keep the zone from the other query.
|
// the failure does not keep the zone from the other query.
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
@@ -112,11 +111,7 @@ func New(params Params) *Server {
|
|||||||
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
|
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
|
||||||
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
|
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
|
||||||
func (s *Server) WaitForReports() {
|
func (s *Server) WaitForReports() {
|
||||||
s.h.reportsMu.Lock()
|
s.h.reports.stopAndWait()
|
||||||
s.h.reportsStopped = true
|
|
||||||
s.h.reportsMu.Unlock()
|
|
||||||
|
|
||||||
s.h.reports.Wait()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handler is the proxy. It holds what every request shares; what belongs to
|
// handler is the proxy. It holds what every request shares; what belongs to
|
||||||
@@ -136,11 +131,8 @@ type handler struct {
|
|||||||
// the time.
|
// the time.
|
||||||
breaker breaker
|
breaker breaker
|
||||||
breakerClock func() time.Time
|
breakerClock func() time.Time
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports are the goroutines sending reports to waf.
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
reports reportGoroutines
|
||||||
reports sync.WaitGroup
|
|
||||||
reportsMu sync.Mutex
|
|
||||||
reportsStopped bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newTransport returns what carries requests to the app. It never goes through
|
// newTransport returns what carries requests to the app. It never goes through
|
||||||
|
|||||||
+33
-8
@@ -8,6 +8,7 @@ import (
|
|||||||
"maps"
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
@@ -210,14 +211,7 @@ func (h *handler) sendReport(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
report *wafprotocol.Report,
|
report *wafprotocol.Report,
|
||||||
) {
|
) {
|
||||||
h.reportsMu.Lock()
|
h.reports.startUnlessStopped(func() {
|
||||||
defer h.reportsMu.Unlock()
|
|
||||||
|
|
||||||
if h.reportsStopped {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.reports.Go(func() {
|
|
||||||
err := h.postReport(context.WithoutCancel(ctx), report)
|
err := h.postReport(context.WithoutCancel(ctx), report)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.processLog.Warn("a report is dropped", "error", err.Error())
|
h.processLog.Warn("a report is dropped", "error", err.Error())
|
||||||
@@ -225,6 +219,37 @@ func (h *handler) sendReport(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportGoroutines are the goroutines sending reports to waf. None starts once
|
||||||
|
// stopAndWait has been called.
|
||||||
|
type reportGoroutines struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
stopped bool
|
||||||
|
running sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// startUnlessStopped runs send on a goroutine of its own, or drops it once
|
||||||
|
// stopAndWait has been called.
|
||||||
|
func (g *reportGoroutines) startUnlessStopped(send func()) {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
|
if g.stopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
g.running.Go(send)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopAndWait stops any more goroutines from starting, and returns once those
|
||||||
|
// running have ended.
|
||||||
|
func (g *reportGoroutines) stopAndWait() {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.stopped = true
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
g.running.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// postReport posts report to waf, and waits for its answer for up to
|
// postReport posts report to waf, and waits for its answer for up to
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||||
func (h *handler) postReport(
|
func (h *handler) postReport(
|
||||||
|
|||||||
@@ -2430,7 +2430,7 @@ func midnight() time.Time {
|
|||||||
// counts, with thresholds fill does not reach.
|
// counts, with thresholds fill does not reach.
|
||||||
func newParams(dir string) state.Params {
|
func newParams(dir string) state.Params {
|
||||||
discard := slog.New(slog.DiscardHandler)
|
discard := slog.New(slog.DiscardHandler)
|
||||||
m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"})
|
m := metrics.New(1, "app")
|
||||||
queue := alerts.New(alerts.Params{
|
queue := alerts.New(alerts.Params{
|
||||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||||
Events: alerts.Events(),
|
Events: alerts.Events(),
|
||||||
|
|||||||
+11
-18
@@ -392,33 +392,26 @@ func newMetrics(
|
|||||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||||
remote *remotelog.Sender,
|
remote *remotelog.Sender,
|
||||||
) *metrics.Metrics {
|
) *metrics.Metrics {
|
||||||
params := metrics.Params{
|
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
m.AddBansAndClients(ledger, limiter, utcNow)
|
||||||
InstanceName: cfg.InstanceName,
|
m.AddRules(ruleFiles)
|
||||||
Now: utcNow,
|
m.AddReputation(lists, dnsbl)
|
||||||
Bans: ledger,
|
|
||||||
Clients: limiter,
|
if cfg.AbuseIPDBKey != "" {
|
||||||
Rules: ruleFiles,
|
m.AddAbuseIPDB(abuseIPDB)
|
||||||
Lists: lists,
|
|
||||||
DNSBL: dnsbl,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A nil pointer in an interface field is not a nil interface, so a part
|
m.AddAlerts(alertQueue)
|
||||||
// that is off is left out rather than set to nil.
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
|
||||||
params.AbuseIPDB = abuseIPDB
|
|
||||||
}
|
|
||||||
|
|
||||||
if lookupFile != nil {
|
if lookupFile != nil {
|
||||||
params.LookupFile = lookupFile
|
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
||||||
}
|
}
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
params.RemoteLog = remote
|
m.AddRemoteLog(remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(params)
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
|
|||||||
+20
-27
@@ -548,11 +548,16 @@ func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
|||||||
params.CoreRuleSet = coreRuleSet
|
params.CoreRuleSet = coreRuleSet
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCounters adds to params the table of clients, the ledger and the
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
// anomaly counters, as sww builds them from params' settings.
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
func addCounters(params *waf.Params) {
|
func addCounters(params *waf.Params) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
|
|
||||||
|
params.Metrics = metrics.New(
|
||||||
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
|
cfg.InstanceName,
|
||||||
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
Limits: ratelimit.Limits{
|
Limits: ratelimit.Limits{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
@@ -582,18 +587,26 @@ func addCounters(params *waf.Params) {
|
|||||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
Alerts: params.Alerts,
|
Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
params.Metrics.AddBansAndClients(
|
||||||
|
params.Ledger,
|
||||||
|
params.Limiter,
|
||||||
|
params.Now,
|
||||||
|
)
|
||||||
|
params.Metrics.AddRules(params.Rules)
|
||||||
}
|
}
|
||||||
|
|
||||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||||
// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as
|
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||||
// sww builds them from params' settings and the parts addCounters added.
|
// them from params' settings and the parts addCounters added. The metrics
|
||||||
|
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
func addOutsideSources(
|
func addOutsideSources(
|
||||||
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||||
) {
|
) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
limiter, ledger := params.Limiter, params.Ledger
|
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
||||||
processLog, alertQueue := params.ProcessLog, params.Alerts
|
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||||
|
|
||||||
params.Lists = reputation.New(reputation.Params{
|
params.Lists = reputation.New(reputation.Params{
|
||||||
@@ -642,9 +655,6 @@ func addOutsideSources(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
m := newMetrics(params)
|
|
||||||
params.Metrics = m
|
|
||||||
|
|
||||||
params.GeoJS = geojs.New(geojs.Params{
|
params.GeoJS = geojs.New(geojs.Params{
|
||||||
URL: geojsURL,
|
URL: geojsURL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
@@ -657,29 +667,12 @@ func addOutsideSources(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
// newMetrics returns the metrics, as sww builds them from params' settings,
|
m.AddReputation(params.Lists, params.DNSBL)
|
||||||
// read from the ledger, the table of clients, the rules, the lists, the
|
|
||||||
// zones' verdicts, and AbuseIPDB's while its key is set.
|
|
||||||
func newMetrics(params *waf.Params) *metrics.Metrics {
|
|
||||||
cfg := params.Config
|
|
||||||
metricsParams := metrics.Params{
|
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
|
||||||
InstanceName: cfg.InstanceName,
|
|
||||||
Now: params.Now,
|
|
||||||
Bans: params.Ledger,
|
|
||||||
Clients: params.Limiter,
|
|
||||||
Rules: params.Rules,
|
|
||||||
Lists: params.Lists,
|
|
||||||
DNSBL: params.DNSBL,
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
if cfg.AbuseIPDBKey != "" {
|
||||||
metricsParams.AbuseIPDB = params.AbuseIPDB
|
m.AddAbuseIPDB(params.AbuseIPDB)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(metricsParams)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newClient returns an HTTP client that sends requests as they are made,
|
// newClient returns an HTTP client that sends requests as they are made,
|
||||||
|
|||||||
Reference in new issue
Block a user