Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer token and a priority and tag by event. The cooldown and the hourly limit stay shared; past them, each destination has its own bounded queue and backoff, and its own sent, failed and dropped counts. alerts.json keeps the alerts waiting by destination; one whose waiting is still a list stops the start, saying what to change. A control character in the ntfy token, or in the instance name ntfy is sent, stops the start. Judgement call: messages also give the detail's file, source, error and mode. Judgement call: alerts_suppressed_total is the same for every destination. Model: opus-5-5
2026-10-07 05:54:34 +02:00
parent 432097ee3f
commit 70a8ea1b92
14 changed files with 1586 additions and 428 deletions
@@ -226,45 +226,48 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
 	)
 }

-// AddAlerts adds the metrics of the alerts sent to
-// SWWAF_ALERT_WEBHOOK_URL, read from queue as the metrics are asked for,
-// with the destination webhook: the alerts sent, the requests to the
-// webhook that failed, and the alerts held back and dropped.
+// AddAlerts adds the metrics of the alerts sent to each destination set,
+// read from queue as the metrics are asked for, by destination: the
+// alerts sent, the requests to the destination that failed, the alerts
+// held back, which are the same for every destination, and those
+// dropped. With no destination set, it adds none.
 func (m *Metrics) AddAlerts(queue *alerts.Queue) {
-	webhook := prometheus.Labels{"destination": "webhook"}
+	for _, name := range queue.DestinationsSet() {
+		destination := prometheus.Labels{"destination": name}

-	m.registry.MustRegister(
-		prometheus.NewCounterFunc(prometheus.CounterOpts{
-			Name:        "smallwebwaf_alerts_sent_total",
-			Help:        "Alerts the destination took.",
-			ConstLabels: webhook,
-		}, func() float64 {
-			return float64(queue.Sent())
-		}),
-		prometheus.NewCounterFunc(prometheus.CounterOpts{
-			Name:        "smallwebwaf_alerts_failed_total",
-			Help:        "Requests to the destination that failed.",
-			ConstLabels: webhook,
-		}, func() float64 {
-			return float64(queue.Failed())
-		}),
-		prometheus.NewCounterFunc(prometheus.CounterOpts{
-			Name: "smallwebwaf_alerts_suppressed_total",
-			Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
-				"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
-			ConstLabels: webhook,
-		}, func() float64 {
-			return float64(queue.Suppressed())
-		}),
-		prometheus.NewCounterFunc(prometheus.CounterOpts{
-			Name: "smallwebwaf_alerts_dropped_total",
-			Help: "Alerts dropped, the oldest first, from a full queue, and alerts " +
-				"given up as the destination refused them.",
-			ConstLabels: webhook,
-		}, func() float64 {
-			return float64(queue.Dropped())
-		}),
-	)
+		m.registry.MustRegister(
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name:        "smallwebwaf_alerts_sent_total",
+				Help:        "Alerts the destination took.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Sent)
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name:        "smallwebwaf_alerts_failed_total",
+				Help:        "Requests to the destination that failed.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Failed)
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name: "smallwebwaf_alerts_suppressed_total",
+				Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
+					"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Suppressed())
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name: "smallwebwaf_alerts_dropped_total",
+				Help: "Alerts dropped, the oldest first, from a full queue, and " +
+					"alerts given up as the destination refused them.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Dropped)
+			}),
+		)
+	}
 }

 // ServeHTTP answers with the metrics in the Prometheus text format.