check / check (push) Waiting to run
Each alert is posted as a message to the Slack incoming webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer token and a priority and tag by event. The cooldown and the hourly limit stay shared; past them, each destination has its own bounded queue and backoff, and its own sent, failed and dropped counts. alerts.json keeps the alerts waiting by destination; one whose waiting is still a list stops the start, saying what to change. A control character in the ntfy token, or in the instance name ntfy is sent, stops the start. Judgement call: messages also give the detail's file, source, error and mode. Judgement call: alerts_suppressed_total is the same for every destination. Model: opus-5-5
265 lines
8.9 KiB
Go
265 lines
8.9 KiB
Go
package proxy_test
|
|
|
|
import (
|
|
"maps"
|
|
"net/http"
|
|
"net/netip"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
)
|
|
|
|
const (
|
|
alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
|
|
alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR"
|
|
// alertInstance is the instance every alert of these tests gives.
|
|
alertInstance = "fsn1app1/gitea"
|
|
)
|
|
|
|
func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, clk, server, queue := startWithAlerts(t, map[string]string{
|
|
rateLimitPerMinute: "1",
|
|
banScopeV4Prefix: "24",
|
|
})
|
|
start := clk.Now()
|
|
|
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
|
|
|
|
netblock := netip.MustParsePrefix("203.0.113.0/24")
|
|
ban := server.Ledger.Bans(netblock)[0]
|
|
|
|
// A request refused under the ban raises no other alert.
|
|
clk.advance(time.Minute)
|
|
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
|
|
|
wantAlerts(t, queue, banAlert(alerts.EventBan, start, client, bans.Ban{
|
|
Netblock: netblock, Cause: bans.CauseLimit,
|
|
Reason: "requests per minute over the limit of 1", Notes: ban.Notes,
|
|
}, requestlog.FormatTime(start.Add(time.Hour))))
|
|
|
|
if ban.Notes.Limit != 1 || ban.Notes.Request.Path != "/" {
|
|
t.Errorf("the alert's notes are %+v, want those of the broken limit", ban.Notes)
|
|
}
|
|
}
|
|
|
|
func TestAttackBanRaisesABanAlertThenAPermanentBanAlert(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, clk, server, queue := startWithAlerts(t, map[string]string{
|
|
rulesDir: writeRules(t, testRules),
|
|
})
|
|
start := clk.Now()
|
|
netblock := netip.MustParsePrefix(client + "/32")
|
|
other := netip.MustParsePrefix(otherClient + "/32")
|
|
|
|
// The probe bans the client for seven days, and its next request makes
|
|
// the ban permanent. The request after that changes nothing.
|
|
s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
|
|
|
attackBan := server.Ledger.Bans(netblock)[0]
|
|
|
|
clk.advance(time.Minute)
|
|
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
|
|
|
permanentBan := server.Ledger.Bans(netblock)[0]
|
|
|
|
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
|
|
|
// Another client's probe after its first ban has run out without a
|
|
// request makes a permanent ban at once.
|
|
s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
|
clk.advance(7 * 24 * time.Hour)
|
|
s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
|
|
|
otherBans := server.Ledger.Bans(other)
|
|
|
|
wantAlerts(t, queue,
|
|
attackAlert(alerts.EventBan, start, client, attackBan,
|
|
requestlog.FormatTime(start.Add(7*24*time.Hour))),
|
|
attackAlert(alerts.EventPermanentBan, start.Add(time.Minute), client,
|
|
permanentBan, "permanent"),
|
|
attackAlert(alerts.EventBan, start.Add(time.Minute), otherClient, otherBans[0],
|
|
requestlog.FormatTime(start.Add(time.Minute+7*24*time.Hour))),
|
|
attackAlert(alerts.EventPermanentBan, start.Add(time.Minute+7*24*time.Hour),
|
|
otherClient, otherBans[1], "permanent"),
|
|
)
|
|
}
|
|
|
|
func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, clk, server, queue := startWithAlerts(t, map[string]string{
|
|
mode: observe,
|
|
rateLimitPerMinute: "2",
|
|
rulesDir: writeRules(t, testRules),
|
|
})
|
|
start := clk.Now()
|
|
|
|
// A ban for a clear sign of attack, which a request under it would make
|
|
// permanent.
|
|
group := netip.MustParsePrefix(ipv6Group)
|
|
attackBan, _ := server.Ledger.BanForAttack(group, start, bans.Notes{RuleID: "probe"})
|
|
|
|
// The third request breaks the limit, and so does the fourth, within the
|
|
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
|
for range 4 {
|
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
}
|
|
|
|
s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
|
|
line := s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
|
|
|
|
// No ban is made, and none made permanent.
|
|
if held := server.Ledger.Snapshot(); len(held) != 1 || held[0] != attackBan ||
|
|
line.BanExpires != requestlog.FormatTime(attackBan.Expires) {
|
|
t.Errorf("the ledger holds %+v, and the log line gives %s, want the ban "+
|
|
"for the attack alone, as it was", held, line.BanExpires)
|
|
}
|
|
|
|
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
|
if len(waiting) != 3 || queue.Suppressed() != 0 {
|
|
t.Fatalf("%d alerts wait and %d are held back, want 3 and 0: %+v",
|
|
len(waiting), queue.Suppressed(), waiting)
|
|
}
|
|
|
|
limitNotes, _ := waiting[0].Detail["notes"].(bans.Notes)
|
|
attackNotes, _ := waiting[1].Detail["notes"].(bans.Notes)
|
|
|
|
if limitNotes.Limit != 2 || limitNotes.Request.Path != "/" ||
|
|
attackNotes.Request.Path != "/.env" {
|
|
t.Errorf("the notes are %+v and %+v, want those of the broken limit and "+
|
|
"of the probe", limitNotes, attackNotes)
|
|
}
|
|
|
|
// Each alert is the one enforce mode would have raised, with mode
|
|
// observe in its detail.
|
|
want := []alerts.Alert{
|
|
banAlert(alerts.EventBan, start, client, bans.Ban{
|
|
Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit,
|
|
Reason: "requests per minute over the limit of 2", Notes: limitNotes,
|
|
}, requestlog.FormatTime(start.Add(time.Hour))),
|
|
attackAlert(alerts.EventBan, start, otherClient, bans.Ban{
|
|
Netblock: netip.MustParsePrefix(otherClient + "/32"), Notes: attackNotes,
|
|
}, requestlog.FormatTime(start.Add(7*24*time.Hour))),
|
|
attackAlert(alerts.EventPermanentBan, start, ipv6Client, attackBan, permanent),
|
|
}
|
|
for _, alert := range want {
|
|
alert.Detail["mode"] = observe
|
|
}
|
|
|
|
wantAlerts(t, queue, want...)
|
|
}
|
|
|
|
func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, _, _, queue := startWithAlerts(t, map[string]string{
|
|
mode: observe,
|
|
rateLimitPerMinute: "2",
|
|
rulesDir: writeRules(t, testRules),
|
|
alertMaxPerHour: "2",
|
|
})
|
|
|
|
// The client's third request breaks the limit, and raises the first
|
|
// alert of the hour. Its fourth is within the cooldown.
|
|
for range 4 {
|
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
}
|
|
|
|
// The other client's first probe raises the second. Its second probe is
|
|
// within the cooldown.
|
|
for range 2 {
|
|
s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
|
|
}
|
|
|
|
// The IPv6 client's third request breaks the limit past the two alerts
|
|
// an hour.
|
|
for range 3 {
|
|
s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
|
|
}
|
|
|
|
// Had the ban been worked out for any of the requests within the
|
|
// cooldown or past the two an hour, its alert would have been raised,
|
|
// held back and counted.
|
|
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
|
if len(waiting) != 2 || queue.Suppressed() != 0 {
|
|
t.Errorf("%d alerts wait and %d are held back, want 2 and 0: %+v",
|
|
len(waiting), queue.Suppressed(), waiting)
|
|
}
|
|
}
|
|
|
|
// startWithAlerts is startWithClock with alerts to a webhook, which is
|
|
// never sent them, and returns the queue they wait in as well.
|
|
func startWithAlerts(
|
|
t *testing.T, env map[string]string,
|
|
) (*sender, *clock, *proxy.Server, *alerts.Queue) {
|
|
t.Helper()
|
|
|
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
|
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
|
|
settings := map[string]string{
|
|
trustedProxies: trustLocalhost,
|
|
alertWebhookURL: "https://alerts.example/smallwebwaf",
|
|
instanceName: alertInstance,
|
|
}
|
|
maps.Copy(settings, env)
|
|
|
|
addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
|
|
|
|
return &sender{t: t, addr: addr, out: out}, clk, server, queue
|
|
}
|
|
|
|
// banAlert returns the alert for event, raised by a request from client at
|
|
// the time raised, for ban, with its netblock, cause, reason and notes,
|
|
// which ends at expires, as the log line gives it.
|
|
func banAlert(
|
|
event string, raised time.Time, client string, ban bans.Ban, expires string,
|
|
) alerts.Alert {
|
|
return alerts.Alert{
|
|
Instance: alertInstance,
|
|
Time: raised,
|
|
Event: event,
|
|
Client: netip.MustParseAddr(client),
|
|
Netblock: ban.Netblock,
|
|
Reason: ban.Reason,
|
|
Detail: map[string]any{
|
|
"cause": ban.Cause, "ban_expires": expires, "notes": ban.Notes,
|
|
},
|
|
}
|
|
}
|
|
|
|
// attackAlert is banAlert for a ban for the probe rule of testRules, with
|
|
// the netblock and the notes of ban.
|
|
func attackAlert(
|
|
event string, raised time.Time, client string, ban bans.Ban, expires string,
|
|
) alerts.Alert {
|
|
return banAlert(event, raised, client, bans.Ban{
|
|
Netblock: ban.Netblock, Cause: bans.CauseAttack, Reason: "matched the rule probe",
|
|
Notes: ban.Notes,
|
|
}, expires)
|
|
}
|
|
|
|
// wantAlerts checks the alerts waiting in queue, in order.
|
|
func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) {
|
|
t.Helper()
|
|
|
|
got := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
|
if len(got) != len(want) {
|
|
t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got)
|
|
}
|
|
|
|
for i := range want {
|
|
if !reflect.DeepEqual(got[i], want[i]) {
|
|
t.Errorf("alert %d is\n%+v\nwant\n%+v", i, got[i], want[i])
|
|
}
|
|
}
|
|
}
|