package proxy_test import ( "bytes" "encoding/json" "maps" "net" "net/http" "net/netip" "strconv" "strings" "testing" "time" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) const ( // siteA and siteB are two sites behind one servewaf. siteA = "a.example" siteB = "b.example" // rproxyPeer is what the tests' questions give as rproxy's TCP peer. rproxyPeer = "172.18.0.2" ) // serveWAF is a servewaf running for one test, on a clock set to midnight. type serveWAF struct { t *testing.T addr string out *output server *proxy.Server clock *clock queue *alerts.Queue } // startServeWAF starts servewaf with the settings in env, clients' AS numbers // and countries looked up at geojsURL. func startServeWAF(t *testing.T, geojsURL string, env map[string]string) *serveWAF { t.Helper() clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} settings := map[string]string{ alertWebhookURL: "https://alerts.example/servewaf", instanceName: alertInstance, } maps.Copy(settings, env) server, out, queue := newServerWith(t, proxy.NewServeWAF, "http://127.0.0.1:8081", geojsURL, clk.Now, settings) listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") if err != nil { t.Fatalf("listen: %v", err) } go func() { _ = server.Serve(listener) }() t.Cleanup(func() { _ = server.Close() }) return &serveWAF{ t: t, addr: listener.Addr().String(), out: out, server: server, clock: clk, queue: queue, } } // question makes rproxy's question about a request for path on host, from the // visitor at from. func (s *serveWAF) question(host, from, path string) *http.Request { s.t.Helper() req := newRequest(s.t, http.MethodGet, s.addr, path, http.NoBody) req.Host = host req.Header.Set(proxy.ClientHeader, from) req.Header.Set(proxy.PeerHeader, rproxyPeer) return req } // ask sends the question about a request for path on siteA from the visitor at // from, and returns the answer. func (s *serveWAF) ask(from, path string) answer { s.t.Helper() return do(s.t, s.question(siteA, from, path)) } // allowed asks about a request for / on siteA, checks that the answer is allow, // and returns the id the answer gives for the report. func (s *serveWAF) allowed(from string) string { s.t.Helper() got := s.ask(from, "/") wantDisposition(s.t, got, http.StatusOK, proxy.DispositionAllow) return got.header.Get(requestIDHeader) } // report sends rproxy's report and returns the answer. func (s *serveWAF) report(report proxy.Report) answer { s.t.Helper() body, err := json.Marshal(report) if err != nil { s.t.Fatalf("encode the report: %v", err) } return do(s.t, newRequest(s.t, http.MethodPost, s.addr, proxy.ReportPath, bytes.NewReader(body))) } // passedOn asks about a request from the visitor at from on siteA, and reports // it as answered 200 with bodyBytes and answerBytes. func (s *serveWAF) passedOn(from string) { s.t.Helper() got := s.report(proxy.Report{ RequestID: s.allowed(from), Status: http.StatusOK, RequestBytes: bodyBytes, ResponseBytes: answerBytes, }) wantStatus(s.t, got, http.StatusNoContent) } // history returns what the table of clients holds of client's history. func (s *serveWAF) history() ratelimit.History { held, _ := s.server.Limiter.Client(netip.MustParsePrefix(client + "/32")) return held.History } // metrics returns what the metrics endpoint answers, for a server started with // the token as SWWAF_METRICS_TOKEN. func (s *serveWAF) metrics() string { s.t.Helper() req := newRequest(s.t, http.MethodGet, s.addr, proxy.MetricsPath, http.NoBody) req.Header.Set("Authorization", bearer) got := do(s.t, req) wantStatus(s.t, got, http.StatusOK) return string(got.body) } // wantDisposition checks an answer's status and disposition. func wantDisposition(t *testing.T, got answer, status int, disposition string) { t.Helper() wantStatus(t, got, status) if answered := got.header.Get(proxy.DispositionHeader); answered != disposition { t.Errorf("answered with disposition %q, want %q", answered, disposition) } } // wantBody checks an answer's body. func wantBody(t *testing.T, got answer, want string) { t.Helper() if string(got.body) != want { t.Errorf("answered with the body %q, want %q", got.body, want) } } func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) { t.Parallel() s := startServeWAF(t, "", nil) req := s.question(siteA, client, "/page?q=1") req.Header.Set(requestIDHeader, "the-id") req.Header.Set("X-Forwarded-Proto", "https") req.Header.Set(proxy.ContentLengthHeader, "1234") got := do(t, req) wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) wantBody(t, got, "") if id := got.header.Get(requestIDHeader); id != "the-id" { t.Errorf("answered with the id %q, want the-id", id) } line := s.out.requestLine(t) wantLine(t, line, http.StatusOK, requestlog.ActionForward) if line.ClientIP != client || line.PeerIP != rproxyPeer || line.Host != siteA || line.Path != "/page" || line.Query != "q=1" || line.RequestID != "the-id" || line.Scheme != "https" || line.ContentLength != 1234 { t.Errorf("log line is %+v", line.Line) } } func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{rateLimitPerMinute: "1"}) s.allowed("2001:db8::1") got := s.ask("2001:db8::2", "/") wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) } func TestQuestionWithoutAnAddressGetsNoDisposition(t *testing.T) { t.Parallel() s := startServeWAF(t, "", nil) got := s.ask("not an address", "/") wantDisposition(t, got, http.StatusBadRequest, "") } func TestQuestionDeniedGetsTheBanResponse(t *testing.T) { t.Parallel() for _, status := range []int{http.StatusForbidden, http.StatusTooManyRequests} { t.Run(strconv.Itoa(status), func(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ denyNets: client, banResponse: strconv.Itoa(status), }) got := s.ask(client, "/") wantDisposition(t, got, status, proxy.DispositionDeny) wantBody(t, got, http.StatusText(status)+"\n") wantLine(t, s.out.requestLine(t), status, requestlog.ActionDenied) }) } } func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ denyNets: client, banResponse: banResponseClose, metricsToken: token, }) got := s.ask(client, "/") wantDisposition(t, got, http.StatusOK, proxy.DispositionClose) wantBody(t, got, "") // The visitor is sent nothing. wantLine(t, s.out.requestLine(t), 0, requestlog.ActionDenied) wantMetric(t, s.metrics(), `smallwebwaf_requests_total{action="denied",instance="`+ alertInstance+`",status_class="none"}`, 1) } func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ rulesDir: writeRules(t, testRules), banResponse: "429", }) got := s.ask(client, "/blocked") wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) wantLine(t, s.out.requestLine(t), http.StatusForbidden, requestlog.ActionRuleBlocked) got = s.ask(client, "/.env") wantDisposition(t, got, http.StatusTooManyRequests, proxy.DispositionDeny) wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests, requestlog.ActionBanned) } func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ metricsToken: token, denyNets: otherClient, }) for _, path := range []string{proxy.HealthPath, proxy.MetricsPath, proxy.ReportPath} { got := s.ask(client, path) wantDisposition(t, got, http.StatusNotFound, proxy.DispositionDeny) } wantLine(t, s.out.requestLine(t), http.StatusNotFound, requestlog.ActionAdmin) got := s.ask(otherClient, proxy.MetricsPath) wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) } func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{mode: observe, denyNets: client}) got := s.ask(client, "/") wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) line := s.out.requestLine(t) wantLine(t, line, http.StatusOK, requestlog.ActionForward) wantWouldAction(t, line, requestlog.ActionDenied) } func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) { t.Parallel() for _, tc := range []struct { setting string wantASN, wantCountry string }{ {"true", "AS64496", "DE"}, {"false", "", ""}, } { t.Run(tc.setting, func(t *testing.T) { t.Parallel() geojsURL, _ := startGeoJS(t) s := startServeWAF(t, geojsURL, map[string]string{ addLookupHeaders: tc.setting, lookupTimeout: "1h", deniedCountries: "kp", }) got := s.ask(fromDE, "/") wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) asn, country := got.header.Get("X-Client-Asn"), got.header.Get("X-Client-Country") if asn != tc.wantASN || country != tc.wantCountry { t.Errorf("answered with the AS number %q and the country %q, want %q and %q", asn, country, tc.wantASN, tc.wantCountry) } }) } } func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) // The request's own question counts no bytes: it is the report that does. id := s.allowed(client) s.allowed(client) if held := s.server.Ledger.Snapshot(); len(held) != 0 { t.Fatalf("banned before any report: %v", held) } // 100 bytes are over the limit of 99. got := s.report(proxy.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, }) wantStatus(t, got, http.StatusNoContent) held := s.server.Ledger.Snapshot() if len(held) != 1 || held[0].Cause != bans.CauseLimit || held[0].Notes.Kind != ratelimit.KindBytes || held[0].Notes.Request.Status != http.StatusBadGateway || held[0].Notes.Request.Host != siteA { t.Errorf("bans held are %+v", held) } got = s.ask(client, "/") wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) if !strings.Contains(s.out.text(), `"msg":"a report's bytes broke a byte limit"`) { t.Error("no message says that a report's bytes broke a byte limit") } } func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) { t.Parallel() geojsURL, _ := startGeoJS(t) // 100 bytes pass half of a limit of 99 and are within a limit of 199. s := startServeWAF(t, geojsURL, map[string]string{ bytesLimitPerMinute: "199", countryBytesPercent: countryDEHalf, lookupTimeout: "1h", }) s.passedOn(fromKP) wantDisposition(t, s.ask(fromKP, "/"), http.StatusOK, proxy.DispositionAllow) s.passedOn(fromDE) wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden, proxy.DispositionDeny) } func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ "SWWAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE": "1000", }) id := s.allowed(client) if requests := s.history().Requests; requests != 0 { t.Errorf("history holds %d requests before the report, want none", requests) } if counters := s.server.Anomalies.Snapshot(); len(counters) != 0 { t.Errorf("anomaly counters %v before the report, want none", counters) } got := s.report(proxy.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, }) wantStatus(t, got, http.StatusNoContent) history := s.history() if history.Requests != 1 || history.Forwarded != 1 || history.RequestBytes != bodyBytes || history.ResponseBytes != answerBytes || history.Responses.Status5xx != 1 { t.Errorf("history is %+v", history) } if counters := s.server.Anomalies.Snapshot(); len(counters) != 1 { t.Errorf("anomaly counters %v, want the client's", counters) } } func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{rateLimitPerMinute: "1"}) s.allowed(client) wantDisposition(t, s.ask(client, "/"), http.StatusForbidden, proxy.DispositionDeny) history := s.history() if history.Requests != 1 || history.Refused != 1 { t.Errorf("history is %+v, want the refused request alone", history) } } func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{metricsToken: token}) id := s.allowed(client) const series = `{action="forward",instance="` + alertInstance + `",status_class="5xx"}` metrics := s.metrics() wantNoSeries(t, metrics, "smallwebwaf_requests_total"+series) wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="`+alertInstance+`"}`, 1) // the scrape itself got := s.report(proxy.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, DurationTotal: 5, DurationUpstreamTotal: 4, }) wantStatus(t, got, http.StatusNoContent) metrics = s.metrics() wantMetric(t, metrics, "smallwebwaf_requests_total"+series, 1) wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+series, bodyBytes) wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+series, answerBytes) wantMetric(t, metrics, `smallwebwaf_upstream_duration_seconds_sum{instance="`+ alertInstance+`"}`, 0.004) } func TestReportForARequestNotHeldIsIgnored(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) got := s.report(proxy.Report{RequestID: "no-such-request", ResponseBytes: 1000}) wantStatus(t, got, http.StatusNoContent) if clients := s.server.Limiter.Snapshot(); len(clients) != 0 { t.Errorf("the table of clients holds %v", clients) } if held := s.server.Ledger.Snapshot(); len(held) != 0 { t.Errorf("bans held are %v", held) } // A second report of the same request does not count it twice. id := s.allowed(client) for range 2 { got = s.report(proxy.Report{RequestID: id, ResponseBytes: answerBytes}) wantStatus(t, got, http.StatusNoContent) } if counted := s.history().ResponseBytes; counted != answerBytes { t.Errorf("history holds %d response bytes, want %d", counted, answerBytes) } } func TestReportThatCannotBeReadIsRefusedAndTheRequestStaysHeld(t *testing.T) { t.Parallel() s := startServeWAF(t, "", nil) id := s.allowed(client) for _, body := range []string{ "not JSON", `{"request_id":"` + id + `","request_bytes":-1}`, `{"request_id":"` + id + `","response_bytes":-1}`, } { got := do(t, newRequest(t, http.MethodPost, s.addr, proxy.ReportPath, strings.NewReader(body))) wantStatus(t, got, http.StatusBadRequest) } got := s.report(proxy.Report{RequestID: id, ResponseBytes: answerBytes}) wantStatus(t, got, http.StatusNoContent) if counted := s.history().ResponseBytes; counted != answerBytes { t.Errorf("history holds %d response bytes, want %d", counted, answerBytes) } } func TestVisitorCannotHaveItsReportTakenForAnothersByChoosingItsID(t *testing.T) { t.Parallel() const ( chosen = "chosen-by-a-visitor" exempt = "/static/x" ) s := startServeWAF(t, "", map[string]string{ bytesLimitPerMinute: byteLimit, rateLimitExemptPaths: "/static/", }) // The request whose bytes are not counted takes the id first. first := s.question(siteA, client, exempt) first.Header.Set(requestIDHeader, chosen) gotFirst := do(t, first) wantDisposition(t, gotFirst, http.StatusOK, proxy.DispositionAllow) second := s.question(siteA, client, "/download") second.Header.Set(requestIDHeader, chosen) gotSecond := do(t, second) wantDisposition(t, gotSecond, http.StatusOK, proxy.DispositionAllow) id := gotSecond.header.Get(requestIDHeader) if gotFirst.header.Get(requestIDHeader) != chosen || id == chosen || id == "" { t.Errorf("answered with the ids %q and %q, want %q and a new one", gotFirst.header.Get(requestIDHeader), id, chosen) } if line := s.out.requestLines(t, 2)[1]; line.RequestID != id { t.Errorf("log line has request_id %q, want %q", line.RequestID, id) } got := s.report(proxy.Report{RequestID: id, ResponseBytes: 1000}) wantStatus(t, got, http.StatusNoContent) if held := s.server.Ledger.Snapshot(); len(held) != 1 { t.Errorf("bans held are %v, want the one the download's bytes make", held) } } func TestIDAJSONReportCannotCarryIsReplaced(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) req := s.question(siteA, client, "/") req.Header.Set(requestIDHeader, "not-utf-8-\xff") got := do(t, req) wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) report := s.report(proxy.Report{ RequestID: got.header.Get(requestIDHeader), ResponseBytes: 1000, }) wantStatus(t, report, http.StatusNoContent) if held := s.server.Ledger.Snapshot(); len(held) != 1 { t.Errorf("bans held are %v, want the one the report's bytes make", held) } } func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{rulesDir: writeRules(t, testRules)}) got := do(t, s.question(siteA, client, "/.env")) wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) got = do(t, s.question(siteB, client, "/")) wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) lines := s.out.requestLines(t, 2) if lines[0].Host != siteA || lines[1].Host != siteB { t.Errorf("log lines have the hosts %q and %q", lines[0].Host, lines[1].Host) } held := s.server.Ledger.Snapshot() if len(held) != 1 || held[0].Notes.Request.Host != siteA { t.Errorf("bans held are %+v, want one whose notes name %s", held, siteA) } waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) == 0 || waiting[0].Host != siteA { t.Errorf("alerts waiting are %+v, want the ban's, naming %s", waiting, siteA) } } func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{rateLimitPerMinute: "1"}) s.allowed(client) got := do(t, s.question(siteB, client, "/")) wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) } func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) { t.Parallel() // Were a check run on its requests, localhost's would be refused. s := startServeWAF(t, "", map[string]string{ metricsToken: token, adminToken: adminSecret, denyNets: localhost, }) for _, tc := range []struct { path, authorization string want int }{ {proxy.HealthPath, "", http.StatusOK}, {proxy.MetricsPath, bearer, http.StatusOK}, {proxy.MetricsPath, "", http.StatusUnauthorized}, {proxy.BansPath, adminBearer, http.StatusOK}, {proxy.BansPath, "", http.StatusUnauthorized}, {proxy.ClientsPath + client, adminBearer, http.StatusOK}, {proxy.ReportPath, "", http.StatusNotFound}, {"/_smallwebwaf/other", "", http.StatusNotFound}, {"/", "", http.StatusNotFound}, } { req := newRequest(t, http.MethodGet, s.addr, tc.path, http.NoBody) req.Header.Set("Authorization", tc.authorization) wantDisposition(t, do(t, req), tc.want, "") // The same request carrying the header is a question about it. req = newRequest(t, http.MethodGet, s.addr, tc.path, http.NoBody) req.Header.Set("Authorization", tc.authorization) req.Header.Set(proxy.ClientHeader, client) if strings.HasPrefix(tc.path, "/_smallwebwaf/") { wantDisposition(t, do(t, req), http.StatusNotFound, proxy.DispositionDeny) } else { wantDisposition(t, do(t, req), http.StatusOK, proxy.DispositionAllow) } } } func TestReportWritesNoLogLine(t *testing.T) { t.Parallel() s := startServeWAF(t, "", nil) s.passedOn(client) // The health check's line comes after the question's. wantStatus(t, get(t, s.addr, proxy.HealthPath), http.StatusOK) lines := s.out.requestLines(t, 2) if len(lines) != 2 || lines[1].Path != proxy.HealthPath { t.Errorf("request log lines are %+v, want the question's and the health check's", lines) } } func TestSettingsOfTheLimitsAndTheProxiesAreIgnored(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ requestMaxBytes: "1", clientHeaderMaxBytes: "5K", clientRequestTimeout: off, clientResponseTimeout: off, trustedProxies: localhost, }) req := newRequest(t, http.MethodPost, s.addr, "/", strings.NewReader("a body")) req.Host = siteA req.Header.Set(proxy.ClientHeader, client) req.Header.Set(proxy.ContentLengthHeader, "1073741824") req.Header.Set(forwardedFor, otherClient) req.Header.Set("X-Long", strings.Repeat("x", 40<<10)) got := do(t, req) wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) line := s.out.requestLine(t) if line.ClientIP != client || line.ContentLength != 1<<30 { t.Errorf("log line has client_ip %q and content_length %d", line.ClientIP, line.ContentLength) } } func TestChecksSeeTheHeadersTheVisitorSent(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ rulesDir: writeRules(t, ` own-header header:X-Smallwebwaf-Client block . announced header:Content-Length block ^70000$ carried header:Content-Length block ^6$ `), }) // What the question's own Content-Length says, 6, is not what the visitor // announced. for _, tc := range []struct { announced string want int }{ {"70000", http.StatusForbidden}, {"", http.StatusOK}, {"6000", http.StatusOK}, } { req := newRequest(t, http.MethodPost, s.addr, "/", strings.NewReader("a body")) req.Header.Set(proxy.ClientHeader, client) if tc.announced != "" { req.Header.Set(proxy.ContentLengthHeader, tc.announced) } wantStatus(t, do(t, req), tc.want) } } func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ wafMode: block, wafBodyLimit: "1M", }) ask := func(contentType, body string) answer { req := newRequest(t, http.MethodPost, s.addr, "/", strings.NewReader(body)) req.Header.Set(proxy.ClientHeader, client) req.Header.Set("Content-Type", contentType) req.Header.Set(proxy.ContentLengthHeader, "100000") return do(t, req) } got := ask(formData, "id=1'%20OR%20'1'='1") wantDisposition(t, got, http.StatusForbidden, proxy.DispositionDeny) // The first 64 KiB of a longer JSON body cannot be parsed, which is no // attack, so the Core Rule Set does not read it. cut := `{"text":"` + strings.Repeat("x", proxy.QuestionBodyMaxBytes) got = ask("application/json", cut[:proxy.QuestionBodyMaxBytes]) wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow) } func TestAnomalyAlertNamesTheSite(t *testing.T) { t.Parallel() s := startServeWAF(t, "", map[string]string{ "SWWAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE": "1", }) for range 2 { s.passedOn(client) } waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || waiting[0].Event != alerts.EventAnomaly || waiting[0].Host != siteA { t.Errorf("alerts waiting are %+v, want an anomaly naming %s", waiting, siteA) } }