package proxy_test import ( "io" "net/http" "net/http/httptest" "net/netip" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) const ( metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name metricsTopN = "SWWAF_METRICS_TOP_N" // token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a // request carries it. token = "0123456789abcdef0123456789abcdef" bearer = "Bearer " + token ) func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) { t.Parallel() var calls atomic.Int32 app := startApp(t, func(http.ResponseWriter, *http.Request) { calls.Add(1) }) addr, out := startProxy(t, app.URL, nil) // An empty token does not match the unset one either. for i, authorization := range []string{bearer, "Bearer ", ""} { req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody) if authorization != "" { req.Header.Set("Authorization", authorization) } wantStatus(t, do(t, req), http.StatusNotFound) wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound, requestlog.ActionAdmin) } if calls.Load() != 0 { t.Errorf("the app was called %d times, want never", calls.Load()) } } func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) { t.Parallel() var calls atomic.Int32 app := startApp(t, func(http.ResponseWriter, *http.Request) { calls.Add(1) }) addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token}) for i, tc := range []struct { method, path, authorization string status int }{ {http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized}, { http.MethodGet, proxy.MetricsPath, "Bearer " + strings.ToUpper(token), http.StatusUnauthorized, }, {http.MethodGet, proxy.MetricsPath, "Basic " + token, http.StatusUnauthorized}, {http.MethodGet, proxy.MetricsPath, bearer, http.StatusOK}, {http.MethodGet, proxy.MetricsPath, "bearer " + token, http.StatusOK}, {http.MethodPost, proxy.MetricsPath, bearer, http.StatusNotFound}, {http.MethodGet, proxy.MetricsPath + "/", bearer, http.StatusNotFound}, {http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound}, {http.MethodPost, proxy.HealthPath, "", http.StatusNotFound}, } { req := newRequest(t, tc.method, addr, tc.path, http.NoBody) if tc.authorization != "" { req.Header.Set("Authorization", tc.authorization) } got := do(t, req) wantStatus(t, got, tc.status) wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin) if tc.status == http.StatusUnauthorized && got.header.Get("WWW-Authenticate") != "Bearer" { t.Errorf("%q was answered without WWW-Authenticate: Bearer", tc.authorization) } if tc.status == http.StatusOK && !strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") { t.Errorf("the metrics are\n%s", got.body) } } if calls.Load() != 0 { t.Errorf("the app was called %d times, want never", calls.Load()) } } func TestMetricsAreAskedForThroughTheChecks(t *testing.T) { t.Parallel() s, _, _ := startWithClock(t, "", map[string]string{ metricsToken: token, rateLimitPerMinute: "1", }) // Asking for the metrics counts toward the client's limit of one // request a minute, so its next request breaks it, and bans it. A // banned client is refused the metrics too. s.scrape(client) s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer, http.StatusForbidden, requestlog.ActionBanned) } func TestMetricsCountTheTraffic(t *testing.T) { t.Parallel() arrived, release := make(chan struct{}), make(chan struct{}) app := startApp(t, func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) if r.URL.Path == "/held" { close(arrived) <-release } _, _ = io.WriteString(w, "hello") }) releaseApp := sync.OnceFunc(func() { close(release) }) t.Cleanup(releaseApp) addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token}) got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))) wantStatus(t, got, http.StatusOK) wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound) out.requestLines(t, 2) forward := `{action="forward",status_class="2xx"}` notFound := `{action="admin",status_class="4xx"}` // The request for the metrics is itself under way. metrics := scrape(t, addr) wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1) wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1) wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3) wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5) wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound, float64(len("Not Found\n"))) wantMetric(t, metrics, "smallwebwaf_request_duration_seconds_count", 2) wantMetric(t, metrics, "smallwebwaf_upstream_duration_seconds_count", 1) wantMetric(t, metrics, "smallwebwaf_requests_in_flight", 1) metric(t, metrics, "go_goroutines") metric(t, metrics, "process_start_time_seconds") // A request the app holds is under way until it ends. httpClient := newClient(t) held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody) ended := make(chan error, 1) go func() { res, err := httpClient.Do(held) if err == nil { err = readAnswer(res).err } ended <- err }() <-arrived wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 2) releaseApp() err := <-ended if err != nil { t.Fatalf("held request: %v", err) } out.requestLines(t, 5) wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 1) } func TestMetricsCountLimitsAndBans(t *testing.T) { t.Parallel() const ( scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS denied = "192.0.2.50" // in SWWAF_DENY_NETS ) s, clk, _ := startWithClock(t, "", map[string]string{ metricsToken: token, rateLimitPerMinute: "1", rateLimitExemptNets: scraper, denyNets: denied, banResponse: "close", limitBanDuration: "1h", maxBanDuration: "2h", }) // SWWAF_BAN_RESPONSE=close sends no status at all. s.get(denied, 0, requestlog.ActionDenied) // A first broken limit bans for an hour. s.get(client, http.StatusOK, requestlog.ActionForward) s.get(client, 0, requestlog.ActionRateLimited) metrics := s.scrape(scraper) wantMetric(t, metrics, `smallwebwaf_requests_total{action="denied",status_class="none"}`, 1) wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 1) wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 1) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 1) wantMetric(t, metrics, "smallwebwaf_active_bans", 1) wantMetric(t, metrics, "smallwebwaf_permanent_bans", 0) clk.advance(time.Hour) wantMetric(t, s.scrape(scraper), "smallwebwaf_active_bans", 0) // A limit broken again right after would ban for three hours, longer // than SWWAF_MAX_BAN_DURATION, so the ban is permanent. s.get(client, http.StatusOK, requestlog.ActionForward) s.get(client, 0, requestlog.ActionRateLimited) metrics = s.scrape(scraper) wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 2) wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 2) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 2) wantMetric(t, metrics, "smallwebwaf_active_bans", 1) wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1) // denied, client, and the scraper as of its earlier requests. wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3) } func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { t.Parallel() const fromFR = "198.51.100.20" app := startApp(t, func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) _, _ = io.WriteString(w, "hello") }) env := map[string]string{ trustedProxies: trustLocalhost, metricsToken: token, metricsTopN: "2", deniedCountries: "kp", } addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env) // The answers are kept before the requests, so that none waits for // GeoJS. server.GeoJS.Load([]lookup.Answer{ keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"), }) lines := 0 send := func(from string, times, status int) { t.Helper() for range times { req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")) req.Header.Set(forwardedFor, from) wantStatus(t, do(t, req), status) // Each is counted before the next is sent, so that the // countries are ranked in the order sent. lines++ out.requestLines(t, lines) } } // With two countries of their own, the third is counted as other. send(fromKP, 3, http.StatusForbidden) send(fromDE, 2, http.StatusOK) send(fromFR, 1, http.StatusOK) metrics := scrape(t, addr) lines++ wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`, 2) wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 1) wantMetric(t, metrics, `smallwebwaf_country_list_refusals_total{country="KP"}`, 3) wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="KP"}`, 0) wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`, 6) wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="KP"}`, float64(3*len("Forbidden\n"))) wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="other"}`, float64(len("hello"))) wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`) // Once FR is busier than DE, it takes DE's place: its series counts // from then on, and DE's is gone. send(fromFR, 3, http.StatusOK) metrics = scrape(t, addr) wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`, 2) wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 2) wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`) wantNoSeries(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`) } func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { t.Parallel() geojs := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) t.Cleanup(geojs.Close) app := startApp(t, func(http.ResponseWriter, *http.Request) {}) addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{ trustedProxies: trustLocalhost, metricsToken: token, deniedCountries: "kp", }) // GeoJS fails, so the client counts as coming from an unknown country, // which SWWAF_DENIED_COUNTRIES does not refuse. req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) req.Header.Set(forwardedFor, fromDE) wantStatus(t, do(t, req), http.StatusOK) // The client stops waiting for GeoJS after a second, so GeoJS's // failure can come after its request has ended. deadline := time.Now().Add(waitLimit) metrics := scrape(t, addr) for metric(t, metrics, "smallwebwaf_geojs_failures_total") == 0 && time.Now().Before(deadline) { time.Sleep(pollInterval) metrics = scrape(t, addr) } wantMetric(t, metrics, "smallwebwaf_geojs_requests_total", 1) wantMetric(t, metrics, "smallwebwaf_geojs_failures_total", 1) wantMetric(t, metrics, "smallwebwaf_geojs_unanswered_total", 1) } // keptAnswer returns GeoJS's answer that the client at addr is in // country, given now. func keptAnswer(addr, country string) lookup.Answer { now := time.Now() return lookup.Answer{ Client: netip.MustParsePrefix(addr + "/32"), Country: country, Answered: now, Used: now, } } // scrape asks smallwebwaf at addr for the metrics, with the token, and // returns them. func scrape(t *testing.T, addr string) string { t.Helper() req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody) req.Header.Set("Authorization", bearer) got := do(t, req) if got.status != http.StatusOK { t.Fatalf("the metrics were answered %d", got.status) } return string(got.body) } // scrape asks for the metrics, with the token, from the client at from, // and returns them. func (s *sender) scrape(from string) string { s.t.Helper() _, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer, http.StatusOK, requestlog.ActionAdmin) return metrics } // metric returns the value of series in metrics, which are in the // Prometheus text format. series is a name and its labels in the order of // their names, such as smallwebwaf_offences_total{kind="limit"}. It fails // the test if there is no such series. func metric(t *testing.T, metrics, series string) float64 { t.Helper() for line := range strings.Lines(metrics) { value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ") if !found { continue } number, err := strconv.ParseFloat(value, 64) if err != nil { t.Fatalf("%s has the value %q", series, value) } return number } t.Fatalf("no series %s in the metrics:\n%s", series, metrics) return 0 } // wantMetric checks the value of series in metrics, as metric reads it. func wantMetric(t *testing.T, metrics, series string, want float64) { t.Helper() got := metric(t, metrics, series) if got != want { t.Errorf("%s is %v, want %v", series, got, want) } } // wantNoSeries checks that metrics have no series series. func wantNoSeries(t *testing.T, metrics, series string) { t.Helper() if strings.Contains(metrics, "\n"+series+" ") { t.Errorf("there is a series %s", series) } } // wantLimitHits checks that the metrics of smallwebwaf at addr count hits // requests that passed the size or time limit of the setting limit, with // no series for it when hits is 0. func wantLimitHits(t *testing.T, addr, limit string, hits int) { t.Helper() series := `smallwebwaf_size_and_time_limit_hits_total{limit="` + limit + `"}` metrics := scrape(t, addr) if hits == 0 { wantNoSeries(t, metrics, series) return } wantMetric(t, metrics, series, float64(hits)) }