smallwebwaf/internal/proxy/metrics_test.go

package proxy_test

import (
	"io"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"strconv"
	"strings"
	"testing"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/lookup"
	"sneak.berlin/go/smallwebwaf/internal/proxy"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
)

const (
	metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
	metricsTopN  = "SWWAF_METRICS_TOP_N"
	// token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a
	// request carries it.
	token  = "0123456789abcdef0123456789abcdef"
	bearer = "Bearer " + token
)

func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
	t.Parallel()

	s, _, _ := startWithClock(t, "", nil)

	// An empty token does not match the unset one either.
	for _, authorization := range []string{bearer, "Bearer ", ""} {
		s.adminRequest(authorization, http.MethodGet, proxy.MetricsPath, "",
			http.StatusNotFound)
	}
}

func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
	t.Parallel()

	s, _, _ := startWithClock(t, "", map[string]string{metricsToken: token})

	for _, tc := range []struct {
		method, path, authorization string
		status                      int
	}{
		{http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized},
		{
			http.MethodGet, proxy.MetricsPath, "Bearer " + strings.ToUpper(token),
			http.StatusUnauthorized,
		},
		{http.MethodGet, proxy.MetricsPath, "Basic " + token, http.StatusUnauthorized},
		{http.MethodGet, proxy.MetricsPath, bearer, http.StatusOK},
		{http.MethodGet, proxy.MetricsPath, "bearer " + token, http.StatusOK},
		{http.MethodPost, proxy.MetricsPath, bearer, http.StatusNotFound},
		{http.MethodGet, proxy.MetricsPath + "/", bearer, http.StatusNotFound},
		{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
		{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
	} {
		got := s.adminRequest(tc.authorization, tc.method, tc.path, "", tc.status)

		if tc.status == http.StatusUnauthorized &&
			got.header.Get("WWW-Authenticate") != "Bearer" {
			t.Errorf("%q was answered without WWW-Authenticate: Bearer",
				tc.authorization)
		}

		if tc.status == http.StatusOK &&
			!strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") {
			t.Errorf("the metrics are\n%s", got.body)
		}
	}
}

func TestMetricsCountTheTraffic(t *testing.T) {
	t.Parallel()

	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
		_, _ = io.Copy(io.Discard, r.Body)
		_, _ = io.WriteString(w, "hello")
	})
	addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
		map[string]string{metricsToken: token})

	got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
	wantStatus(t, got, http.StatusOK)
	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
	out.endedLines(t, 2)

	forward := `{action="forward",instance="app",status_class="2xx"}`
	notFound := `{action="admin",instance="app",status_class="4xx"}`

	// The request for the metrics is itself under way.
	metrics := scrape(t, server.Addr)
	wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
	wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
	wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5)
	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound,
		float64(len("Not Found\n")))
	wantMetric(t, metrics,
		`smallwebwaf_request_duration_seconds_count{instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_upstream_duration_seconds_count{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
	metric(t, metrics, `go_goroutines{instance="app"}`)
	metric(t, metrics, `process_start_time_seconds{instance="app"}`)
}

func TestMetricsCountLimitsAndBans(t *testing.T) {
	t.Parallel()

	const denied = "192.0.2.50" // in SWWAF_DENY_NETS

	s, clk, _ := startWithClock(t, "", map[string]string{
		metricsToken:       token,
		rateLimitPerMinute: "1",
		denyNets:           denied,
		banResponse:        banResponseClose,
		limitBanDuration:   "1h",
		maxBanDuration:     "2h",
	})

	// SWWAF_BAN_RESPONSE=close sends no status at all.
	s.get(denied, 0, requestlog.ActionDenied)

	// A first broken limit bans for an hour.
	s.get(client, http.StatusOK, requestlog.ActionForward)
	s.get(client, 0, requestlog.ActionRateLimited)

	metrics := s.scrape()
	wantMetric(t, metrics,
		`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
		`kind="requests",window="minute"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)

	clk.advance(time.Hour)
	wantMetric(t, s.scrape(), `smallwebwaf_active_bans{instance="app"}`, 0)

	// A limit broken again right after would ban for three hours, longer
	// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
	s.get(client, http.StatusOK, requestlog.ActionForward)
	s.get(client, 0, requestlog.ActionRateLimited)

	metrics = s.scrape()
	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
		`kind="requests",window="minute"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
	// denied and client.
	wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 2)
}

func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
	t.Parallel()

	s, clk, server := startWithClock(t, "", map[string]string{
		metricsToken: token,
		adminToken:   adminSecret,
	})

	const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`

	wantMetric(t, s.scrape(), admins, 0)

	// As an admin's edit of bans.json that adds a ban is taken in.
	server.Ledger.LoadEdit([]bans.Ban{{
		Netblock: netip.MustParsePrefix(client + "/32"),
		Start:    clk.Now(),
	}})

	wantMetric(t, s.scrape(), admins, 1)

	// And a ban made through the endpoint.
	s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
	wantMetric(t, s.scrape(), admins, 2)
}

func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
	t.Parallel()

	const fromFR = "198.51.100.20"

	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
		_, _ = io.Copy(io.Discard, r.Body)
		_, _ = io.WriteString(w, "hello")
	})
	env := map[string]string{
		trustedProxies:  trustLocalhost,
		metricsToken:    token,
		metricsTopN:     "2",
		deniedCountries: "kp",
	}
	geojsURL, _ := startGeoJS(t)
	addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)

	// The answers are kept before the requests, so that none waits for
	// GeoJS.
	server.GeoJS.Load([]lookup.Answer{
		keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"),
	})

	lines := 0
	send := func(from string, times, status int) {
		t.Helper()

		for range times {
			req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))
			req.Header.Set(forwardedFor, from)
			wantStatus(t, do(t, req), status)

			// Each is counted before the next is sent, so that the
			// countries are ranked in the order sent.
			lines++
			out.endedLines(t, lines)
		}
	}

	// With two countries of their own, the third is counted as other.
	send(fromKP, 3, http.StatusForbidden)
	send(fromDE, 2, http.StatusOK)
	send(fromFR, 1, http.StatusOK)

	metrics := scrape(t, server.Addr)
	lines++

	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="DE",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 1)
	wantMetric(t, metrics,
		`smallwebwaf_country_list_refusals_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="KP",instance="app"}`, 0)
	wantMetric(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`, 6)
	wantMetric(t, metrics,
		`smallwebwaf_country_response_bytes_total{country="KP",instance="app"}`,
		float64(3*len("Forbidden\n")))
	wantMetric(t, metrics,
		`smallwebwaf_country_response_bytes_total{country="other",instance="app"}`,
		float64(len("hello")))
	wantNoSeries(t, metrics,
		`smallwebwaf_country_requests_total{country="FR",instance="app"}`)

	// Once FR is busier than DE, it takes DE's place: its series counts
	// from then on, and DE's is gone.
	send(fromFR, 3, http.StatusOK)

	metrics = scrape(t, server.Addr)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="FR",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 2)
	wantNoSeries(t, metrics,
		`smallwebwaf_country_requests_total{country="DE",instance="app"}`)
	wantNoSeries(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
}

func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
	t.Parallel()

	geojsURL, _ := startGeoJS(t)
	s, clk, server := startWithClock(t, geojsURL, map[string]string{
		metricsToken: token,
		metricsTopN:  "1",
	})

	// The answers are kept before the requests, so that GeoJS gives none
	// of its own. Each client is in an AS of its own.
	answer := func(addr, asn string) lookup.Answer {
		return lookup.Answer{
			Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
			Answered: clk.Now(), Used: clk.Now(),
		}
	}
	server.GeoJS.Load([]lookup.Answer{
		answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
	})

	// With one AS number of its own, the other is counted as other.
	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	s.get(fromKP, http.StatusOK, requestlog.ActionForward)

	metrics := s.scrape()
	wantMetric(t, metrics,
		`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1)
	wantMetric(t, metrics,
		`smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0)
	wantMetric(t, metrics,
		`smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0)
	wantNoSeries(t, metrics,
		`smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`)
}

func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
	t.Parallel()

	geojs := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusServiceUnavailable)
		}))
	t.Cleanup(geojs.Close)

	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
	addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
		map[string]string{
			trustedProxies:  trustLocalhost,
			metricsToken:    token,
			deniedCountries: "kp",
		})

	// GeoJS fails, so the client counts as coming from an unknown country,
	// which SWWAF_DENIED_COUNTRIES does not refuse.
	req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
	req.Header.Set(forwardedFor, fromDE)
	wantStatus(t, do(t, req), http.StatusOK)

	// The client stops waiting for GeoJS once SWWAF_LOOKUP_TIMEOUT is up, so
	// GeoJS's failure can come after its request has ended.
	deadline := time.Now().Add(waitLimit)
	metrics := scrape(t, server.Addr)

	for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
		time.Now().Before(deadline) {
		time.Sleep(pollInterval)

		metrics = scrape(t, server.Addr)
	}

	wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_geojs_unanswered_total{instance="app"}`, 1)
}

// keptAnswer returns GeoJS's answer that the client at addr is in
// country, given now.
func keptAnswer(addr, country string) lookup.Answer {
	now := time.Now()

	return lookup.Answer{
		Client: netip.MustParsePrefix(addr + "/32"), Country: country,
		Answered: now, Used: now,
	}
}

// scrape asks servewaf at addr for the metrics, with the token, and returns
// them.
func scrape(t *testing.T, addr string) string {
	t.Helper()

	req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
	req.Header.Set("Authorization", bearer)

	got := do(t, req)
	if got.status != http.StatusOK {
		t.Fatalf("the metrics were answered %d", got.status)
	}

	return string(got.body)
}

// scrape asks for the metrics, with the token, and returns them.
func (s *sender) scrape() string {
	s.t.Helper()

	return string(s.adminRequest(bearer, http.MethodGet, proxy.MetricsPath, "",
		http.StatusOK).body)
}

// metric returns the value of series in metrics, which are in the
// Prometheus text format. series is a name and its labels in the order of
// their names, such as
// smallwebwaf_offences_total{instance="app",kind="limit"}. It fails the
// test if there is no such series.
func metric(t *testing.T, metrics, series string) float64 {
	t.Helper()

	for line := range strings.Lines(metrics) {
		value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ")
		if !found {
			continue
		}

		number, err := strconv.ParseFloat(value, 64)
		if err != nil {
			t.Fatalf("%s has the value %q", series, value)
		}

		return number
	}

	t.Fatalf("no series %s in the metrics:\n%s", series, metrics)

	return 0
}

// wantMetric checks the value of series in metrics, as metric reads it.
func wantMetric(t *testing.T, metrics, series string, want float64) {
	t.Helper()

	got := metric(t, metrics, series)
	if got != want {
		t.Errorf("%s is %v, want %v", series, got, want)
	}
}

// wantNoSeries checks that metrics have no series series.
func wantNoSeries(t *testing.T, metrics, series string) {
	t.Helper()

	if strings.Contains(metrics, "\n"+series+" ") {
		t.Errorf("there is a series %s", series)
	}
}