smallwebwaf/internal/proxy/metrics_test.go

package proxy_test

import (
	"io"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/lookup"
	"sneak.berlin/go/smallwebwaf/internal/proxy"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
)

const (
	metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
	metricsTopN  = "SWWAF_METRICS_TOP_N"
	// token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a
	// request carries it.
	token  = "0123456789abcdef0123456789abcdef"
	bearer = "Bearer " + token
)

func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
	t.Parallel()

	var calls atomic.Int32

	app := startApp(t, func(http.ResponseWriter, *http.Request) {
		calls.Add(1)
	})
	addr, out := startProxy(t, app.URL, nil)

	// An empty token does not match the unset one either.
	for i, authorization := range []string{bearer, "Bearer ", ""} {
		req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
		if authorization != "" {
			req.Header.Set("Authorization", authorization)
		}

		wantStatus(t, do(t, req), http.StatusNotFound)
		wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound,
			requestlog.ActionAdmin)
	}

	if calls.Load() != 0 {
		t.Errorf("the app was called %d times, want never", calls.Load())
	}
}

func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
	t.Parallel()

	var calls atomic.Int32

	app := startApp(t, func(http.ResponseWriter, *http.Request) {
		calls.Add(1)
	})
	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})

	for i, tc := range []struct {
		method, path, authorization string
		status                      int
	}{
		{http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized},
		{
			http.MethodGet, proxy.MetricsPath, "Bearer " + strings.ToUpper(token),
			http.StatusUnauthorized,
		},
		{http.MethodGet, proxy.MetricsPath, "Basic " + token, http.StatusUnauthorized},
		{http.MethodGet, proxy.MetricsPath, bearer, http.StatusOK},
		{http.MethodGet, proxy.MetricsPath, "bearer " + token, http.StatusOK},
		{http.MethodPost, proxy.MetricsPath, bearer, http.StatusNotFound},
		{http.MethodGet, proxy.MetricsPath + "/", bearer, http.StatusNotFound},
		{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
		{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
	} {
		req := newRequest(t, tc.method, addr, tc.path, http.NoBody)
		if tc.authorization != "" {
			req.Header.Set("Authorization", tc.authorization)
		}

		got := do(t, req)
		wantStatus(t, got, tc.status)
		wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin)

		if tc.status == http.StatusUnauthorized &&
			got.header.Get("WWW-Authenticate") != "Bearer" {
			t.Errorf("%q was answered without WWW-Authenticate: Bearer",
				tc.authorization)
		}

		if tc.status == http.StatusOK &&
			!strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") {
			t.Errorf("the metrics are\n%s", got.body)
		}
	}

	if calls.Load() != 0 {
		t.Errorf("the app was called %d times, want never", calls.Load())
	}
}

func TestMetricsAreAskedForThroughTheChecks(t *testing.T) {
	t.Parallel()

	s, _, _ := startWithClock(t, "", map[string]string{
		metricsToken:       token,
		rateLimitPerMinute: "1",
	})

	// Asking for the metrics counts toward the client's limit of one
	// request a minute, so its next request breaks it, and bans it. A
	// banned client is refused the metrics too.
	s.scrape(client)
	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
	s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer,
		http.StatusForbidden, requestlog.ActionBanned)
}

func TestMetricsCountTheTraffic(t *testing.T) {
	t.Parallel()

	arrived, release := make(chan struct{}), make(chan struct{})
	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
		_, _ = io.Copy(io.Discard, r.Body)

		if r.URL.Path == "/held" {
			close(arrived)
			<-release
		}

		_, _ = io.WriteString(w, "hello")
	})
	releaseApp := sync.OnceFunc(func() { close(release) })
	t.Cleanup(releaseApp)

	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})

	got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
	wantStatus(t, got, http.StatusOK)
	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
	out.requestLines(t, 2)

	forward := `{action="forward",instance="app",status_class="2xx"}`
	notFound := `{action="admin",instance="app",status_class="4xx"}`

	// The request for the metrics is itself under way.
	metrics := scrape(t, addr)
	wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
	wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
	wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5)
	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound,
		float64(len("Not Found\n")))
	wantMetric(t, metrics,
		`smallwebwaf_request_duration_seconds_count{instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_upstream_duration_seconds_count{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
	metric(t, metrics, `go_goroutines{instance="app"}`)
	metric(t, metrics, `process_start_time_seconds{instance="app"}`)

	// A request the app holds is under way until it ends.
	httpClient := newClient(t)
	held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody)
	ended := make(chan error, 1)

	go func() {
		res, err := httpClient.Do(held)
		if err == nil {
			err = readAnswer(res).err
		}

		ended <- err
	}()

	<-arrived
	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 2)
	releaseApp()

	err := <-ended
	if err != nil {
		t.Fatalf("held request: %v", err)
	}

	out.requestLines(t, 5)
	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 1)
}

func TestMetricsCountLimitsAndBans(t *testing.T) {
	t.Parallel()

	const (
		scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
		denied  = "192.0.2.50"  // in SWWAF_DENY_NETS
	)

	s, clk, _ := startWithClock(t, "", map[string]string{
		metricsToken:        token,
		rateLimitPerMinute:  "1",
		rateLimitExemptNets: scraper,
		denyNets:            denied,
		banResponse:         "close",
		limitBanDuration:    "1h",
		maxBanDuration:      "2h",
	})

	// SWWAF_BAN_RESPONSE=close sends no status at all.
	s.get(denied, 0, requestlog.ActionDenied)

	// A first broken limit bans for an hour.
	s.get(client, http.StatusOK, requestlog.ActionForward)
	s.get(client, 0, requestlog.ActionRateLimited)

	metrics := s.scrape(scraper)
	wantMetric(t, metrics,
		`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
		`kind="requests",window="minute"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)

	clk.advance(time.Hour)
	wantMetric(t, s.scrape(scraper), `smallwebwaf_active_bans{instance="app"}`, 0)

	// A limit broken again right after would ban for three hours, longer
	// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
	s.get(client, http.StatusOK, requestlog.ActionForward)
	s.get(client, 0, requestlog.ActionRateLimited)

	metrics = s.scrape(scraper)
	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
		`kind="requests",window="minute"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
	// denied, client, and the scraper as of its earlier requests.
	wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 3)
}

func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
	t.Parallel()

	const scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS

	s, clk, server := startWithClock(t, "", map[string]string{
		metricsToken:        token,
		adminToken:          adminSecret,
		rateLimitExemptNets: scraper,
	})

	const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`

	wantMetric(t, s.scrape(scraper), admins, 0)

	// As an admin's edit of bans.json that adds a ban is taken in.
	server.Ledger.LoadEdit([]bans.Ban{{
		Netblock: netip.MustParsePrefix(client + "/32"),
		Start:    clk.Now(),
	}})

	wantMetric(t, s.scrape(scraper), admins, 1)

	// And a ban made through the endpoint.
	s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
	wantMetric(t, s.scrape(scraper), admins, 2)
}

func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
	t.Parallel()

	const fromFR = "198.51.100.20"

	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
		_, _ = io.Copy(io.Discard, r.Body)
		_, _ = io.WriteString(w, "hello")
	})
	env := map[string]string{
		trustedProxies:  trustLocalhost,
		metricsToken:    token,
		metricsTopN:     "2",
		deniedCountries: "kp",
	}
	geojsURL, _ := startGeoJS(t)
	addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)

	// The answers are kept before the requests, so that none waits for
	// GeoJS.
	server.GeoJS.Load([]lookup.Answer{
		keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"),
	})

	lines := 0
	send := func(from string, times, status int) {
		t.Helper()

		for range times {
			req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))
			req.Header.Set(forwardedFor, from)
			wantStatus(t, do(t, req), status)

			// Each is counted before the next is sent, so that the
			// countries are ranked in the order sent.
			lines++
			out.requestLines(t, lines)
		}
	}

	// With two countries of their own, the third is counted as other.
	send(fromKP, 3, http.StatusForbidden)
	send(fromDE, 2, http.StatusOK)
	send(fromFR, 1, http.StatusOK)

	metrics := scrape(t, addr)
	lines++

	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="DE",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 1)
	wantMetric(t, metrics,
		`smallwebwaf_country_list_refusals_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="KP",instance="app"}`, 0)
	wantMetric(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`, 6)
	wantMetric(t, metrics,
		`smallwebwaf_country_response_bytes_total{country="KP",instance="app"}`,
		float64(3*len("Forbidden\n")))
	wantMetric(t, metrics,
		`smallwebwaf_country_response_bytes_total{country="other",instance="app"}`,
		float64(len("hello")))
	wantNoSeries(t, metrics,
		`smallwebwaf_country_requests_total{country="FR",instance="app"}`)

	// Once FR is busier than DE, it takes DE's place: its series counts
	// from then on, and DE's is gone.
	send(fromFR, 3, http.StatusOK)

	metrics = scrape(t, addr)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="FR",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 2)
	wantNoSeries(t, metrics,
		`smallwebwaf_country_requests_total{country="DE",instance="app"}`)
	wantNoSeries(t, metrics,
		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
}

func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
	t.Parallel()

	geojsURL, _ := startGeoJS(t)
	s, clk, server := startWithClock(t, geojsURL, map[string]string{
		metricsToken: token,
		metricsTopN:  "1",
	})

	// The answers are kept before the requests, so that GeoJS gives none
	// of its own. Each client is in an AS of its own.
	answer := func(addr, asn string) lookup.Answer {
		return lookup.Answer{
			Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
			Answered: clk.Now(), Used: clk.Now(),
		}
	}
	server.GeoJS.Load([]lookup.Answer{
		answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
	})

	// With one AS number of its own, the other is counted as other. The
	// metrics are asked for from a private address, which has no AS number.
	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	s.get(fromKP, http.StatusOK, requestlog.ActionForward)

	metrics := s.scrape("10.0.0.9")
	wantMetric(t, metrics,
		`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
	wantMetric(t, metrics,
		`smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1)
	wantMetric(t, metrics,
		`smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0)
	wantMetric(t, metrics,
		`smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0)
	wantNoSeries(t, metrics,
		`smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`)
}

func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
	t.Parallel()

	geojs := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusServiceUnavailable)
		}))
	t.Cleanup(geojs.Close)

	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
	addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{
		trustedProxies:  trustLocalhost,
		metricsToken:    token,
		deniedCountries: "kp",
	})

	// GeoJS fails, so the client counts as coming from an unknown country,
	// which SWWAF_DENIED_COUNTRIES does not refuse.
	req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
	req.Header.Set(forwardedFor, fromDE)
	wantStatus(t, do(t, req), http.StatusOK)

	// The client stops waiting for GeoJS after a second, so GeoJS's
	// failure can come after its request has ended.
	deadline := time.Now().Add(waitLimit)
	metrics := scrape(t, addr)

	for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
		time.Now().Before(deadline) {
		time.Sleep(pollInterval)

		metrics = scrape(t, addr)
	}

	wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`, 1)
	wantMetric(t, metrics, `smallwebwaf_geojs_unanswered_total{instance="app"}`, 1)
}

// keptAnswer returns GeoJS's answer that the client at addr is in
// country, given now.
func keptAnswer(addr, country string) lookup.Answer {
	now := time.Now()

	return lookup.Answer{
		Client: netip.MustParsePrefix(addr + "/32"), Country: country,
		Answered: now, Used: now,
	}
}

// scrape asks smallwebwaf at addr for the metrics, with the token, and
// returns them.
func scrape(t *testing.T, addr string) string {
	t.Helper()

	req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
	req.Header.Set("Authorization", bearer)

	got := do(t, req)
	if got.status != http.StatusOK {
		t.Fatalf("the metrics were answered %d", got.status)
	}

	return string(got.body)
}

// scrape asks for the metrics, with the token, from the client at from,
// and returns them.
func (s *sender) scrape(from string) string {
	s.t.Helper()

	_, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer,
		http.StatusOK, requestlog.ActionAdmin)

	return metrics
}

// metric returns the value of series in metrics, which are in the
// Prometheus text format. series is a name and its labels in the order of
// their names, such as
// smallwebwaf_offences_total{instance="app",kind="limit"}. It fails the
// test if there is no such series.
func metric(t *testing.T, metrics, series string) float64 {
	t.Helper()

	for line := range strings.Lines(metrics) {
		value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ")
		if !found {
			continue
		}

		number, err := strconv.ParseFloat(value, 64)
		if err != nil {
			t.Fatalf("%s has the value %q", series, value)
		}

		return number
	}

	t.Fatalf("no series %s in the metrics:\n%s", series, metrics)

	return 0
}

// wantMetric checks the value of series in metrics, as metric reads it.
func wantMetric(t *testing.T, metrics, series string, want float64) {
	t.Helper()

	got := metric(t, metrics, series)
	if got != want {
		t.Errorf("%s is %v, want %v", series, got, want)
	}
}

// wantNoSeries checks that metrics have no series series.
func wantNoSeries(t *testing.T, metrics, series string) {
	t.Helper()

	if strings.Contains(metrics, "\n"+series+" ") {
		t.Errorf("there is a series %s", series)
	}
}

// wantLimitHits checks that the metrics of smallwebwaf at addr count hits
// requests that passed the size or time limit of the setting limit, with
// no series for it when hits is 0.
func wantLimitHits(t *testing.T, addr, limit string, hits int) {
	t.Helper()

	series := `smallwebwaf_size_and_time_limit_hits_total{instance="app",limit="` +
		limit + `"}`
	metrics := scrape(t, addr)

	if hits == 0 {
		wantNoSeries(t, metrics, series)

		return
	}

	wantMetric(t, metrics, series, float64(hits))
}