sneak
6acde5c3bc
check / check (push) Waiting to run
The package passes nothing to an app since rproxy took that over, so it is named after the command it serves. The rproxy tests call their stand-in for servewaf standIn, since a local servewaf would hide the package. No change of behaviour. Model: opus-5-5
432 lines
14 KiB
Go
432 lines
14 KiB
Go
package servewaf_test
|
|
|
|
import (
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
"sneak.berlin/go/smallwebwaf/internal/servewaf"
|
|
)
|
|
|
|
const (
|
|
metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
|
|
metricsTopN = "SWWAF_METRICS_TOP_N"
|
|
// token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a
|
|
// request carries it.
|
|
token = "0123456789abcdef0123456789abcdef"
|
|
bearer = "Bearer " + token
|
|
)
|
|
|
|
func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, _, _ := startWithClock(t, "", nil)
|
|
|
|
// An empty token does not match the unset one either.
|
|
for _, authorization := range []string{bearer, "Bearer ", ""} {
|
|
s.adminRequest(authorization, http.MethodGet, servewaf.MetricsPath, "",
|
|
http.StatusNotFound)
|
|
}
|
|
}
|
|
|
|
func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, _, _ := startWithClock(t, "", map[string]string{metricsToken: token})
|
|
|
|
for _, tc := range []struct {
|
|
method, path, authorization string
|
|
status int
|
|
}{
|
|
{http.MethodGet, servewaf.MetricsPath, "", http.StatusUnauthorized},
|
|
{
|
|
http.MethodGet, servewaf.MetricsPath, "Bearer " + strings.ToUpper(token),
|
|
http.StatusUnauthorized,
|
|
},
|
|
{http.MethodGet, servewaf.MetricsPath, "Basic " + token, http.StatusUnauthorized},
|
|
{http.MethodGet, servewaf.MetricsPath, bearer, http.StatusOK},
|
|
{http.MethodGet, servewaf.MetricsPath, "bearer " + token, http.StatusOK},
|
|
{http.MethodPost, servewaf.MetricsPath, bearer, http.StatusNotFound},
|
|
{http.MethodGet, servewaf.MetricsPath + "/", bearer, http.StatusNotFound},
|
|
{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
|
|
{http.MethodPost, servewaf.HealthPath, "", http.StatusNotFound},
|
|
} {
|
|
got := s.adminRequest(tc.authorization, tc.method, tc.path, "", tc.status)
|
|
|
|
if tc.status == http.StatusUnauthorized &&
|
|
got.header.Get("WWW-Authenticate") != "Bearer" {
|
|
t.Errorf("%q was answered without WWW-Authenticate: Bearer",
|
|
tc.authorization)
|
|
}
|
|
|
|
if tc.status == http.StatusOK &&
|
|
!strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") {
|
|
t.Errorf("the metrics are\n%s", got.body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMetricsCountTheTraffic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
_, _ = io.WriteString(w, "hello")
|
|
})
|
|
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
|
|
map[string]string{metricsToken: token})
|
|
|
|
got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
|
|
wantStatus(t, got, http.StatusOK)
|
|
wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
|
|
out.endedLines(t, 2)
|
|
|
|
forward := `{action="forward",instance="app",status_class="2xx"}`
|
|
notFound := `{action="admin",instance="app",status_class="4xx"}`
|
|
|
|
// The request for the metrics is itself under way.
|
|
metrics := scrape(t, server.Addr)
|
|
wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
|
|
wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
|
|
wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
|
|
wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5)
|
|
wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound,
|
|
float64(len("Not Found\n")))
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_request_duration_seconds_count{instance="app"}`, 2)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_upstream_duration_seconds_count{instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
|
|
metric(t, metrics, `go_goroutines{instance="app"}`)
|
|
metric(t, metrics, `process_start_time_seconds{instance="app"}`)
|
|
}
|
|
|
|
func TestMetricsCountLimitsAndBans(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
|
|
|
s, clk, _ := startWithClock(t, "", map[string]string{
|
|
metricsToken: token,
|
|
rateLimitPerMinute: "1",
|
|
denyNets: denied,
|
|
banResponse: banResponseClose,
|
|
limitBanDuration: "1h",
|
|
maxBanDuration: "2h",
|
|
})
|
|
|
|
// SWWAF_BAN_RESPONSE=close sends no status at all.
|
|
s.get(denied, 0, requestlog.ActionDenied)
|
|
|
|
// A first broken limit bans for an hour.
|
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
s.get(client, 0, requestlog.ActionRateLimited)
|
|
|
|
metrics := s.scrape()
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
|
`kind="requests",window="minute"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)
|
|
|
|
clk.advance(time.Hour)
|
|
wantMetric(t, s.scrape(), `smallwebwaf_active_bans{instance="app"}`, 0)
|
|
|
|
// A limit broken again right after would ban for three hours, longer
|
|
// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
|
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
s.get(client, 0, requestlog.ActionRateLimited)
|
|
|
|
metrics = s.scrape()
|
|
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
|
`kind="requests",window="minute"}`, 2)
|
|
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
|
|
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
|
|
wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
|
|
// denied and client.
|
|
wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 2)
|
|
}
|
|
|
|
func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, clk, server := startWithClock(t, "", map[string]string{
|
|
metricsToken: token,
|
|
adminToken: adminSecret,
|
|
})
|
|
|
|
const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`
|
|
|
|
wantMetric(t, s.scrape(), admins, 0)
|
|
|
|
// As an admin's edit of bans.json that adds a ban is taken in.
|
|
server.Ledger.LoadEdit([]bans.Ban{{
|
|
Netblock: netip.MustParsePrefix(client + "/32"),
|
|
Start: clk.Now(),
|
|
}})
|
|
|
|
wantMetric(t, s.scrape(), admins, 1)
|
|
|
|
// And a ban made through the endpoint.
|
|
s.admin(http.MethodPost, servewaf.BansPath, banOtherClient, http.StatusOK)
|
|
wantMetric(t, s.scrape(), admins, 2)
|
|
}
|
|
|
|
func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const fromFR = "198.51.100.20"
|
|
|
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
_, _ = io.WriteString(w, "hello")
|
|
})
|
|
env := map[string]string{
|
|
trustedProxies: trustLocalhost,
|
|
metricsToken: token,
|
|
metricsTopN: "2",
|
|
deniedCountries: "kp",
|
|
}
|
|
geojsURL, _ := startGeoJS(t)
|
|
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
|
|
|
|
// The answers are kept before the requests, so that none waits for
|
|
// GeoJS.
|
|
server.GeoJS.Load([]lookup.Answer{
|
|
keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"),
|
|
})
|
|
|
|
lines := 0
|
|
send := func(from string, times, status int) {
|
|
t.Helper()
|
|
|
|
for range times {
|
|
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))
|
|
req.Header.Set(forwardedFor, from)
|
|
wantStatus(t, do(t, req), status)
|
|
|
|
// Each is counted before the next is sent, so that the
|
|
// countries are ranked in the order sent.
|
|
lines++
|
|
out.endedLines(t, lines)
|
|
}
|
|
}
|
|
|
|
// With two countries of their own, the third is counted as other.
|
|
send(fromKP, 3, http.StatusForbidden)
|
|
send(fromDE, 2, http.StatusOK)
|
|
send(fromFR, 1, http.StatusOK)
|
|
|
|
metrics := scrape(t, server.Addr)
|
|
lines++
|
|
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="DE",instance="app"}`, 2)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="other",instance="app"}`, 1)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_list_refusals_total{country="KP",instance="app"}`, 3)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_request_bytes_total{country="KP",instance="app"}`, 0)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`, 6)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_response_bytes_total{country="KP",instance="app"}`,
|
|
float64(3*len("Forbidden\n")))
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_response_bytes_total{country="other",instance="app"}`,
|
|
float64(len("hello")))
|
|
wantNoSeries(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="FR",instance="app"}`)
|
|
|
|
// Once FR is busier than DE, it takes DE's place: its series counts
|
|
// from then on, and DE's is gone.
|
|
send(fromFR, 3, http.StatusOK)
|
|
|
|
metrics = scrape(t, server.Addr)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="FR",instance="app"}`, 2)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="other",instance="app"}`, 2)
|
|
wantNoSeries(t, metrics,
|
|
`smallwebwaf_country_requests_total{country="DE",instance="app"}`)
|
|
wantNoSeries(t, metrics,
|
|
`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
|
|
}
|
|
|
|
func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geojsURL, _ := startGeoJS(t)
|
|
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
|
metricsToken: token,
|
|
metricsTopN: "1",
|
|
})
|
|
|
|
// The answers are kept before the requests, so that GeoJS gives none
|
|
// of its own. Each client is in an AS of its own.
|
|
answer := func(addr, asn string) lookup.Answer {
|
|
return lookup.Answer{
|
|
Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
|
|
Answered: clk.Now(), Used: clk.Now(),
|
|
}
|
|
}
|
|
server.GeoJS.Load([]lookup.Answer{
|
|
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
|
|
})
|
|
|
|
// With one AS number of its own, the other is counted as other.
|
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
|
s.get(fromKP, http.StatusOK, requestlog.ActionForward)
|
|
|
|
metrics := s.scrape()
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0)
|
|
wantMetric(t, metrics,
|
|
`smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0)
|
|
wantNoSeries(t, metrics,
|
|
`smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`)
|
|
}
|
|
|
|
func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geojs := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}))
|
|
t.Cleanup(geojs.Close)
|
|
|
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
|
addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
|
|
map[string]string{
|
|
trustedProxies: trustLocalhost,
|
|
metricsToken: token,
|
|
deniedCountries: "kp",
|
|
})
|
|
|
|
// GeoJS fails, so the client counts as coming from an unknown country,
|
|
// which SWWAF_DENIED_COUNTRIES does not refuse.
|
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
|
req.Header.Set(forwardedFor, fromDE)
|
|
wantStatus(t, do(t, req), http.StatusOK)
|
|
|
|
// The client stops waiting for GeoJS once SWWAF_LOOKUP_TIMEOUT is up, so
|
|
// GeoJS's failure can come after its request has ended.
|
|
deadline := time.Now().Add(waitLimit)
|
|
metrics := scrape(t, server.Addr)
|
|
|
|
for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
|
|
time.Now().Before(deadline) {
|
|
time.Sleep(pollInterval)
|
|
|
|
metrics = scrape(t, server.Addr)
|
|
}
|
|
|
|
wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`, 1)
|
|
wantMetric(t, metrics, `smallwebwaf_geojs_unanswered_total{instance="app"}`, 1)
|
|
}
|
|
|
|
// keptAnswer returns GeoJS's answer that the client at addr is in
|
|
// country, given now.
|
|
func keptAnswer(addr, country string) lookup.Answer {
|
|
now := time.Now()
|
|
|
|
return lookup.Answer{
|
|
Client: netip.MustParsePrefix(addr + "/32"), Country: country,
|
|
Answered: now, Used: now,
|
|
}
|
|
}
|
|
|
|
// scrape asks servewaf at addr for the metrics, with the token, and returns
|
|
// them.
|
|
func scrape(t *testing.T, addr string) string {
|
|
t.Helper()
|
|
|
|
req := newRequest(t, http.MethodGet, addr, servewaf.MetricsPath, http.NoBody)
|
|
req.Header.Set("Authorization", bearer)
|
|
|
|
got := do(t, req)
|
|
if got.status != http.StatusOK {
|
|
t.Fatalf("the metrics were answered %d", got.status)
|
|
}
|
|
|
|
return string(got.body)
|
|
}
|
|
|
|
// scrape asks for the metrics, with the token, and returns them.
|
|
func (s *sender) scrape() string {
|
|
s.t.Helper()
|
|
|
|
return string(s.adminRequest(bearer, http.MethodGet, servewaf.MetricsPath, "",
|
|
http.StatusOK).body)
|
|
}
|
|
|
|
// metric returns the value of series in metrics, which are in the
|
|
// Prometheus text format. series is a name and its labels in the order of
|
|
// their names, such as
|
|
// smallwebwaf_offences_total{instance="app",kind="limit"}. It fails the
|
|
// test if there is no such series.
|
|
func metric(t *testing.T, metrics, series string) float64 {
|
|
t.Helper()
|
|
|
|
for line := range strings.Lines(metrics) {
|
|
value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ")
|
|
if !found {
|
|
continue
|
|
}
|
|
|
|
number, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
t.Fatalf("%s has the value %q", series, value)
|
|
}
|
|
|
|
return number
|
|
}
|
|
|
|
t.Fatalf("no series %s in the metrics:\n%s", series, metrics)
|
|
|
|
return 0
|
|
}
|
|
|
|
// wantMetric checks the value of series in metrics, as metric reads it.
|
|
func wantMetric(t *testing.T, metrics, series string, want float64) {
|
|
t.Helper()
|
|
|
|
got := metric(t, metrics, series)
|
|
if got != want {
|
|
t.Errorf("%s is %v, want %v", series, got, want)
|
|
}
|
|
}
|
|
|
|
// wantNoSeries checks that metrics have no series series.
|
|
func wantNoSeries(t *testing.T, metrics, series string) {
|
|
t.Helper()
|
|
|
|
if strings.Contains(metrics, "\n"+series+" ") {
|
|
t.Errorf("there is a series %s", series)
|
|
}
|
|
}
|