Network lists: always allowed, exempt from rate limits, always refused (closes #19)

Adds SWWAF_ALLOW_NETS, SWWAF_RATE_LIMIT_EXEMPT_NETS and SWWAF_DENY_NETS,
read like SWWAF_TRUSTED_PROXIES and empty by default, and checked against
the client's own address before its country is looked up. A client in
SWWAF_ALLOW_NETS skips the country lists and the rate limits and is not
looked up. One in SWWAF_DENY_NETS is refused with 403, logged as denied
and not counted. One in SWWAF_RATE_LIMIT_EXEMPT_NETS is neither counted
nor refused by the rate limits. SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES now
refuses a private, loopback or link-local client unless SWWAF_ALLOW_NETS
lists it.

Judgement call: an address in both SWWAF_ALLOW_NETS and SWWAF_DENY_NETS is let through.
Judgement call: the size and time limits still apply to SWWAF_ALLOW_NETS.

Model: opus-5-5

internal/config/config.go

@@ -45,6 +45,14 @@ type Config struct {
 	// ResponseMaxBytes is the largest response body
 	// (SWWAF_RESPONSE_MAX_BYTES).
 	ResponseMaxBytes int64
+	// AllowNets are the netblocks whose clients skip every check
+	// (SWWAF_ALLOW_NETS). RateLimitExemptNets are those whose clients the
+	// rate limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_NETS).
+	// DenyNets are those whose clients are always refused
+	// (SWWAF_DENY_NETS).
+	AllowNets           []netip.Prefix
+	RateLimitExemptNets []netip.Prefix
+	DenyNets            []netip.Prefix
 	// RateLimitPerMinute, RateLimitPerHour and RateLimitPerDay are the
 	// most requests a client may make in a minute, an hour and a day
 	// (SWWAF_RATE_LIMIT_PER_MINUTE, SWWAF_RATE_LIMIT_PER_HOUR and
@@ -112,6 +120,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"),
 		RequestMaxBytes:         env.size("SWWAF_REQUEST_MAX_BYTES", "100M"),
 		ResponseMaxBytes:        env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"),
+		AllowNets:               env.netblocks("SWWAF_ALLOW_NETS", ""),
+		RateLimitExemptNets:     env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
+		DenyNets:                env.netblocks("SWWAF_DENY_NETS", ""),
 		RateLimitPerMinute:      env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"),
 		RateLimitPerHour:        env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"),
 		RateLimitPerDay:         env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),

internal/config/config_test.go

@@ -25,6 +25,9 @@ const (
 	upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT"
 	requestMaxBytes         = "SWWAF_REQUEST_MAX_BYTES"
 	responseMaxBytes        = "SWWAF_RESPONSE_MAX_BYTES"
+	allowNets               = "SWWAF_ALLOW_NETS"
+	rateLimitExemptNets     = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
+	denyNets                = "SWWAF_DENY_NETS"
 	rateLimitPerMinute      = "SWWAF_RATE_LIMIT_PER_MINUTE"
 	rateLimitPerHour        = "SWWAF_RATE_LIMIT_PER_HOUR"
 	rateLimitPerDay         = "SWWAF_RATE_LIMIT_PER_DAY"
@@ -81,6 +84,9 @@ func TestDefaults(t *testing.T) {
 
 	wantNetblocks(t, cfg.TrustedProxies,
 		"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")
+	wantNetblocks(t, cfg.AllowNets)
+	wantNetblocks(t, cfg.RateLimitExemptNets)
+	wantNetblocks(t, cfg.DenyNets)
 	wantCountries(t, deniedCountries, cfg.DeniedCountries)
 	wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries)
 }
@@ -98,6 +104,9 @@ func TestValuesAsSet(t *testing.T) {
 		upstreamResponseTimeout: off,
 		requestMaxBytes:         "512K",
 		responseMaxBytes:        "1234",
+		allowNets:               "192.0.2.7",
+		rateLimitExemptNets:     "2001:db8::/48, 10.9.8.7",
+		denyNets:                "198.51.100.0/24",
 		rateLimitPerMinute:      "60",
 		rateLimitPerHour:        "600",
 		rateLimitPerDay:         "6000",
@@ -123,6 +132,9 @@ func TestValuesAsSet(t *testing.T) {
 	}
 
 	wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32")
+	wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
+	wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
+	wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
 	wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK")
 	wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
 }
@@ -205,6 +217,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{trustedProxies, "traefik"},
 		{trustedProxies, "10.0.0.0/8,,192.168.0.0/16"},
 		{trustedProxies, "fe80::1%eth0"},
+		{allowNets, "192.0.2.0/24,monitoring"},
+		{rateLimitExemptNets, "2001:db8::/129"},
+		{denyNets, "198.51.100.0/24,"},
 		{clientRequestTimeout, "60"},
 		{clientRequestTimeout, ""},
 		{clientResponseTimeout, "1y"},
@@ -279,6 +294,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		upstreamResponseTimeout: "30m",
 		requestMaxBytes:         "100M",
 		responseMaxBytes:        "5G",
+		allowNets:               "",
+		rateLimitExemptNets:     "",
+		denyNets:                "",
 		rateLimitPerMinute:      "1000",
 		rateLimitPerHour:        "10000",
 		rateLimitPerDay:         "50000",

internal/proxy/countries.go

@@ -9,9 +9,9 @@ import (
 // countryDenied reports whether the country lists refuse the request.
 // The client's country is looked up only while a list is set, and never
 // for a client on a private, loopback or link-local address, which has
-// no country and which neither list checks. A client whose country
-// cannot be found is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
-// ctx is the request's own context.
+// no country. A client without a country, or whose country cannot be
+// found, is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. ctx is
+// the request's own context.
 func (rq *request) countryDenied(ctx context.Context) bool {
 	denied := rq.h.config.DeniedCountries
 	allowed := rq.h.config.ExclusivelyAllowedCountries
@@ -20,11 +20,11 @@ func (rq *request) countryDenied(ctx context.Context) bool {
 		return false
 	}
 
-	if !hasCountry(rq.client) {
-		return false
+	var country string
+	if hasCountry(rq.client) {
+		country = rq.h.geojs.Country(ctx, clientGroup(rq.client))
 	}
 
-	country := rq.h.geojs.Country(ctx, clientGroup(rq.client))
 	rq.line.Country = country
 
 	if slices.Contains(denied, country) {

internal/proxy/countries_test.go

@@ -185,7 +185,7 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
 		{"no country list is set", nil, []string{fromKP, fromDE}},
 		{
 			"private, loopback and link-local addresses",
-			map[string]string{allowedCountries: "de"},
+			map[string]string{deniedCountries: "kp"},
 			[]string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"},
 		},
 	} {
@@ -223,6 +223,47 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) {
 	}
 }
 
+func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name      string
+		allowNets string
+		status    int
+		action    string
+	}{
+		{
+			"not in SWWAF_ALLOW_NETS", "",
+			http.StatusForbidden, requestlog.ActionCountryDenied,
+		},
+		{
+			"in SWWAF_ALLOW_NETS", "10.0.0.7,fd00::/8",
+			http.StatusOK, requestlog.ActionForward,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+			geojsURL, asked := startGeoJS(t)
+			addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
+				trustedProxies:   trustLocalhost,
+				allowedCountries: "de",
+				allowNets:        tc.allowNets,
+			})
+
+			wantAnswers(t, addr, out, []sentRequest{
+				{"10.0.0.7", tc.status, tc.action},
+				{"fd00::5", tc.status, tc.action},
+			})
+
+			if len(asked()) != 0 {
+				t.Errorf("GeoJS was asked about %v, want nothing", asked())
+			}
+		})
+	}
+}
+
 // startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP
 // and no other address. It returns its URL, and what returns the
 // addresses it has been asked about.

internal/proxy/proxy_test.go

@@ -51,6 +51,9 @@ const (
 	requestMaxBytes         = "SWWAF_REQUEST_MAX_BYTES"
 	responseMaxBytes        = "SWWAF_RESPONSE_MAX_BYTES"
 	trustedProxies          = "SWWAF_TRUSTED_PROXIES"
+	allowNets               = "SWWAF_ALLOW_NETS"
+	rateLimitExemptNets     = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
+	denyNets                = "SWWAF_DENY_NETS"
 	rateLimitPerMinute      = "SWWAF_RATE_LIMIT_PER_MINUTE"
 	deniedCountries         = "SWWAF_DENIED_COUNTRIES"
 	allowedCountries        = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"

internal/proxy/request.go

@@ -103,29 +103,44 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
 
 // check is the one place where a request can be refused once its client
 // is known, before its body is read or anything reaches the app. It
-// returns nil to let the request through. The country lists come first,
-// and a request they refuse is not counted for the rate limits; then the
-// rate limits, so that every other request is counted, one refused for
-// its size too. ctx is the request's own context.
+// returns nil to let the request through. A client in SWWAF_ALLOW_NETS
+// skips every check but the size limit. For any other client,
+// SWWAF_DENY_NETS comes first, so that a client it refuses is not looked
+// up, and then the country lists; a request either refuses is not counted
+// for the rate limits. Then come the rate limits, unless the client is in
+// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted,
+// one refused for its size too. ctx is the request's own context.
 func (rq *request) check(ctx context.Context) *refusal {
-	if rq.countryDenied(ctx) {
+	cfg := rq.h.config
+	allowed := isInside(rq.client, cfg.AllowNets)
+
+	if !allowed && isInside(rq.client, cfg.DenyNets) {
+		return &refusal{
+			status: http.StatusForbidden,
+			action: requestlog.ActionDenied,
+		}
+	}
+
+	if !allowed && rq.countryDenied(ctx) {
 		return &refusal{
 			status: http.StatusForbidden,
 			action: requestlog.ActionCountryDenied,
 		}
 	}
 
-	limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start)
-	if limitHit != "" {
-		rq.line.LimitHit = limitHit
+	if !allowed && !isInside(rq.client, cfg.RateLimitExemptNets) {
+		limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start)
+		if limitHit != "" {
+			rq.line.LimitHit = limitHit
 
-		return &refusal{
-			status: http.StatusTooManyRequests,
-			action: requestlog.ActionRateLimited,
+			return &refusal{
+				status: http.StatusTooManyRequests,
+				action: requestlog.ActionRateLimited,
+			}
 		}
 	}
 
-	maxBytes := rq.h.config.RequestMaxBytes
+	maxBytes := cfg.RequestMaxBytes
 	if maxBytes > 0 && rq.in.ContentLength > maxBytes {
 		return &refusal{
 			status: http.StatusRequestEntityTooLarge,

internal/proxy/staticlists_test.go

@@ -0,0 +1,163 @@
+package proxy_test
+
+import (
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// The rate limits count an IPv6 client by its /64, so these two addresses
+// are one client for them. The static lists match each address on its own,
+// and the tests list listedAddr alone.
+const (
+	listedAddr   = "2001:db8::1"
+	unlistedAddr = "2001:db8::2"
+)
+
+func TestAllowNetsSkipEveryCheckButTheSizeLimit(t *testing.T) {
+	t.Parallel()
+
+	var calls atomic.Int32
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {
+		calls.Add(1)
+	})
+	geojsURL, asked := startGeoJS(t)
+	// fromKP is in SWWAF_ALLOW_NETS, and in SWWAF_DENY_NETS too, which
+	// comes after it.
+	addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
+		trustedProxies:     trustLocalhost,
+		allowNets:          "198.51.100.0/24",
+		denyNets:           fromKP,
+		deniedCountries:    "kp",
+		rateLimitPerMinute: "1",
+		requestMaxBytes:    "1K",
+	})
+
+	// Neither SWWAF_DENY_NETS, the country lists nor the limit of one
+	// request a minute refuses the client, and its country is not looked
+	// up.
+	wantAnswers(t, addr, out, []sentRequest{
+		{fromKP, http.StatusOK, requestlog.ActionForward},
+		{fromKP, http.StatusOK, requestlog.ActionForward},
+	})
+
+	if len(asked()) != 0 {
+		t.Errorf("GeoJS was asked about %v, want nothing", asked())
+	}
+
+	// The size limit still applies.
+	body := strings.NewReader(strings.Repeat("a", 2<<10))
+	req := newRequest(t, http.MethodPost, addr, "/", body)
+	req.Header.Set(forwardedFor, fromKP)
+	wantStatus(t, do(t, req), http.StatusRequestEntityTooLarge)
+	wantLine(t, out.requestLines(t, 3)[2],
+		http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge)
+
+	if calls.Load() != 2 {
+		t.Errorf("the app was called %d times, want 2", calls.Load())
+	}
+}
+
+func TestDenyNetsRefuseBeforeTheLookupAndTheBody(t *testing.T) {
+	t.Parallel()
+
+	var calls atomic.Int32
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {
+		calls.Add(1)
+	})
+	geojsURL, asked := startGeoJS(t)
+	addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
+		trustedProxies:  trustLocalhost,
+		denyNets:        "203.0.113.0/24",
+		deniedCountries: "kp",
+	})
+
+	req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
+	req.Header.Set(forwardedFor, fromDE)
+	wantStatus(t, do(t, req), http.StatusForbidden)
+
+	line := out.requestLine(t)
+	wantLine(t, line, http.StatusForbidden, requestlog.ActionDenied)
+
+	if line.RequestBytes != 0 {
+		t.Errorf("log line has request_bytes %d, want 0", line.RequestBytes)
+	}
+
+	if len(asked()) != 0 {
+		t.Errorf("GeoJS was asked about %v, want nothing", asked())
+	}
+
+	if calls.Load() != 0 {
+		t.Errorf("the app was called %d times, want none", calls.Load())
+	}
+}
+
+func TestRequestRefusedByDenyNetsIsNotCounted(t *testing.T) {
+	t.Parallel()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	addr, out := startProxy(t, app.URL, map[string]string{
+		trustedProxies:     trustLocalhost,
+		denyNets:           listedAddr,
+		rateLimitPerMinute: "1",
+	})
+
+	// listedAddr's refused requests are not counted, so the first request
+	// from unlistedAddr is within the limit of one a minute.
+	wantAnswers(t, addr, out, []sentRequest{
+		{listedAddr, http.StatusForbidden, requestlog.ActionDenied},
+		{listedAddr, http.StatusForbidden, requestlog.ActionDenied},
+		{unlistedAddr, http.StatusOK, requestlog.ActionForward},
+		{unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited},
+	})
+}
+
+func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) {
+	t.Parallel()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	geojsURL, _ := startGeoJS(t)
+	addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
+		trustedProxies:      trustLocalhost,
+		rateLimitExemptNets: listedAddr + "," + fromKP,
+		deniedCountries:     "kp",
+		rateLimitPerMinute:  "1",
+	})
+
+	// listedAddr's requests are neither refused nor counted, so the first
+	// request from unlistedAddr is within the limit of one a minute. The
+	// country lists still refuse an exempt client.
+	wantAnswers(t, addr, out, []sentRequest{
+		{listedAddr, http.StatusOK, requestlog.ActionForward},
+		{listedAddr, http.StatusOK, requestlog.ActionForward},
+		{unlistedAddr, http.StatusOK, requestlog.ActionForward},
+		{unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited},
+		{fromKP, http.StatusForbidden, requestlog.ActionCountryDenied},
+	})
+}
+
+// sentRequest is a GET request from client, as X-Forwarded-For names it,
+// and the status and log line action it should get.
+type sentRequest struct {
+	client string
+	status int
+	action string
+}
+
+// wantAnswers sends requests to smallwebwaf at addr in turn, as the first
+// it is sent, and checks each one's answer and log line.
+func wantAnswers(t *testing.T, addr string, out *output, requests []sentRequest) {
+	t.Helper()
+
+	for i, sent := range requests {
+		req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+		req.Header.Set(forwardedFor, sent.client)
+		wantStatus(t, do(t, req), sent.status)
+		wantLine(t, out.requestLines(t, i+1)[i], sent.status, sent.action)
+	}
+}

internal/requestlog/requestlog.go

@@ -26,6 +26,9 @@ const (
 	// ActionRateLimited is a request refused because it took its client
 	// over a rate limit, or came while the client was over one.
 	ActionRateLimited = "rate_limited"
+	// ActionDenied is a request refused because its client is in
+	// SWWAF_DENY_NETS.
+	ActionDenied = "denied"
 	// ActionCountryDenied is a request refused for its client's country.
 	ActionCountryDenied = "country_denied"
 	// ActionAdmin is a request smallwebwaf answered at one of its own

internal/smallwebwaf/smallwebwaf_test.go

@@ -186,6 +186,9 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) {
 		"SWWAF_UPSTREAM_RESPONSE_TIMEOUT":     "30m",
 		"SWWAF_REQUEST_MAX_BYTES":             "100M",
 		"SWWAF_RESPONSE_MAX_BYTES":            "5G",
+		"SWWAF_ALLOW_NETS":                    "",
+		"SWWAF_RATE_LIMIT_EXEMPT_NETS":        "",
+		"SWWAF_DENY_NETS":                     "",
 		"SWWAF_RATE_LIMIT_PER_MINUTE":         "1000",
 		"SWWAF_RATE_LIMIT_PER_HOUR":           "10000",
 		"SWWAF_RATE_LIMIT_PER_DAY":            "50000",