Country allow and deny lists, looked up through GeoJS (closes #44)

SWWAF_DENIED_COUNTRIES and SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES refuse a
request with 403 before its body is read or rate-limited, logged as
country_denied. internal/lookup asks GeoJS only while a list is set, 200
waiting clients per request, one at a time, keeping answers 7 days. A
redirect, or an answer leaving an address out, is a failure; GeoJS is then
left alone a second, doubling to five minutes. Private, loopback and
link-local clients are never sent.

Deviation, per the issue: no SWWAF_LOOKUP_SOURCE or SWWAF_LOOKUP_TIMEOUT; 403, not SWWAF_BAN_RESPONSE.
Deviation: GeoJS's country endpoint, not geo.json.
Judgement call: an IPv6 /64 is asked about by its first address; at most 10,000 clients wait.
Judgement call: config.go lists the ISO 3166-1 codes; no widely used library holds them.

Model: opus-5-5

README.md

@@ -119,10 +119,11 @@ it, and the effective settings are logged at start.
 Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are
 bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024
 bytes). Rate limits are whole numbers of requests. Netblocks are in CIDR form,
-and a bare address stands for itself alone. Countries are two-letter ISO codes
-in either case (`de` and `DE` are the same); a code that is not a country code,
-such as `nk` (North Korea is `kp`), stops the start, and so does a code on both
-country lists. `off` switches a timeout, a size limit or a rate limit off.
+and a bare address stands for itself alone. Countries are the two-letter codes
+ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are
+the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn
+`su`, stops the start, and so does a code on both country lists. `off` switches
+a timeout, a size limit or a rate limit off.
 
 Several limits are fixed rather than settings. The request line and headers may
 take up to 32 KiB, above which the answer is `431` and nothing reaches the app.
@@ -397,9 +398,9 @@ from an unknown country until the answer arrives. The addresses waiting are
 asked about together, up to 200 in one request, one request at a time; at most
 10,000 visitors wait, and one more counts as coming from an unknown country
 until there is room. While GeoJS fails, visitors with a kept answer are
-unaffected, new ones count as coming from an unknown country, and GeoJS is asked
-again a second later, then twice as long after each failure in a row, up to five
-minutes.
+unaffected and new ones count as coming from an unknown country. GeoJS is then
+left alone for a second, twice as long after each further failure up to five
+minutes, and asked again by the next request that needs it.
 
 In the full design, `smallwebwaf` looks up the AS number and country of every
 client, for the request log, the metrics and the ban notes, and for the country

go.mod

@@ -2,7 +2,4 @@ module sneak.berlin/go/smallwebwaf
 
 go 1.26.0
 
-require (
-	github.com/hashicorp/golang-lru/v2 v2.0.7
-	golang.org/x/text v0.42.0
-)
+require github.com/hashicorp/golang-lru/v2 v2.0.7

go.sum

@@ -1,4 +1,2 @@
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
-golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=

internal/config/config.go

@@ -15,8 +15,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"golang.org/x/text/language"
 )
 
 // Config is smallwebwaf's settings. A timeout, size or rate limit of zero
@@ -379,27 +377,57 @@ func parseNetblock(value string) (netip.Prefix, error) {
 	return netip.PrefixFrom(addr, addr.BitLen()), nil
 }
 
-// parseCountries reads a comma-separated list of two-letter ISO 3166-1
-// country codes in either case, and returns them in capitals.
+// countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK,
+// the code in common use for Kosovo. golang.org/x/text/language cannot
+// check them: it also takes withdrawn codes such as su, and reserved ones
+// such as ac, as countries.
+const countryCodes = `
+AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ
+BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ
+CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ
+DE DJ DK DM DO DZ
+EC EE EG EH ER ES ET
+FI FJ FK FM FO FR
+GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY
+HK HM HN HR HT HU
+ID IE IL IM IN IO IQ IR IS IT
+JE JM JO JP
+KE KG KH KI KM KN KP KR KW KY KZ
+LA LB LC LI LK LR LS LT LU LV LY
+MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ
+NA NC NE NF NG NI NL NO NP NR NU NZ
+OM
+PA PE PF PG PH PK PL PM PN PR PS PT PW PY
+QA
+RE RO RS RU RW
+SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ
+TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ
+UA UG UM US UY UZ
+VA VC VE VG VI VN VU
+WF WS
+XK
+YE YT
+ZA ZM ZW
+`
+
+// parseCountries reads a comma-separated list of country codes in either
+// case, and returns them in capitals.
 func parseCountries(value string) ([]string, error) {
 	items, err := parseList(value)
 	if err != nil {
 		return nil, err
 	}
 
+	known := strings.Fields(countryCodes)
 	countries := make([]string, 0, len(items))
 
 	for _, item := range items {
-		// ParseRegion also takes three-letter and numeric codes, groups of
-		// countries such as eu, and codes replaced by another, such as uk
-		// by gb, which GeoJS never gives.
-		region, err := language.ParseRegion(item)
-		if err != nil || len(item) != 2 || !region.IsCountry() ||
-			region.Canonicalize() != region {
+		country := strings.ToUpper(item)
+		if !slices.Contains(known, country) {
 			return nil, fmt.Errorf("%q %w", item, errNotCountry)
 		}
 
-		countries = append(countries, region.String())
+		countries = append(countries, country)
 	}
 
 	return countries, nil

internal/config/config_test.go

@@ -229,6 +229,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{deniedCountries, "408"},
 		{deniedCountries, "k"},
 		{deniedCountries, "eu"},
+		{deniedCountries, "un"},
+		{deniedCountries, "su"},
+		{allowedCountries, "ac"},
 		{allowedCountries, "uk"},
 		{allowedCountries, "zz"},
 		{allowedCountries, "de,germany"},

internal/lookup/lookup.go

@@ -49,7 +49,10 @@ const (
 	maxResponseBytes = 1 << 20
 )
 
-var errStatus = errors.New("GeoJS answered")
+var (
+	errStatus  = errors.New("GeoJS answered")
+	errLeftOut = errors.New("GeoJS's answer left out")
+)
 
 // Params are what New needs.
 type Params struct {
@@ -68,6 +71,9 @@ type GeoJS struct {
 	url        string
 	now        func() time.Time
 	processLog *slog.Logger
+	// httpClient follows no redirect, so that visitors' addresses go to
+	// GeoJS alone: a redirect is a failure.
+	httpClient *http.Client
 
 	mu      sync.Mutex
 	answers *simplelru.LRU[netip.Prefix, answer]
@@ -110,8 +116,13 @@ func New(params Params) *GeoJS {
 		url:        params.URL,
 		now:        params.Now,
 		processLog: params.ProcessLog,
-		answers:    answers,
-		waiting:    map[netip.Prefix]*wait{},
+		httpClient: &http.Client{
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+		answers: answers,
+		waiting: map[netip.Prefix]*wait{},
 	}
 }
 
@@ -260,10 +271,11 @@ func (g *GeoJS) nextClients() []netip.Prefix {
 }
 
 // keep notes how a request to GeoJS about clients ended, and reports
-// whether GeoJS answered. Each client asked about gets its answer, with
-// no country for one GeoJS gave none for. After a failure GeoJS is left
-// alone for a while, and every waiting client stops waiting and is
-// asked about once GeoJS is asked again.
+// whether GeoJS answered about all of them. Each client whose address
+// GeoJS's answer names gets its answer, with no country when GeoJS gave
+// none. An answer that leaves an address out is a failure. After a
+// failure GeoJS is left alone for a while, and every client still waiting
+// stops waiting and is asked about once GeoJS is asked again.
 func (g *GeoJS) keep(
 	clients []netip.Prefix, countries map[netip.Addr]string, err error,
 ) bool {
@@ -271,6 +283,24 @@ func (g *GeoJS) keep(
 	defer g.mu.Unlock()
 
 	now := g.now()
+	leftOut := 0
+
+	for _, client := range clients {
+		country, named := countries[client.Addr()]
+		if !named {
+			leftOut++
+
+			continue
+		}
+
+		g.answers.Add(client, answer{country: country, received: now})
+		close(g.waiting[client].asked)
+		delete(g.waiting, client)
+	}
+
+	if err == nil && leftOut > 0 {
+		err = fmt.Errorf("%w %d of %d addresses", errLeftOut, leftOut, len(clients))
+	}
 
 	if err != nil {
 		g.retryDelay = min(max(retryDelayFactor*g.retryDelay, firstRetryDelay),
@@ -293,17 +323,11 @@ func (g *GeoJS) keep(
 
 	g.retryDelay = 0
 
-	for _, client := range clients {
-		g.answers.Add(client, answer{country: countries[client.Addr()], received: now})
-		close(g.waiting[client].asked)
-		delete(g.waiting, client)
-	}
-
 	return true
 }
 
 // request asks GeoJS about clients in one request, and returns the
-// country it gave for each address it answered for.
+// country it gave, in capitals, for each address its answer names.
 func (g *GeoJS) request(
 	ctx context.Context, clients []netip.Prefix,
 ) (map[netip.Addr]string, error) {
@@ -322,7 +346,7 @@ func (g *GeoJS) request(
 		return nil, fmt.Errorf("make the request to GeoJS: %w", err)
 	}
 
-	res, err := http.DefaultClient.Do(req)
+	res, err := g.httpClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("ask GeoJS: %w", err)
 	}
@@ -350,7 +374,7 @@ func (g *GeoJS) request(
 	for _, item := range answers {
 		addr, err := netip.ParseAddr(item.IP)
 		if err == nil {
-			countries[addr] = item.Country
+			countries[addr] = strings.ToUpper(item.Country)
 		}
 	}
 

internal/lookup/lookup_test.go

@@ -21,6 +21,9 @@ const (
 	germany = "DE"
 	// unplaced is the address it cannot place.
 	unplaced = "192.0.2.1"
+	// leftOut is the address it leaves out of its answer when
+	// answeringWithoutLeftOut.
+	leftOut = "203.0.113.7"
 	// timeout is how long a new client waits for its answer.
 	timeout = time.Second
 	// waitLimit bounds how long a test waits for what should happen.
@@ -60,7 +63,18 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	geojs, clock, g := start(t)
 	client := netip.MustParsePrefix("203.0.113.9/32")
 
-	// GeoJS does not answer: the request to it is abandoned.
+	// The client comes while GeoJS is asked about an earlier client, which
+	// it answers most of a second later. It is then asked about the client
+	// and does not answer: that request is abandoned a second after it
+	// began, well after the client's wait is over.
+	geojs.set(answeringSlowly)
+
+	var earlier sync.WaitGroup
+
+	earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
+	defer earlier.Wait()
+
+	waitForRequests(t, geojs, 1)
 	geojs.set(hanging)
 
 	began := time.Now()
@@ -68,7 +82,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	wantCountry(t, g, client, "")
 
 	took := time.Since(began)
-	if took < timeout || took > timeout+waitLimit/2 {
+	if took < timeout || took > timeout+timeout/2 {
 		t.Errorf("waited %s for the answer, want %s", took, timeout)
 	}
 
@@ -85,19 +99,73 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	// Once GeoJS answers, the client is asked about again in the
 	// background, and has its country.
 	geojs.set(answering)
+	waitForCountry(t, g, clock, client, germany)
+}
 
-	deadline := time.Now().Add(waitLimit)
-	for g.Country(t.Context(), client) != germany {
-		if time.Now().After(deadline) {
-			t.Fatalf("no answer after %s", waitLimit)
-		}
+func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
+	t.Parallel()
 
-		clock.advance(time.Minute)
-		time.Sleep(pollInterval)
+	for _, tc := range []struct {
+		name    string
+		answers int
+		// named is whether the answer names the other client asked about.
+		named bool
+	}{
+		{"null", answeringNull, false},
+		{"empty list", answeringEmptyList, false},
+		{"list without " + leftOut, answeringWithoutLeftOut, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			geojs, clock, g := start(t)
+			other := netip.MustParsePrefix("203.0.113.1/32")
+			client := netip.MustParsePrefix(leftOut + "/32")
+
+			// GeoJS fails, and is left alone for a second while the client
+			// comes too, so that the next request asks about both.
+			geojs.set(failing)
+			wantCountry(t, g, other, "")
+			wantCountry(t, g, client, "")
+
+			geojs.set(tc.answers)
+			clock.advance(time.Second)
+			wantCountry(t, g, other, "")
+			waitForRequests(t, geojs, 2)
+
+			// The answer counts as a failure, and the client is asked about
+			// again, with the other client only if the answer left it out too.
+			geojs.set(answering)
+			waitForCountry(t, g, clock, client, germany)
+			wantCountry(t, g, other, germany)
+			wantRequests(t, geojs, 3)
+
+			if tc.named {
+				wantAsked(t, geojs, 2, leftOut)
+			} else {
+				wantAsked(t, geojs, 2, leftOut, "203.0.113.1")
+			}
+		})
 	}
+}
 
-	wantRequests(t, geojs, 2)
-	wantAsked(t, geojs, 1, "203.0.113.9")
+func TestRedirectCountsAsFailure(t *testing.T) {
+	t.Parallel()
+
+	geojs, _, g := start(t)
+	geojs.set(redirecting)
+
+	wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
+	wantRequests(t, geojs, 1)
+}
+
+func TestCountryIsKeptInCapitals(t *testing.T) {
+	t.Parallel()
+
+	geojs, _, g := start(t)
+	geojs.set(answeringInLowerCase)
+
+	wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany)
 }
 
 func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) {
@@ -231,9 +299,15 @@ func TestAtMost10000ClientsWait(t *testing.T) {
 
 // How the stand-in for GeoJS answers.
 const (
-	answering = iota
-	failing   // with 503
-	hanging   // not at all, until the request is abandoned
+	answering               = iota
+	answeringSlowly         // most of a second later
+	answeringInLowerCase    // with each country in lower case
+	answeringWithoutLeftOut // with a list that leaves leftOut out
+	answeringEmptyList      // with []
+	answeringNull           // with null
+	failing                 // with 503
+	hanging                 // not at all, until the request is abandoned
+	redirecting             // with a redirect to itself
 )
 
 // standIn is a stand-in for GeoJS. It notes the addresses each request
@@ -264,20 +338,45 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		<-r.Context().Done()
 
 		return
+	case redirecting:
+		http.Redirect(w, r, "/", http.StatusFound)
+
+		return
+	case answeringSlowly:
+		select {
+		case <-time.After(timeout * 4 / 5):
+		case <-r.Context().Done():
+			return
+		}
 	}
 
 	list := make([]map[string]string, 0, len(addrs))
 
 	for _, addr := range addrs {
 		country := germany
-		if addr == unplaced {
+
+		switch {
+		case addr == unplaced:
 			country = ""
+		case addr == leftOut && answers == answeringWithoutLeftOut:
+			continue
+		case answers == answeringInLowerCase:
+			country = strings.ToLower(germany)
 		}
 
 		list = append(list, map[string]string{"ip": addr, "country": country})
 	}
 
-	err := json.NewEncoder(w).Encode(list)
+	var answer any = list
+
+	switch answers {
+	case answeringEmptyList:
+		answer = []string{}
+	case answeringNull:
+		answer = nil
+	}
+
+	err := json.NewEncoder(w).Encode(answer)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
@@ -409,3 +508,22 @@ func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string {
 
 	return nil
 }
+
+// waitForCountry waits for g to give client the country want, moving the
+// clock on a minute at a time, so that GeoJS is asked again after a
+// failure.
+func waitForCountry(
+	t *testing.T, g *lookup.GeoJS, clock *testClock, client netip.Prefix, want string,
+) {
+	t.Helper()
+
+	deadline := time.Now().Add(waitLimit)
+	for g.Country(t.Context(), client) != want {
+		if time.Now().After(deadline) {
+			t.Fatalf("%s is not in %q after %s", client, want, waitLimit)
+		}
+
+		clock.advance(time.Minute)
+		time.Sleep(pollInterval)
+	}
+}