Country allow and deny lists, looked up through GeoJS (closes #44)

SWWAF_DENIED_COUNTRIES and SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES refuse a
request with 403 before its body is read and before the rate limits count
it, logged as country_denied; every log line gains country. The new
internal/lookup asks GeoJS only while a list is set, one request at a
time carrying up to 200 waiting clients, keeps answers 7 days (at most
100,000), and after a failure waits a second, doubling to five minutes.
Private, loopback and link-local clients have no country and are never
sent. Codes are checked with golang.org/x/text/language.

Deviation from SPEC.md, per the issue: no SWWAF_LOOKUP_SOURCE or SWWAF_LOOKUP_TIMEOUT; 403, not SWWAF_BAN_RESPONSE.
Deviation: GeoJS's country endpoint, not geo.json, since only the country is needed.
Judgement call: an IPv6 /64 is asked about by its first address; at most 10,000 clients wait.
Deviation: go.mod and go.sum hand-written; no make target tidies them.

Model: opus-5-5

README.md

@@ -119,11 +119,10 @@ it, and the effective settings are logged at start.
 Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are
 bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024
 bytes). Rate limits are whole numbers of requests. Netblocks are in CIDR form,
-and a bare address stands for itself alone. Countries are the two-letter codes
-ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are
-the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn
-`su`, stops the start, and so does a code on both country lists. `off` switches
-a timeout, a size limit or a rate limit off.
+and a bare address stands for itself alone. Countries are two-letter ISO codes
+in either case (`de` and `DE` are the same); a code that is not a country code,
+such as `nk` (North Korea is `kp`), stops the start, and so does a code on both
+country lists. `off` switches a timeout, a size limit or a rate limit off.
 
 Several limits are fixed rather than settings. The request line and headers may
 take up to 32 KiB, above which the answer is `431` and nothing reaches the app.
@@ -398,9 +397,9 @@ from an unknown country until the answer arrives. The addresses waiting are
 asked about together, up to 200 in one request, one request at a time; at most
 10,000 visitors wait, and one more counts as coming from an unknown country
 until there is room. While GeoJS fails, visitors with a kept answer are
-unaffected and new ones count as coming from an unknown country. GeoJS is then
-left alone for a second, twice as long after each further failure up to five
-minutes, and asked again by the next request that needs it.
+unaffected, new ones count as coming from an unknown country, and GeoJS is asked
+again a second later, then twice as long after each failure in a row, up to five
+minutes.
 
 In the full design, `smallwebwaf` looks up the AS number and country of every
 client, for the request log, the metrics and the ban notes, and for the country

go.mod

@@ -2,4 +2,7 @@ module sneak.berlin/go/smallwebwaf
 
 go 1.26.0
 
-require github.com/hashicorp/golang-lru/v2 v2.0.7
+require (
+	github.com/hashicorp/golang-lru/v2 v2.0.7
+	golang.org/x/text v0.42.0
+)

go.sum

@@ -1,2 +1,4 @@
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=

internal/config/config.go

@@ -15,6 +15,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"golang.org/x/text/language"
 )
 
 // Config is smallwebwaf's settings. A timeout, size or rate limit of zero
@@ -377,57 +379,27 @@ func parseNetblock(value string) (netip.Prefix, error) {
 	return netip.PrefixFrom(addr, addr.BitLen()), nil
 }
 
-// countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK,
-// the code in common use for Kosovo. golang.org/x/text/language cannot
-// check them: it also takes withdrawn codes such as su, and reserved ones
-// such as ac, as countries.
-const countryCodes = `
-AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ
-BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ
-CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ
-DE DJ DK DM DO DZ
-EC EE EG EH ER ES ET
-FI FJ FK FM FO FR
-GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY
-HK HM HN HR HT HU
-ID IE IL IM IN IO IQ IR IS IT
-JE JM JO JP
-KE KG KH KI KM KN KP KR KW KY KZ
-LA LB LC LI LK LR LS LT LU LV LY
-MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ
-NA NC NE NF NG NI NL NO NP NR NU NZ
-OM
-PA PE PF PG PH PK PL PM PN PR PS PT PW PY
-QA
-RE RO RS RU RW
-SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ
-TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ
-UA UG UM US UY UZ
-VA VC VE VG VI VN VU
-WF WS
-XK
-YE YT
-ZA ZM ZW
-`
-
-// parseCountries reads a comma-separated list of country codes in either
-// case, and returns them in capitals.
+// parseCountries reads a comma-separated list of two-letter ISO 3166-1
+// country codes in either case, and returns them in capitals.
 func parseCountries(value string) ([]string, error) {
 	items, err := parseList(value)
 	if err != nil {
 		return nil, err
 	}
 
-	known := strings.Fields(countryCodes)
 	countries := make([]string, 0, len(items))
 
 	for _, item := range items {
-		country := strings.ToUpper(item)
-		if !slices.Contains(known, country) {
+		// ParseRegion also takes three-letter and numeric codes, groups of
+		// countries such as eu, and codes replaced by another, such as uk
+		// by gb, which GeoJS never gives.
+		region, err := language.ParseRegion(item)
+		if err != nil || len(item) != 2 || !region.IsCountry() ||
+			region.Canonicalize() != region {
 			return nil, fmt.Errorf("%q %w", item, errNotCountry)
 		}
 
-		countries = append(countries, country)
+		countries = append(countries, region.String())
 	}
 
 	return countries, nil

internal/config/config_test.go

@@ -229,9 +229,6 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{deniedCountries, "408"},
 		{deniedCountries, "k"},
 		{deniedCountries, "eu"},
-		{deniedCountries, "un"},
-		{deniedCountries, "su"},
-		{allowedCountries, "ac"},
 		{allowedCountries, "uk"},
 		{allowedCountries, "zz"},
 		{allowedCountries, "de,germany"},

internal/lookup/lookup.go

@@ -49,10 +49,7 @@ const (
 	maxResponseBytes = 1 << 20
 )
 
-var (
-	errStatus  = errors.New("GeoJS answered")
-	errLeftOut = errors.New("GeoJS's answer left out")
-)
+var errStatus = errors.New("GeoJS answered")
 
 // Params are what New needs.
 type Params struct {
@@ -71,9 +68,6 @@ type GeoJS struct {
 	url        string
 	now        func() time.Time
 	processLog *slog.Logger
-	// httpClient follows no redirect, so that visitors' addresses go to
-	// GeoJS alone: a redirect is a failure.
-	httpClient *http.Client
 
 	mu      sync.Mutex
 	answers *simplelru.LRU[netip.Prefix, answer]
@@ -116,13 +110,8 @@ func New(params Params) *GeoJS {
 		url:        params.URL,
 		now:        params.Now,
 		processLog: params.ProcessLog,
-		httpClient: &http.Client{
-			CheckRedirect: func(*http.Request, []*http.Request) error {
-				return http.ErrUseLastResponse
-			},
-		},
-		answers: answers,
-		waiting: map[netip.Prefix]*wait{},
+		answers:    answers,
+		waiting:    map[netip.Prefix]*wait{},
 	}
 }
 
@@ -271,11 +260,10 @@ func (g *GeoJS) nextClients() []netip.Prefix {
 }
 
 // keep notes how a request to GeoJS about clients ended, and reports
-// whether GeoJS answered about all of them. Each client whose address
-// GeoJS's answer names gets its answer, with no country when GeoJS gave
-// none. An answer that leaves an address out is a failure. After a
-// failure GeoJS is left alone for a while, and every client still waiting
-// stops waiting and is asked about once GeoJS is asked again.
+// whether GeoJS answered. Each client asked about gets its answer, with
+// no country for one GeoJS gave none for. After a failure GeoJS is left
+// alone for a while, and every waiting client stops waiting and is
+// asked about once GeoJS is asked again.
 func (g *GeoJS) keep(
 	clients []netip.Prefix, countries map[netip.Addr]string, err error,
 ) bool {
@@ -283,24 +271,6 @@ func (g *GeoJS) keep(
 	defer g.mu.Unlock()
 
 	now := g.now()
-	leftOut := 0
-
-	for _, client := range clients {
-		country, named := countries[client.Addr()]
-		if !named {
-			leftOut++
-
-			continue
-		}
-
-		g.answers.Add(client, answer{country: country, received: now})
-		close(g.waiting[client].asked)
-		delete(g.waiting, client)
-	}
-
-	if err == nil && leftOut > 0 {
-		err = fmt.Errorf("%w %d of %d addresses", errLeftOut, leftOut, len(clients))
-	}
 
 	if err != nil {
 		g.retryDelay = min(max(retryDelayFactor*g.retryDelay, firstRetryDelay),
@@ -323,11 +293,17 @@ func (g *GeoJS) keep(
 
 	g.retryDelay = 0
 
+	for _, client := range clients {
+		g.answers.Add(client, answer{country: countries[client.Addr()], received: now})
+		close(g.waiting[client].asked)
+		delete(g.waiting, client)
+	}
+
 	return true
 }
 
 // request asks GeoJS about clients in one request, and returns the
-// country it gave, in capitals, for each address its answer names.
+// country it gave for each address it answered for.
 func (g *GeoJS) request(
 	ctx context.Context, clients []netip.Prefix,
 ) (map[netip.Addr]string, error) {
@@ -346,7 +322,7 @@ func (g *GeoJS) request(
 		return nil, fmt.Errorf("make the request to GeoJS: %w", err)
 	}
 
-	res, err := g.httpClient.Do(req)
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("ask GeoJS: %w", err)
 	}
@@ -374,7 +350,7 @@ func (g *GeoJS) request(
 	for _, item := range answers {
 		addr, err := netip.ParseAddr(item.IP)
 		if err == nil {
-			countries[addr] = strings.ToUpper(item.Country)
+			countries[addr] = item.Country
 		}
 	}
 

internal/lookup/lookup_test.go

@@ -21,9 +21,6 @@ const (
 	germany = "DE"
 	// unplaced is the address it cannot place.
 	unplaced = "192.0.2.1"
-	// leftOut is the address it leaves out of its answer when
-	// answeringWithoutLeftOut.
-	leftOut = "203.0.113.7"
 	// timeout is how long a new client waits for its answer.
 	timeout = time.Second
 	// waitLimit bounds how long a test waits for what should happen.
@@ -63,18 +60,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	geojs, clock, g := start(t)
 	client := netip.MustParsePrefix("203.0.113.9/32")
 
-	// The client comes while GeoJS is asked about an earlier client, which
-	// it answers most of a second later. It is then asked about the client
-	// and does not answer: that request is abandoned a second after it
-	// began, well after the client's wait is over.
-	geojs.set(answeringSlowly)
-
-	var earlier sync.WaitGroup
-
-	earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
-	defer earlier.Wait()
-
-	waitForRequests(t, geojs, 1)
+	// GeoJS does not answer: the request to it is abandoned.
 	geojs.set(hanging)
 
 	began := time.Now()
@@ -82,7 +68,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	wantCountry(t, g, client, "")
 
 	took := time.Since(began)
-	if took < timeout || took > timeout+timeout/2 {
+	if took < timeout || took > timeout+waitLimit/2 {
 		t.Errorf("waited %s for the answer, want %s", took, timeout)
 	}
 
@@ -99,73 +85,19 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
 	// Once GeoJS answers, the client is asked about again in the
 	// background, and has its country.
 	geojs.set(answering)
-	waitForCountry(t, g, clock, client, germany)
-}
 
-func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
-	t.Parallel()
+	deadline := time.Now().Add(waitLimit)
+	for g.Country(t.Context(), client) != germany {
+		if time.Now().After(deadline) {
+			t.Fatalf("no answer after %s", waitLimit)
+		}
 
-	for _, tc := range []struct {
-		name    string
-		answers int
-		// named is whether the answer names the other client asked about.
-		named bool
-	}{
-		{"null", answeringNull, false},
-		{"empty list", answeringEmptyList, false},
-		{"list without " + leftOut, answeringWithoutLeftOut, true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			geojs, clock, g := start(t)
-			other := netip.MustParsePrefix("203.0.113.1/32")
-			client := netip.MustParsePrefix(leftOut + "/32")
-
-			// GeoJS fails, and is left alone for a second while the client
-			// comes too, so that the next request asks about both.
-			geojs.set(failing)
-			wantCountry(t, g, other, "")
-			wantCountry(t, g, client, "")
-
-			geojs.set(tc.answers)
-			clock.advance(time.Second)
-			wantCountry(t, g, other, "")
-			waitForRequests(t, geojs, 2)
-
-			// The answer counts as a failure, and the client is asked about
-			// again, with the other client only if the answer left it out too.
-			geojs.set(answering)
-			waitForCountry(t, g, clock, client, germany)
-			wantCountry(t, g, other, germany)
-			wantRequests(t, geojs, 3)
-
-			if tc.named {
-				wantAsked(t, geojs, 2, leftOut)
-			} else {
-				wantAsked(t, geojs, 2, leftOut, "203.0.113.1")
-			}
-		})
+		clock.advance(time.Minute)
+		time.Sleep(pollInterval)
 	}
-}
 
-func TestRedirectCountsAsFailure(t *testing.T) {
-	t.Parallel()
-
-	geojs, _, g := start(t)
-	geojs.set(redirecting)
-
-	wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
-	wantRequests(t, geojs, 1)
-}
-
-func TestCountryIsKeptInCapitals(t *testing.T) {
-	t.Parallel()
-
-	geojs, _, g := start(t)
-	geojs.set(answeringInLowerCase)
-
-	wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany)
+	wantRequests(t, geojs, 2)
+	wantAsked(t, geojs, 1, "203.0.113.9")
 }
 
 func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) {
@@ -299,15 +231,9 @@ func TestAtMost10000ClientsWait(t *testing.T) {
 
 // How the stand-in for GeoJS answers.
 const (
-	answering               = iota
-	answeringSlowly         // most of a second later
-	answeringInLowerCase    // with each country in lower case
-	answeringWithoutLeftOut // with a list that leaves leftOut out
-	answeringEmptyList      // with []
-	answeringNull           // with null
-	failing                 // with 503
-	hanging                 // not at all, until the request is abandoned
-	redirecting             // with a redirect to itself
+	answering = iota
+	failing   // with 503
+	hanging   // not at all, until the request is abandoned
 )
 
 // standIn is a stand-in for GeoJS. It notes the addresses each request
@@ -338,45 +264,20 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		<-r.Context().Done()
 
 		return
-	case redirecting:
-		http.Redirect(w, r, "/", http.StatusFound)
-
-		return
-	case answeringSlowly:
-		select {
-		case <-time.After(timeout * 4 / 5):
-		case <-r.Context().Done():
-			return
-		}
 	}
 
 	list := make([]map[string]string, 0, len(addrs))
 
 	for _, addr := range addrs {
 		country := germany
-
-		switch {
-		case addr == unplaced:
+		if addr == unplaced {
 			country = ""
-		case addr == leftOut && answers == answeringWithoutLeftOut:
-			continue
-		case answers == answeringInLowerCase:
-			country = strings.ToLower(germany)
 		}
 
 		list = append(list, map[string]string{"ip": addr, "country": country})
 	}
 
-	var answer any = list
-
-	switch answers {
-	case answeringEmptyList:
-		answer = []string{}
-	case answeringNull:
-		answer = nil
-	}
-
-	err := json.NewEncoder(w).Encode(answer)
+	err := json.NewEncoder(w).Encode(list)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
@@ -508,22 +409,3 @@ func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string {
 
 	return nil
 }
-
-// waitForCountry waits for g to give client the country want, moving the
-// clock on a minute at a time, so that GeoJS is asked again after a
-// failure.
-func waitForCountry(
-	t *testing.T, g *lookup.GeoJS, clock *testClock, client netip.Prefix, want string,
-) {
-	t.Helper()
-
-	deadline := time.Now().Add(waitLimit)
-	for g.Country(t.Context(), client) != want {
-		if time.Now().After(deadline) {
-			t.Fatalf("%s is not in %q after %s", client, want, waitLimit)
-		}
-
-		clock.advance(time.Minute)
-		time.Sleep(pollInterval)
-	}
-}