Country allow and deny lists, looked up through GeoJS (closes #44)

SWWAF_DENIED_COUNTRIES and SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES refuse a
request with 403 before its body is read and before the rate limits count
it, logged as country_denied; every log line gains country. The new
internal/lookup asks GeoJS only while a list is set, one request at a
time carrying up to 200 waiting clients, keeps answers 7 days (at most
100,000), and after a failure waits a second, doubling to five minutes.
Private, loopback and link-local clients have no country and are never
sent. Codes are checked with golang.org/x/text/language.

Deviation from SPEC.md, per the issue: no SWWAF_LOOKUP_SOURCE or SWWAF_LOOKUP_TIMEOUT; 403, not SWWAF_BAN_RESPONSE.
Deviation: GeoJS's country endpoint, not geo.json, since only the country is needed.
Judgement call: an IPv6 /64 is asked about by its first address; at most 10,000 clients wait.
Deviation: go.mod and go.sum hand-written; no make target tidies them.

Model: opus-5-5

internal/lookup/lookup.go

@@ -0,0 +1,358 @@
+// Package lookup looks up each client's country through the GeoJS web
+// service, and keeps the answers in memory, for at most 100,000 clients
+// and for 7 days each.
+package lookup
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/netip"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/hashicorp/golang-lru/v2/simplelru"
+)
+
+// URL is GeoJS's country endpoint. Asked about several addresses at once,
+// comma separated in its ip parameter, it answers with a list.
+const URL = "https://get.geojs.io/v1/ip/country.json"
+
+const (
+	// keepFor is how long an answer is used instead of asking GeoJS again.
+	keepFor = 7 * 24 * time.Hour
+	// maxAnswers is how many answers are kept. Past it, the one used
+	// longest ago is dropped.
+	maxAnswers = 100000
+	// maxWaiting is how many clients may wait to be asked about. Past it,
+	// a new client counts as not found and is not asked about until there
+	// is room, so that a swarm of new addresses while GeoJS is down cannot
+	// fill the memory.
+	maxWaiting = 10000
+	// maxPerRequest is how many addresses one request to GeoJS asks about.
+	maxPerRequest = 200
+	// timeout is how long a new client waits for its answer, and how long
+	// a request to GeoJS may take before it is abandoned.
+	timeout = time.Second
+	// After a failure GeoJS is not asked again for a second, and for
+	// retryDelayFactor times as long after each further failure in a row,
+	// up to five minutes.
+	firstRetryDelay  = time.Second
+	retryDelayFactor = 2
+	maxRetryDelay    = 5 * time.Minute
+	// maxResponseBytes is the most of GeoJS's answer that is read.
+	maxResponseBytes = 1 << 20
+)
+
+var errStatus = errors.New("GeoJS answered")
+
+// Params are what New needs.
+type Params struct {
+	// URL is where GeoJS is asked, normally URL.
+	URL string
+	// Now tells the time, normally time.Now.
+	Now func() time.Time
+	// ProcessLog receives GeoJS's failures.
+	ProcessLog *slog.Logger
+}
+
+// GeoJS looks up clients' countries through GeoJS. At most one request
+// to GeoJS is under way at a time, and it asks about every client waiting,
+// up to maxPerRequest. It is safe for concurrent use.
+type GeoJS struct {
+	url        string
+	now        func() time.Time
+	processLog *slog.Logger
+
+	mu      sync.Mutex
+	answers *simplelru.LRU[netip.Prefix, answer]
+	// waiting are the clients without an answer: those to ask GeoJS about,
+	// and those it is being asked about.
+	waiting map[netip.Prefix]*wait
+	// asking is true while a request to GeoJS is under way.
+	asking bool
+	// retryDelay is how long GeoJS is left alone after its last failure,
+	// zero after an answer; retryAt is when it may be asked again.
+	retryDelay time.Duration
+	retryAt    time.Time
+}
+
+// answer is what GeoJS said about a client: its country, "" when GeoJS
+// cannot place it, and when GeoJS said so.
+type answer struct {
+	country  string
+	received time.Time
+}
+
+// wait is a client waiting for its answer.
+type wait struct {
+	// asked is closed when the client gets its answer, and closed and
+	// replaced each time GeoJS fails before then.
+	asked chan struct{}
+	// late is true once the client has gone without an answer, for a
+	// whole timeout or because GeoJS failed: its requests no longer wait.
+	late bool
+}
+
+// New returns a GeoJS with no answer kept yet.
+func New(params Params) *GeoJS {
+	answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil)
+	if err != nil {
+		panic(err) // NewLRU fails only for a size below one
+	}
+
+	return &GeoJS{
+		url:        params.URL,
+		now:        params.Now,
+		processLog: params.ProcessLog,
+		answers:    answers,
+		waiting:    map[netip.Prefix]*wait{},
+	}
+}
+
+// Country returns the country GeoJS places client in, as a two-letter
+// code in capitals, or "" when the country cannot be found: GeoJS cannot
+// place the client, or has not answered in time. An answer is kept for 7
+// days. Without one, a client waits up to timeout for it, unless it has
+// gone without one before; until GeoJS answers, the client is asked about
+// again in the background. ctx is the context of the client's request,
+// and ends the wait when it ends.
+//
+// GeoJS is asked about the client's first address, which is the client's
+// own address for IPv4, and an address in the same place for an IPv6 /64.
+func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
+	country, asked := g.answerOrWait(ctx, client)
+	if asked == nil {
+		return country
+	}
+
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	select {
+	case <-asked:
+	case <-timer.C:
+	case <-ctx.Done():
+	}
+
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	country, found := g.kept(client)
+
+	w, waiting := g.waiting[client]
+	if !found && waiting {
+		w.late = true
+	}
+
+	return country
+}
+
+// answerOrWait returns client's kept answer if it has one. Otherwise it
+// puts the client among those waiting if there is room, has GeoJS asked
+// about them if it can be, and returns what to wait on for the answer, or
+// nil when there is nothing to wait for.
+func (g *GeoJS) answerOrWait(
+	ctx context.Context, client netip.Prefix,
+) (string, <-chan struct{}) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	country, found := g.kept(client)
+	if found {
+		return country, nil
+	}
+
+	w, waiting := g.waiting[client]
+	if !waiting && len(g.waiting) < maxWaiting {
+		w = &wait{asked: make(chan struct{})}
+		g.waiting[client] = w
+	}
+
+	g.ask(ctx)
+
+	if w == nil {
+		return "", nil // too many clients wait already
+	}
+
+	if !g.asking {
+		// GeoJS is left alone after a failure, so no answer can come.
+		w.late = true
+	}
+
+	if w.late {
+		return "", nil
+	}
+
+	return "", w.asked
+}
+
+// kept returns client's answer, if one was received less than keepFor
+// ago.
+func (g *GeoJS) kept(client netip.Prefix) (string, bool) {
+	kept, found := g.answers.Get(client)
+	if !found || g.now().Sub(kept.received) >= keepFor {
+		return "", false
+	}
+
+	return kept.country, true
+}
+
+// ask starts asking GeoJS about the waiting clients, unless a request to
+// it is under way or it is left alone after a failure. The requests to
+// GeoJS are for every client waiting, so they go on when the client's
+// request whose ctx is given ends.
+func (g *GeoJS) ask(ctx context.Context) {
+	if g.asking || g.now().Before(g.retryAt) {
+		return
+	}
+
+	g.asking = true
+
+	go g.askAboutWaiting(context.WithoutCancel(ctx))
+}
+
+// askAboutWaiting asks GeoJS about the waiting clients, one request at a
+// time, until none is left or GeoJS fails.
+func (g *GeoJS) askAboutWaiting(ctx context.Context) {
+	for {
+		clients := g.nextClients()
+		if len(clients) == 0 {
+			return
+		}
+
+		countries, err := g.request(ctx, clients)
+		if !g.keep(clients, countries, err) {
+			return
+		}
+	}
+}
+
+// nextClients returns up to maxPerRequest of the waiting clients. When
+// none is waiting, it returns none and notes that no request to GeoJS is
+// under way.
+func (g *GeoJS) nextClients() []netip.Prefix {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	if len(g.waiting) == 0 {
+		g.asking = false
+
+		return nil
+	}
+
+	clients := make([]netip.Prefix, 0, min(len(g.waiting), maxPerRequest))
+
+	for client := range g.waiting {
+		if len(clients) == maxPerRequest {
+			break
+		}
+
+		clients = append(clients, client)
+	}
+
+	return clients
+}
+
+// keep notes how a request to GeoJS about clients ended, and reports
+// whether GeoJS answered. Each client asked about gets its answer, with
+// no country for one GeoJS gave none for. After a failure GeoJS is left
+// alone for a while, and every waiting client stops waiting and is
+// asked about once GeoJS is asked again.
+func (g *GeoJS) keep(
+	clients []netip.Prefix, countries map[netip.Addr]string, err error,
+) bool {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	now := g.now()
+
+	if err != nil {
+		g.retryDelay = min(max(retryDelayFactor*g.retryDelay, firstRetryDelay),
+			maxRetryDelay)
+		g.retryAt = now.Add(g.retryDelay)
+		g.asking = false
+
+		for _, w := range g.waiting {
+			close(w.asked)
+
+			w.asked = make(chan struct{})
+			w.late = true
+		}
+
+		g.processLog.Warn("asking GeoJS failed",
+			"error", err.Error(), "asking_again_in", g.retryDelay.String())
+
+		return false
+	}
+
+	g.retryDelay = 0
+
+	for _, client := range clients {
+		g.answers.Add(client, answer{country: countries[client.Addr()], received: now})
+		close(g.waiting[client].asked)
+		delete(g.waiting, client)
+	}
+
+	return true
+}
+
+// request asks GeoJS about clients in one request, and returns the
+// country it gave for each address it answered for.
+func (g *GeoJS) request(
+	ctx context.Context, clients []netip.Prefix,
+) (map[netip.Addr]string, error) {
+	addrs := make([]string, 0, len(clients))
+
+	for _, client := range clients {
+		addrs = append(addrs, client.Addr().String())
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
+		g.url+"?ip="+strings.Join(addrs, ","), http.NoBody)
+	if err != nil {
+		return nil, fmt.Errorf("make the request to GeoJS: %w", err)
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("ask GeoJS: %w", err)
+	}
+
+	defer func() {
+		_ = res.Body.Close()
+	}()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w %s", errStatus, res.Status)
+	}
+
+	var answers []struct {
+		IP      string `json:"ip"`
+		Country string `json:"country"`
+	}
+
+	err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers)
+	if err != nil {
+		return nil, fmt.Errorf("read GeoJS's answer: %w", err)
+	}
+
+	countries := make(map[netip.Addr]string, len(answers))
+
+	for _, item := range answers {
+		addr, err := netip.ParseAddr(item.IP)
+		if err == nil {
+			countries[addr] = item.Country
+		}
+	}
+
+	return countries, nil
+}

internal/lookup/lookup_test.go

@@ -0,0 +1,411 @@
+package lookup_test
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
+)
+
+const (
+	// germany is where the stand-in for GeoJS places every address but
+	// unplaced.
+	germany = "DE"
+	// unplaced is the address it cannot place.
+	unplaced = "192.0.2.1"
+	// timeout is how long a new client waits for its answer.
+	timeout = time.Second
+	// waitLimit bounds how long a test waits for what should happen.
+	waitLimit = 10 * time.Second
+	// pollInterval is how often a test looks again.
+	pollInterval = 10 * time.Millisecond
+	// week is how long an answer is kept.
+	week = 7 * 24 * time.Hour
+)
+
+func TestKeptAnswerIsUsedFor7DaysThenAskedAgain(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	placed := netip.MustParsePrefix("203.0.113.9/32")
+	notPlaced := netip.MustParsePrefix(unplaced + "/32")
+
+	wantCountry(t, g, placed, germany)
+	wantCountry(t, g, notPlaced, "")
+	wantRequests(t, geojs, 2)
+
+	// An answer without a country is kept too.
+	clock.advance(week - time.Second)
+	wantCountry(t, g, placed, germany)
+	wantCountry(t, g, notPlaced, "")
+	wantRequests(t, geojs, 2)
+
+	clock.advance(time.Second)
+	wantCountry(t, g, placed, germany)
+	wantRequests(t, geojs, 3)
+	wantAsked(t, geojs, 2, "203.0.113.9")
+}
+
+func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	client := netip.MustParsePrefix("203.0.113.9/32")
+
+	// GeoJS does not answer: the request to it is abandoned.
+	geojs.set(hanging)
+
+	began := time.Now()
+
+	wantCountry(t, g, client, "")
+
+	took := time.Since(began)
+	if took < timeout || took > timeout+waitLimit/2 {
+		t.Errorf("waited %s for the answer, want %s", took, timeout)
+	}
+
+	// Its next request does not wait.
+	began = time.Now()
+
+	wantCountry(t, g, client, "")
+
+	took = time.Since(began)
+	if took > timeout/2 {
+		t.Errorf("waited %s again, want no wait", took)
+	}
+
+	// Once GeoJS answers, the client is asked about again in the
+	// background, and has its country.
+	geojs.set(answering)
+
+	deadline := time.Now().Add(waitLimit)
+	for g.Country(t.Context(), client) != germany {
+		if time.Now().After(deadline) {
+			t.Fatalf("no answer after %s", waitLimit)
+		}
+
+		clock.advance(time.Minute)
+		time.Sleep(pollInterval)
+	}
+
+	wantRequests(t, geojs, 2)
+	wantAsked(t, geojs, 1, "203.0.113.9")
+}
+
+func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+
+	// GeoJS fails, and is then left alone for a second, while three more
+	// clients come. An IPv6 client is a /64, and GeoJS is asked about its
+	// first address.
+	geojs.set(failing)
+	wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "")
+	wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "")
+	wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "")
+	wantRequests(t, geojs, 1)
+
+	geojs.set(answering)
+	clock.advance(time.Second)
+	wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany)
+	wantRequests(t, geojs, 2)
+	wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3")
+}
+
+func TestKeptAnswersUnaffectedWhileGeoJSFailsAndAskedAgainWithBackoff(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	clients := newClients()
+	kept := clients()
+
+	wantCountry(t, g, kept, germany)
+
+	geojs.set(failing)
+	wantCountry(t, g, kept, germany)
+	wantRequests(t, geojs, 1)
+
+	// Each failure leaves GeoJS alone twice as long as the one before, up
+	// to five minutes. New clients meanwhile count as not found.
+	requests := 1
+
+	for _, delay := range []time.Duration{
+		time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second,
+		16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second,
+		256 * time.Second, 5 * time.Minute, 5 * time.Minute,
+	} {
+		wantCountry(t, g, clients(), "")
+
+		requests++
+		wantRequests(t, geojs, requests)
+
+		clock.advance(delay - time.Millisecond)
+		wantCountry(t, g, clients(), "")
+		wantRequests(t, geojs, requests)
+
+		clock.advance(time.Millisecond)
+	}
+
+	// Once GeoJS answers again, it is asked about every client waiting.
+	geojs.set(answering)
+	wantCountry(t, g, clients(), germany)
+	wantRequests(t, geojs, requests+1)
+
+	asked := waitForRequests(t, geojs, requests+1)
+	if len(asked[requests]) != 23 {
+		t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests]))
+	}
+}
+
+func TestAtMost200AddressesInOneRequest(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	clients := newClients()
+	first := clients()
+
+	// 201 clients wait while GeoJS is left alone after a failure.
+	geojs.set(failing)
+	wantCountry(t, g, first, "")
+
+	for range 200 {
+		wantCountry(t, g, clients(), "")
+	}
+
+	// The first one's next request has GeoJS asked again.
+	geojs.set(answering)
+	clock.advance(time.Second)
+	wantCountry(t, g, first, "")
+
+	asked := waitForRequests(t, geojs, 3)
+	if len(asked[1]) != 200 || len(asked[2]) != 1 {
+		t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1",
+			len(asked[1]), len(asked[2]))
+	}
+}
+
+func TestAtMost10000ClientsWait(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	clients := newClients()
+	first := clients()
+
+	// 10,000 clients wait while GeoJS is left alone after a failure, and
+	// one more cannot join them.
+	geojs.set(failing)
+	wantCountry(t, g, first, "")
+
+	for range 9999 {
+		wantCountry(t, g, clients(), "")
+	}
+
+	extra := clients()
+	wantCountry(t, g, extra, "")
+
+	// The first one's next request has GeoJS asked about the 10,000, 200
+	// at a time, and not about the one more.
+	geojs.set(answering)
+	clock.advance(time.Second)
+	wantCountry(t, g, first, "")
+
+	asked := waitForRequests(t, geojs, 51)
+	for i, request := range asked {
+		if slices.Contains(request, extra.Addr().String()) {
+			t.Errorf("request %d asked about %s", i, extra.Addr())
+		}
+	}
+
+	// With room among those waiting, it is asked about.
+	wantCountry(t, g, extra, germany)
+}
+
+// How the stand-in for GeoJS answers.
+const (
+	answering = iota
+	failing   // with 503
+	hanging   // not at all, until the request is abandoned
+)
+
+// standIn is a stand-in for GeoJS. It notes the addresses each request
+// asks about.
+type standIn struct {
+	server *httptest.Server
+
+	mu       sync.Mutex
+	answers  int
+	requests [][]string
+}
+
+// ServeHTTP answers a request about the addresses in its ip parameter.
+func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	addrs := strings.Split(r.URL.Query().Get("ip"), ",")
+
+	s.mu.Lock()
+	s.requests = append(s.requests, addrs)
+	answers := s.answers
+	s.mu.Unlock()
+
+	switch answers {
+	case failing:
+		w.WriteHeader(http.StatusServiceUnavailable)
+
+		return
+	case hanging:
+		<-r.Context().Done()
+
+		return
+	}
+
+	list := make([]map[string]string, 0, len(addrs))
+
+	for _, addr := range addrs {
+		country := germany
+		if addr == unplaced {
+			country = ""
+		}
+
+		list = append(list, map[string]string{"ip": addr, "country": country})
+	}
+
+	err := json.NewEncoder(w).Encode(list)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+// set sets how the stand-in answers.
+func (s *standIn) set(answers int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.answers = answers
+}
+
+// asked returns the addresses each request has asked about so far.
+func (s *standIn) asked() [][]string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	return slices.Clone(s.requests)
+}
+
+// testClock is a clock the test sets.
+type testClock struct {
+	mu  sync.Mutex
+	now time.Time
+}
+
+// Now tells the time.
+func (c *testClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	return c.now
+}
+
+// advance moves the clock on by d.
+func (c *testClock) advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.now = c.now.Add(d)
+}
+
+// start starts a stand-in for GeoJS that answers, and returns it, a
+// clock, and a GeoJS asking it by that clock.
+func start(t *testing.T) (*standIn, *testClock, *lookup.GeoJS) {
+	t.Helper()
+
+	geojs := &standIn{}
+	geojs.server = httptest.NewServer(geojs)
+	t.Cleanup(geojs.server.Close)
+
+	clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
+	g := lookup.New(lookup.Params{
+		URL:        geojs.server.URL,
+		Now:        clock.Now,
+		ProcessLog: slog.New(slog.DiscardHandler),
+	})
+
+	return geojs, clock, g
+}
+
+// newClients returns what returns a new IPv4 client each time it is
+// called.
+func newClients() func() netip.Prefix {
+	addr := netip.MustParseAddr("10.0.0.0")
+
+	return func() netip.Prefix {
+		addr = addr.Next()
+
+		return netip.PrefixFrom(addr, addr.BitLen())
+	}
+}
+
+// wantCountry checks the country g gives client.
+func wantCountry(t *testing.T, g *lookup.GeoJS, client netip.Prefix, want string) {
+	t.Helper()
+
+	got := g.Country(t.Context(), client)
+	if got != want {
+		t.Errorf("%s is in %q, want %q", client, got, want)
+	}
+}
+
+// wantRequests checks how many requests GeoJS has had.
+func wantRequests(t *testing.T, geojs *standIn, want int) {
+	t.Helper()
+
+	got := len(geojs.asked())
+	if got != want {
+		t.Errorf("GeoJS had %d requests, want %d", got, want)
+	}
+}
+
+// wantAsked checks the addresses request i asked about, in any order.
+func wantAsked(t *testing.T, geojs *standIn, i int, want ...string) {
+	t.Helper()
+
+	asked := geojs.asked()
+	if len(asked) <= i {
+		t.Fatalf("GeoJS had %d requests, want more than %d", len(asked), i)
+	}
+
+	got := slices.Sorted(slices.Values(asked[i]))
+
+	slices.Sort(want)
+
+	if !slices.Equal(got, want) {
+		t.Errorf("request %d asked about %v, want %v", i, got, want)
+	}
+}
+
+// waitForRequests waits for GeoJS to have had count requests, and returns
+// the addresses each asked about.
+func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string {
+	t.Helper()
+
+	deadline := time.Now().Add(waitLimit)
+	for time.Now().Before(deadline) {
+		asked := geojs.asked()
+		if len(asked) >= count {
+			return asked
+		}
+
+		time.Sleep(pollInterval)
+	}
+
+	t.Fatalf("fewer than %d requests to GeoJS after %s", count, waitLimit)
+
+	return nil
+}