AS number and country looked up for every client (closes #95)

GeoJS's geo.json is asked about every new visitor unless
SWWAF_LOOKUP_SOURCE is off. A request waits for its client's first
answer only while a country list or SWWAF_ADD_LOOKUP_HEADERS needs it;
otherwise the answer reaches the client's history and ban notes when it
comes. The AS number and name go beside the country in the request log,
history, ban notes, alerts and lookups.json, with metrics by AS number;
64512 counts as unknown. A client's own X-Client-ASN and
X-Client-Country never reach the app, whatever the setting says, and
make example-app sends no address to GeoJS.

Judgement call: AS numbers are written AS64496, as SPEC's settings write them.
Judgement call: SWWAF_LOOKUP_TIMEOUT is added, default 1s, and cannot be off.

Model: opus-5-5

internal/lookup/lookup.go

@@ -1,7 +1,7 @@
-// Package lookup looks up each client's country through the GeoJS web
-// service, and keeps the answers in memory, for at most 100,000 clients
-// and for 7 days each. The answers are written to lookups.json and read
-// from it by the state package.
+// Package lookup looks up each client's AS number and country through
+// the GeoJS web service, and keeps the answers in memory, for at most
+// 100,000 clients and for 7 days each. The answers are written to
+// lookups.json and read from it by the state package.
 package lookup
 
 import (
@@ -14,6 +14,7 @@ import (
 	"net/http"
 	"net/netip"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -23,9 +24,10 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 )
 
-// URL is GeoJS's country endpoint. Asked about several addresses at once,
-// comma separated in its ip parameter, it answers with a list.
-const URL = "https://get.geojs.io/v1/ip/country.json"
+// URL is GeoJS's endpoint for an address's place and network. Asked about
+// several addresses at once, comma separated in its ip parameter, it
+// answers with a list.
+const URL = "https://get.geojs.io/v1/ip/geo.json"
 
 const (
 	// keepFor is how long an answer is used instead of asking GeoJS again.
@@ -40,9 +42,8 @@ const (
 	maxWaiting = 10000
 	// maxPerRequest is how many addresses one request to GeoJS asks about.
 	maxPerRequest = 200
-	// timeout is how long a new client waits for its answer, and how long
-	// a request to GeoJS may take before it is abandoned.
-	timeout = time.Second
+	// unknownASN is the AS number GeoJS gives when it knows none.
+	unknownASN = 64512
 	// After a failure GeoJS is not asked again for a second, and for
 	// retryDelayFactor times as long after each further failure in a row,
 	// up to five minutes.
@@ -62,6 +63,16 @@ var (
 type Params struct {
 	// URL is where GeoJS is asked, normally URL.
 	URL string
+	// Timeout is how long a request waits for its client's first answer,
+	// and how long a request to GeoJS may take before it is abandoned
+	// (SWWAF_LOOKUP_TIMEOUT).
+	Timeout time.Duration
+	// Wait is true when a setting needs each request's answer before the
+	// request goes on. Otherwise no request waits for one.
+	Wait bool
+	// Answered, unless nil, is given each answer GeoJS gives, once it is
+	// kept.
+	Answered func(Answer)
 	// Now tells the time, normally time.Now.
 	Now func() time.Time
 	// ProcessLog receives GeoJS's failures.
@@ -73,11 +84,14 @@ type Params struct {
 	Alerts *alerts.Queue
 }
 
-// GeoJS looks up clients' countries through GeoJS. At most one request
-// to GeoJS is under way at a time, and it asks about every client waiting,
-// up to maxPerRequest. It is safe for concurrent use.
+// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
+// one request to GeoJS is under way at a time, and it asks about every
+// client waiting, up to maxPerRequest. It is safe for concurrent use.
 type GeoJS struct {
 	url        string
+	timeout    time.Duration
+	wait       bool
+	answered   func(Answer)
 	now        func() time.Time
 	processLog *slog.Logger
 	metrics    *metrics.Metrics
@@ -100,10 +114,16 @@ type GeoJS struct {
 }
 
 // Answer is what GeoJS said about a client, as lookups.json holds it: its
-// country, "" when GeoJS cannot place it, when GeoJS said so, and when
-// the answer was last used.
+// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows
+// no AS number for it; its country, "" when GeoJS cannot place it; when
+// GeoJS said so, and when the answer was last used. The zero Answer is
+// that of a client with no answer.
+//
+//nolint:tagliatelle // the state files use snake_case, as the request log does
 type Answer struct {
 	Client   netip.Prefix `json:"client"`
+	ASN      string       `json:"asn"`
+	ASName   string       `json:"as_name"`
 	Country  string       `json:"country"`
 	Answered time.Time    `json:"answered"`
 	Used     time.Time    `json:"used"`
@@ -128,6 +148,9 @@ func New(params Params) *GeoJS {
 
 	return &GeoJS{
 		url:        params.URL,
+		timeout:    params.Timeout,
+		wait:       params.Wait,
+		answered:   params.Answered,
 		now:        params.Now,
 		processLog: params.ProcessLog,
 		metrics:    params.Metrics,
@@ -142,23 +165,23 @@ func New(params Params) *GeoJS {
 	}
 }
 
-// Country returns the country GeoJS places client in, as a two-letter
-// code in capitals, or "" when the country cannot be found: GeoJS cannot
-// place the client, or has not answered in time. An answer is kept for 7
-// days. Without one, a client waits up to timeout for it, unless it has
-// gone without one before; until GeoJS answers, the client is asked about
-// again in the background. ctx is the context of the client's request,
-// and ends the wait when it ends.
+// LookUp returns the answer GeoJS gave about client, with its country as
+// a two-letter code in capitals, or the zero Answer when there is none
+// yet. An answer is kept for 7 days. Without one, the client is asked
+// about in the background, and, while Wait is set, the request waits up
+// to Timeout for the answer, unless the client has gone without one
+// before. ctx is the context of the client's request, and ends the wait
+// when it ends.
 //
 // GeoJS is asked about the client's first address, which is the client's
 // own address for IPv4, and an address in the same place for an IPv6 /64.
-func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
-	country, asked := g.answerOrWait(ctx, client)
+func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
+	answer, asked := g.answerOrWait(ctx, client)
 	if asked == nil {
-		return country
+		return answer
 	}
 
-	timer := time.NewTimer(timeout)
+	timer := time.NewTimer(g.timeout)
 	defer timer.Stop()
 
 	select {
@@ -170,7 +193,7 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
 	g.mu.Lock()
 	defer g.mu.Unlock()
 
-	country, found := g.kept(client)
+	answer, found := g.kept(client)
 	if !found {
 		g.metrics.GeoJSUnanswered.Inc()
 	}
@@ -180,7 +203,15 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
 		w.late = true
 	}
 
-	return country
+	return answer
+}
+
+// Kept returns client's answer, if one is kept, without asking GeoJS.
+func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	return g.kept(client)
 }
 
 // Snapshot returns every answer kept, sorted by client, as lookups.json
@@ -232,13 +263,13 @@ func (g *GeoJS) Load(answers []Answer) {
 // nil when there is nothing to wait for.
 func (g *GeoJS) answerOrWait(
 	ctx context.Context, client netip.Prefix,
-) (string, <-chan struct{}) {
+) (Answer, <-chan struct{}) {
 	g.mu.Lock()
 	defer g.mu.Unlock()
 
-	country, found := g.kept(client)
+	answer, found := g.kept(client)
 	if found {
-		return country, nil
+		return answer, nil
 	}
 
 	w, waiting := g.waiting[client]
@@ -249,10 +280,14 @@ func (g *GeoJS) answerOrWait(
 
 	g.ask(ctx)
 
+	if !g.wait {
+		return Answer{}, nil // the answer is not needed before the request goes on
+	}
+
 	if w == nil {
 		g.metrics.GeoJSUnanswered.Inc()
 
-		return "", nil // too many clients wait already
+		return Answer{}, nil // too many clients wait already
 	}
 
 	if !g.asking {
@@ -263,25 +298,25 @@ func (g *GeoJS) answerOrWait(
 	if w.late {
 		g.metrics.GeoJSUnanswered.Inc()
 
-		return "", nil
+		return Answer{}, nil
 	}
 
-	return "", w.asked
+	return Answer{}, w.asked
 }
 
 // kept returns client's answer, if GeoJS gave it less than keepFor ago,
 // and notes that it was used.
-func (g *GeoJS) kept(client netip.Prefix) (string, bool) {
+func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
 	now := g.now()
 
 	kept, found := g.answers.Get(client)
 	if !found || now.Sub(kept.Answered) >= keepFor {
-		return "", false
+		return Answer{}, false
 	}
 
 	kept.Used = now
 
-	return kept.Country, true
+	return *kept, true
 }
 
 // ask starts asking GeoJS about the waiting clients, unless a request to
@@ -299,7 +334,8 @@ func (g *GeoJS) ask(ctx context.Context) {
 }
 
 // askAboutWaiting asks GeoJS about the waiting clients, one request at a
-// time, until none is left or GeoJS fails.
+// time, until none is left or GeoJS fails. Each answer kept is given to
+// Answered, outside the lock, since Answered takes locks of its own.
 func (g *GeoJS) askAboutWaiting(ctx context.Context) {
 	for {
 		clients := g.nextClients()
@@ -307,8 +343,16 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
 			return
 		}
 
-		countries, err := g.request(ctx, clients)
-		if !g.keep(clients, countries, err) {
+		given, err := g.request(ctx, clients)
+		kept, answered := g.keep(clients, given, err)
+
+		if g.answered != nil {
+			for _, answer := range kept {
+				g.answered(answer)
+			}
+		}
+
+		if !answered {
 			return
 		}
 	}
@@ -340,32 +384,35 @@ func (g *GeoJS) nextClients() []netip.Prefix {
 	return clients
 }
 
-// keep notes how a request to GeoJS about clients ended, and reports
-// whether GeoJS answered about all of them. Each client whose address
-// GeoJS's answer names gets its answer, with no country when GeoJS gave
-// none. An answer that leaves an address out is a failure. After a
-// failure GeoJS is left alone for a while, and every client still waiting
-// stops waiting and is asked about once GeoJS is asked again.
+// keep notes how a request to GeoJS about clients ended, given being the
+// answer for each address GeoJS's answer names. It returns the answers it
+// kept, and reports whether GeoJS answered about all of the clients. Each
+// client whose address GeoJS's answer names gets its answer. An answer
+// that leaves an address out is a failure. After a failure GeoJS is left
+// alone for a while, and every client still waiting stops waiting and is
+// asked about once GeoJS is asked again.
 func (g *GeoJS) keep(
-	clients []netip.Prefix, countries map[netip.Addr]string, err error,
-) bool {
+	clients []netip.Prefix, given map[netip.Addr]Answer, err error,
+) ([]Answer, bool) {
 	g.mu.Lock()
 	defer g.mu.Unlock()
 
 	now := g.now()
+	kept := make([]Answer, 0, len(clients))
 	leftOut := 0
 
 	for _, client := range clients {
-		country, named := countries[client.Addr()]
+		answer, named := given[client.Addr()]
 		if !named {
 			leftOut++
 
 			continue
 		}
 
-		g.answers.Add(client, &Answer{
-			Client: client, Country: country, Answered: now, Used: now,
-		})
+		answer.Client, answer.Answered, answer.Used = client, now, now
+		g.answers.Add(client, &answer)
+		kept = append(kept, answer)
+
 		close(g.waiting[client].asked)
 		delete(g.waiting, client)
 	}
@@ -400,26 +447,28 @@ func (g *GeoJS) keep(
 			},
 		})
 
-		return false
+		return kept, false
 	}
 
 	g.retryDelay = 0
 
-	return true
+	return kept, true
 }
 
-// request asks GeoJS about clients in one request, and returns the
-// country it gave, in capitals, for each address its answer names.
+// request asks GeoJS about clients in one request, and returns the answer
+// for each address GeoJS's answer names: its AS number and the AS's name,
+// both "" for the AS number 64512, which GeoJS gives when it knows none,
+// and its country, in capitals.
 func (g *GeoJS) request(
 	ctx context.Context, clients []netip.Prefix,
-) (map[netip.Addr]string, error) {
+) (map[netip.Addr]Answer, error) {
 	addrs := make([]string, 0, len(clients))
 
 	for _, client := range clients {
 		addrs = append(addrs, client.Addr().String())
 	}
 
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+	ctx, cancel := context.WithTimeout(ctx, g.timeout)
 	defer cancel()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.url, http.NoBody)
@@ -446,9 +495,12 @@ func (g *GeoJS) request(
 		return nil, fmt.Errorf("%w %s", errStatus, res.Status)
 	}
 
+	//nolint:tagliatelle // GeoJS's own names
 	var answers []struct {
-		IP      string `json:"ip"`
-		Country string `json:"country"`
+		IP          string `json:"ip"`
+		ASN         int64  `json:"asn"`
+		ASName      string `json:"organization_name"`
+		CountryCode string `json:"country_code"`
 	}
 
 	err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers)
@@ -456,14 +508,22 @@ func (g *GeoJS) request(
 		return nil, fmt.Errorf("read GeoJS's answer: %w", err)
 	}
 
-	countries := make(map[netip.Addr]string, len(answers))
+	given := make(map[netip.Addr]Answer, len(answers))
 
 	for _, item := range answers {
 		addr, err := netip.ParseAddr(item.IP)
-		if err == nil {
-			countries[addr] = strings.ToUpper(item.Country)
+		if err != nil {
+			continue
 		}
+
+		answer := Answer{Country: strings.ToUpper(item.CountryCode)}
+		if item.ASN != 0 && item.ASN != unknownASN {
+			answer.ASN = "AS" + strconv.FormatInt(item.ASN, 10)
+			answer.ASName = item.ASName
+		}
+
+		given[addr] = answer
 	}
 
-	return countries, nil
+	return given, nil
 }