Look clients up in the IPinfo Lite file with SWWAF_LOOKUP_SOURCE=file (closes #22)

SWWAF_LOOKUP_SOURCE=file looks every client up in the file
SWWAF_LOOKUP_DB_PATH names, without GeoJS. file without the path, the
path with another source, or a file that cannot be read stops the start.
The file is read whole into memory, so overwriting it in place cannot
disturb a lookup, and read again 2 seconds after its last change; a
replacement that cannot be read is logged, counted and sent as a
file_error alert, and the old one stays in use. Metrics give when it
was read and the failed reads. Tests write their databases through
internal/lookup/lookuptest.

Deviation: go.mod and go.sum written by hand; go runs only through make.
Judgement call: the 2-second wait, as the rule files have.

Model: opus-5-5

internal/proxy/lookup.go

@@ -18,26 +18,32 @@ const (
 	countryHeader = "X-Client-Country"
 )
 
-// lookUp looks up the client's AS number and country, and notes them for
-// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a
-// private, loopback or link-local address, which no lookup can place.
-// While a setting needs the answer, a new client's request waits for it.
-// ctx is the request's own context.
+// lookUp looks up the client's AS number and country, in the lookup
+// database or through GeoJS, and notes them for the log line, unless
+// SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
+// link-local address, which no lookup can place. The lookup database
+// answers at once. With GeoJS, while a setting needs the answer, a new
+// client's request waits for it. ctx is the request's own context.
 func (rq *request) lookUp(ctx context.Context) {
 	if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
 		return
 	}
 
-	answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
+	if rq.h.config.LookupSource == "file" {
+		rq.lookupAnswer = rq.h.lookupFile.LookUp(clientGroup(rq.client))
+	} else {
+		rq.lookupAnswer = rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
+	}
+
 	rq.lookedUp = true
-	rq.line.ASN = answer.ASN
-	rq.line.ASName = answer.ASName
-	rq.line.Country = answer.Country
+	rq.line.ASN = rq.lookupAnswer.ASN
+	rq.line.ASName = rq.lookupAnswer.ASName
+	rq.line.Country = rq.lookupAnswer.Country
 }
 
-// addLookup adds answer, GeoJS's answer about a client, to the client's
-// history, and to the notes of the bans on its netblock that have no AS
-// number, AS name or country yet.
+// addLookup adds answer, an answer about a client from the lookup
+// database or GeoJS, to the client's history, and to the notes of the bans
+// on its netblock that have no AS number, AS name or country yet.
 func (h *handler) addLookup(answer lookup.Answer) {
 	h.limiter.AddLookup(answer.Client, answer.Answered,
 		answer.ASN, answer.ASName, answer.Country)

internal/proxy/lookup_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"path/filepath"
 	"slices"
 	"sync"
 	"testing"
@@ -13,6 +14,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
@@ -166,6 +168,64 @@ func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
 	}
 }
 
+func TestClientsAreLookedUpInTheLookupDatabaseAndGeoJSIsNotAsked(t *testing.T) {
+	t.Parallel()
+
+	geojsURL, asked := startGeoJS(t)
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+		fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"},
+	})
+	s, clk, server := startWithClock(t, geojsURL, map[string]string{
+		lookupSource:       "file",
+		lookupDBPath:       path,
+		allowedCountries:   "DE",
+		rateLimitPerMinute: "1",
+	})
+
+	// fromDE's second request breaks the limit and bans it. The list
+	// refuses fromKP, and unplaced, which the file does not hold.
+	de := asnAndCountry{asnDE, asNameDE, "DE"}
+
+	for _, tc := range []struct {
+		line logLine
+		want asnAndCountry
+	}{
+		{s.get(fromDE, http.StatusOK, requestlog.ActionForward), de},
+		{s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited), de},
+		{
+			s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied),
+			asnAndCountry{asnKP, asNameKP, "KP"},
+		},
+		{
+			s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied),
+			asnAndCountry{},
+		},
+	} {
+		got := asnAndCountry{tc.line.ASN, tc.line.ASName, tc.line.Country}
+		if got != tc.want {
+			t.Errorf("log line has %+v, want %+v", got, tc.want)
+		}
+	}
+
+	h := historyOf(t, server, fromDE)
+	if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != de ||
+		!h.LookedUp.Equal(clk.Now()) {
+		t.Errorf("history has %+v, looked up at %s; want %+v, at %s",
+			got, h.LookedUp, de, clk.Now())
+	}
+
+	notes := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32"))[0].Notes
+	if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de {
+		t.Errorf("the ban's notes have %+v, want %+v", got, de)
+	}
+
+	if len(asked()) != 0 {
+		t.Errorf("GeoJS was asked about %v, want nothing", asked())
+	}
+}
+
 func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) {
 	t.Parallel()
 

internal/proxy/proxy.go

@@ -54,9 +54,12 @@ type Params struct {
 	RequestLog io.Writer
 	// ProcessLog receives the process's own messages.
 	ProcessLog *slog.Logger
-	// GeoJSURL is where clients' AS numbers and countries are looked up,
-	// normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off.
+	// GeoJSURL is where clients' AS numbers and countries are looked up
+	// while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
 	GeoJSURL string
+	// LookupFile is the lookup database they are looked up in while
+	// SWWAF_LOOKUP_SOURCE is file, and nil otherwise.
+	LookupFile *lookup.File
 	// Now tells the time by which requests are counted for the rate
 	// limits, bans are made and run out, and GeoJS's answers are kept,
 	// normally time.Now in UTC, the time the state files give.
@@ -70,14 +73,16 @@ type Params struct {
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
-// whose state the state files keep, and the metrics.
+// whose state the state files keep, the lookup database, nil unless
+// SWWAF_LOOKUP_SOURCE is file, and the metrics.
 type Server struct {
 	*http.Server
 
-	Ledger  *bans.Ledger
-	Limiter *ratelimit.Limiter
-	GeoJS   *lookup.GeoJS
-	Metrics *metrics.Metrics
+	Ledger     *bans.Ledger
+	Limiter    *ratelimit.Limiter
+	GeoJS      *lookup.GeoJS
+	LookupFile *lookup.File
+	Metrics    *metrics.Metrics
 }
 
 // New returns the server smallwebwaf runs: each request it reads passes
@@ -109,8 +114,9 @@ func New(params Params) *Server {
 			AttackBanDuration:    params.Config.AttackBanDuration,
 			MaxBans:              params.Config.MaxBans,
 		}),
-		rules:  params.Rules,
-		alerts: params.Alerts,
+		lookupFile: params.LookupFile,
+		rules:      params.Rules,
+		alerts:     params.Alerts,
 	}
 	h.geojs = lookup.New(lookup.Params{
 		URL:     params.GeoJSURL,
@@ -142,10 +148,11 @@ func New(params Params) *Server {
 			MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10),
 			ErrorLog:       errorLog,
 		},
-		Ledger:  h.ledger,
-		Limiter: h.limiter,
-		GeoJS:   h.geojs,
-		Metrics: m,
+		Ledger:     h.ledger,
+		Limiter:    h.limiter,
+		GeoJS:      h.geojs,
+		LookupFile: h.lookupFile,
+		Metrics:    m,
 	}
 }
 
@@ -162,6 +169,7 @@ type handler struct {
 	limiter    *ratelimit.Limiter
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
+	lookupFile *lookup.File
 	rules      *rules.Files
 	alerts     *alerts.Queue
 }

internal/proxy/proxy_test.go

@@ -16,6 +16,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 	"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -67,6 +68,7 @@ const (
 	rateLimitPerDay         = "SWWAF_RATE_LIMIT_PER_DAY"
 	rateLimitExemptPaths    = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
 	lookupSource            = "SWWAF_LOOKUP_SOURCE"
+	lookupDBPath            = "SWWAF_LOOKUP_DB_PATH"
 	lookupTimeout           = "SWWAF_LOOKUP_TIMEOUT"
 	addLookupHeaders        = "SWWAF_ADD_LOOKUP_HEADERS"
 	deniedCountries         = "SWWAF_DENIED_COUNTRIES"
@@ -265,7 +267,8 @@ func startProxyWithAlerts(
 // proxy raises, as the settings in env make it. No alert is sent from the
 // queue: they wait in it, for the test to look at. With no geojsURL, there
 // is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
-// is off unless env sets it.
+// is off unless env sets it. While it is file, the lookup database
+// SWWAF_LOOKUP_DB_PATH names is read.
 func newProxy(
 	t *testing.T, appURL, geojsURL string, now func() time.Time,
 	env map[string]string,
@@ -310,11 +313,23 @@ func newProxy(
 		ProcessLog: processLog,
 	})
 
+	var lookupFile *lookup.File
+
+	if cfg.LookupSource == "file" {
+		lookupFile, err = lookup.OpenFile(lookup.FileParams{
+			Path: cfg.LookupDBPath, Now: now, ProcessLog: processLog, Alerts: alertQueue,
+		})
+		if err != nil {
+			t.Fatalf("lookup database: %v", err)
+		}
+	}
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: out,
 		ProcessLog: processLog,
 		GeoJSURL:   geojsURL,
+		LookupFile: lookupFile,
 		Now:        now,
 		Rules:      ruleFiles,
 		Alerts:     alertQueue,

internal/proxy/request.go

@@ -15,6 +15,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
@@ -49,9 +50,11 @@ type request struct {
 	peer        netip.Addr
 	peerTrusted bool
 	// lookedUp is true once the client's AS number and country have been
-	// looked up, whether or not an answer was there.
-	lookedUp bool
-	start    time.Time
+	// looked up, whether or not an answer was there, and lookupAnswer is
+	// what the lookup gave then, the zero Answer while GeoJS had given none.
+	lookedUp     bool
+	lookupAnswer lookup.Answer
+	start        time.Time
 	// checked is when the checks were done, and upstreamStart when the
 	// request was handed to the app.
 	checked       time.Time
@@ -488,10 +491,11 @@ func timing(start, end time.Time) *float64 {
 }
 
 // addToHistory adds the request, which has ended, to its client's
-// history, and then, for a client that was looked up, the answer kept
-// about it to that history and to the notes of the bans on its netblock:
-// an answer that came during the request may have come before either was
-// there, and one that comes later is added when it comes.
+// history, and then, for a client that was looked up, the lookup
+// database's answer about it, or the answer from GeoJS kept about it, to
+// that history and to the notes of the bans on its netblock: an answer
+// may have come before either was there, and one from GeoJS that comes
+// later is added when it comes.
 func (rq *request) addToHistory() {
 	var requestBytes int64
 	if rq.body != nil {
@@ -514,6 +518,13 @@ func (rq *request) addToHistory() {
 		return
 	}
 
+	// The lookup database's answer was there at once.
+	if rq.h.config.LookupSource == "file" {
+		rq.h.addLookup(rq.lookupAnswer)
+
+		return
+	}
+
 	answer, kept := rq.h.geojs.Kept(group)
 	if kept {
 		rq.h.addLookup(answer)