Look clients up in the IPinfo Lite file with SWWAF_LOOKUP_SOURCE=file (closes #22)

SWWAF_LOOKUP_SOURCE=file looks every client up in the file
SWWAF_LOOKUP_DB_PATH names, without GeoJS. file without the path, the
path with another source, or a file that cannot be read stops the start.
The file is read whole into memory, so overwriting it in place cannot
disturb a lookup, and read again 2 seconds after its last change; a
replacement that cannot be read is logged, counted and sent as a
file_error alert, and the old one stays in use. Metrics give when it
was read and the failed reads. Tests write their databases through
internal/lookup/lookuptest.

Deviation: go.mod and go.sum written by hand; go runs only through make.
Judgement call: the 2-second wait, as the rule files have.

Model: opus-5-5

internal/alerts/alerts.go

@@ -49,7 +49,8 @@ const (
 	// EventSourceFailure is GeoJS failing or refusing smallwebwaf.
 	EventSourceFailure = "source_failure"
 	// EventFileError is a rule file or state file edited while smallwebwaf
-	// runs that does not parse, or a state file that cannot be written.
+	// runs that does not parse, a replacement of the lookup database that
+	// cannot be read, or a state file that cannot be written.
 	EventFileError = "file_error"
 	// EventSummary is the summary of the alerts an hour held back past
 	// SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it.

internal/bans/bans.go

@@ -416,10 +416,10 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban {
 }
 
 // AddLookup gives the notes of netblock's bans that have no AS number, AS
-// name or country yet those of a client in it, as GeoJS answered about
-// it. It is not a request from netblock, and leaves when it was last seen
-// unchanged. It does not have bans.json written at once: the notes are
-// written with its next write, as the counts in them are.
+// name or country yet those of a client in it, as the lookup answered
+// about it. It is not a request from netblock, and leaves when it was last
+// seen unchanged. It does not have bans.json written at once: the notes
+// are written with its next write, as the counts in them are.
 func (l *Ledger) AddLookup(netblock netip.Prefix, asn, asName, country string) {
 	l.mu.Lock()
 	defer l.mu.Unlock()

internal/config/config.go

@@ -92,12 +92,16 @@ type Config struct {
 	// Each starts with /.
 	RateLimitExemptPaths []string
 	// LookupSource is where each client's AS number and country are
-	// looked up (SWWAF_LOOKUP_SOURCE): geojs, or off for nowhere. A request
-	// waits up to LookupTimeout for its client's first answer while a
-	// setting needs it (SWWAF_LOOKUP_TIMEOUT), which cannot be off.
-	// AddLookupHeaders is true when the app is passed the client's AS
-	// number and country in headers (SWWAF_ADD_LOOKUP_HEADERS).
+	// looked up (SWWAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
+	// LookupDBPath is the lookup database, the IPinfo Lite file looked up
+	// in while LookupSource is file (SWWAF_LOOKUP_DB_PATH), and "" for any
+	// other source. A request waits up to LookupTimeout for its client's
+	// first answer from GeoJS while a setting needs it
+	// (SWWAF_LOOKUP_TIMEOUT), which cannot be off. AddLookupHeaders is true
+	// when the app is passed the client's AS number and country in headers
+	// (SWWAF_ADD_LOOKUP_HEADERS).
 	LookupSource     string
+	LookupDBPath     string
 	LookupTimeout    time.Duration
 	AddLookupHeaders bool
 	// DeniedCountries are the countries whose clients are refused
@@ -201,6 +205,10 @@ type Config struct {
 // off.
 const off = "off"
 
+// fileSource is the SWWAF_LOOKUP_SOURCE that looks clients up in the
+// lookup database, the file SWWAF_LOOKUP_DB_PATH names.
+const fileSource = "file"
+
 const (
 	day      = 24 * time.Hour
 	kibibyte = 1 << 10
@@ -242,8 +250,10 @@ var (
 		"is taken out of every request by Go's HTTP server, so it can never " +
 			"be logged")
 	errOnBothLists          = errors.New("is in SWWAF_DENIED_COUNTRIES too")
-	errNotLookupSource      = errors.New("is not geojs or off")
+	errNotLookupSource      = errors.New("is not geojs, file or off")
 	errNeedsLookups         = errors.New("it needs each client looked up")
+	errNeedsDBPath          = errors.New("it names the file to look clients up in")
+	errDBPathUnused         = errors.New("only file reads it")
 	errNotOver4K            = errors.New("is not a size of more than 4K, such as 32K")
 	errNotDurationAboveZero = errors.New(
 		"is not a duration above zero, such as 1h or 7d")
@@ -312,6 +322,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		RateLimitPerDay:         env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),
 		RateLimitExemptPaths:    env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
 		LookupSource:            env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
+		LookupDBPath:            env.value("SWWAF_LOOKUP_DB_PATH", ""),
 		LookupTimeout:           env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"),
 		AddLookupHeaders:        env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"),
 		DeniedCountries:         env.countries("SWWAF_DENIED_COUNTRIES", ""),
@@ -353,6 +364,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		cfg.InstanceName, cfg.LogRemoteURL != nil)
 
 	env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
+	env.checkLookupDBPath(cfg)
 	env.checkCountriesAndLookups(cfg)
 
 	if env.err != nil {
@@ -556,16 +568,30 @@ func (e *environment) countries(name, defaultValue string) []string {
 }
 
 // lookupSource reads the setting that is where clients are looked up:
-// geojs, or off.
+// geojs, file, or off.
 func (e *environment) lookupSource(name, defaultValue string) string {
 	source := e.value(name, defaultValue)
-	if source != "geojs" && source != off {
+	if source != "geojs" && source != fileSource && source != off {
 		e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
 	}
 
 	return source
 }
 
+// checkLookupDBPath refuses SWWAF_LOOKUP_SOURCE=file without
+// SWWAF_LOOKUP_DB_PATH, and SWWAF_LOOKUP_DB_PATH with any other source:
+// one source at a time.
+func (e *environment) checkLookupDBPath(cfg *Config) {
+	switch {
+	case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
+		e.check("SWWAF_LOOKUP_SOURCE", fmt.Errorf(
+			"is file while SWWAF_LOOKUP_DB_PATH is unset; %w", errNeedsDBPath))
+	case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
+		e.check("SWWAF_LOOKUP_DB_PATH", fmt.Errorf(
+			"is set while SWWAF_LOOKUP_SOURCE is %s; %w", cfg.LookupSource, errDBPathUnused))
+	}
+}
+
 // checkCountriesAndLookups refuses a country on both country lists, and,
 // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
 // up: the country lists and SWWAF_ADD_LOOKUP_HEADERS.

internal/config/config_test.go

@@ -41,6 +41,7 @@ const (
 	rateLimitPerDay         = "SWWAF_RATE_LIMIT_PER_DAY"
 	rateLimitExemptPaths    = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
 	lookupSource            = "SWWAF_LOOKUP_SOURCE"
+	lookupDBPath            = "SWWAF_LOOKUP_DB_PATH"
 	lookupTimeout           = "SWWAF_LOOKUP_TIMEOUT"
 	addLookupHeaders        = "SWWAF_ADD_LOOKUP_HEADERS"
 	deniedCountries         = "SWWAF_DENIED_COUNTRIES"
@@ -123,8 +124,12 @@ const off = "off"
 // enabled is true, as a setting's value.
 const enabled = "true"
 
-// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE.
-const defaultLookupSource = "geojs"
+// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE, and
+// fileSource the source that is the lookup database.
+const (
+	defaultLookupSource = "geojs"
+	fileSource          = "file"
+)
 
 // environment is a set of environment variables, for FromEnvironment.
 type environment map[string]string
@@ -802,6 +807,47 @@ func TestLookupSettingsAsSet(t *testing.T) {
 	})
 }
 
+func TestLookupDBPathGoesWithTheFileSourceAlone(t *testing.T) {
+	t.Parallel()
+
+	const path = "/var/lib/ipinfo/ipinfo_lite.mmdb"
+
+	cfg := fromEnvironment(t, environment{lookupSource: fileSource, lookupDBPath: path})
+	if cfg.LookupSource != fileSource || cfg.LookupDBPath != path {
+		t.Errorf("lookups from %q in %q, want file in %q",
+			cfg.LookupSource, cfg.LookupDBPath, path)
+	}
+
+	for _, tc := range []struct {
+		env  environment
+		want string
+	}{
+		{
+			environment{lookupSource: fileSource},
+			lookupSource + ": is file while " + lookupDBPath +
+				" is unset; it names the file to look clients up in",
+		},
+		{
+			environment{lookupSource: fileSource, lookupDBPath: ""},
+			lookupSource + ": is file while " + lookupDBPath +
+				" is unset; it names the file to look clients up in",
+		},
+		{
+			environment{lookupDBPath: path},
+			lookupDBPath + ": is set while " + lookupSource + " is geojs; only file reads it",
+		},
+		{
+			environment{lookupSource: off, lookupDBPath: path},
+			lookupDBPath + ": is set while " + lookupSource + " is off; only file reads it",
+		},
+	} {
+		_, err := config.FromEnvironment(tc.env.lookupEnv)
+		if err == nil || err.Error() != tc.want {
+			t.Errorf("settings %v: error %v, want %s", tc.env, err, tc.want)
+		}
+	}
+}
+
 func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
 	t.Parallel()
 
@@ -950,7 +996,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{rateLimitPerHour, "1.5"},
 		{rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"},
 		{rateLimitExemptPaths, "/assets/,,/static/"},
-		{lookupSource, "file"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
+		{lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
 		{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
 		{addLookupHeaders, "yes"},
 		{deniedCountries, "nk"},
@@ -1241,6 +1287,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		rateLimitPerDay:         "50000",
 		rateLimitExemptPaths:    "",
 		lookupSource:            defaultLookupSource,
+		lookupDBPath:            "",
 		lookupTimeout:           "1s",
 		addLookupHeaders:        "false",
 		deniedCountries:         "",

internal/lookup/file.go

@@ -0,0 +1,233 @@
+package lookup
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/oschwald/maxminddb-golang/v2"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+)
+
+// quietTime is how long the lookup database must go without a change
+// before it is read again, so that a file still being copied in is read
+// only once whole.
+const quietTime = 2 * time.Second
+
+// FileParams are what OpenFile needs.
+type FileParams struct {
+	// Path is the lookup database, the IPinfo Lite file in its .mmdb form
+	// (SWWAF_LOOKUP_DB_PATH).
+	Path string
+	// Now tells the time, normally time.Now.
+	Now func() time.Time
+	// ProcessLog receives each reading of the file, and why a replacement
+	// of it cannot be read.
+	ProcessLog *slog.Logger
+	// Alerts receive a file_error alert for each replacement that cannot
+	// be read.
+	Alerts *alerts.Queue
+}
+
+// File looks up clients' AS numbers and countries in the lookup database,
+// held in memory, and reads it again when it is replaced. It is safe for
+// concurrent use.
+type File struct {
+	params FileParams
+
+	mu sync.Mutex
+	// reader is the database in use, and lastRead when it was read.
+	// readFailures are the replacements that could not be read.
+	reader       *maxminddb.Reader
+	lastRead     time.Time
+	readFailures int
+}
+
+// record is what the lookup database holds about a network, of the fields
+// smallwebwaf reads.
+type record struct {
+	ASN         string `maxminddb:"asn"`
+	ASName      string `maxminddb:"as_name"`
+	CountryCode string `maxminddb:"country_code"`
+}
+
+// OpenFile reads the lookup database. A file that cannot be read, or that
+// is not a .mmdb file, is an error.
+func OpenFile(params FileParams) (*File, error) {
+	reader, err := read(params.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	f := &File{params: params}
+	f.use(reader)
+
+	return f, nil
+}
+
+// LookUp returns what the lookup database says about client: its AS
+// number, such as AS64496, the AS's name, and its country, such as DE,
+// each "" when the database does not give it, as for an address missing
+// from it. The database is asked about the client's first address, as
+// GeoJS is.
+func (f *File) LookUp(client netip.Prefix) Answer {
+	f.mu.Lock()
+	reader := f.reader
+	f.mu.Unlock()
+
+	var found record
+
+	// A record that cannot be decoded places the client nowhere, as a
+	// missing one does.
+	err := reader.Lookup(client.Addr()).Decode(&found)
+	if err != nil {
+		found = record{}
+	}
+
+	return Answer{
+		Client:   client,
+		ASN:      found.ASN,
+		ASName:   found.ASName,
+		Country:  found.CountryCode,
+		Answered: f.params.Now(),
+	}
+}
+
+// LastRead returns when the lookup database in use was read.
+func (f *File) LastRead() time.Time {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.lastRead
+}
+
+// ReadFailures returns how many replacements of the lookup database could
+// not be read.
+func (f *File) ReadFailures() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.readFailures
+}
+
+// Watch watches the directory of the lookup database until ctx is done,
+// and reads the file again once it has gone without a change for
+// quietTime, after it is replaced, written or removed, and after Watch
+// starts watching. If the directory cannot be watched, that is logged, and
+// the database read at start stays in use.
+func (f *File) Watch(ctx context.Context) {
+	watcher, err := fsnotify.NewWatcher()
+	if err == nil {
+		defer func() {
+			_ = watcher.Close()
+		}()
+
+		err = watcher.Add(filepath.Dir(f.params.Path))
+	}
+
+	if err != nil {
+		f.params.ProcessLog.Error("cannot watch the lookup database for replacements",
+			"error", err.Error())
+
+		return
+	}
+
+	f.params.ProcessLog.Info("watching the lookup database for replacements",
+		"file", f.params.Path)
+
+	f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
+}
+
+// readAfterChanges reads the lookup database again once quietTime has
+// passed without a change to it from events, until ctx is done, and logs
+// the errors from errs. A change to another file in its directory does not
+// count. The wait starts at once, as if for a change, so that a file
+// replaced after OpenFile read it, and before its directory was watched,
+// is read too.
+func (f *File) readAfterChanges(
+	ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
+) {
+	path := filepath.Clean(f.params.Path)
+
+	quiet := time.NewTimer(quietTime)
+	defer quiet.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case event := <-events:
+			if filepath.Clean(event.Name) == path {
+				quiet.Reset(quietTime)
+			}
+		case <-quiet.C:
+			f.readAgain()
+		case err := <-errs:
+			f.params.ProcessLog.Warn("watching the lookup database failed",
+				"error", err.Error())
+		}
+	}
+}
+
+// readAgain reads the lookup database again, in place of the one in use,
+// or, if it cannot be read, counts that, raises a file_error alert for it
+// and logs it, and the one in use stays in use.
+func (f *File) readAgain() {
+	reader, err := read(f.params.Path)
+	if err != nil {
+		const kept = "the lookup database cannot be read, " +
+			"and the one read before stays in use"
+
+		f.mu.Lock()
+		f.readFailures++
+		f.mu.Unlock()
+
+		// Raised before it is logged, so that the alert is there once the
+		// log line is.
+		f.params.Alerts.Raise(alerts.Alert{
+			Event:  alerts.EventFileError,
+			Reason: kept,
+			Detail: map[string]any{"file": f.params.Path, "error": err.Error()},
+		})
+		f.params.ProcessLog.Error(kept, "error", err.Error())
+
+		return
+	}
+
+	f.use(reader)
+}
+
+// use puts reader in use, in place of the database read before, and logs
+// that the file was read.
+func (f *File) use(reader *maxminddb.Reader) {
+	f.mu.Lock()
+	f.reader = reader
+	f.lastRead = f.params.Now()
+	f.mu.Unlock()
+
+	f.params.ProcessLog.Info("read the lookup database", "file", f.params.Path)
+}
+
+// read reads the lookup database at path. The whole file is read into
+// memory, rather than mapped into it as the reader can, so that a file
+// overwritten in place cannot change, or end, under a lookup.
+func read(path string) (*maxminddb.Reader, error) {
+	data, err := os.ReadFile(path) //nolint:gosec // the file the admin names
+	if err != nil {
+		return nil, fmt.Errorf("SWWAF_LOOKUP_DB_PATH cannot be read: %w", err)
+	}
+
+	reader, err := maxminddb.OpenBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("SWWAF_LOOKUP_DB_PATH %s is not a .mmdb file: %w", path, err)
+	}
+
+	return reader, nil
+}

internal/lookup/file_internal_test.go

@@ -0,0 +1,379 @@
+package lookup
+
+import (
+	"context"
+	"log/slog"
+	"net/netip"
+	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
+)
+
+// testNetblock is the netblock the tests' lookup databases place, and
+// testClient a client in it.
+const (
+	testNetblock = "203.0.113.0/24"
+	testClient   = "203.0.113.9/32"
+)
+
+func TestFilePlacesClientsAndCountsAnAddressMissingFromItAsUnknown(t *testing.T) {
+	t.Parallel()
+
+	germany := lookuptest.Network{ASN: "AS64496", ASName: "Example Net", Country: "DE"}
+	northKorea := lookuptest.Network{ASN: "AS64511", ASName: "Other Net", Country: "KP"}
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		testNetblock:    germany,
+		"2001:db8::/32": northKorea,
+	})
+
+	now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
+
+	f, err := OpenFile(FileParams{
+		Path:       path,
+		Now:        func() time.Time { return now },
+		ProcessLog: slog.New(slog.DiscardHandler),
+		Alerts:     newQueue(),
+	})
+	if err != nil {
+		t.Fatalf("open %s: %v", path, err)
+	}
+
+	for client, want := range map[string]lookuptest.Network{
+		testClient: germany,
+		// An IPv6 client is its /64.
+		"2001:db8:1:2::/64": northKorea,
+		"198.51.100.7/32":   {},
+	} {
+		prefix := netip.MustParsePrefix(client)
+
+		got := f.LookUp(prefix)
+		if got != (Answer{
+			Client: prefix, ASN: want.ASN, ASName: want.ASName, Country: want.Country,
+			Answered: now,
+		}) {
+			t.Errorf("%s has the answer %+v, want %+v, answered %s", client, got, want, now)
+		}
+	}
+}
+
+func TestRecordThatCannotBeReadPlacesTheClientNowhere(t *testing.T) {
+	t.Parallel()
+
+	// The AS number is a number, where a string belongs. The writer writes
+	// a record's fields in the order of their names, so as_name is read
+	// before the AS number fails.
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.WriteRecords(t, path, map[string]mmdbtype.Map{
+		testNetblock: {
+			"asn":          mmdbtype.Uint32(64496),
+			"as_name":      mmdbtype.String("Example Net"),
+			"country_code": mmdbtype.String("DE"),
+		},
+	})
+
+	f := openFile(t, path, newQueue())
+
+	answer := f.LookUp(netip.MustParsePrefix(testClient))
+	if answer.ASN != "" || answer.ASName != "" || answer.Country != "" {
+		t.Errorf("%s is placed %+v, want nowhere", testClient, answer)
+	}
+}
+
+func TestFileThatCannotBeReadIsAnError(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	missing := filepath.Join(dir, "missing.mmdb")
+	notDatabase := filepath.Join(dir, "not.mmdb")
+	writeFile(t, notDatabase, "not a lookup database\n")
+
+	for path, want := range map[string]string{
+		missing: "SWWAF_LOOKUP_DB_PATH cannot be read: open " + missing +
+			": no such file or directory",
+		notDatabase: "SWWAF_LOOKUP_DB_PATH " + notDatabase +
+			" is not a .mmdb file: error opening database: invalid MaxMind DB file",
+	} {
+		_, err := OpenFile(FileParams{
+			Path:       path,
+			Now:        time.Now,
+			ProcessLog: slog.New(slog.DiscardHandler),
+			Alerts:     newQueue(),
+		})
+		if err == nil || err.Error() != want {
+			t.Errorf("opening %s failed with %v, want %s", path, err, want)
+		}
+	}
+}
+
+// The tests below run readAfterChanges in a synctest bubble, where time is
+// a clock of the test's own: time.Sleep moves it on at once, and
+// synctest.Wait returns once readAfterChanges waits again, so that every
+// reading due by then is done. The test sends the changes itself, as the
+// watch of a directory cannot run in a bubble.
+
+func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "ipinfo_lite.mmdb")
+		writeDatabase(t, path, "DE")
+
+		queue := newQueue()
+		f := openFile(t, path, queue)
+		changes := watch(t, f)
+
+		other := filepath.Join(dir, "replacement.mmdb")
+		writeDatabase(t, other, "KP")
+
+		replacement, err := os.ReadFile(other) //nolint:gosec // a file the test wrote
+		if err != nil {
+			t.Fatalf("read %s: %v", other, err)
+		}
+
+		// The file in use is overwritten in place, and keeps giving what
+		// it gave. Its first part alone is not a .mmdb file.
+		file, err := os.Create(path) //nolint:gosec // a file the test wrote
+		if err != nil {
+			t.Fatalf("create %s: %v", path, err)
+		}
+
+		defer func() {
+			_ = file.Close()
+		}()
+
+		half := len(replacement) / 2
+		write(t, file, replacement[:half])
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		time.Sleep(quietTime - time.Nanosecond)
+		synctest.Wait()
+		wantCountry(t, f, "DE")
+
+		// The second part starts the wait again.
+		write(t, file, replacement[half:])
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		time.Sleep(quietTime - time.Nanosecond)
+		synctest.Wait()
+		wantCountry(t, f, "DE")
+
+		time.Sleep(time.Nanosecond)
+		synctest.Wait()
+		wantCountry(t, f, "KP")
+
+		if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
+			t.Errorf("read at %s, with %d failures; want read now, with none",
+				f.LastRead(), f.ReadFailures())
+		}
+
+		wantAlerts(t, queue)
+	})
+}
+
+func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+		writeDatabase(t, path, "DE")
+
+		queue := newQueue()
+		f := openFile(t, path, queue)
+		read := f.LastRead()
+		changes := watch(t, f)
+
+		writeFile(t, path, "not a lookup database\n")
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		// Long after, the replacement has been read once.
+		time.Sleep(time.Hour)
+		synctest.Wait()
+		wantCountry(t, f, "DE")
+
+		if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
+			t.Errorf("read at %s, with %d failures; want read at %s, with one",
+				f.LastRead(), f.ReadFailures(), read)
+		}
+
+		wantAlerts(t, queue, alerts.Alert{
+			Time:   read.Add(quietTime),
+			Event:  alerts.EventFileError,
+			Reason: "the lookup database cannot be read, and the one read before stays in use",
+			Detail: map[string]any{
+				"file": path,
+				"error": "SWWAF_LOOKUP_DB_PATH " + path + " is not a .mmdb file: " +
+					"error opening database: invalid MaxMind DB file",
+			},
+		})
+	})
+}
+
+func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "ipinfo_lite.mmdb")
+		writeDatabase(t, path, "DE")
+
+		f := openFile(t, path, newQueue())
+		changes := watch(t, f)
+
+		// The wait that starts with the watch ends with a reading.
+		time.Sleep(quietTime)
+		synctest.Wait()
+		writeDatabase(t, path, "KP")
+
+		changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
+
+		time.Sleep(quietTime)
+		synctest.Wait()
+		wantCountry(t, f, "DE")
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		time.Sleep(quietTime)
+		synctest.Wait()
+		wantCountry(t, f, "KP")
+	})
+}
+
+func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+		writeDatabase(t, path, "DE")
+
+		f := openFile(t, path, newQueue())
+
+		// Saved after OpenFile read the file, and before its directory was
+		// watched, so that no change is seen for it.
+		writeDatabase(t, path, "KP")
+		watch(t, f)
+		time.Sleep(quietTime)
+		synctest.Wait()
+		wantCountry(t, f, "KP")
+	})
+}
+
+// newQueue returns a queue of alerts for a webhook that is never sent
+// them, so that they wait in it for the test to look at.
+func newQueue() *alerts.Queue {
+	return alerts.New(alerts.Params{
+		WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
+		Events:     alerts.Events(),
+		Cooldown:   15 * time.Minute,
+		Now:        time.Now,
+	})
+}
+
+// writeDatabase writes a lookup database at path that places testNetblock
+// in country, and no other address.
+func writeDatabase(t *testing.T, path, country string) {
+	t.Helper()
+
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		testNetblock: {ASN: "AS64496", ASName: "Example Net", Country: country},
+	})
+}
+
+// openFile opens the lookup database at path, which raises its alerts to
+// queue.
+func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
+	t.Helper()
+
+	f, err := OpenFile(FileParams{
+		Path:       path,
+		Now:        time.Now,
+		ProcessLog: slog.New(slog.DiscardHandler),
+		Alerts:     queue,
+	})
+	if err != nil {
+		t.Fatalf("open %s: %v", path, err)
+	}
+
+	return f
+}
+
+// watch runs f's readAfterChanges until the test ends, and returns the
+// channel that sends it changes.
+func watch(t *testing.T, f *File) chan<- fsnotify.Event {
+	t.Helper()
+
+	changes := make(chan fsnotify.Event)
+	ctx, stop := context.WithCancel(t.Context())
+	stopped := make(chan struct{})
+
+	go func() {
+		f.readAfterChanges(ctx, changes, nil)
+		close(stopped)
+	}()
+
+	t.Cleanup(func() {
+		stop()
+		<-stopped
+	})
+
+	return changes
+}
+
+// wantCountry checks the country f gives testClient.
+func wantCountry(t *testing.T, f *File, want string) {
+	t.Helper()
+
+	got := f.LookUp(netip.MustParsePrefix(testClient)).Country
+	if got != want {
+		t.Errorf("%s is in %q, want %q", testClient, got, want)
+	}
+}
+
+// wantAlerts checks the alerts waiting in queue, and that it held none
+// back.
+func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) {
+	t.Helper()
+
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != len(want) || (len(want) > 0 && !reflect.DeepEqual(waiting, want)) {
+		t.Errorf("alerts waiting %+v, want %+v", waiting, want)
+	}
+
+	if queue.Suppressed() != 0 {
+		t.Errorf("%d alerts held back, want none", queue.Suppressed())
+	}
+}
+
+// writeFile writes content to the file at path.
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+
+	err := os.WriteFile(path, []byte(content), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// write writes data to the end of file.
+func write(t *testing.T, file *os.File, data []byte) {
+	t.Helper()
+
+	_, err := file.Write(data)
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+}

internal/lookup/lookup.go

@@ -1,6 +1,7 @@
-// Package lookup looks up each client's AS number and country through
-// the GeoJS web service, and keeps the answers in memory, for at most
-// 100,000 clients and for 7 days each. The answers are written to
+// Package lookup looks up each client's AS number and country, through
+// the GeoJS web service or in the lookup database, the IPinfo Lite file
+// SWWAF_LOOKUP_DB_PATH names. GeoJS's answers are kept in memory, for at
+// most 100,000 clients and for 7 days each, and are written to
 // lookups.json and read from it by the state package.
 package lookup
 
@@ -113,11 +114,12 @@ type GeoJS struct {
 	retryAt    time.Time
 }
 
-// Answer is what GeoJS said about a client, as lookups.json holds it: its
-// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows
-// no AS number for it; its country, "" when GeoJS cannot place it; when
-// GeoJS said so, and when the answer was last used. The zero Answer is
-// that of a client with no answer.
+// Answer is what GeoJS or the lookup database said about a client: its AS
+// number, such as AS64496, and the AS's name, both "" when the source knows
+// no AS number for it; its country, "" when the source cannot place it;
+// when the source said so; and, for GeoJS's answers, which lookups.json
+// holds, when the answer was last used. The zero Answer is that of a
+// client with no answer.
 //
 //nolint:tagliatelle // the state files use snake_case, as the request log does
 type Answer struct {

internal/lookup/lookuptest/lookuptest.go

@@ -0,0 +1,83 @@
+// Package lookuptest writes lookup databases, IPinfo Lite files in their
+// .mmdb form, for the tests of the packages that read them.
+package lookuptest
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+)
+
+// fileMode is the mode of the files written: read and written by their
+// owner alone.
+const fileMode = 0o600
+
+// Network is what a lookup database holds about a netblock, of the fields
+// smallwebwaf reads: its AS number, such as AS64496, the AS's name, and
+// its country, such as DE.
+type Network struct {
+	ASN     string
+	ASName  string
+	Country string
+}
+
+// Write writes a lookup database at path that places each netblock in
+// networks, such as 203.0.113.0/24, as its Network says, and no other
+// address.
+func Write(tb testing.TB, path string, networks map[string]Network) {
+	tb.Helper()
+
+	records := make(map[string]mmdbtype.Map, len(networks))
+	for netblock, network := range networks {
+		records[netblock] = mmdbtype.Map{
+			"asn":          mmdbtype.String(network.ASN),
+			"as_name":      mmdbtype.String(network.ASName),
+			"country_code": mmdbtype.String(network.Country),
+		}
+	}
+
+	WriteRecords(tb, path, records)
+}
+
+// WriteRecords writes a lookup database at path that holds each record in
+// records for its netblock, and nothing for any other address.
+func WriteRecords(tb testing.TB, path string, records map[string]mmdbtype.Map) {
+	tb.Helper()
+
+	tree, err := mmdbwriter.New(mmdbwriter.Options{
+		DatabaseType: "ipinfo_lite",
+		// The tests' clients are in the netblocks kept for documentation.
+		IncludeReservedNetworks: true,
+	})
+	if err != nil {
+		tb.Fatalf("new lookup database: %v", err)
+	}
+
+	for netblock, record := range records {
+		_, network, err := net.ParseCIDR(netblock)
+		if err != nil {
+			tb.Fatalf("netblock %q: %v", netblock, err)
+		}
+
+		err = tree.Insert(network, record)
+		if err != nil {
+			tb.Fatalf("insert %s: %v", netblock, err)
+		}
+	}
+
+	var database bytes.Buffer
+
+	_, err = tree.WriteTo(&database)
+	if err != nil {
+		tb.Fatalf("write the lookup database: %v", err)
+	}
+
+	err = os.WriteFile(path, database.Bytes(), fileMode)
+	if err != nil {
+		tb.Fatalf("write %s: %v", path, err)
+	}
+}

internal/metrics/metrics.go

@@ -229,6 +229,28 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
 	)
 }
 
+// AddLookupFile adds the metrics of the lookup database, read as the
+// metrics are asked for: when the file in use was read, which lastRead
+// returns, and the replacements of it that could not be read, which
+// readFailures returns. The lookup package's File, which has both, cannot
+// be named here: that package counts GeoJS's requests in these metrics.
+func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() int) {
+	m.registry.MustRegister(
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_lookup_database_last_read_timestamp_seconds",
+			Help: "When the lookup database in use was read, in seconds since 1970.",
+		}, func() float64 {
+			return float64(lastRead().Unix())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name: "smallwebwaf_lookup_database_read_failures_total",
+			Help: "Replacements of the lookup database that could not be read.",
+		}, func() float64 {
+			return float64(readFailures())
+		}),
+	)
+}
+
 // AddAlerts adds the metrics of the alerts sent to each destination set,
 // read from queue as the metrics are asked for, by destination: the
 // alerts sent, the requests to the destination that failed, the alerts

internal/proxy/lookup.go

@@ -18,26 +18,32 @@ const (
 	countryHeader = "X-Client-Country"
 )
 
-// lookUp looks up the client's AS number and country, and notes them for
-// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a
-// private, loopback or link-local address, which no lookup can place.
-// While a setting needs the answer, a new client's request waits for it.
-// ctx is the request's own context.
+// lookUp looks up the client's AS number and country, in the lookup
+// database or through GeoJS, and notes them for the log line, unless
+// SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
+// link-local address, which no lookup can place. The lookup database
+// answers at once. With GeoJS, while a setting needs the answer, a new
+// client's request waits for it. ctx is the request's own context.
 func (rq *request) lookUp(ctx context.Context) {
 	if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
 		return
 	}
 
-	answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
+	if rq.h.config.LookupSource == "file" {
+		rq.lookupAnswer = rq.h.lookupFile.LookUp(clientGroup(rq.client))
+	} else {
+		rq.lookupAnswer = rq.h.geojs.LookUp(ctx, clientGroup(rq.client))
+	}
+
 	rq.lookedUp = true
-	rq.line.ASN = answer.ASN
-	rq.line.ASName = answer.ASName
-	rq.line.Country = answer.Country
+	rq.line.ASN = rq.lookupAnswer.ASN
+	rq.line.ASName = rq.lookupAnswer.ASName
+	rq.line.Country = rq.lookupAnswer.Country
 }
 
-// addLookup adds answer, GeoJS's answer about a client, to the client's
-// history, and to the notes of the bans on its netblock that have no AS
-// number, AS name or country yet.
+// addLookup adds answer, an answer about a client from the lookup
+// database or GeoJS, to the client's history, and to the notes of the bans
+// on its netblock that have no AS number, AS name or country yet.
 func (h *handler) addLookup(answer lookup.Answer) {
 	h.limiter.AddLookup(answer.Client, answer.Answered,
 		answer.ASN, answer.ASName, answer.Country)

internal/proxy/lookup_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"path/filepath"
 	"slices"
 	"sync"
 	"testing"
@@ -13,6 +14,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
@@ -166,6 +168,64 @@ func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
 	}
 }
 
+func TestClientsAreLookedUpInTheLookupDatabaseAndGeoJSIsNotAsked(t *testing.T) {
+	t.Parallel()
+
+	geojsURL, asked := startGeoJS(t)
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+		fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"},
+	})
+	s, clk, server := startWithClock(t, geojsURL, map[string]string{
+		lookupSource:       "file",
+		lookupDBPath:       path,
+		allowedCountries:   "DE",
+		rateLimitPerMinute: "1",
+	})
+
+	// fromDE's second request breaks the limit and bans it. The list
+	// refuses fromKP, and unplaced, which the file does not hold.
+	de := asnAndCountry{asnDE, asNameDE, "DE"}
+
+	for _, tc := range []struct {
+		line logLine
+		want asnAndCountry
+	}{
+		{s.get(fromDE, http.StatusOK, requestlog.ActionForward), de},
+		{s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited), de},
+		{
+			s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied),
+			asnAndCountry{asnKP, asNameKP, "KP"},
+		},
+		{
+			s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied),
+			asnAndCountry{},
+		},
+	} {
+		got := asnAndCountry{tc.line.ASN, tc.line.ASName, tc.line.Country}
+		if got != tc.want {
+			t.Errorf("log line has %+v, want %+v", got, tc.want)
+		}
+	}
+
+	h := historyOf(t, server, fromDE)
+	if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != de ||
+		!h.LookedUp.Equal(clk.Now()) {
+		t.Errorf("history has %+v, looked up at %s; want %+v, at %s",
+			got, h.LookedUp, de, clk.Now())
+	}
+
+	notes := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32"))[0].Notes
+	if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de {
+		t.Errorf("the ban's notes have %+v, want %+v", got, de)
+	}
+
+	if len(asked()) != 0 {
+		t.Errorf("GeoJS was asked about %v, want nothing", asked())
+	}
+}
+
 func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) {
 	t.Parallel()
 

internal/proxy/proxy.go

@@ -54,9 +54,12 @@ type Params struct {
 	RequestLog io.Writer
 	// ProcessLog receives the process's own messages.
 	ProcessLog *slog.Logger
-	// GeoJSURL is where clients' AS numbers and countries are looked up,
-	// normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off.
+	// GeoJSURL is where clients' AS numbers and countries are looked up
+	// while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
 	GeoJSURL string
+	// LookupFile is the lookup database they are looked up in while
+	// SWWAF_LOOKUP_SOURCE is file, and nil otherwise.
+	LookupFile *lookup.File
 	// Now tells the time by which requests are counted for the rate
 	// limits, bans are made and run out, and GeoJS's answers are kept,
 	// normally time.Now in UTC, the time the state files give.
@@ -70,14 +73,16 @@ type Params struct {
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
-// whose state the state files keep, and the metrics.
+// whose state the state files keep, the lookup database, nil unless
+// SWWAF_LOOKUP_SOURCE is file, and the metrics.
 type Server struct {
 	*http.Server
 
-	Ledger  *bans.Ledger
-	Limiter *ratelimit.Limiter
-	GeoJS   *lookup.GeoJS
-	Metrics *metrics.Metrics
+	Ledger     *bans.Ledger
+	Limiter    *ratelimit.Limiter
+	GeoJS      *lookup.GeoJS
+	LookupFile *lookup.File
+	Metrics    *metrics.Metrics
 }
 
 // New returns the server smallwebwaf runs: each request it reads passes
@@ -109,8 +114,9 @@ func New(params Params) *Server {
 			AttackBanDuration:    params.Config.AttackBanDuration,
 			MaxBans:              params.Config.MaxBans,
 		}),
-		rules:  params.Rules,
-		alerts: params.Alerts,
+		lookupFile: params.LookupFile,
+		rules:      params.Rules,
+		alerts:     params.Alerts,
 	}
 	h.geojs = lookup.New(lookup.Params{
 		URL:     params.GeoJSURL,
@@ -142,10 +148,11 @@ func New(params Params) *Server {
 			MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10),
 			ErrorLog:       errorLog,
 		},
-		Ledger:  h.ledger,
-		Limiter: h.limiter,
-		GeoJS:   h.geojs,
-		Metrics: m,
+		Ledger:     h.ledger,
+		Limiter:    h.limiter,
+		GeoJS:      h.geojs,
+		LookupFile: h.lookupFile,
+		Metrics:    m,
 	}
 }
 
@@ -162,6 +169,7 @@ type handler struct {
 	limiter    *ratelimit.Limiter
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
+	lookupFile *lookup.File
 	rules      *rules.Files
 	alerts     *alerts.Queue
 }

internal/proxy/proxy_test.go

@@ -16,6 +16,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 	"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -67,6 +68,7 @@ const (
 	rateLimitPerDay         = "SWWAF_RATE_LIMIT_PER_DAY"
 	rateLimitExemptPaths    = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
 	lookupSource            = "SWWAF_LOOKUP_SOURCE"
+	lookupDBPath            = "SWWAF_LOOKUP_DB_PATH"
 	lookupTimeout           = "SWWAF_LOOKUP_TIMEOUT"
 	addLookupHeaders        = "SWWAF_ADD_LOOKUP_HEADERS"
 	deniedCountries         = "SWWAF_DENIED_COUNTRIES"
@@ -265,7 +267,8 @@ func startProxyWithAlerts(
 // proxy raises, as the settings in env make it. No alert is sent from the
 // queue: they wait in it, for the test to look at. With no geojsURL, there
 // is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
-// is off unless env sets it.
+// is off unless env sets it. While it is file, the lookup database
+// SWWAF_LOOKUP_DB_PATH names is read.
 func newProxy(
 	t *testing.T, appURL, geojsURL string, now func() time.Time,
 	env map[string]string,
@@ -310,11 +313,23 @@ func newProxy(
 		ProcessLog: processLog,
 	})
 
+	var lookupFile *lookup.File
+
+	if cfg.LookupSource == "file" {
+		lookupFile, err = lookup.OpenFile(lookup.FileParams{
+			Path: cfg.LookupDBPath, Now: now, ProcessLog: processLog, Alerts: alertQueue,
+		})
+		if err != nil {
+			t.Fatalf("lookup database: %v", err)
+		}
+	}
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: out,
 		ProcessLog: processLog,
 		GeoJSURL:   geojsURL,
+		LookupFile: lookupFile,
 		Now:        now,
 		Rules:      ruleFiles,
 		Alerts:     alertQueue,

internal/proxy/request.go

@@ -15,6 +15,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
@@ -49,9 +50,11 @@ type request struct {
 	peer        netip.Addr
 	peerTrusted bool
 	// lookedUp is true once the client's AS number and country have been
-	// looked up, whether or not an answer was there.
-	lookedUp bool
-	start    time.Time
+	// looked up, whether or not an answer was there, and lookupAnswer is
+	// what the lookup gave then, the zero Answer while GeoJS had given none.
+	lookedUp     bool
+	lookupAnswer lookup.Answer
+	start        time.Time
 	// checked is when the checks were done, and upstreamStart when the
 	// request was handed to the app.
 	checked       time.Time
@@ -488,10 +491,11 @@ func timing(start, end time.Time) *float64 {
 }
 
 // addToHistory adds the request, which has ended, to its client's
-// history, and then, for a client that was looked up, the answer kept
-// about it to that history and to the notes of the bans on its netblock:
-// an answer that came during the request may have come before either was
-// there, and one that comes later is added when it comes.
+// history, and then, for a client that was looked up, the lookup
+// database's answer about it, or the answer from GeoJS kept about it, to
+// that history and to the notes of the bans on its netblock: an answer
+// may have come before either was there, and one from GeoJS that comes
+// later is added when it comes.
 func (rq *request) addToHistory() {
 	var requestBytes int64
 	if rq.body != nil {
@@ -514,6 +518,13 @@ func (rq *request) addToHistory() {
 		return
 	}
 
+	// The lookup database's answer was there at once.
+	if rq.h.config.LookupSource == "file" {
+		rq.h.addLookup(rq.lookupAnswer)
+
+		return
+	}
+
 	answer, kept := rq.h.geojs.Kept(group)
 	if kept {
 		rq.h.addLookup(answer)

internal/ratelimit/ratelimit.go

@@ -68,8 +68,8 @@ type History struct {
 	LastSeen  time.Time `json:"last_seen"`
 	// ASN, ASName and Country are the client's AS number, AS name and
 	// country as last looked up, each empty when the lookup could not
-	// find it, and LookedUp is when GeoJS gave that answer; all are empty
-	// while the client never was looked up.
+	// find it, and LookedUp is when the lookup gave that answer; all are
+	// empty while the client never was looked up.
 	ASN      string    `json:"asn,omitempty"`
 	ASName   string    `json:"as_name,omitempty"`
 	Country  string    `json:"country,omitempty"`
@@ -230,7 +230,8 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
 }
 
 // AddLookup gives client's history its AS number, AS name and country, as
-// GeoJS gave them at lookedUp, if the table of clients holds the client.
+// the lookup gave them at lookedUp, if the table of clients holds the
+// client.
 // It does not make the client the most recently seen.
 func (l *Limiter) AddLookup(
 	client netip.Prefix, lookedUp time.Time, asn, asName, country string,

internal/smallwebwaf/smallwebwaf.go

@@ -1,6 +1,7 @@
 // Package smallwebwaf runs the smallwebwaf process: it reads the settings,
-// the rule files and the state files, serves requests until it is told to
-// stop, and then stops in an orderly way, writing the state files.
+// the rule files, the lookup database and the state files, serves requests
+// until it is told to stop, and then stops in an orderly way, writing the
+// state files.
 package smallwebwaf
 
 import (
@@ -64,9 +65,9 @@ func Main(version string) int {
 	})
 }
 
-// Run reads the settings, the rule files and the state files, then serves
-// requests until ctx is done. It returns the process's exit status, 1
-// when smallwebwaf cannot start.
+// Run reads the settings, the rule files, the lookup database and the
+// state files, then serves requests until ctx is done. It returns the
+// process's exit status, 1 when smallwebwaf cannot start.
 func Run(ctx context.Context, params Params) int {
 	processLog := requestlog.NewProcessLogger(params.Stdout,
 		config.InstanceName(params.LookupEnv))
@@ -110,21 +111,17 @@ func Run(ctx context.Context, params Params) int {
 		return 1
 	}
 
-	server := proxy.New(proxy.Params{
-		Config:     cfg,
-		RequestLog: stdout,
-		ProcessLog: processLog,
-		GeoJSURL:   lookup.URL,
-		Now:        now,
-		Rules:      ruleFiles,
-		Alerts:     alertQueue,
-	})
+	server, err := newServer(cfg, stdout, processLog, now, ruleFiles, alertQueue)
+	if err != nil {
+		processLog.Error("cannot use the lookup database", "error", err.Error())
+
+		return 1
+	}
+
 	if remote != nil {
 		server.Metrics.AddRemoteLog(remote)
 	}
 
-	server.Metrics.AddAlerts(alertQueue)
-
 	files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
 	if err != nil {
 		processLog.Error("cannot use the state files", "error", err.Error())
@@ -145,7 +142,49 @@ func Run(ctx context.Context, params Params) int {
 		"address", listener.Addr().String(),
 		"settings", cfg)
 
-	return serve(ctx, server.Server, listener, files, ruleFiles, alertQueue, processLog)
+	return serve(ctx, server, listener, files, ruleFiles, alertQueue, processLog)
+}
+
+// newServer returns the server smallwebwaf runs, with the metrics of the
+// alerts, after reading the lookup database while SWWAF_LOOKUP_SOURCE is
+// file. A lookup database that cannot be read is an error.
+func newServer(
+	cfg *config.Config, stdout io.Writer, processLog *slog.Logger,
+	now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
+) (*proxy.Server, error) {
+	var lookupFile *lookup.File
+
+	if cfg.LookupSource == "file" {
+		var err error
+
+		lookupFile, err = lookup.OpenFile(lookup.FileParams{
+			Path:       cfg.LookupDBPath,
+			Now:        now,
+			ProcessLog: processLog,
+			Alerts:     alertQueue,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	server := proxy.New(proxy.Params{
+		Config:     cfg,
+		RequestLog: stdout,
+		ProcessLog: processLog,
+		GeoJSURL:   lookup.URL,
+		LookupFile: lookupFile,
+		Now:        now,
+		Rules:      ruleFiles,
+		Alerts:     alertQueue,
+	})
+	server.Metrics.AddAlerts(alertQueue)
+
+	if lookupFile != nil {
+		server.Metrics.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
+	}
+
+	return server, nil
 }
 
 // loadStateFiles reads the state files into the parts of server and into
@@ -227,11 +266,12 @@ func startSending(
 
 // serve serves requests on listener, writes the state files as they are
 // due, takes in an admin's edits of them, reads the rule files again as
-// they change, and sends the alerts, until ctx is done. Then it gives the
-// requests in progress shutdownTimeout to finish, and writes every state
-// file, alerts.json with the alerts still waiting.
+// they change, and the lookup database when it is replaced, and sends the
+// alerts, until ctx is done. Then it gives the requests in progress
+// shutdownTimeout to finish, and writes every state file, alerts.json with
+// the alerts still waiting.
 func serve(
-	ctx context.Context, server *http.Server, listener net.Listener,
+	ctx context.Context, server *proxy.Server, listener net.Listener,
 	files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
 	processLog *slog.Logger,
 ) int {
@@ -247,6 +287,11 @@ func serve(
 	written := inBackground(func() { files.Run(writing) })
 	watched := inBackground(func() { files.Watch(writing) })
 	rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
+	lookupFileWatched := inBackground(func() {
+		if server.LookupFile != nil {
+			server.LookupFile.Watch(writing)
+		}
+	})
 	alertsSent := inBackground(func() { alertQueue.Run(writing) })
 
 	select {
@@ -289,6 +334,7 @@ func serve(
 	<-written
 	<-watched
 	<-rulesWatched
+	<-lookupFileWatched
 	<-alertsSent
 
 	err = files.WriteAll()

internal/smallwebwaf/smallwebwaf_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
 	"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
 )
 
@@ -38,6 +39,8 @@ const (
 	stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
 	rateLimitPerDay      = "SWWAF_RATE_LIMIT_PER_DAY"
 	rulesDir             = "SWWAF_RULES_DIR"
+	lookupSource         = "SWWAF_LOOKUP_SOURCE"
+	lookupDBPath         = "SWWAF_LOOKUP_DB_PATH"
 	instanceName         = "SWWAF_INSTANCE_NAME"
 	adminToken           = "SWWAF_ADMIN_TOKEN"   //nolint:gosec // the setting's name
 	metricsToken         = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
@@ -48,6 +51,8 @@ const (
 	instance = "fsn1app1/gitea"
 	// greeting is what the tests' app answers.
 	greeting = "hello from the app"
+	// placed is the client the tests' lookup databases place.
+	placed = "203.0.113.9"
 )
 
 // output collects what smallwebwaf writes on stdout.
@@ -495,6 +500,107 @@ func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) {
 		": no such file or directory")
 }
 
+func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
+	t.Parallel()
+
+	const token = "0123456789abcdef0123456789abcdef"
+
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	writeLookupDatabase(t, path, "DE")
+
+	env := map[string]string{
+		listenAddr:               localhost + ":0",
+		upstreamURL:              startApp(t),
+		stateDir:                 t.TempDir(),
+		rulesDir:                 t.TempDir(),
+		trustedProxies:           localhost + "/32",
+		metricsToken:             token,
+		instanceName:             instance,
+		lookupSource:             "file",
+		lookupDBPath:             path,
+		"SWWAF_DENIED_COUNTRIES": "kp",
+		// The requests sent until a replacement takes effect, and those
+		// for the metrics, must not break a rate limit, whose ban would
+		// refuse them too.
+		"SWWAF_RATE_LIMIT_EXEMPT_NETS": placed + "," + localhost,
+	}
+	began := time.Now()
+	// Each replacement is written beside the file and renamed over it, as
+	// a refresh is.
+	replacement := path + ".new"
+
+	var lastRead float64
+
+	runUntilStopped(t, env, func(url string) {
+		wantStatus(t, url, placed, http.StatusOK)
+
+		// smallwebwaf reads the file again once it watches its directory,
+		// which may be after the first replacement. Once that has taken
+		// effect, only the watch can show the next. Each takes as long as
+		// it takes, so that a slow test process cannot fail the test.
+		writeLookupDatabase(t, replacement, "KP")
+		rename(t, replacement, path)
+
+		for statusFrom(t, url, placed) != http.StatusForbidden {
+			time.Sleep(pollInterval)
+		}
+
+		writeLookupDatabase(t, replacement, "DE")
+		rename(t, replacement, path)
+
+		for statusFrom(t, url, placed) != http.StatusOK {
+			time.Sleep(pollInterval)
+		}
+
+		// One that cannot be read leaves the file read before in use.
+		err := os.WriteFile(replacement, []byte("not a lookup database\n"), 0o600)
+		if err != nil {
+			t.Fatalf("write %s: %v", replacement, err)
+		}
+
+		rename(t, replacement, path)
+
+		metrics := metricsWith(t, url+"_smallwebwaf/metrics", token,
+			`smallwebwaf_lookup_database_read_failures_total{instance="fsn1app1/gitea"} 1`)
+		lastRead = seriesValue(t, metrics,
+			`smallwebwaf_lookup_database_last_read_timestamp_seconds{instance="fsn1app1/gitea"}`)
+
+		wantStatus(t, url, placed, http.StatusOK)
+	})
+
+	if lastRead < float64(began.Unix()) || lastRead > float64(time.Now().Unix()) {
+		t.Errorf("the lookup database was last read at %v, want a time in the test", lastRead)
+	}
+}
+
+func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+
+	// If it starts instead, it is stopped after waitLimit.
+	ctx, stop := context.WithTimeout(t.Context(), waitLimit)
+	defer stop()
+
+	out := &output{}
+
+	status := run(ctx, map[string]string{
+		listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
+		lookupSource: "file", lookupDBPath: path,
+	}, out)
+	if status != 1 {
+		t.Fatalf("exit status %d, want 1", status)
+	}
+
+	want := "SWWAF_LOOKUP_DB_PATH cannot be read: open " + path +
+		": no such file or directory"
+
+	line := out.line(t, "msg", "cannot use the lookup database")
+	if line["error"] != want {
+		t.Errorf("start refused with %v, want the error %q", line, want)
+	}
+}
+
 func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) {
 	t.Parallel()
 
@@ -1046,6 +1152,50 @@ func metricsWith(t *testing.T, url, token string, series ...string) string {
 	}
 }
 
+// seriesValue returns the value of series, such as
+// name{instance="app"}, in metrics.
+func seriesValue(t *testing.T, metrics, series string) float64 {
+	t.Helper()
+
+	for line := range strings.Lines(metrics) {
+		value, found := strings.CutPrefix(strings.TrimSpace(line), series+" ")
+		if !found {
+			continue
+		}
+
+		number, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			t.Fatalf("%s has the value %q: %v", series, value, err)
+		}
+
+		return number
+	}
+
+	t.Fatalf("no %s in the metrics:\n%s", series, metrics)
+
+	return 0
+}
+
+// rename renames the file at from to, replacing any file there.
+func rename(t *testing.T, from, to string) {
+	t.Helper()
+
+	err := os.Rename(from, to)
+	if err != nil {
+		t.Fatalf("rename %s to %s: %v", from, to, err)
+	}
+}
+
+// writeLookupDatabase writes a lookup database at path that places the
+// client placed in country, and no other address.
+func writeLookupDatabase(t *testing.T, path, country string) {
+	t.Helper()
+
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		placed + "/32": {ASN: "AS64496", ASName: "Example Net", Country: country},
+	})
+}
+
 // askAsAdmin sends a request with method to url, with body and
 // adminSecret, and checks that it is answered 200.
 func askAsAdmin(t *testing.T, method, url, body string) {