Add best-effort geolocation to the IP lookup from geojs and DB-IP City Lite (closes #239)

Each result gains `geo`. One request to geojs carries every address, 1 second in all. On a timeout, an error or an answer with no country, city or coordinates, the lookup reads DB-IP City Lite, downloaded monthly into the state directory. geojs answers stay in memory (250,000, least recently used dropped) and in a new `cache.db` table for 7 days.

Judgement call: a reserved address goes to neither source, and its `geo` is null.
Judgement call: `updated` for DB-IP is when the copy was downloaded.
Deviation: the cache is SQLite, by sneak's words.

Model: opus-5-5

internal/cachedb/schema.sql

@@ -6,3 +6,19 @@ CREATE TABLE IF NOT EXISTS hostnames (
     names TEXT NOT NULL,
     expires_at INTEGER NOT NULL
 );
+
+-- geolocations holds the answers of geojs. A part geojs does not know is NULL.
+-- fetched_at is in seconds since 1970.
+CREATE TABLE IF NOT EXISTS geolocations (
+    address TEXT PRIMARY KEY,
+    country_code TEXT,
+    country TEXT,
+    continent_code TEXT,
+    region TEXT,
+    city TEXT,
+    latitude REAL,
+    longitude REAL,
+    accuracy_radius_km INTEGER,
+    time_zone TEXT,
+    fetched_at INTEGER NOT NULL
+);

internal/geo/cache.go

@@ -0,0 +1,253 @@
+package geo
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"maps"
+	"time"
+)
+
+const (
+	// entryLifetime is how long a geojs answer is kept after its fetch.
+	entryLifetime = 7 * 24 * time.Hour
+
+	// flushInterval is how often new answers are written to cache.db.
+	flushInterval = 5 * time.Second
+
+	// sweepInterval is how often expired answers are deleted.
+	sweepInterval = 10 * time.Minute
+
+	// rowColumns are the columns of the geolocations table in the order
+	// scanRow reads them.
+	rowColumns = "address, country_code, country, continent_code, region, " +
+		"city, latitude, longitude, accuracy_radius_km, time_zone, fetched_at"
+)
+
+// rowScanner is the Scan method of *sql.Row and *sql.Rows.
+type rowScanner interface {
+	Scan(dest ...any) error
+}
+
+// cached returns the current answer held for address, from memory, then from
+// the answers waiting to be written, then from cache.db. It returns nil when
+// there is none.
+func (l *Locator) cached(ctx context.Context, address string) *Location {
+	found, ok := l.entries.Get(address)
+	if ok {
+		if !l.expired(found) {
+			return found
+		}
+
+		l.entries.Remove(address)
+	}
+
+	l.pendingMu.Lock()
+	waiting := l.pending[address]
+	l.pendingMu.Unlock()
+
+	if waiting != nil && !l.expired(waiting) {
+		l.entries.Add(address, waiting)
+
+		return waiting
+	}
+
+	stored, err := l.readRow(ctx, address)
+	if err != nil {
+		l.logger.Warn("Failed to read a geolocation cache row",
+			"address", address, "error", err)
+
+		return nil
+	}
+
+	if stored != nil {
+		l.entries.Add(address, stored)
+	}
+
+	return stored
+}
+
+// expired says whether the answer is at the end of its lifetime.
+func (l *Locator) expired(answer *Location) bool {
+	return !l.now().Before(answer.Updated.Add(entryLifetime))
+}
+
+// cutoff returns the fetch time at or before which an answer has expired.
+func (l *Locator) cutoff() int64 {
+	return l.now().Add(-entryLifetime).Unix()
+}
+
+// store keeps an answer in memory and queues it for cache.db. While cache.db
+// cannot be written, the queue stops growing at maxEntries answers.
+func (l *Locator) store(address string, answer *Location) {
+	l.entries.Add(address, answer)
+
+	l.pendingMu.Lock()
+	defer l.pendingMu.Unlock()
+
+	if len(l.pending) < l.maxEntries {
+		l.pending[address] = answer
+	}
+}
+
+// keepCache writes new answers and deletes expired ones until ctx ends.
+func (l *Locator) keepCache(ctx context.Context) {
+	flushes := time.NewTicker(flushInterval)
+	defer flushes.Stop()
+
+	sweeps := time.NewTicker(sweepInterval)
+	defer sweeps.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			l.flush(context.WithoutCancel(ctx))
+
+			return
+		case <-flushes.C:
+			l.flush(ctx)
+		case <-sweeps.C:
+			l.sweep(ctx)
+		}
+	}
+}
+
+// load reads the answers that have not expired into memory, at most maxEntries
+// of them, the newest last so that they are the last to be dropped.
+func (l *Locator) load(ctx context.Context) error {
+	rows, err := l.db.QueryContext(ctx,
+		"SELECT "+rowColumns+" FROM (SELECT "+rowColumns+" FROM geolocations "+
+			"WHERE fetched_at > ? ORDER BY fetched_at DESC LIMIT ?) "+
+			"ORDER BY fetched_at",
+		l.cutoff(), l.maxEntries)
+	if err != nil {
+		return fmt.Errorf("reading the geolocation cache: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		address, answer, err := scanRow(rows)
+		if err != nil {
+			return fmt.Errorf("reading a geolocation cache row: %w", err)
+		}
+
+		l.entries.Add(address, answer)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return fmt.Errorf("reading the geolocation cache: %w", err)
+	}
+
+	return nil
+}
+
+// readRow returns the current answer for address in cache.db, or nil when it
+// has none.
+func (l *Locator) readRow(
+	ctx context.Context, address string,
+) (*Location, error) {
+	row := l.db.QueryRowContext(ctx,
+		"SELECT "+rowColumns+" FROM geolocations "+
+			"WHERE address = ? AND fetched_at > ?",
+		address, l.cutoff())
+
+	_, answer, err := scanRow(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil //nolint:nilnil // no row is not a failure
+	}
+
+	return answer, err
+}
+
+// scanRow reads one row of the geolocations table.
+func scanRow(row rowScanner) (string, *Location, error) {
+	var (
+		address string
+		fetched int64
+		answer  Location
+	)
+
+	err := row.Scan(&address, &answer.CountryCode, &answer.Country,
+		&answer.ContinentCode, &answer.Region, &answer.City, &answer.Latitude,
+		&answer.Longitude, &answer.AccuracyRadiusKm, &answer.TimeZone, &fetched)
+	if err != nil {
+		return "", nil, err
+	}
+
+	answer.Source = sourceGeoJS
+	answer.Updated = time.Unix(fetched, 0).UTC()
+
+	return address, &answer, nil
+}
+
+// flush writes the pending answers to cache.db. Those it cannot write stay
+// pending, unless a newer answer for the address has arrived meanwhile.
+func (l *Locator) flush(ctx context.Context) {
+	l.pendingMu.Lock()
+	batch := l.pending
+	l.pending = make(map[string]*Location)
+	l.pendingMu.Unlock()
+
+	if len(batch) == 0 {
+		return
+	}
+
+	err := l.writeRows(ctx, batch)
+	if err == nil {
+		return
+	}
+
+	l.logger.Error("Failed to write the geolocation cache", "error", err)
+
+	l.pendingMu.Lock()
+	defer l.pendingMu.Unlock()
+
+	maps.Insert(batch, maps.All(l.pending))
+	l.pending = batch
+}
+
+// writeRows writes batch to cache.db in one transaction.
+func (l *Locator) writeRows(
+	ctx context.Context, batch map[string]*Location,
+) error {
+	tx, err := l.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	for address, answer := range batch {
+		_, err = tx.ExecContext(ctx,
+			"INSERT OR REPLACE INTO geolocations ("+rowColumns+") "+
+				"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+			address, answer.CountryCode, answer.Country, answer.ContinentCode,
+			answer.Region, answer.City, answer.Latitude, answer.Longitude,
+			answer.AccuracyRadiusKm, answer.TimeZone, answer.Updated.Unix())
+		if err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+// sweep deletes the expired answers from memory and cache.db.
+func (l *Locator) sweep(ctx context.Context) {
+	for _, address := range l.entries.Keys() {
+		answer, ok := l.entries.Peek(address)
+		if ok && l.expired(answer) {
+			l.entries.Remove(address)
+		}
+	}
+
+	_, err := l.db.ExecContext(ctx,
+		"DELETE FROM geolocations WHERE fetched_at <= ?", l.cutoff())
+	if err != nil {
+		l.logger.Error("Failed to delete expired geolocation cache rows",
+			"error", err)
+	}
+}

internal/geo/database.go

@@ -0,0 +1,349 @@
+package geo
+
+import (
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/oschwald/maxminddb-golang/v2"
+)
+
+const (
+	// DatabaseDownloadFormat is the URL of DB-IP's City Lite database, with %s
+	// for the year and month.
+	DatabaseDownloadFormat = "https://download.db-ip.com/free/" +
+		"dbip-city-lite-%s.mmdb.gz"
+
+	// databaseMonthFormat is the year and month in the URL.
+	databaseMonthFormat = "2006-01"
+
+	// databaseFileName is the copy of the database in the state directory.
+	databaseFileName = "dbip-city-lite.mmdb"
+
+	// databaseDownloadSuffix ends the name of the file a download is written
+	// to, until it has opened.
+	databaseDownloadSuffix = ".download"
+
+	// maxDatabaseSize is the largest database accepted once unpacked, in
+	// bytes.
+	maxDatabaseSize = 200 << 20
+
+	// databaseDownloadTimeout limits one download.
+	databaseDownloadTimeout = 10 * time.Minute
+
+	// databaseCheckInterval is how often RouteWatch checks whether a new
+	// month's database is due. A new month's file appears some time on the
+	// first of the month, so a failed download is tried again this much later.
+	databaseCheckInterval = time.Hour
+)
+
+var (
+	errDatabaseStatus  = errors.New("unexpected HTTP status from DB-IP")
+	errDatabaseTooLong = errors.New("database is over the size limit")
+)
+
+// dbipName is the part of a database record that holds an English name.
+type dbipName struct {
+	Names struct {
+		English string `maxminddb:"en"`
+	} `maxminddb:"names"`
+}
+
+// dbipRecord is the part of a database record RouteWatch reads.
+type dbipRecord struct {
+	City      dbipName `maxminddb:"city"`
+	Continent struct {
+		Code string `maxminddb:"code"`
+	} `maxminddb:"continent"`
+	Country struct {
+		dbipName
+
+		ISOCode string `maxminddb:"iso_code"`
+	} `maxminddb:"country"`
+	Location struct {
+		Latitude  *float64 `maxminddb:"latitude"`
+		Longitude *float64 `maxminddb:"longitude"`
+	} `maxminddb:"location"`
+	Subdivisions []dbipName `maxminddb:"subdivisions"`
+}
+
+// location returns the record as a Location fetched at the time given. The
+// first subdivision is the region. The database has no accuracy radius and no
+// time zone.
+func (r *dbipRecord) location(fetched time.Time) *Location {
+	found := &Location{
+		CountryCode:   nilIfEmpty(r.Country.ISOCode),
+		Country:       nilIfEmpty(r.Country.Names.English),
+		ContinentCode: nilIfEmpty(r.Continent.Code),
+		City:          nilIfEmpty(r.City.Names.English),
+		Latitude:      r.Location.Latitude,
+		Longitude:     r.Location.Longitude,
+		Source:        sourceDBIP,
+		Updated:       fetched,
+	}
+
+	if len(r.Subdivisions) > 0 {
+		found.Region = nilIfEmpty(r.Subdivisions[0].Names.English)
+	}
+
+	return found
+}
+
+// nilIfEmpty returns a pointer to text, or nil when text is empty.
+func nilIfEmpty(text string) *string {
+	if text == "" {
+		return nil
+	}
+
+	return &text
+}
+
+// fromDatabase returns where the copy of the database places addr. It returns
+// nil when there is no copy yet or the copy has no record for addr. It also
+// returns nil, and logs the error, when reading the copy fails.
+func (l *Locator) fromDatabase(addr netip.Addr) *Location {
+	l.databaseMu.RLock()
+	defer l.databaseMu.RUnlock()
+
+	if l.database == nil {
+		return nil
+	}
+
+	result := l.database.Lookup(addr)
+
+	err := result.Err()
+	if err != nil {
+		l.logger.Error("Failed to look up an address in the database",
+			"address", addr, "error", err)
+
+		return nil
+	}
+
+	if !result.Found() {
+		return nil
+	}
+
+	var record dbipRecord
+
+	err = result.Decode(&record)
+	if err != nil {
+		l.logger.Error("Failed to read a database record",
+			"address", addr, "error", err)
+
+		return nil
+	}
+
+	return record.location(l.databaseFetched)
+}
+
+// keepDatabaseCurrent downloads the database when it is due, at once and then
+// every databaseCheckInterval, until ctx ends.
+func (l *Locator) keepDatabaseCurrent(ctx context.Context) {
+	l.downloadDatabaseIfDue(ctx)
+
+	checks := time.NewTicker(databaseCheckInterval)
+	defer checks.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-checks.C:
+			l.downloadDatabaseIfDue(ctx)
+		}
+	}
+}
+
+// downloadDatabaseIfDue downloads this month's database, unless the copy in
+// use was downloaded this month. A failure leaves the copy as it is.
+func (l *Locator) downloadDatabaseIfDue(ctx context.Context) {
+	now := l.now().UTC()
+
+	if !l.databaseDue(now) {
+		return
+	}
+
+	err := l.downloadDatabase(ctx, now)
+	if err != nil && ctx.Err() == nil {
+		l.logger.Error("Database download failed; keeping the previous copy",
+			"error", err)
+	}
+}
+
+// databaseDue says whether there is no open copy or it is from an earlier
+// month than now.
+func (l *Locator) databaseDue(now time.Time) bool {
+	l.databaseMu.RLock()
+	defer l.databaseMu.RUnlock()
+
+	return l.database == nil ||
+		l.databaseFetched.Format(databaseMonthFormat) !=
+			now.Format(databaseMonthFormat)
+}
+
+// downloadDatabase fetches the database of now's month. The file replaces the
+// copy, on disk and in use, only once it has opened.
+func (l *Locator) downloadDatabase(ctx context.Context, now time.Time) error {
+	ctx, cancel := context.WithTimeout(ctx, databaseDownloadTimeout)
+	defer cancel()
+
+	url := fmt.Sprintf(l.databaseURLFormat, now.Format(databaseMonthFormat))
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	request.Header.Set(userAgentHeader, userAgent)
+
+	response, err := l.client.Do(request)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("%w: %s", errDatabaseStatus, response.Status)
+	}
+
+	return l.replaceDatabase(response.Body, now)
+}
+
+// replaceDatabase unpacks the gzip stream body beside the copy, then makes the
+// result the copy once it has opened. Whatever fails, the file it wrote is
+// removed and the copy stays.
+func (l *Locator) replaceDatabase(body io.Reader, now time.Time) error {
+	root, err := os.OpenRoot(l.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	downloadName := databaseFileName + databaseDownloadSuffix
+
+	err = unpackDatabase(root, downloadName, body, l.maxDatabaseBytes)
+	if err != nil {
+		_ = root.Remove(downloadName)
+
+		return err
+	}
+
+	opened, err := maxminddb.Open(filepath.Join(l.dir, downloadName))
+	if err != nil {
+		_ = root.Remove(downloadName)
+
+		return fmt.Errorf("opening the downloaded database: %w", err)
+	}
+
+	err = root.Rename(downloadName, databaseFileName)
+	if err != nil {
+		_ = opened.Close()
+		_ = root.Remove(downloadName)
+
+		return err
+	}
+
+	l.useDatabase(opened, now.Truncate(time.Second))
+	l.logger.Info("Downloaded a new copy of the database")
+
+	return nil
+}
+
+// unpackDatabase writes the gzip stream body to the file name in root, up to
+// limit bytes once unpacked.
+func unpackDatabase(
+	root *os.Root, name string, body io.Reader, limit int64,
+) error {
+	unpacked, err := gzip.NewReader(body)
+	if err != nil {
+		return fmt.Errorf("unpacking the database: %w", err)
+	}
+
+	f, err := root.Create(name)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = f.Close() }()
+
+	written, err := io.Copy(f, io.LimitReader(unpacked, limit+1))
+	if err != nil {
+		return fmt.Errorf("unpacking the database: %w", err)
+	}
+
+	if written > limit {
+		return errDatabaseTooLong
+	}
+
+	return nil
+}
+
+// openDatabaseCopy opens the copy left by an earlier run, if there is one. A
+// copy that does not open is treated as no copy, so it is downloaded again.
+func (l *Locator) openDatabaseCopy() {
+	path := filepath.Join(l.dir, databaseFileName)
+
+	info, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return
+	}
+
+	if err != nil {
+		l.logger.Error("Failed to read the database copy", "error", err)
+
+		return
+	}
+
+	opened, err := maxminddb.Open(path)
+	if err != nil {
+		l.logger.Error("Failed to open the database copy", "error", err)
+
+		return
+	}
+
+	l.useDatabase(opened, info.ModTime().UTC().Truncate(time.Second))
+}
+
+// useDatabase makes opened the copy in use and closes the one it replaces.
+func (l *Locator) useDatabase(opened *maxminddb.Reader, fetched time.Time) {
+	l.databaseMu.Lock()
+	defer l.databaseMu.Unlock()
+
+	if l.database != nil {
+		err := l.database.Close()
+		if err != nil {
+			l.logger.Error("Failed to close the old database copy", "error", err)
+		}
+	}
+
+	l.database = opened
+	l.databaseFetched = fetched
+}
+
+// closeDatabase closes the copy in use, after which no source can answer from
+// it.
+func (l *Locator) closeDatabase() {
+	l.databaseMu.Lock()
+	defer l.databaseMu.Unlock()
+
+	if l.database == nil {
+		return
+	}
+
+	err := l.database.Close()
+	if err != nil {
+		l.logger.Error("Failed to close the database copy", "error", err)
+	}
+
+	l.database = nil
+}

internal/geo/geojs.go

@@ -0,0 +1,151 @@
+package geo
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/netip"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	// GeoJSEndpoint is the URL of the geojs geo endpoint.
+	GeoJSEndpoint = "https://get.geojs.io/v1/ip/geo.json"
+
+	// geojsTimeLimit limits one request to geojs.
+	geojsTimeLimit = time.Second
+
+	// geojsAddressSeparator separates the addresses of one request.
+	geojsAddressSeparator = ","
+
+	// geojsUnknown is what geojs sends for a value it does not know.
+	geojsUnknown = "nil"
+
+	// maxGeoJSAnswerBytes is far more than an answer for the most addresses
+	// an IP lookup can ask about.
+	maxGeoJSAnswerBytes = 1 << 20
+)
+
+var errGeoJSStatus = errors.New("unexpected HTTP status from geojs")
+
+// geojsAnswer is one address's entry in an answer from geojs.
+//
+//nolint:tagliatelle // the keys geojs sends
+type geojsAnswer struct {
+	IP            string `json:"ip"`
+	CountryCode   string `json:"country_code"`
+	Country       string `json:"country"`
+	ContinentCode string `json:"continent_code"`
+	Region        string `json:"region"`
+	City          string `json:"city"`
+	Latitude      string `json:"latitude"`
+	Longitude     string `json:"longitude"`
+	Accuracy      *int   `json:"accuracy"`
+	TimeZone      string `json:"timezone"`
+}
+
+// askGeoJS asks geojs about addresses in one request, and returns its answers
+// by address. An address geojs left out, or gave no country, city or
+// coordinates for, has no entry.
+func (l *Locator) askGeoJS(
+	ctx context.Context, addresses []string,
+) (map[string]*Location, error) {
+	ctx, cancel := context.WithTimeout(ctx, l.timeLimit)
+	defer cancel()
+
+	// A comma and a colon are legal in a query, and the escaped forms are
+	// not what geojs documents.
+	url := l.geojsURL + "?ip=" + strings.Join(addresses, geojsAddressSeparator)
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Set(userAgentHeader, userAgent)
+
+	response, err := l.client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: %d", errGeoJSStatus, response.StatusCode)
+	}
+
+	var parsed []geojsAnswer
+
+	err = json.NewDecoder(io.LimitReader(response.Body, maxGeoJSAnswerBytes)).
+		Decode(&parsed)
+	if err != nil {
+		return nil, fmt.Errorf("reading the geojs answer: %w", err)
+	}
+
+	fetched := l.now().UTC().Truncate(time.Second)
+	answers := make(map[string]*Location, len(parsed))
+
+	for _, answer := range parsed {
+		addr, err := netip.ParseAddr(answer.IP)
+		if err != nil {
+			continue
+		}
+
+		location := answer.location(fetched)
+		if hasLocation(location) {
+			answers[plainAddress(addr)] = location
+		}
+	}
+
+	return answers, nil
+}
+
+// hasLocation says whether found has a country, a city or a coordinate.
+func hasLocation(found *Location) bool {
+	return found.CountryCode != nil || found.Country != nil ||
+		found.City != nil || found.Latitude != nil || found.Longitude != nil
+}
+
+// location returns the answer as a Location fetched at the time given.
+func (a geojsAnswer) location(fetched time.Time) *Location {
+	return &Location{
+		CountryCode:      knownText(a.CountryCode),
+		Country:          knownText(a.Country),
+		ContinentCode:    knownText(a.ContinentCode),
+		Region:           knownText(a.Region),
+		City:             knownText(a.City),
+		Latitude:         knownNumber(a.Latitude),
+		Longitude:        knownNumber(a.Longitude),
+		AccuracyRadiusKm: a.Accuracy,
+		TimeZone:         knownText(a.TimeZone),
+		Source:           sourceGeoJS,
+		Updated:          fetched,
+	}
+}
+
+// knownText returns a pointer to text, or nil when geojs does not know it.
+func knownText(text string) *string {
+	if text == "" || text == geojsUnknown {
+		return nil
+	}
+
+	return &text
+}
+
+// knownNumber returns text as a number, or nil when geojs does not know it. NaN
+// and infinity parse, but JSON cannot carry them.
+func knownNumber(text string) *float64 {
+	number, err := strconv.ParseFloat(text, 64)
+	if err != nil || math.IsNaN(number) || math.IsInf(number, 0) {
+		return nil
+	}
+
+	return &number
+}

internal/geo/locator.go

@@ -0,0 +1,232 @@
+// Package geo finds where IP addresses are. It asks geojs.io, and keeps those
+// answers in memory and in cache.db. When geojs.io gives no location for an
+// address, it reads a copy of DB-IP's City Lite database, which it downloads
+// once a month.
+package geo
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/netip"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/oschwald/maxminddb-golang/v2"
+)
+
+// The values of Location.Source.
+const (
+	sourceGeoJS = "geojs"
+	sourceDBIP  = "dbip"
+)
+
+const (
+	// maxEntries limits how many geojs answers are held in memory.
+	maxEntries = 250_000
+
+	// userAgentHeader and userAgent name routewatch to geojs and DB-IP.
+	userAgentHeader = "User-Agent"
+	userAgent       = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
+)
+
+// Params is what New needs.
+type Params struct {
+	// DB is the open cache.db.
+	DB *sql.DB
+
+	// Dir is the directory that holds the copy of the database.
+	Dir string
+
+	// GeoJSURL is the URL of the geojs geo endpoint, GeoJSEndpoint in use.
+	GeoJSURL string
+
+	// DatabaseURLFormat is the URL of the database with %s for the year and
+	// month, DatabaseDownloadFormat in use.
+	DatabaseURLFormat string
+
+	Logger *slog.Logger
+}
+
+// Location is where an address is. A part the source does not know is nil.
+//
+//nolint:tagliatelle // snake_case keys, as the Go style guide asks
+type Location struct {
+	CountryCode      *string  `json:"country_code"`
+	Country          *string  `json:"country"`
+	ContinentCode    *string  `json:"continent_code"`
+	Region           *string  `json:"region"`
+	City             *string  `json:"city"`
+	Latitude         *float64 `json:"latitude"`
+	Longitude        *float64 `json:"longitude"`
+	AccuracyRadiusKm *int     `json:"accuracy_radius_km"`
+	TimeZone         *string  `json:"time_zone"`
+
+	// Source is geojs or dbip.
+	Source string `json:"source"`
+
+	// Updated is when the value was fetched: the geojs answer, or the copy of
+	// the database.
+	Updated time.Time `json:"updated"`
+}
+
+// Locator answers where addresses are. Start loads its cache and starts its
+// goroutines.
+type Locator struct {
+	db                *sql.DB
+	dir               string
+	geojsURL          string
+	databaseURLFormat string
+	timeLimit         time.Duration
+	maxEntries        int
+	maxDatabaseBytes  int64
+	logger            *slog.Logger
+	client            *http.Client
+	now               func() time.Time
+
+	// entries holds geojs answers by address, and drops the least recently
+	// used first.
+	entries *lru.Cache[string, *Location]
+
+	// pending holds the answers that cache.db does not have yet.
+	pendingMu sync.Mutex
+	pending   map[string]*Location
+
+	// database is the open copy, nil until one has opened, and databaseFetched
+	// is when that copy was downloaded. A lookup holds the read lock, so that
+	// a new copy never closes the old one under it.
+	databaseMu      sync.RWMutex
+	database        *maxminddb.Reader
+	databaseFetched time.Time
+
+	// stop ends the goroutines Start runs, and done is closed when they have
+	// returned.
+	stop context.CancelFunc
+	done chan struct{}
+}
+
+// New returns a Locator with an empty cache. Start loads it.
+func New(params Params) (*Locator, error) {
+	entries, err := lru.New[string, *Location](maxEntries)
+	if err != nil {
+		return nil, fmt.Errorf("creating the geolocation cache: %w", err)
+	}
+
+	return &Locator{
+		db:                params.DB,
+		dir:               params.Dir,
+		geojsURL:          params.GeoJSURL,
+		databaseURLFormat: params.DatabaseURLFormat,
+		timeLimit:         geojsTimeLimit,
+		maxEntries:        maxEntries,
+		maxDatabaseBytes:  maxDatabaseSize,
+		logger:            params.Logger.With("component", "geo"),
+		client:            &http.Client{},
+		now:               time.Now,
+		entries:           entries,
+		pending:           make(map[string]*Location),
+		done:              make(chan struct{}),
+	}, nil
+}
+
+// Lookup returns where each of addrs is, in the same order. The entry for an
+// address neither source answers for is nil. Addresses with no current cached
+// answer go to geojs in one request. The returned Locations are shared with
+// the cache and must not be changed.
+func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
+	locations := make([]*Location, len(addrs))
+
+	var unanswered []string
+
+	for i, addr := range addrs {
+		address := plainAddress(addr)
+
+		locations[i] = l.cached(ctx, address)
+		if locations[i] == nil {
+			unanswered = append(unanswered, address)
+		}
+	}
+
+	if len(unanswered) == 0 {
+		return locations
+	}
+
+	answers, err := l.askGeoJS(ctx, unanswered)
+	if err != nil {
+		l.logger.Warn("geojs lookup failed; using the database", "error", err)
+	}
+
+	for i, addr := range addrs {
+		if locations[i] != nil {
+			continue
+		}
+
+		address := plainAddress(addr)
+
+		answer, ok := answers[address]
+		if ok {
+			l.store(address, answer)
+			locations[i] = answer
+
+			continue
+		}
+
+		locations[i] = l.fromDatabase(addr)
+	}
+
+	return locations
+}
+
+// plainAddress returns addr as the requests to geojs and the cache name it: an
+// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
+// chooses the zone, so it must not reach the request.
+func plainAddress(addr netip.Addr) string {
+	return addr.Unmap().WithZone("").String()
+}
+
+// Start loads the cached answers that have not expired and opens the copy of
+// the database, if there is one. It then starts the goroutines that write new
+// answers, delete expired ones and download the database. They run until Stop:
+// the end of ctx does not stop them.
+func (l *Locator) Start(ctx context.Context) error {
+	err := l.load(ctx)
+	if err != nil {
+		return err
+	}
+
+	l.logger.Info("Geolocation cache loaded", "entries", l.entries.Len())
+
+	l.openDatabaseCopy()
+
+	ctx, l.stop = context.WithCancel(context.WithoutCancel(ctx))
+
+	go func() {
+		defer close(l.done)
+		defer l.closeDatabase()
+
+		var workers sync.WaitGroup
+
+		workers.Go(func() { l.keepCache(ctx) })
+		workers.Go(func() { l.keepDatabaseCurrent(ctx) })
+		workers.Wait()
+	}()
+
+	return nil
+}
+
+// Stop cancels a download in progress, writes the answers cache.db does not
+// have yet, and waits until the goroutines have returned or ctx ends.
+func (l *Locator) Stop(ctx context.Context) {
+	l.stop()
+
+	select {
+	case <-l.done:
+		l.logger.Info("Geolocation stopped")
+	case <-ctx.Done():
+		l.logger.Warn("Stop deadline passed before the geolocation cache " +
+			"was written")
+	}
+}

internal/routewatch/app.go

@@ -19,6 +19,7 @@ import (
 	"git.eeqj.de/sneak/routewatch/internal/cachedb"
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/geo"
 	"git.eeqj.de/sneak/routewatch/internal/iplist"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
 	"git.eeqj.de/sneak/routewatch/internal/metrics"
@@ -67,6 +68,7 @@ type RouteWatch struct {
 	ipLists        *iplist.IPLists
 	cacheDB        *sql.DB
 	ptrCache       *ptr.Cache
+	geoLocator     *geo.Locator
 }
 
 // ipListsDirName is the directory in the state directory that holds the IP
@@ -160,6 +162,11 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
 		return err
 	}
 
+	err = rw.startGeoLocator(ctx)
+	if err != nil {
+		return err
+	}
+
 	// Start HTTP server
 	err = rw.server.Start(ctx)
 	if err != nil {
@@ -178,8 +185,9 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
 
 // Shutdown stops all services in order: the streamer, the handlers, which
 // write what they hold, the WHOIS fetcher, the database maintainer, the
-// IP list downloads, the HTTP server and the PTR cache, and last the database,
-// which it checkpoints and closes. Each stage waits at most until ctx ends, so
+// IP list downloads, the HTTP server, the geolocation cache and the PTR cache,
+// and last the database, which it checkpoints and closes. Each stage waits at
+// most until ctx ends, so
 // Shutdown returns soon after: a handler logs what it could not write by
 // then, and the HTTP server closes the connections of requests still
 // running.
@@ -239,6 +247,10 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
 			"error", err)
 	}
 
+	if rw.geoLocator != nil {
+		rw.geoLocator.Stop(ctx)
+	}
+
 	rw.stopPTRCache(ctx)
 
 	rw.closeDatabase(ctx)
@@ -283,6 +295,32 @@ func (rw *RouteWatch) startPTRCache(ctx context.Context) error {
 	return nil
 }
 
+// startGeoLocator loads the geolocation cache from cache.db, which
+// startPTRCache has opened, and gives the server the locator. The HTTP handlers
+// read it without a lock, so it is set before they run.
+func (rw *RouteWatch) startGeoLocator(ctx context.Context) error {
+	locator, err := geo.New(geo.Params{
+		DB:                rw.cacheDB,
+		Dir:               rw.config.StateDir,
+		GeoJSURL:          geo.GeoJSEndpoint,
+		DatabaseURLFormat: geo.DatabaseDownloadFormat,
+		Logger:            rw.logger.Logger,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = locator.Start(ctx)
+	if err != nil {
+		return err
+	}
+
+	rw.geoLocator = locator
+	rw.server.SetGeoLocator(locator)
+
+	return nil
+}
+
 // stopPTRCache stops the PTR cache, which writes the answers cache.db does not
 // have yet, and closes cache.db, unless ctx has ended and a write may still be
 // running.

internal/server/handlers.go

@@ -17,6 +17,7 @@ import (
 
 	"git.eeqj.de/sneak/routewatch/internal/bogon"
 	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/geo"
 	"git.eeqj.de/sneak/routewatch/internal/iplist"
 	"git.eeqj.de/sneak/routewatch/internal/metrics"
 	"git.eeqj.de/sneak/routewatch/internal/ristypes"
@@ -764,9 +765,9 @@ type IPLookupResponse struct {
 }
 
 // IPLookupResult is the answer for one address: its PTR names, its flags, its
-// live route and that route's AS, and the IP lists that contain it. Every
-// result has every field. The fields of IPInfo are null when the address has
-// no live route.
+// live route and that route's AS, the IP lists that contain it, and where it
+// is. Every result has every field. The fields of IPInfo are null when the
+// address has no live route.
 //
 //nolint:tagliatelle // snake_case keys, as the Go style guide asks
 type IPLookupResult struct {
@@ -784,6 +785,9 @@ type IPLookupResult struct {
 
 	// IPLists has one entry for each IP list the address is on.
 	IPLists []iplist.Listing `json:"ip_lists"`
+
+	// Geo is where the address is. It is nil when no source answers.
+	Geo *geo.Location `json:"geo"`
 }
 
 // IPLookupFlags are the yes-or-no facts about an address. Every result has
@@ -839,6 +843,8 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 			return
 		}
 
+		locations := s.geoLocations(ctx, ipsToLookup)
+
 		// Track ASNs that need WHOIS refresh
 		refreshASNs := make(map[int]bool)
 
@@ -874,7 +880,9 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 				return
 			}
 
-			response.Results = append(response.Results, s.ipLookupResult(ip, ptrs, ipInfo))
+			result := s.ipLookupResult(ip, ptrs, ipInfo)
+			result.Geo = locations[ip]
+			response.Results = append(response.Results, result)
 
 			if ipInfo.NeedsWHOISRefresh {
 				refreshASNs[*ipInfo.ASN] = true
@@ -1029,6 +1037,45 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
 	}
 }
 
+// geoLocations returns where ips are, by address. An address that does not
+// parse has no entry. Nor has a reserved one, which is sent to no outside
+// source and has no location.
+func (s *Server) geoLocations(
+	ctx context.Context, ips []string,
+) map[string]*geo.Location {
+	if s.geoLocator == nil {
+		return nil
+	}
+
+	var (
+		kept  []string
+		addrs []netip.Addr
+	)
+
+	for _, ip := range ips {
+		addr, err := netip.ParseAddr(ip)
+		if err != nil || isBogon(ip) {
+			continue
+		}
+
+		kept = append(kept, ip)
+		addrs = append(addrs, addr)
+	}
+
+	if len(addrs) == 0 {
+		return nil
+	}
+
+	answers := s.geoLocator.Lookup(ctx, addrs)
+	locations := make(map[string]*geo.Location, len(kept))
+
+	for i, ip := range kept {
+		locations[ip] = answers[i]
+	}
+
+	return locations
+}
+
 // isBogon says whether the address ip is reserved, as package bogon tells.
 func isBogon(ip string) bool {
 	addr, err := netip.ParseAddr(ip)

internal/server/handlers_internal_test.go

@@ -15,6 +15,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"testing/synctest"
@@ -22,6 +23,7 @@ import (
 
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/geo"
 	"git.eeqj.de/sneak/routewatch/internal/iplist"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
 	"git.eeqj.de/sneak/routewatch/internal/metrics"
@@ -1613,6 +1615,131 @@ func TestIPLookupFlags(t *testing.T) {
 	}
 }
 
+// fakeGeoLocator answers every address with location, and keeps the addresses
+// of each call. It is safe for the concurrent subtests that share it.
+type fakeGeoLocator struct {
+	location *geo.Location
+
+	mu    sync.Mutex
+	calls [][]netip.Addr
+}
+
+func (f *fakeGeoLocator) Lookup(
+	_ context.Context, addrs []netip.Addr,
+) []*geo.Location {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	f.calls = append(f.calls, addrs)
+
+	answers := make([]*geo.Location, len(addrs))
+	for i := range addrs {
+		answers[i] = f.location
+	}
+
+	return answers
+}
+
+// TestIPLookupGivesTheGeoObject looks up two public addresses. They go to the
+// locator in one call, and each result has the geo object under the keys the
+// API documents. These addresses are public, because a reserved one is not
+// looked up.
+func TestIPLookupGivesTheGeoObject(t *testing.T) {
+	t.Parallel()
+
+	s := newServerWithTestRoutes(t)
+	useFakeResolver(s, &fakeResolver{})
+
+	locator := &fakeGeoLocator{location: &geo.Location{
+		CountryCode: new("DE"),
+		City:        new("Berlin"),
+		Latitude:    new(52.5),
+		Source:      "geojs",
+		Updated:     testRouteTime(0),
+	}}
+	s.SetGeoLocator(locator)
+
+	path := "/api/v1/ip/1.1.1.1,8.8.8.8"
+
+	results := resultFields(t, get(t, s, path).Body.Bytes())
+	if len(results) != 2 {
+		t.Fatalf("%s: %d results, want 2", path, len(results))
+	}
+
+	if len(locator.calls) != 1 || len(locator.calls[0]) != 2 {
+		t.Errorf("locator calls %v, want one with both addresses", locator.calls)
+	}
+
+	var got map[string]json.RawMessage
+
+	err := json.Unmarshal(results[0]["geo"], &got)
+	if err != nil {
+		t.Fatalf("decoding geo %s: %v", results[0]["geo"], err)
+	}
+
+	wantKeys := []string{
+		"accuracy_radius_km", "city", "continent_code", "country",
+		"country_code", "latitude", "longitude", "region", "source",
+		"time_zone", "updated",
+	}
+	if keys := slices.Sorted(maps.Keys(got)); !slices.Equal(keys, wantKeys) {
+		t.Errorf("geo keys %v, want %v", keys, wantKeys)
+	}
+
+	if string(got["city"]) != `"Berlin"` || string(got["latitude"]) != "52.5" ||
+		string(got["region"]) != jsonNull {
+		t.Errorf("wrong geo %s", results[0]["geo"])
+	}
+}
+
+// TestIPLookupOfAReservedAddressHasNoGeo looks up a reserved address alone and
+// beside a public one. The reserved one never reaches the locator, and its geo
+// is null.
+func TestIPLookupOfAReservedAddressHasNoGeo(t *testing.T) {
+	t.Parallel()
+
+	s := newServerWithTestRoutes(t)
+	useFakeResolver(s, &fakeResolver{})
+
+	locator := &fakeGeoLocator{location: &geo.Location{Source: "geojs"}}
+	s.SetGeoLocator(locator)
+
+	results := resultFields(t, get(t, s, "/api/v1/ip/"+addrInTestPrefixV4).Body.Bytes())
+	if len(results) != 1 || string(results[0]["geo"]) != jsonNull {
+		t.Fatalf("results %v, want one with geo null", results)
+	}
+
+	if len(locator.calls) != 0 {
+		t.Errorf("locator calls %v, want none", locator.calls)
+	}
+
+	results = resultFields(t,
+		get(t, s, "/api/v1/ip/"+addrInTestPrefixV4+",1.1.1.1").Body.Bytes())
+	if len(results) != 2 || string(results[0]["geo"]) != jsonNull ||
+		string(results[1]["geo"]) == jsonNull {
+		t.Errorf("results %v, want geo only for the public address", results)
+	}
+
+	if len(locator.calls) != 1 || len(locator.calls[0]) != 1 {
+		t.Errorf("locator calls %v, want one with one address", locator.calls)
+	}
+}
+
+// TestIPLookupGeoIsNullWhenNoSourceAnswers checks an address the locator has
+// no answer for.
+func TestIPLookupGeoIsNullWhenNoSourceAnswers(t *testing.T) {
+	t.Parallel()
+
+	s := newServerWithTestRoutes(t)
+	useFakeResolver(s, &fakeResolver{})
+	s.SetGeoLocator(&fakeGeoLocator{})
+
+	results := resultFields(t, get(t, s, "/api/v1/ip/1.1.1.1").Body.Bytes())
+	if len(results) != 1 || string(results[0]["geo"]) != jsonNull {
+		t.Errorf("results %v, want one with geo null", results)
+	}
+}
+
 // TestIPLookupReportsVPNProviders looks up addresses on no list, on a list
 // that is not a VPN list, on one VPN list and on two. is_vpn says whether any
 // VPN list holds the address, and vpn_providers names only those lists. An

internal/server/server.go

@@ -24,6 +24,7 @@ import (
 
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/geo"
 	"git.eeqj.de/sneak/routewatch/internal/iplist"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
 	"git.eeqj.de/sneak/routewatch/internal/routetable"
@@ -63,6 +64,12 @@ type PTRResolver interface {
 	LookupAddr(ctx context.Context, addr string) ([]string, error)
 }
 
+// GeoLocator finds where addresses are, as *geo.Locator does. It returns one
+// entry for each address, nil when no source answers.
+type GeoLocator interface {
+	Lookup(ctx context.Context, addrs []netip.Addr) []*geo.Location
+}
+
 // Server provides HTTP endpoints for status monitoring
 type Server struct {
 	router     *chi.Mux
@@ -86,6 +93,10 @@ type Server struct {
 	// ptrResolver looks up the IP lookup's PTR names. SetPTRResolver sets it.
 	ptrResolver PTRResolver
 
+	// geoLocator finds where the IP lookup's addresses are. SetGeoLocator
+	// sets it.
+	geoLocator GeoLocator
+
 	// cards is the copy of the home page's cards that the pages serve, nil
 	// until the server first works them out
 	cards atomic.Pointer[homeCards]
@@ -209,6 +220,12 @@ func (s *Server) SetPTRResolver(resolver PTRResolver) {
 	s.ptrResolver = resolver
 }
 
+// SetGeoLocator sets what finds where the IP lookup's addresses are. Call it
+// before Start: the handlers read it without a lock.
+func (s *Server) SetGeoLocator(locator GeoLocator) {
+	s.geoLocator = locator
+}
+
 // SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups. Call it
 // before Start: the handlers read the fetcher without a lock.
 func (s *Server) SetASNFetcher(fetcher ASNFetcher) {

internal/templates/index.html

@@ -544,6 +544,8 @@
         <span>{{appLicense}}</span>
         <span class="separator">|</span>
         <span><a href="{{appGitCommitURL}}">{{appGitRevision}}</a></span>
+        <span class="separator">|</span>
+        <span>IP Geolocation by <a href="https://db-ip.com">DB-IP</a></span>
     </footer>
 
     <script nonce="{{.CSPNonce}}">

internal/templates/templates_test.go

@@ -67,6 +67,23 @@ func TestFooterLicense(t *testing.T) {
 	}
 }
 
+// TestHomePageCreditsDBIP expects the home page's footer to carry the credit
+// to DB-IP.
+func TestHomePageCreditsDBIP(t *testing.T) {
+	t.Parallel()
+
+	const attribution = `IP Geolocation by <a href="https://db-ip.com">DB-IP</a>`
+
+	_, footer, found := strings.Cut(renderPage(t, templates.IndexTemplate()), "<footer")
+	if !found {
+		t.Fatal("the page has no footer")
+	}
+
+	if !strings.Contains(footer, attribution) {
+		t.Errorf("footer does not credit DB-IP: %q", footer)
+	}
+}
+
 // TestFreshnessIndicator renders the home and status pages and expects each to
 // have, right after the status element, the element where the page's script
 // shows its spinner or "stale".