Add best-effort geolocation to the IP lookup from geojs and DB-IP City Lite (closes #239)
check / check (push) Successful in 3m58s
check / check (push) Successful in 3m58s
Each result gains `geo`. One request to geojs carries every address, 1 second in all. On a timeout, an error or an answer with no country, city or coordinates, the lookup reads DB-IP City Lite, downloaded monthly into the state directory. geojs answers stay in memory (250,000, least recently used dropped) and in a new `cache.db` table for 7 days. Judgement call: a reserved address goes to neither source, and its `geo` is null. Judgement call: `updated` for DB-IP is when the copy was downloaded. Deviation: the cache is SQLite, by sneak's words. Model: opus-5-5
This commit is contained in:
sneak committed
1 parent
b2553f044d
commit
c8ae355787
16 files changed
+2372
-22
No files matched your search
@@ -0,0 +1,253 @@
|
||||
package geo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// entryLifetime is how long a geojs answer is kept after its fetch.
|
||||
entryLifetime = 7 * 24 * time.Hour
|
||||
|
||||
// flushInterval is how often new answers are written to cache.db.
|
||||
flushInterval = 5 * time.Second
|
||||
|
||||
// sweepInterval is how often expired answers are deleted.
|
||||
sweepInterval = 10 * time.Minute
|
||||
|
||||
// rowColumns are the columns of the geolocations table in the order
|
||||
// scanRow reads them.
|
||||
rowColumns = "address, country_code, country, continent_code, region, " +
|
||||
"city, latitude, longitude, accuracy_radius_km, time_zone, fetched_at"
|
||||
)
|
||||
|
||||
// rowScanner is the Scan method of *sql.Row and *sql.Rows.
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// cached returns the current answer held for address, from memory, then from
|
||||
// the answers waiting to be written, then from cache.db. It returns nil when
|
||||
// there is none.
|
||||
func (l *Locator) cached(ctx context.Context, address string) *Location {
|
||||
found, ok := l.entries.Get(address)
|
||||
if ok {
|
||||
if !l.expired(found) {
|
||||
return found
|
||||
}
|
||||
|
||||
l.entries.Remove(address)
|
||||
}
|
||||
|
||||
l.pendingMu.Lock()
|
||||
waiting := l.pending[address]
|
||||
l.pendingMu.Unlock()
|
||||
|
||||
if waiting != nil && !l.expired(waiting) {
|
||||
l.entries.Add(address, waiting)
|
||||
|
||||
return waiting
|
||||
}
|
||||
|
||||
stored, err := l.readRow(ctx, address)
|
||||
if err != nil {
|
||||
l.logger.Warn("Failed to read a geolocation cache row",
|
||||
"address", address, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if stored != nil {
|
||||
l.entries.Add(address, stored)
|
||||
}
|
||||
|
||||
return stored
|
||||
}
|
||||
|
||||
// expired says whether the answer is at the end of its lifetime.
|
||||
func (l *Locator) expired(answer *Location) bool {
|
||||
return !l.now().Before(answer.Updated.Add(entryLifetime))
|
||||
}
|
||||
|
||||
// cutoff returns the fetch time at or before which an answer has expired.
|
||||
func (l *Locator) cutoff() int64 {
|
||||
return l.now().Add(-entryLifetime).Unix()
|
||||
}
|
||||
|
||||
// store keeps an answer in memory and queues it for cache.db. While cache.db
|
||||
// cannot be written, the queue stops growing at maxEntries answers.
|
||||
func (l *Locator) store(address string, answer *Location) {
|
||||
l.entries.Add(address, answer)
|
||||
|
||||
l.pendingMu.Lock()
|
||||
defer l.pendingMu.Unlock()
|
||||
|
||||
if len(l.pending) < l.maxEntries {
|
||||
l.pending[address] = answer
|
||||
}
|
||||
}
|
||||
|
||||
// keepCache writes new answers and deletes expired ones until ctx ends.
|
||||
func (l *Locator) keepCache(ctx context.Context) {
|
||||
flushes := time.NewTicker(flushInterval)
|
||||
defer flushes.Stop()
|
||||
|
||||
sweeps := time.NewTicker(sweepInterval)
|
||||
defer sweeps.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
l.flush(context.WithoutCancel(ctx))
|
||||
|
||||
return
|
||||
case <-flushes.C:
|
||||
l.flush(ctx)
|
||||
case <-sweeps.C:
|
||||
l.sweep(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// load reads the answers that have not expired into memory, at most maxEntries
|
||||
// of them, the newest last so that they are the last to be dropped.
|
||||
func (l *Locator) load(ctx context.Context) error {
|
||||
rows, err := l.db.QueryContext(ctx,
|
||||
"SELECT "+rowColumns+" FROM (SELECT "+rowColumns+" FROM geolocations "+
|
||||
"WHERE fetched_at > ? ORDER BY fetched_at DESC LIMIT ?) "+
|
||||
"ORDER BY fetched_at",
|
||||
l.cutoff(), l.maxEntries)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the geolocation cache: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
address, answer, err := scanRow(rows)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading a geolocation cache row: %w", err)
|
||||
}
|
||||
|
||||
l.entries.Add(address, answer)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the geolocation cache: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readRow returns the current answer for address in cache.db, or nil when it
|
||||
// has none.
|
||||
func (l *Locator) readRow(
|
||||
ctx context.Context, address string,
|
||||
) (*Location, error) {
|
||||
row := l.db.QueryRowContext(ctx,
|
||||
"SELECT "+rowColumns+" FROM geolocations "+
|
||||
"WHERE address = ? AND fetched_at > ?",
|
||||
address, l.cutoff())
|
||||
|
||||
_, answer, err := scanRow(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // no row is not a failure
|
||||
}
|
||||
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// scanRow reads one row of the geolocations table.
|
||||
func scanRow(row rowScanner) (string, *Location, error) {
|
||||
var (
|
||||
address string
|
||||
fetched int64
|
||||
answer Location
|
||||
)
|
||||
|
||||
err := row.Scan(&address, &answer.CountryCode, &answer.Country,
|
||||
&answer.ContinentCode, &answer.Region, &answer.City, &answer.Latitude,
|
||||
&answer.Longitude, &answer.AccuracyRadiusKm, &answer.TimeZone, &fetched)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
answer.Source = sourceGeoJS
|
||||
answer.Updated = time.Unix(fetched, 0).UTC()
|
||||
|
||||
return address, &answer, nil
|
||||
}
|
||||
|
||||
// flush writes the pending answers to cache.db. Those it cannot write stay
|
||||
// pending, unless a newer answer for the address has arrived meanwhile.
|
||||
func (l *Locator) flush(ctx context.Context) {
|
||||
l.pendingMu.Lock()
|
||||
batch := l.pending
|
||||
l.pending = make(map[string]*Location)
|
||||
l.pendingMu.Unlock()
|
||||
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
err := l.writeRows(ctx, batch)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
l.logger.Error("Failed to write the geolocation cache", "error", err)
|
||||
|
||||
l.pendingMu.Lock()
|
||||
defer l.pendingMu.Unlock()
|
||||
|
||||
maps.Insert(batch, maps.All(l.pending))
|
||||
l.pending = batch
|
||||
}
|
||||
|
||||
// writeRows writes batch to cache.db in one transaction.
|
||||
func (l *Locator) writeRows(
|
||||
ctx context.Context, batch map[string]*Location,
|
||||
) error {
|
||||
tx, err := l.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
for address, answer := range batch {
|
||||
_, err = tx.ExecContext(ctx,
|
||||
"INSERT OR REPLACE INTO geolocations ("+rowColumns+") "+
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
address, answer.CountryCode, answer.Country, answer.ContinentCode,
|
||||
answer.Region, answer.City, answer.Latitude, answer.Longitude,
|
||||
answer.AccuracyRadiusKm, answer.TimeZone, answer.Updated.Unix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// sweep deletes the expired answers from memory and cache.db.
|
||||
func (l *Locator) sweep(ctx context.Context) {
|
||||
for _, address := range l.entries.Keys() {
|
||||
answer, ok := l.entries.Peek(address)
|
||||
if ok && l.expired(answer) {
|
||||
l.entries.Remove(address)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := l.db.ExecContext(ctx,
|
||||
"DELETE FROM geolocations WHERE fetched_at <= ?", l.cutoff())
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to delete expired geolocation cache rows",
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package geo
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/oschwald/maxminddb-golang/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
// DatabaseDownloadFormat is the URL of DB-IP's City Lite database, with %s
|
||||
// for the year and month.
|
||||
DatabaseDownloadFormat = "https://download.db-ip.com/free/" +
|
||||
"dbip-city-lite-%s.mmdb.gz"
|
||||
|
||||
// databaseMonthFormat is the year and month in the URL.
|
||||
databaseMonthFormat = "2006-01"
|
||||
|
||||
// databaseFileName is the copy of the database in the state directory.
|
||||
databaseFileName = "dbip-city-lite.mmdb"
|
||||
|
||||
// databaseDownloadSuffix ends the name of the file a download is written
|
||||
// to, until it has opened.
|
||||
databaseDownloadSuffix = ".download"
|
||||
|
||||
// maxDatabaseSize is the largest database accepted once unpacked, in
|
||||
// bytes.
|
||||
maxDatabaseSize = 200 << 20
|
||||
|
||||
// databaseDownloadTimeout limits one download.
|
||||
databaseDownloadTimeout = 10 * time.Minute
|
||||
|
||||
// databaseCheckInterval is how often RouteWatch checks whether a new
|
||||
// month's database is due. A new month's file appears some time on the
|
||||
// first of the month, so a failed download is tried again this much later.
|
||||
databaseCheckInterval = time.Hour
|
||||
)
|
||||
|
||||
var (
|
||||
errDatabaseStatus = errors.New("unexpected HTTP status from DB-IP")
|
||||
errDatabaseTooLong = errors.New("database is over the size limit")
|
||||
)
|
||||
|
||||
// dbipName is the part of a database record that holds an English name.
|
||||
type dbipName struct {
|
||||
Names struct {
|
||||
English string `maxminddb:"en"`
|
||||
} `maxminddb:"names"`
|
||||
}
|
||||
|
||||
// dbipRecord is the part of a database record RouteWatch reads.
|
||||
type dbipRecord struct {
|
||||
City dbipName `maxminddb:"city"`
|
||||
Continent struct {
|
||||
Code string `maxminddb:"code"`
|
||||
} `maxminddb:"continent"`
|
||||
Country struct {
|
||||
dbipName
|
||||
|
||||
ISOCode string `maxminddb:"iso_code"`
|
||||
} `maxminddb:"country"`
|
||||
Location struct {
|
||||
Latitude *float64 `maxminddb:"latitude"`
|
||||
Longitude *float64 `maxminddb:"longitude"`
|
||||
} `maxminddb:"location"`
|
||||
Subdivisions []dbipName `maxminddb:"subdivisions"`
|
||||
}
|
||||
|
||||
// location returns the record as a Location fetched at the time given. The
|
||||
// first subdivision is the region. The database has no accuracy radius and no
|
||||
// time zone.
|
||||
func (r *dbipRecord) location(fetched time.Time) *Location {
|
||||
found := &Location{
|
||||
CountryCode: nilIfEmpty(r.Country.ISOCode),
|
||||
Country: nilIfEmpty(r.Country.Names.English),
|
||||
ContinentCode: nilIfEmpty(r.Continent.Code),
|
||||
City: nilIfEmpty(r.City.Names.English),
|
||||
Latitude: r.Location.Latitude,
|
||||
Longitude: r.Location.Longitude,
|
||||
Source: sourceDBIP,
|
||||
Updated: fetched,
|
||||
}
|
||||
|
||||
if len(r.Subdivisions) > 0 {
|
||||
found.Region = nilIfEmpty(r.Subdivisions[0].Names.English)
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
// nilIfEmpty returns a pointer to text, or nil when text is empty.
|
||||
func nilIfEmpty(text string) *string {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &text
|
||||
}
|
||||
|
||||
// fromDatabase returns where the copy of the database places addr. It returns
|
||||
// nil when there is no copy yet or the copy has no record for addr. It also
|
||||
// returns nil, and logs the error, when reading the copy fails.
|
||||
func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
||||
l.databaseMu.RLock()
|
||||
defer l.databaseMu.RUnlock()
|
||||
|
||||
if l.database == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := l.database.Lookup(addr)
|
||||
|
||||
err := result.Err()
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to look up an address in the database",
|
||||
"address", addr, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !result.Found() {
|
||||
return nil
|
||||
}
|
||||
|
||||
var record dbipRecord
|
||||
|
||||
err = result.Decode(&record)
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to read a database record",
|
||||
"address", addr, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return record.location(l.databaseFetched)
|
||||
}
|
||||
|
||||
// keepDatabaseCurrent downloads the database when it is due, at once and then
|
||||
// every databaseCheckInterval, until ctx ends.
|
||||
func (l *Locator) keepDatabaseCurrent(ctx context.Context) {
|
||||
l.downloadDatabaseIfDue(ctx)
|
||||
|
||||
checks := time.NewTicker(databaseCheckInterval)
|
||||
defer checks.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-checks.C:
|
||||
l.downloadDatabaseIfDue(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// downloadDatabaseIfDue downloads this month's database, unless the copy in
|
||||
// use was downloaded this month. A failure leaves the copy as it is.
|
||||
func (l *Locator) downloadDatabaseIfDue(ctx context.Context) {
|
||||
now := l.now().UTC()
|
||||
|
||||
if !l.databaseDue(now) {
|
||||
return
|
||||
}
|
||||
|
||||
err := l.downloadDatabase(ctx, now)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
l.logger.Error("Database download failed; keeping the previous copy",
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// databaseDue says whether there is no open copy or it is from an earlier
|
||||
// month than now.
|
||||
func (l *Locator) databaseDue(now time.Time) bool {
|
||||
l.databaseMu.RLock()
|
||||
defer l.databaseMu.RUnlock()
|
||||
|
||||
return l.database == nil ||
|
||||
l.databaseFetched.Format(databaseMonthFormat) !=
|
||||
now.Format(databaseMonthFormat)
|
||||
}
|
||||
|
||||
// downloadDatabase fetches the database of now's month. The file replaces the
|
||||
// copy, on disk and in use, only once it has opened.
|
||||
func (l *Locator) downloadDatabase(ctx context.Context, now time.Time) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, databaseDownloadTimeout)
|
||||
defer cancel()
|
||||
|
||||
url := fmt.Sprintf(l.databaseURLFormat, now.Format(databaseMonthFormat))
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
request.Header.Set(userAgentHeader, userAgent)
|
||||
|
||||
response, err := l.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%w: %s", errDatabaseStatus, response.Status)
|
||||
}
|
||||
|
||||
return l.replaceDatabase(response.Body, now)
|
||||
}
|
||||
|
||||
// replaceDatabase unpacks the gzip stream body beside the copy, then makes the
|
||||
// result the copy once it has opened. Whatever fails, the file it wrote is
|
||||
// removed and the copy stays.
|
||||
func (l *Locator) replaceDatabase(body io.Reader, now time.Time) error {
|
||||
root, err := os.OpenRoot(l.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
downloadName := databaseFileName + databaseDownloadSuffix
|
||||
|
||||
err = unpackDatabase(root, downloadName, body, l.maxDatabaseBytes)
|
||||
if err != nil {
|
||||
_ = root.Remove(downloadName)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
opened, err := maxminddb.Open(filepath.Join(l.dir, downloadName))
|
||||
if err != nil {
|
||||
_ = root.Remove(downloadName)
|
||||
|
||||
return fmt.Errorf("opening the downloaded database: %w", err)
|
||||
}
|
||||
|
||||
err = root.Rename(downloadName, databaseFileName)
|
||||
if err != nil {
|
||||
_ = opened.Close()
|
||||
_ = root.Remove(downloadName)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
l.useDatabase(opened, now.Truncate(time.Second))
|
||||
l.logger.Info("Downloaded a new copy of the database")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// unpackDatabase writes the gzip stream body to the file name in root, up to
|
||||
// limit bytes once unpacked.
|
||||
func unpackDatabase(
|
||||
root *os.Root, name string, body io.Reader, limit int64,
|
||||
) error {
|
||||
unpacked, err := gzip.NewReader(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unpacking the database: %w", err)
|
||||
}
|
||||
|
||||
f, err := root.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
written, err := io.Copy(f, io.LimitReader(unpacked, limit+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unpacking the database: %w", err)
|
||||
}
|
||||
|
||||
if written > limit {
|
||||
return errDatabaseTooLong
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// openDatabaseCopy opens the copy left by an earlier run, if there is one. A
|
||||
// copy that does not open is treated as no copy, so it is downloaded again.
|
||||
func (l *Locator) openDatabaseCopy() {
|
||||
path := filepath.Join(l.dir, databaseFileName)
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to read the database copy", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
opened, err := maxminddb.Open(path)
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to open the database copy", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
l.useDatabase(opened, info.ModTime().UTC().Truncate(time.Second))
|
||||
}
|
||||
|
||||
// useDatabase makes opened the copy in use and closes the one it replaces.
|
||||
func (l *Locator) useDatabase(opened *maxminddb.Reader, fetched time.Time) {
|
||||
l.databaseMu.Lock()
|
||||
defer l.databaseMu.Unlock()
|
||||
|
||||
if l.database != nil {
|
||||
err := l.database.Close()
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to close the old database copy", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
l.database = opened
|
||||
l.databaseFetched = fetched
|
||||
}
|
||||
|
||||
// closeDatabase closes the copy in use, after which no source can answer from
|
||||
// it.
|
||||
func (l *Locator) closeDatabase() {
|
||||
l.databaseMu.Lock()
|
||||
defer l.databaseMu.Unlock()
|
||||
|
||||
if l.database == nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := l.database.Close()
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to close the database copy", "error", err)
|
||||
}
|
||||
|
||||
l.database = nil
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package geo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// GeoJSEndpoint is the URL of the geojs geo endpoint.
|
||||
GeoJSEndpoint = "https://get.geojs.io/v1/ip/geo.json"
|
||||
|
||||
// geojsTimeLimit limits one request to geojs.
|
||||
geojsTimeLimit = time.Second
|
||||
|
||||
// geojsAddressSeparator separates the addresses of one request.
|
||||
geojsAddressSeparator = ","
|
||||
|
||||
// geojsUnknown is what geojs sends for a value it does not know.
|
||||
geojsUnknown = "nil"
|
||||
|
||||
// maxGeoJSAnswerBytes is far more than an answer for the most addresses
|
||||
// an IP lookup can ask about.
|
||||
maxGeoJSAnswerBytes = 1 << 20
|
||||
)
|
||||
|
||||
var errGeoJSStatus = errors.New("unexpected HTTP status from geojs")
|
||||
|
||||
// geojsAnswer is one address's entry in an answer from geojs.
|
||||
//
|
||||
//nolint:tagliatelle // the keys geojs sends
|
||||
type geojsAnswer struct {
|
||||
IP string `json:"ip"`
|
||||
CountryCode string `json:"country_code"`
|
||||
Country string `json:"country"`
|
||||
ContinentCode string `json:"continent_code"`
|
||||
Region string `json:"region"`
|
||||
City string `json:"city"`
|
||||
Latitude string `json:"latitude"`
|
||||
Longitude string `json:"longitude"`
|
||||
Accuracy *int `json:"accuracy"`
|
||||
TimeZone string `json:"timezone"`
|
||||
}
|
||||
|
||||
// askGeoJS asks geojs about addresses in one request, and returns its answers
|
||||
// by address. An address geojs left out, or gave no country, city or
|
||||
// coordinates for, has no entry.
|
||||
func (l *Locator) askGeoJS(
|
||||
ctx context.Context, addresses []string,
|
||||
) (map[string]*Location, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, l.timeLimit)
|
||||
defer cancel()
|
||||
|
||||
// A comma and a colon are legal in a query, and the escaped forms are
|
||||
// not what geojs documents.
|
||||
url := l.geojsURL + "?ip=" + strings.Join(addresses, geojsAddressSeparator)
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
request.Header.Set(userAgentHeader, userAgent)
|
||||
|
||||
response, err := l.client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w: %d", errGeoJSStatus, response.StatusCode)
|
||||
}
|
||||
|
||||
var parsed []geojsAnswer
|
||||
|
||||
err = json.NewDecoder(io.LimitReader(response.Body, maxGeoJSAnswerBytes)).
|
||||
Decode(&parsed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the geojs answer: %w", err)
|
||||
}
|
||||
|
||||
fetched := l.now().UTC().Truncate(time.Second)
|
||||
answers := make(map[string]*Location, len(parsed))
|
||||
|
||||
for _, answer := range parsed {
|
||||
addr, err := netip.ParseAddr(answer.IP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
location := answer.location(fetched)
|
||||
if hasLocation(location) {
|
||||
answers[plainAddress(addr)] = location
|
||||
}
|
||||
}
|
||||
|
||||
return answers, nil
|
||||
}
|
||||
|
||||
// hasLocation says whether found has a country, a city or a coordinate.
|
||||
func hasLocation(found *Location) bool {
|
||||
return found.CountryCode != nil || found.Country != nil ||
|
||||
found.City != nil || found.Latitude != nil || found.Longitude != nil
|
||||
}
|
||||
|
||||
// location returns the answer as a Location fetched at the time given.
|
||||
func (a geojsAnswer) location(fetched time.Time) *Location {
|
||||
return &Location{
|
||||
CountryCode: knownText(a.CountryCode),
|
||||
Country: knownText(a.Country),
|
||||
ContinentCode: knownText(a.ContinentCode),
|
||||
Region: knownText(a.Region),
|
||||
City: knownText(a.City),
|
||||
Latitude: knownNumber(a.Latitude),
|
||||
Longitude: knownNumber(a.Longitude),
|
||||
AccuracyRadiusKm: a.Accuracy,
|
||||
TimeZone: knownText(a.TimeZone),
|
||||
Source: sourceGeoJS,
|
||||
Updated: fetched,
|
||||
}
|
||||
}
|
||||
|
||||
// knownText returns a pointer to text, or nil when geojs does not know it.
|
||||
func knownText(text string) *string {
|
||||
if text == "" || text == geojsUnknown {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &text
|
||||
}
|
||||
|
||||
// knownNumber returns text as a number, or nil when geojs does not know it. NaN
|
||||
// and infinity parse, but JSON cannot carry them.
|
||||
func knownNumber(text string) *float64 {
|
||||
number, err := strconv.ParseFloat(text, 64)
|
||||
if err != nil || math.IsNaN(number) || math.IsInf(number, 0) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &number
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// Package geo finds where IP addresses are. It asks geojs.io, and keeps those
|
||||
// answers in memory and in cache.db. When geojs.io gives no location for an
|
||||
// address, it reads a copy of DB-IP's City Lite database, which it downloads
|
||||
// once a month.
|
||||
package geo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/oschwald/maxminddb-golang/v2"
|
||||
)
|
||||
|
||||
// The values of Location.Source.
|
||||
const (
|
||||
sourceGeoJS = "geojs"
|
||||
sourceDBIP = "dbip"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxEntries limits how many geojs answers are held in memory.
|
||||
maxEntries = 250_000
|
||||
|
||||
// userAgentHeader and userAgent name routewatch to geojs and DB-IP.
|
||||
userAgentHeader = "User-Agent"
|
||||
userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
|
||||
)
|
||||
|
||||
// Params is what New needs.
|
||||
type Params struct {
|
||||
// DB is the open cache.db.
|
||||
DB *sql.DB
|
||||
|
||||
// Dir is the directory that holds the copy of the database.
|
||||
Dir string
|
||||
|
||||
// GeoJSURL is the URL of the geojs geo endpoint, GeoJSEndpoint in use.
|
||||
GeoJSURL string
|
||||
|
||||
// DatabaseURLFormat is the URL of the database with %s for the year and
|
||||
// month, DatabaseDownloadFormat in use.
|
||||
DatabaseURLFormat string
|
||||
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// Location is where an address is. A part the source does not know is nil.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case keys, as the Go style guide asks
|
||||
type Location struct {
|
||||
CountryCode *string `json:"country_code"`
|
||||
Country *string `json:"country"`
|
||||
ContinentCode *string `json:"continent_code"`
|
||||
Region *string `json:"region"`
|
||||
City *string `json:"city"`
|
||||
Latitude *float64 `json:"latitude"`
|
||||
Longitude *float64 `json:"longitude"`
|
||||
AccuracyRadiusKm *int `json:"accuracy_radius_km"`
|
||||
TimeZone *string `json:"time_zone"`
|
||||
|
||||
// Source is geojs or dbip.
|
||||
Source string `json:"source"`
|
||||
|
||||
// Updated is when the value was fetched: the geojs answer, or the copy of
|
||||
// the database.
|
||||
Updated time.Time `json:"updated"`
|
||||
}
|
||||
|
||||
// Locator answers where addresses are. Start loads its cache and starts its
|
||||
// goroutines.
|
||||
type Locator struct {
|
||||
db *sql.DB
|
||||
dir string
|
||||
geojsURL string
|
||||
databaseURLFormat string
|
||||
timeLimit time.Duration
|
||||
maxEntries int
|
||||
maxDatabaseBytes int64
|
||||
logger *slog.Logger
|
||||
client *http.Client
|
||||
now func() time.Time
|
||||
|
||||
// entries holds geojs answers by address, and drops the least recently
|
||||
// used first.
|
||||
entries *lru.Cache[string, *Location]
|
||||
|
||||
// pending holds the answers that cache.db does not have yet.
|
||||
pendingMu sync.Mutex
|
||||
pending map[string]*Location
|
||||
|
||||
// database is the open copy, nil until one has opened, and databaseFetched
|
||||
// is when that copy was downloaded. A lookup holds the read lock, so that
|
||||
// a new copy never closes the old one under it.
|
||||
databaseMu sync.RWMutex
|
||||
database *maxminddb.Reader
|
||||
databaseFetched time.Time
|
||||
|
||||
// stop ends the goroutines Start runs, and done is closed when they have
|
||||
// returned.
|
||||
stop context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// New returns a Locator with an empty cache. Start loads it.
|
||||
func New(params Params) (*Locator, error) {
|
||||
entries, err := lru.New[string, *Location](maxEntries)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating the geolocation cache: %w", err)
|
||||
}
|
||||
|
||||
return &Locator{
|
||||
db: params.DB,
|
||||
dir: params.Dir,
|
||||
geojsURL: params.GeoJSURL,
|
||||
databaseURLFormat: params.DatabaseURLFormat,
|
||||
timeLimit: geojsTimeLimit,
|
||||
maxEntries: maxEntries,
|
||||
maxDatabaseBytes: maxDatabaseSize,
|
||||
logger: params.Logger.With("component", "geo"),
|
||||
client: &http.Client{},
|
||||
now: time.Now,
|
||||
entries: entries,
|
||||
pending: make(map[string]*Location),
|
||||
done: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Lookup returns where each of addrs is, in the same order. The entry for an
|
||||
// address neither source answers for is nil. Addresses with no current cached
|
||||
// answer go to geojs in one request. The returned Locations are shared with
|
||||
// the cache and must not be changed.
|
||||
func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
||||
locations := make([]*Location, len(addrs))
|
||||
|
||||
var unanswered []string
|
||||
|
||||
for i, addr := range addrs {
|
||||
address := plainAddress(addr)
|
||||
|
||||
locations[i] = l.cached(ctx, address)
|
||||
if locations[i] == nil {
|
||||
unanswered = append(unanswered, address)
|
||||
}
|
||||
}
|
||||
|
||||
if len(unanswered) == 0 {
|
||||
return locations
|
||||
}
|
||||
|
||||
answers, err := l.askGeoJS(ctx, unanswered)
|
||||
if err != nil {
|
||||
l.logger.Warn("geojs lookup failed; using the database", "error", err)
|
||||
}
|
||||
|
||||
for i, addr := range addrs {
|
||||
if locations[i] != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
address := plainAddress(addr)
|
||||
|
||||
answer, ok := answers[address]
|
||||
if ok {
|
||||
l.store(address, answer)
|
||||
locations[i] = answer
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
locations[i] = l.fromDatabase(addr)
|
||||
}
|
||||
|
||||
return locations
|
||||
}
|
||||
|
||||
// plainAddress returns addr as the requests to geojs and the cache name it: an
|
||||
// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
|
||||
// chooses the zone, so it must not reach the request.
|
||||
func plainAddress(addr netip.Addr) string {
|
||||
return addr.Unmap().WithZone("").String()
|
||||
}
|
||||
|
||||
// Start loads the cached answers that have not expired and opens the copy of
|
||||
// the database, if there is one. It then starts the goroutines that write new
|
||||
// answers, delete expired ones and download the database. They run until Stop:
|
||||
// the end of ctx does not stop them.
|
||||
func (l *Locator) Start(ctx context.Context) error {
|
||||
err := l.load(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l.logger.Info("Geolocation cache loaded", "entries", l.entries.Len())
|
||||
|
||||
l.openDatabaseCopy()
|
||||
|
||||
ctx, l.stop = context.WithCancel(context.WithoutCancel(ctx))
|
||||
|
||||
go func() {
|
||||
defer close(l.done)
|
||||
defer l.closeDatabase()
|
||||
|
||||
var workers sync.WaitGroup
|
||||
|
||||
workers.Go(func() { l.keepCache(ctx) })
|
||||
workers.Go(func() { l.keepDatabaseCurrent(ctx) })
|
||||
workers.Wait()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop cancels a download in progress, writes the answers cache.db does not
|
||||
// have yet, and waits until the goroutines have returned or ctx ends.
|
||||
func (l *Locator) Stop(ctx context.Context) {
|
||||
l.stop()
|
||||
|
||||
select {
|
||||
case <-l.done:
|
||||
l.logger.Info("Geolocation stopped")
|
||||
case <-ctx.Done():
|
||||
l.logger.Warn("Stop deadline passed before the geolocation cache " +
|
||||
"was written")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user