1 Commits
Author SHA1 Message Date
sneak c5bd4a63d0 Add best-effort geolocation to the IP lookup from geojs and DB-IP City Lite (closes #239)
check / check (push) Canceled after 1m16s
Each result gains `geo`. One request to geojs carries every address, 1 second in all. On a timeout, an error or an answer with no country, city or coordinates, the lookup reads DB-IP City Lite, downloaded monthly into the state directory. geojs answers stay in memory (250,000, least recently used dropped) and in a new `cache.db` table for 7 days.

Judgement call: a reserved address goes to neither source, and its `geo` is null.
Judgement call: `updated` for DB-IP is when the copy was downloaded.
Deviation: the cache is SQLite, by sneak's words.

Model: opus-5-5
2026-11-10 13:38:03 +00:00
6 changed files with 147 additions and 19 deletions

No files matched your search

+9 -5
View File
@@ -510,9 +510,13 @@ sources:
- [geojs](https://www.geojs.io/) is asked first. One request carries every - [geojs](https://www.geojs.io/) is asked first. One request carries every
address of the lookup, and it has 1 second to answer. address of the lookup, and it has 1 second to answer.
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when - [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
geojs fails or does not answer in time. It is a database file that RouteWatch geojs has no answer for the address. It is a database file that RouteWatch
reads itself. reads itself.
geojs has no answer when its request fails or runs out of time. An answer with
no country, city or coordinates also counts as no answer, and RouteWatch does
not cache it.
The `geo` object of a result has these fields: The `geo` object of a result has these fields:
- `country_code`: the two-letter country code - `country_code`: the two-letter country code
@@ -534,8 +538,8 @@ goes to neither source, so its `geo` is null.
What goes outside: geojs receives each address that the cache does not hold. For What goes outside: geojs receives each address that the cache does not hold. For
`/ip/` and the home page, that address is the visitor's own. A lookup answered `/ip/` and the home page, that address is the visitor's own. A lookup answered
from the cache or from the database sends nothing. DB-IP receives only the from the cache sends nothing. DB-IP receives only the monthly request for its
monthly request for its file. file.
How RouteWatch keeps the answers of geojs: How RouteWatch keeps the answers of geojs:
@@ -561,8 +565,8 @@ How RouteWatch keeps the database:
The kernel loads the parts that are read, and can drop them again when memory The kernel loads the parts that are read, and can drop them again when memory
is short. is short.
DB-IP's licence requires an attribution, so the home page's footer says "IP The home page's footer says "IP Geolocation by DB-IP" with a link to
Geolocation by DB-IP" with a link to [db-ip.com](https://db-ip.com). [db-ip.com](https://db-ip.com).
## Configuration ## Configuration
+12 -2
View File
@@ -107,7 +107,8 @@ func nilIfEmpty(text string) *string {
} }
// fromDatabase returns where the copy of the database places addr. It returns // fromDatabase returns where the copy of the database places addr. It returns
// nil when there is no copy yet or the copy has no record for addr. // nil when there is no copy yet or the copy has no record for addr. It also
// returns nil, and logs the error, when reading the copy fails.
func (l *Locator) fromDatabase(addr netip.Addr) *Location { func (l *Locator) fromDatabase(addr netip.Addr) *Location {
l.databaseMu.RLock() l.databaseMu.RLock()
defer l.databaseMu.RUnlock() defer l.databaseMu.RUnlock()
@@ -117,13 +118,22 @@ func (l *Locator) fromDatabase(addr netip.Addr) *Location {
} }
result := l.database.Lookup(addr) result := l.database.Lookup(addr)
err := result.Err()
if err != nil {
l.logger.Error("Failed to look up an address in the database",
"address", addr, "error", err)
return nil
}
if !result.Found() { if !result.Found() {
return nil return nil
} }
var record dbipRecord var record dbipRecord
err := result.Decode(&record) err = result.Decode(&record)
if err != nil { if err != nil {
l.logger.Error("Failed to read a database record", l.logger.Error("Failed to read a database record",
"address", addr, "error", err) "address", addr, "error", err)
+12 -2
View File
@@ -51,7 +51,8 @@ type geojsAnswer struct {
} }
// askGeoJS asks geojs about addresses in one request, and returns its answers // askGeoJS asks geojs about addresses in one request, and returns its answers
// by address. An address geojs left out has no entry. // by address. An address geojs left out, or gave no country, city or
// coordinates for, has no entry.
func (l *Locator) askGeoJS( func (l *Locator) askGeoJS(
ctx context.Context, addresses []string, ctx context.Context, addresses []string,
) (map[string]*Location, error) { ) (map[string]*Location, error) {
@@ -97,12 +98,21 @@ func (l *Locator) askGeoJS(
continue continue
} }
answers[addr.Unmap().String()] = answer.location(fetched) location := answer.location(fetched)
if hasLocation(location) {
answers[plainAddress(addr)] = location
}
} }
return answers, nil return answers, nil
} }
// hasLocation says whether found has a country, a city or a coordinate.
func hasLocation(found *Location) bool {
return found.CountryCode != nil || found.Country != nil ||
found.City != nil || found.Latitude != nil || found.Longitude != nil
}
// location returns the answer as a Location fetched at the time given. // location returns the answer as a Location fetched at the time given.
func (a geojsAnswer) location(fetched time.Time) *Location { func (a geojsAnswer) location(fetched time.Time) *Location {
return &Location{ return &Location{
+12 -4
View File
@@ -1,6 +1,7 @@
// Package geo finds where IP addresses are. It asks geojs.io, and keeps those // Package geo finds where IP addresses are. It asks geojs.io, and keeps those
// answers in memory and in cache.db. When geojs.io does not answer in time, it // answers in memory and in cache.db. When geojs.io gives no location for an
// reads a copy of DB-IP's City Lite database, which it downloads once a month. // address, it reads a copy of DB-IP's City Lite database, which it downloads
// once a month.
package geo package geo
import ( import (
@@ -141,7 +142,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
var unanswered []string var unanswered []string
for i, addr := range addrs { for i, addr := range addrs {
address := addr.Unmap().String() address := plainAddress(addr)
locations[i] = l.cached(ctx, address) locations[i] = l.cached(ctx, address)
if locations[i] == nil { if locations[i] == nil {
@@ -163,7 +164,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
continue continue
} }
address := addr.Unmap().String() address := plainAddress(addr)
answer, ok := answers[address] answer, ok := answers[address]
if ok { if ok {
@@ -179,6 +180,13 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
return locations return locations
} }
// plainAddress returns addr as the requests to geojs and the cache name it: an
// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
// chooses the zone, so it must not reach the request.
func plainAddress(addr netip.Addr) string {
return addr.Unmap().WithZone("").String()
}
// Start loads the cached answers that have not expired and opens the copy of // Start loads the cached answers that have not expired and opens the copy of
// the database, if there is one. It then starts the goroutines that write new // the database, if there is one. It then starts the goroutines that write new
// answers, delete expired ones and download the database. They run until Stop: // answers, delete expired ones and download the database. They run until Stop:
+100 -4
View File
@@ -53,9 +53,18 @@ const (
"country": "Germany", "country_code": "DE", "ip": "` + testAddress + `", "country": "Germany", "country_code": "DE", "ip": "` + testAddress + `",
"latitude": "52.6171", "longitude": "13.1207", "latitude": "52.6171", "longitude": "13.1207",
"region": "Brandenburg", "timezone": "Europe/Berlin"}, "region": "Brandenburg", "timezone": "Europe/Berlin"},
{"ip": "` + otherAddress + `", "latitude": "nil", "longitude": "nil"} {"country": "Germany", "ip": "` + otherAddress + `",
"latitude": "nil", "longitude": "nil"}
]` ]`
geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]` geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]`
// geojsNoLocation has no country, city or coordinates, as geojs sends for
// some addresses. Its other parts do not count as a location.
geojsNoLocation = `[{"accuracy": 1000, "area_code": "0", "asn": 64496,
"continent_code": "EU", "ip": "` + testAddress + `",
"latitude": "nil", "longitude": "nil",
"organization": "AS64496 Example", "organization_name": "Example",
"region": "Brandenburg", "timezone": "Europe/Berlin"}]`
) )
// fixedTime returns the time a test's clock starts at. // fixedTime returns the time a test's clock starts at.
@@ -315,9 +324,8 @@ func TestLookupAsksGeoJSOnceForEveryAddress(t *testing.T) {
} }
} }
// TestLookupGivesUnknownPartsAsNil checks an answer whose coordinates are the // TestLookupGivesUnknownPartsAsNil checks an answer with only a country name,
// string "nil" and which has no other part: the parts are nil, and the answer // whose coordinates are the string "nil": the other parts are nil.
// is still an answer.
func TestLookupGivesUnknownPartsAsNil(t *testing.T) { func TestLookupGivesUnknownPartsAsNil(t *testing.T) {
t.Parallel() t.Parallel()
@@ -368,6 +376,7 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
{name: "error status", status: http.StatusBadGateway}, {name: "error status", status: http.StatusBadGateway},
{name: "unreadable answer", body: "not JSON"}, {name: "unreadable answer", body: "not JSON"},
{name: "address left out", body: "[]"}, {name: "address left out", body: "[]"},
{name: "answer with no location", body: geojsNoLocation},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -411,6 +420,93 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
} }
} }
// TestAnyCountryCityOrCoordinateIsAnAnswer checks that each of those parts,
// alone, makes an answer from geojs that is returned and kept.
func TestAnyCountryCityOrCoordinateIsAnAnswer(t *testing.T) {
t.Parallel()
parts := []string{
`"country": "Germany"`,
`"country_code": "DE"`,
`"city": "` + testCity + `"`,
`"latitude": "52.5"`,
`"longitude": "13.5"`,
}
for _, part := range parts {
t.Run(part, func(t *testing.T) {
t.Parallel()
fake := newFakeGeoJS(t, `[{"ip": "`+testAddress+`", `+part+`}]`)
dir := t.TempDir()
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
got := lookupOne(t, locator, testAddress)
if got == nil || got.Source != sourceGeoJS {
t.Fatalf("location %+v, want the answer from geojs", got)
}
if !locator.entries.Contains(testAddress) {
t.Error("the answer is not in memory")
}
})
}
}
// TestLookupSendsNoZoneToGeoJS looks up an address with a zone, which is text
// a client chooses. geojs receives only the plain address, and the answer is
// kept under it.
func TestLookupSendsNoZoneToGeoJS(t *testing.T) {
t.Parallel()
fake := newFakeGeoJS(t, geojsTwoAddresses)
dir := t.TempDir()
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
got := lookupOne(t, locator, otherAddress+"%x&callback=evil")
requests := fake.requestsSeen()
if len(requests) != 1 || requests[0] != otherAddress {
t.Errorf("requests %v, want one naming only %s", requests, otherAddress)
}
if got == nil || got.Source != sourceGeoJS {
t.Errorf("location %+v, want the answer from geojs", got)
}
if !locator.entries.Contains(otherAddress) {
t.Error("the answer is not kept under the plain address")
}
}
// TestFailedDatabaseLookupIsLogged closes the copy under the locator, so that
// the lookup in it fails. The location is nil and the failure is logged.
func TestFailedDatabaseLookupIsLogged(t *testing.T) {
t.Parallel()
fake := newFakeGeoJS(t, "[]")
dir := t.TempDir()
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
giveDatabase(t, locator)
err := locator.database.Close()
if err != nil {
t.Fatalf("closing the copy: %v", err)
}
var logged bytes.Buffer
locator.logger = slog.New(slog.NewTextHandler(&logged, nil))
if got := locator.fromDatabase(parseAddresses(t, testAddress)[0]); got != nil {
t.Errorf("location %+v from a failed lookup, want nil", got)
}
if !strings.Contains(logged.String(), "address="+testAddress) {
t.Errorf("the failure was not logged: %q", logged.String())
}
}
// TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database // TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database
// copy, and then with a copy that does not cover the address. // copy, and then with a copy that does not cover the address.
func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) { func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) {
+2 -2
View File
@@ -67,8 +67,8 @@ func TestFooterLicense(t *testing.T) {
} }
} }
// TestHomePageCreditsDBIP expects the home page's footer to carry the // TestHomePageCreditsDBIP expects the home page's footer to carry the credit
// attribution the DB-IP licence requires. // to DB-IP.
func TestHomePageCreditsDBIP(t *testing.T) { func TestHomePageCreditsDBIP(t *testing.T) {
t.Parallel() t.Parallel()