Add best-effort geolocation to the IP lookup from geojs and DB-IP City Lite (closes #239)

Each result gains `geo`. One request to geojs carries every address, 1 second in all. On a timeout, an error or an answer with no country, city or coordinates, the lookup reads DB-IP City Lite, downloaded monthly into the state directory. geojs answers stay in memory (250,000, least recently used dropped) and in a new `cache.db` table for 7 days.

Judgement call: a reserved address goes to neither source, and its `geo` is null.
Judgement call: `updated` for DB-IP is when the copy was downloaded.
Deviation: the cache is SQLite, by sneak's words.

Model: opus-5-5

README.md

@@ -510,9 +510,13 @@ sources:
 - [geojs](https://www.geojs.io/) is asked first. One request carries every
   address of the lookup, and it has 1 second to answer.
 - [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
-  geojs fails or does not answer in time. It is a database file that RouteWatch
+  geojs has no answer for the address. It is a database file that RouteWatch
   reads itself.
 
+geojs has no answer when its request fails or runs out of time. An answer with
+no country, city or coordinates also counts as no answer, and RouteWatch does
+not cache it.
+
 The `geo` object of a result has these fields:
 
 - `country_code`: the two-letter country code
@@ -534,8 +538,8 @@ goes to neither source, so its `geo` is null.
 
 What goes outside: geojs receives each address that the cache does not hold. For
 `/ip/` and the home page, that address is the visitor's own. A lookup answered
-from the cache or from the database sends nothing. DB-IP receives only the
-monthly request for its file.
+from the cache sends nothing. DB-IP receives only the monthly request for its
+file.
 
 How RouteWatch keeps the answers of geojs:
 
@@ -561,8 +565,8 @@ How RouteWatch keeps the database:
   The kernel loads the parts that are read, and can drop them again when memory
   is short.
 
-DB-IP's licence requires an attribution, so the home page's footer says "IP
-Geolocation by DB-IP" with a link to [db-ip.com](https://db-ip.com).
+The home page's footer says "IP Geolocation by DB-IP" with a link to
+[db-ip.com](https://db-ip.com).
 
 ## Configuration
 

internal/geo/database.go

@@ -107,7 +107,8 @@ func nilIfEmpty(text string) *string {
 }
 
 // fromDatabase returns where the copy of the database places addr. It returns
-// nil when there is no copy yet or the copy has no record for addr.
+// nil when there is no copy yet or the copy has no record for addr. It also
+// returns nil, and logs the error, when reading the copy fails.
 func (l *Locator) fromDatabase(addr netip.Addr) *Location {
 	l.databaseMu.RLock()
 	defer l.databaseMu.RUnlock()
@@ -117,13 +118,22 @@ func (l *Locator) fromDatabase(addr netip.Addr) *Location {
 	}
 
 	result := l.database.Lookup(addr)
+
+	err := result.Err()
+	if err != nil {
+		l.logger.Error("Failed to look up an address in the database",
+			"address", addr, "error", err)
+
+		return nil
+	}
+
 	if !result.Found() {
 		return nil
 	}
 
 	var record dbipRecord
 
-	err := result.Decode(&record)
+	err = result.Decode(&record)
 	if err != nil {
 		l.logger.Error("Failed to read a database record",
 			"address", addr, "error", err)

internal/geo/geojs.go

@@ -51,7 +51,8 @@ type geojsAnswer struct {
 }
 
 // askGeoJS asks geojs about addresses in one request, and returns its answers
-// by address. An address geojs left out has no entry.
+// by address. An address geojs left out, or gave no country, city or
+// coordinates for, has no entry.
 func (l *Locator) askGeoJS(
 	ctx context.Context, addresses []string,
 ) (map[string]*Location, error) {
@@ -97,12 +98,21 @@ func (l *Locator) askGeoJS(
 			continue
 		}
 
-		answers[addr.Unmap().String()] = answer.location(fetched)
+		location := answer.location(fetched)
+		if hasLocation(location) {
+			answers[plainAddress(addr)] = location
+		}
 	}
 
 	return answers, nil
 }
 
+// hasLocation says whether found has a country, a city or a coordinate.
+func hasLocation(found *Location) bool {
+	return found.CountryCode != nil || found.Country != nil ||
+		found.City != nil || found.Latitude != nil || found.Longitude != nil
+}
+
 // location returns the answer as a Location fetched at the time given.
 func (a geojsAnswer) location(fetched time.Time) *Location {
 	return &Location{

internal/geo/locator.go

@@ -1,6 +1,7 @@
 // Package geo finds where IP addresses are. It asks geojs.io, and keeps those
-// answers in memory and in cache.db. When geojs.io does not answer in time, it
-// reads a copy of DB-IP's City Lite database, which it downloads once a month.
+// answers in memory and in cache.db. When geojs.io gives no location for an
+// address, it reads a copy of DB-IP's City Lite database, which it downloads
+// once a month.
 package geo
 
 import (
@@ -141,7 +142,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
 	var unanswered []string
 
 	for i, addr := range addrs {
-		address := addr.Unmap().String()
+		address := plainAddress(addr)
 
 		locations[i] = l.cached(ctx, address)
 		if locations[i] == nil {
@@ -163,7 +164,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
 			continue
 		}
 
-		address := addr.Unmap().String()
+		address := plainAddress(addr)
 
 		answer, ok := answers[address]
 		if ok {
@@ -179,6 +180,13 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
 	return locations
 }
 
+// plainAddress returns addr as the requests to geojs and the cache name it: an
+// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
+// chooses the zone, so it must not reach the request.
+func plainAddress(addr netip.Addr) string {
+	return addr.Unmap().WithZone("").String()
+}
+
 // Start loads the cached answers that have not expired and opens the copy of
 // the database, if there is one. It then starts the goroutines that write new
 // answers, delete expired ones and download the database. They run until Stop:

internal/geo/locator_internal_test.go

@@ -53,9 +53,18 @@ const (
 		 "country": "Germany", "country_code": "DE", "ip": "` + testAddress + `",
 		 "latitude": "52.6171", "longitude": "13.1207",
 		 "region": "Brandenburg", "timezone": "Europe/Berlin"},
-		{"ip": "` + otherAddress + `", "latitude": "nil", "longitude": "nil"}
+		{"country": "Germany", "ip": "` + otherAddress + `",
+		 "latitude": "nil", "longitude": "nil"}
 	]`
 	geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]`
+
+	// geojsNoLocation has no country, city or coordinates, as geojs sends for
+	// some addresses. Its other parts do not count as a location.
+	geojsNoLocation = `[{"accuracy": 1000, "area_code": "0", "asn": 64496,
+		"continent_code": "EU", "ip": "` + testAddress + `",
+		"latitude": "nil", "longitude": "nil",
+		"organization": "AS64496 Example", "organization_name": "Example",
+		"region": "Brandenburg", "timezone": "Europe/Berlin"}]`
 )
 
 // fixedTime returns the time a test's clock starts at.
@@ -315,9 +324,8 @@ func TestLookupAsksGeoJSOnceForEveryAddress(t *testing.T) {
 	}
 }
 
-// TestLookupGivesUnknownPartsAsNil checks an answer whose coordinates are the
-// string "nil" and which has no other part: the parts are nil, and the answer
-// is still an answer.
+// TestLookupGivesUnknownPartsAsNil checks an answer with only a country name,
+// whose coordinates are the string "nil": the other parts are nil.
 func TestLookupGivesUnknownPartsAsNil(t *testing.T) {
 	t.Parallel()
 
@@ -368,6 +376,7 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
 		{name: "error status", status: http.StatusBadGateway},
 		{name: "unreadable answer", body: "not JSON"},
 		{name: "address left out", body: "[]"},
+		{name: "answer with no location", body: geojsNoLocation},
 	}
 
 	for _, tt := range tests {
@@ -411,6 +420,93 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
 	}
 }
 
+// TestAnyCountryCityOrCoordinateIsAnAnswer checks that each of those parts,
+// alone, makes an answer from geojs that is returned and kept.
+func TestAnyCountryCityOrCoordinateIsAnAnswer(t *testing.T) {
+	t.Parallel()
+
+	parts := []string{
+		`"country": "Germany"`,
+		`"country_code": "DE"`,
+		`"city": "` + testCity + `"`,
+		`"latitude": "52.5"`,
+		`"longitude": "13.5"`,
+	}
+
+	for _, part := range parts {
+		t.Run(part, func(t *testing.T) {
+			t.Parallel()
+
+			fake := newFakeGeoJS(t, `[{"ip": "`+testAddress+`", `+part+`}]`)
+			dir := t.TempDir()
+			locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
+
+			got := lookupOne(t, locator, testAddress)
+			if got == nil || got.Source != sourceGeoJS {
+				t.Fatalf("location %+v, want the answer from geojs", got)
+			}
+
+			if !locator.entries.Contains(testAddress) {
+				t.Error("the answer is not in memory")
+			}
+		})
+	}
+}
+
+// TestLookupSendsNoZoneToGeoJS looks up an address with a zone, which is text
+// a client chooses. geojs receives only the plain address, and the answer is
+// kept under it.
+func TestLookupSendsNoZoneToGeoJS(t *testing.T) {
+	t.Parallel()
+
+	fake := newFakeGeoJS(t, geojsTwoAddresses)
+	dir := t.TempDir()
+	locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
+
+	got := lookupOne(t, locator, otherAddress+"%x&callback=evil")
+
+	requests := fake.requestsSeen()
+	if len(requests) != 1 || requests[0] != otherAddress {
+		t.Errorf("requests %v, want one naming only %s", requests, otherAddress)
+	}
+
+	if got == nil || got.Source != sourceGeoJS {
+		t.Errorf("location %+v, want the answer from geojs", got)
+	}
+
+	if !locator.entries.Contains(otherAddress) {
+		t.Error("the answer is not kept under the plain address")
+	}
+}
+
+// TestFailedDatabaseLookupIsLogged closes the copy under the locator, so that
+// the lookup in it fails. The location is nil and the failure is logged.
+func TestFailedDatabaseLookupIsLogged(t *testing.T) {
+	t.Parallel()
+
+	fake := newFakeGeoJS(t, "[]")
+	dir := t.TempDir()
+	locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
+	giveDatabase(t, locator)
+
+	err := locator.database.Close()
+	if err != nil {
+		t.Fatalf("closing the copy: %v", err)
+	}
+
+	var logged bytes.Buffer
+
+	locator.logger = slog.New(slog.NewTextHandler(&logged, nil))
+
+	if got := locator.fromDatabase(parseAddresses(t, testAddress)[0]); got != nil {
+		t.Errorf("location %+v from a failed lookup, want nil", got)
+	}
+
+	if !strings.Contains(logged.String(), "address="+testAddress) {
+		t.Errorf("the failure was not logged: %q", logged.String())
+	}
+}
+
 // TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database
 // copy, and then with a copy that does not cover the address.
 func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) {

internal/templates/templates_test.go

@@ -67,8 +67,8 @@ func TestFooterLicense(t *testing.T) {
 	}
 }
 
-// TestHomePageCreditsDBIP expects the home page's footer to carry the
-// attribution the DB-IP licence requires.
+// TestHomePageCreditsDBIP expects the home page's footer to carry the credit
+// to DB-IP.
 func TestHomePageCreditsDBIP(t *testing.T) {
 	t.Parallel()