Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5bd4a63d0 |
No files matched your search
@@ -510,9 +510,13 @@ sources:
|
|||||||
- [geojs](https://www.geojs.io/) is asked first. One request carries every
|
- [geojs](https://www.geojs.io/) is asked first. One request carries every
|
||||||
address of the lookup, and it has 1 second to answer.
|
address of the lookup, and it has 1 second to answer.
|
||||||
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
|
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
|
||||||
geojs fails or does not answer in time. It is a database file that RouteWatch
|
geojs has no answer for the address. It is a database file that RouteWatch
|
||||||
reads itself.
|
reads itself.
|
||||||
|
|
||||||
|
geojs has no answer when its request fails or runs out of time. An answer with
|
||||||
|
no country, city or coordinates also counts as no answer, and RouteWatch does
|
||||||
|
not cache it.
|
||||||
|
|
||||||
The `geo` object of a result has these fields:
|
The `geo` object of a result has these fields:
|
||||||
|
|
||||||
- `country_code`: the two-letter country code
|
- `country_code`: the two-letter country code
|
||||||
@@ -534,8 +538,8 @@ goes to neither source, so its `geo` is null.
|
|||||||
|
|
||||||
What goes outside: geojs receives each address that the cache does not hold. For
|
What goes outside: geojs receives each address that the cache does not hold. For
|
||||||
`/ip/` and the home page, that address is the visitor's own. A lookup answered
|
`/ip/` and the home page, that address is the visitor's own. A lookup answered
|
||||||
from the cache or from the database sends nothing. DB-IP receives only the
|
from the cache sends nothing. DB-IP receives only the monthly request for its
|
||||||
monthly request for its file.
|
file.
|
||||||
|
|
||||||
How RouteWatch keeps the answers of geojs:
|
How RouteWatch keeps the answers of geojs:
|
||||||
|
|
||||||
@@ -561,8 +565,8 @@ How RouteWatch keeps the database:
|
|||||||
The kernel loads the parts that are read, and can drop them again when memory
|
The kernel loads the parts that are read, and can drop them again when memory
|
||||||
is short.
|
is short.
|
||||||
|
|
||||||
DB-IP's licence requires an attribution, so the home page's footer says "IP
|
The home page's footer says "IP Geolocation by DB-IP" with a link to
|
||||||
Geolocation by DB-IP" with a link to [db-ip.com](https://db-ip.com).
|
[db-ip.com](https://db-ip.com).
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
|
|||||||
@@ -107,7 +107,8 @@ func nilIfEmpty(text string) *string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// fromDatabase returns where the copy of the database places addr. It returns
|
// fromDatabase returns where the copy of the database places addr. It returns
|
||||||
// nil when there is no copy yet or the copy has no record for addr.
|
// nil when there is no copy yet or the copy has no record for addr. It also
|
||||||
|
// returns nil, and logs the error, when reading the copy fails.
|
||||||
func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
||||||
l.databaseMu.RLock()
|
l.databaseMu.RLock()
|
||||||
defer l.databaseMu.RUnlock()
|
defer l.databaseMu.RUnlock()
|
||||||
@@ -117,13 +118,22 @@ func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
|||||||
}
|
}
|
||||||
|
|
||||||
result := l.database.Lookup(addr)
|
result := l.database.Lookup(addr)
|
||||||
|
|
||||||
|
err := result.Err()
|
||||||
|
if err != nil {
|
||||||
|
l.logger.Error("Failed to look up an address in the database",
|
||||||
|
"address", addr, "error", err)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if !result.Found() {
|
if !result.Found() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var record dbipRecord
|
var record dbipRecord
|
||||||
|
|
||||||
err := result.Decode(&record)
|
err = result.Decode(&record)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.logger.Error("Failed to read a database record",
|
l.logger.Error("Failed to read a database record",
|
||||||
"address", addr, "error", err)
|
"address", addr, "error", err)
|
||||||
|
|||||||
+12
-2
@@ -51,7 +51,8 @@ type geojsAnswer struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// askGeoJS asks geojs about addresses in one request, and returns its answers
|
// askGeoJS asks geojs about addresses in one request, and returns its answers
|
||||||
// by address. An address geojs left out has no entry.
|
// by address. An address geojs left out, or gave no country, city or
|
||||||
|
// coordinates for, has no entry.
|
||||||
func (l *Locator) askGeoJS(
|
func (l *Locator) askGeoJS(
|
||||||
ctx context.Context, addresses []string,
|
ctx context.Context, addresses []string,
|
||||||
) (map[string]*Location, error) {
|
) (map[string]*Location, error) {
|
||||||
@@ -97,12 +98,21 @@ func (l *Locator) askGeoJS(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
answers[addr.Unmap().String()] = answer.location(fetched)
|
location := answer.location(fetched)
|
||||||
|
if hasLocation(location) {
|
||||||
|
answers[plainAddress(addr)] = location
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return answers, nil
|
return answers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hasLocation says whether found has a country, a city or a coordinate.
|
||||||
|
func hasLocation(found *Location) bool {
|
||||||
|
return found.CountryCode != nil || found.Country != nil ||
|
||||||
|
found.City != nil || found.Latitude != nil || found.Longitude != nil
|
||||||
|
}
|
||||||
|
|
||||||
// location returns the answer as a Location fetched at the time given.
|
// location returns the answer as a Location fetched at the time given.
|
||||||
func (a geojsAnswer) location(fetched time.Time) *Location {
|
func (a geojsAnswer) location(fetched time.Time) *Location {
|
||||||
return &Location{
|
return &Location{
|
||||||
|
|||||||
+12
-4
@@ -1,6 +1,7 @@
|
|||||||
// Package geo finds where IP addresses are. It asks geojs.io, and keeps those
|
// Package geo finds where IP addresses are. It asks geojs.io, and keeps those
|
||||||
// answers in memory and in cache.db. When geojs.io does not answer in time, it
|
// answers in memory and in cache.db. When geojs.io gives no location for an
|
||||||
// reads a copy of DB-IP's City Lite database, which it downloads once a month.
|
// address, it reads a copy of DB-IP's City Lite database, which it downloads
|
||||||
|
// once a month.
|
||||||
package geo
|
package geo
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -141,7 +142,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
|||||||
var unanswered []string
|
var unanswered []string
|
||||||
|
|
||||||
for i, addr := range addrs {
|
for i, addr := range addrs {
|
||||||
address := addr.Unmap().String()
|
address := plainAddress(addr)
|
||||||
|
|
||||||
locations[i] = l.cached(ctx, address)
|
locations[i] = l.cached(ctx, address)
|
||||||
if locations[i] == nil {
|
if locations[i] == nil {
|
||||||
@@ -163,7 +164,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
address := addr.Unmap().String()
|
address := plainAddress(addr)
|
||||||
|
|
||||||
answer, ok := answers[address]
|
answer, ok := answers[address]
|
||||||
if ok {
|
if ok {
|
||||||
@@ -179,6 +180,13 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
|||||||
return locations
|
return locations
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// plainAddress returns addr as the requests to geojs and the cache name it: an
|
||||||
|
// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
|
||||||
|
// chooses the zone, so it must not reach the request.
|
||||||
|
func plainAddress(addr netip.Addr) string {
|
||||||
|
return addr.Unmap().WithZone("").String()
|
||||||
|
}
|
||||||
|
|
||||||
// Start loads the cached answers that have not expired and opens the copy of
|
// Start loads the cached answers that have not expired and opens the copy of
|
||||||
// the database, if there is one. It then starts the goroutines that write new
|
// the database, if there is one. It then starts the goroutines that write new
|
||||||
// answers, delete expired ones and download the database. They run until Stop:
|
// answers, delete expired ones and download the database. They run until Stop:
|
||||||
|
|||||||
@@ -53,9 +53,18 @@ const (
|
|||||||
"country": "Germany", "country_code": "DE", "ip": "` + testAddress + `",
|
"country": "Germany", "country_code": "DE", "ip": "` + testAddress + `",
|
||||||
"latitude": "52.6171", "longitude": "13.1207",
|
"latitude": "52.6171", "longitude": "13.1207",
|
||||||
"region": "Brandenburg", "timezone": "Europe/Berlin"},
|
"region": "Brandenburg", "timezone": "Europe/Berlin"},
|
||||||
{"ip": "` + otherAddress + `", "latitude": "nil", "longitude": "nil"}
|
{"country": "Germany", "ip": "` + otherAddress + `",
|
||||||
|
"latitude": "nil", "longitude": "nil"}
|
||||||
]`
|
]`
|
||||||
geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]`
|
geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]`
|
||||||
|
|
||||||
|
// geojsNoLocation has no country, city or coordinates, as geojs sends for
|
||||||
|
// some addresses. Its other parts do not count as a location.
|
||||||
|
geojsNoLocation = `[{"accuracy": 1000, "area_code": "0", "asn": 64496,
|
||||||
|
"continent_code": "EU", "ip": "` + testAddress + `",
|
||||||
|
"latitude": "nil", "longitude": "nil",
|
||||||
|
"organization": "AS64496 Example", "organization_name": "Example",
|
||||||
|
"region": "Brandenburg", "timezone": "Europe/Berlin"}]`
|
||||||
)
|
)
|
||||||
|
|
||||||
// fixedTime returns the time a test's clock starts at.
|
// fixedTime returns the time a test's clock starts at.
|
||||||
@@ -315,9 +324,8 @@ func TestLookupAsksGeoJSOnceForEveryAddress(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLookupGivesUnknownPartsAsNil checks an answer whose coordinates are the
|
// TestLookupGivesUnknownPartsAsNil checks an answer with only a country name,
|
||||||
// string "nil" and which has no other part: the parts are nil, and the answer
|
// whose coordinates are the string "nil": the other parts are nil.
|
||||||
// is still an answer.
|
|
||||||
func TestLookupGivesUnknownPartsAsNil(t *testing.T) {
|
func TestLookupGivesUnknownPartsAsNil(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -368,6 +376,7 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
|||||||
{name: "error status", status: http.StatusBadGateway},
|
{name: "error status", status: http.StatusBadGateway},
|
||||||
{name: "unreadable answer", body: "not JSON"},
|
{name: "unreadable answer", body: "not JSON"},
|
||||||
{name: "address left out", body: "[]"},
|
{name: "address left out", body: "[]"},
|
||||||
|
{name: "answer with no location", body: geojsNoLocation},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -411,6 +420,93 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestAnyCountryCityOrCoordinateIsAnAnswer checks that each of those parts,
|
||||||
|
// alone, makes an answer from geojs that is returned and kept.
|
||||||
|
func TestAnyCountryCityOrCoordinateIsAnAnswer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
parts := []string{
|
||||||
|
`"country": "Germany"`,
|
||||||
|
`"country_code": "DE"`,
|
||||||
|
`"city": "` + testCity + `"`,
|
||||||
|
`"latitude": "52.5"`,
|
||||||
|
`"longitude": "13.5"`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, part := range parts {
|
||||||
|
t.Run(part, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fake := newFakeGeoJS(t, `[{"ip": "`+testAddress+`", `+part+`}]`)
|
||||||
|
dir := t.TempDir()
|
||||||
|
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||||
|
|
||||||
|
got := lookupOne(t, locator, testAddress)
|
||||||
|
if got == nil || got.Source != sourceGeoJS {
|
||||||
|
t.Fatalf("location %+v, want the answer from geojs", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !locator.entries.Contains(testAddress) {
|
||||||
|
t.Error("the answer is not in memory")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLookupSendsNoZoneToGeoJS looks up an address with a zone, which is text
|
||||||
|
// a client chooses. geojs receives only the plain address, and the answer is
|
||||||
|
// kept under it.
|
||||||
|
func TestLookupSendsNoZoneToGeoJS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fake := newFakeGeoJS(t, geojsTwoAddresses)
|
||||||
|
dir := t.TempDir()
|
||||||
|
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||||
|
|
||||||
|
got := lookupOne(t, locator, otherAddress+"%x&callback=evil")
|
||||||
|
|
||||||
|
requests := fake.requestsSeen()
|
||||||
|
if len(requests) != 1 || requests[0] != otherAddress {
|
||||||
|
t.Errorf("requests %v, want one naming only %s", requests, otherAddress)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got == nil || got.Source != sourceGeoJS {
|
||||||
|
t.Errorf("location %+v, want the answer from geojs", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !locator.entries.Contains(otherAddress) {
|
||||||
|
t.Error("the answer is not kept under the plain address")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFailedDatabaseLookupIsLogged closes the copy under the locator, so that
|
||||||
|
// the lookup in it fails. The location is nil and the failure is logged.
|
||||||
|
func TestFailedDatabaseLookupIsLogged(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fake := newFakeGeoJS(t, "[]")
|
||||||
|
dir := t.TempDir()
|
||||||
|
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||||
|
giveDatabase(t, locator)
|
||||||
|
|
||||||
|
err := locator.database.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("closing the copy: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var logged bytes.Buffer
|
||||||
|
|
||||||
|
locator.logger = slog.New(slog.NewTextHandler(&logged, nil))
|
||||||
|
|
||||||
|
if got := locator.fromDatabase(parseAddresses(t, testAddress)[0]); got != nil {
|
||||||
|
t.Errorf("location %+v from a failed lookup, want nil", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(logged.String(), "address="+testAddress) {
|
||||||
|
t.Errorf("the failure was not logged: %q", logged.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database
|
// TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database
|
||||||
// copy, and then with a copy that does not cover the address.
|
// copy, and then with a copy that does not cover the address.
|
||||||
func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) {
|
func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) {
|
||||||
|
|||||||
@@ -67,8 +67,8 @@ func TestFooterLicense(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHomePageCreditsDBIP expects the home page's footer to carry the
|
// TestHomePageCreditsDBIP expects the home page's footer to carry the credit
|
||||||
// attribution the DB-IP licence requires.
|
// to DB-IP.
|
||||||
func TestHomePageCreditsDBIP(t *testing.T) {
|
func TestHomePageCreditsDBIP(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user