package geo import ( "bytes" "compress/gzip" "context" "database/sql" "errors" "io/fs" "log/slog" "net" "net/http" "net/http/httptest" "net/netip" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "git.eeqj.de/sneak/routewatch/internal/cachedb" lru "github.com/hashicorp/golang-lru/v2" "github.com/maxmind/mmdbwriter" "github.com/maxmind/mmdbwriter/mmdbtype" ) // The addresses and networks of the tests, from the documentation ranges. const ( testAddress = "192.0.2.10" otherAddress = "2001:db8::10" thirdAddress = "203.0.113.10" coveredNetwork = "192.0.2.0/24" testCity = "Testville" databaseCity = "Databaseville" databaseMonth = "2026-01" databasePath = "/dbip-city-lite-" + databaseMonth + ".mmdb.gz" geojsPath = "/geo.json" // testTimeLimit replaces geojsTimeLimit in the tests that use a real // server, so that a stall on a busy host cannot use it up. testTimeLimit = 2 * time.Minute // shortTimeLimit is the limit of the test in which geojs never answers. shortTimeLimit = 50 * time.Millisecond ) // The bodies the fake geojs sends. const ( geojsTwoAddresses = `[ {"accuracy": 20, "city": "` + testCity + `", "continent_code": "EU", "country": "Germany", "country_code": "DE", "ip": "` + testAddress + `", "latitude": "52.6171", "longitude": "13.1207", "region": "Brandenburg", "timezone": "Europe/Berlin"}, {"ip": "` + otherAddress + `", "latitude": "nil", "longitude": "nil"} ]` geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]` ) // fixedTime returns the time a test's clock starts at. func fixedTime() time.Time { return time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) } // parseAddresses returns the given addresses parsed. func parseAddresses(t *testing.T, addresses ...string) []netip.Addr { t.Helper() addrs := make([]netip.Addr, len(addresses)) for i, address := range addresses { addr, err := netip.ParseAddr(address) if err != nil { t.Fatalf("parsing %s: %v", address, err) } addrs[i] = addr } return addrs } // fakeGeoJS is a geojs server. It sends body, or status when that is set, or // waits for the request to end when hang is true. It keeps each request's // address list. type fakeGeoJS struct { server *httptest.Server body string status int hang bool mu sync.Mutex requests []string } // newFakeGeoJS starts a fake geojs that sends body. func newFakeGeoJS(t *testing.T, body string) *fakeGeoJS { t.Helper() fake := &fakeGeoJS{body: body} fake.server = httptest.NewServer(http.HandlerFunc(fake.serve)) t.Cleanup(fake.server.Close) return fake } func (f *fakeGeoJS) serve(w http.ResponseWriter, r *http.Request) { f.mu.Lock() f.requests = append(f.requests, r.URL.Query().Get("ip")) f.mu.Unlock() if f.hang { <-r.Context().Done() return } if f.status != 0 { w.WriteHeader(f.status) return } _, _ = w.Write([]byte(f.body)) } // requestsSeen returns the address list of each request so far. func (f *fakeGeoJS) requestsSeen() []string { f.mu.Lock() defer f.mu.Unlock() return append([]string{}, f.requests...) } // openCacheDB opens a new cache.db in a temporary directory. func openCacheDB(t *testing.T, dir string) *sql.DB { t.Helper() db, err := cachedb.Open(t.Context(), dir) if err != nil { t.Fatalf("opening cache.db: %v", err) } t.Cleanup(func() { _ = db.Close() }) return db } // newTestLocator returns a Locator that asks fake and uses db and dir, with // its clock stopped at fixedTime. func newTestLocator( t *testing.T, fake *fakeGeoJS, db *sql.DB, dir string, ) *Locator { t.Helper() locator, err := New(Params{ DB: db, Dir: dir, GeoJSURL: fake.server.URL + geojsPath, DatabaseURLFormat: fake.server.URL + "/dbip-city-lite-%s.mmdb.gz", Logger: slog.New(slog.DiscardHandler), }) if err != nil { t.Fatalf("New: %v", err) } locator.timeLimit = testTimeLimit locator.now = fixedTime t.Cleanup(locator.closeDatabase) return locator } // lookupOne returns the location of one address. func lookupOne(t *testing.T, locator *Locator, address string) *Location { t.Helper() return locator.Lookup(t.Context(), parseAddresses(t, address))[0] } // databaseFile returns an MMDB file with the shape of City Lite. It covers // coveredNetwork, with no time zone or accuracy radius, as the real file has // none. func databaseFile(t *testing.T) []byte { t.Helper() tree, err := mmdbwriter.New(mmdbwriter.Options{ DatabaseType: "DBIP-City-Lite", IncludeReservedNetworks: true, }) if err != nil { t.Fatalf("creating the test database: %v", err) } _, network, err := net.ParseCIDR(coveredNetwork) if err != nil { t.Fatalf("parsing the network: %v", err) } names := func(name string) mmdbtype.Map { return mmdbtype.Map{"names": mmdbtype.Map{"en": mmdbtype.String(name)}} } country := names("Germany") country["iso_code"] = mmdbtype.String("DE") err = tree.Insert(network, mmdbtype.Map{ "city": names(databaseCity), "continent": mmdbtype.Map{"code": mmdbtype.String("EU")}, "country": country, "subdivisions": mmdbtype.Slice{names("Brandenburg")}, "location": mmdbtype.Map{ "latitude": mmdbtype.Float64(52.5), "longitude": mmdbtype.Float64(13.5), }, }) if err != nil { t.Fatalf("inserting the test record: %v", err) } var file bytes.Buffer _, err = tree.WriteTo(&file) if err != nil { t.Fatalf("writing the test database: %v", err) } return file.Bytes() } // gzipped returns data compressed with gzip. func gzipped(t *testing.T, data []byte) []byte { t.Helper() var packed bytes.Buffer writer := gzip.NewWriter(&packed) _, err := writer.Write(data) if err != nil { t.Fatalf("compressing: %v", err) } err = writer.Close() if err != nil { t.Fatalf("compressing: %v", err) } return packed.Bytes() } // giveDatabase puts a copy of the test database in the locator's directory, and // opens it. func giveDatabase(t *testing.T, locator *Locator) { t.Helper() path := filepath.Join(locator.dir, databaseFileName) err := os.WriteFile(path, databaseFile(t), 0o600) if err != nil { t.Fatalf("writing the database copy: %v", err) } locator.openDatabaseCopy() } // requireCity checks where a location says it is, and its source. func requireCity(t *testing.T, got *Location, city, source string) { t.Helper() if got == nil { t.Fatal("no location") } if got.City == nil || *got.City != city { t.Errorf("city %v, want %s", got.City, city) } if got.Source != source { t.Errorf("source %s, want %s", got.Source, source) } } // TestLookupAsksGeoJSOnceForEveryAddress looks up two addresses, and expects a // single request that names both, and each answer in its own place. func TestLookupAsksGeoJSOnceForEveryAddress(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsTwoAddresses) dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) got := locator.Lookup(t.Context(), parseAddresses(t, otherAddress, testAddress)) requests := fake.requestsSeen() if len(requests) != 1 || requests[0] != otherAddress+","+testAddress { t.Fatalf("requests %v, want one naming both addresses", requests) } first := got[1] requireCity(t, first, testCity, sourceGeoJS) if *first.CountryCode != "DE" || *first.Country != "Germany" || *first.ContinentCode != "EU" || *first.Region != "Brandenburg" || *first.Latitude != 52.6171 || *first.Longitude != 13.1207 || *first.AccuracyRadiusKm != 20 || *first.TimeZone != "Europe/Berlin" { t.Errorf("wrong location %+v", first) } if !first.Updated.Equal(fixedTime()) { t.Errorf("updated %v, want %v", first.Updated, fixedTime()) } } // TestLookupGivesUnknownPartsAsNil checks an answer whose coordinates are the // string "nil" and which has no other part: the parts are nil, and the answer // is still an answer. func TestLookupGivesUnknownPartsAsNil(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsTwoAddresses) dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) got := lookupOne(t, locator, otherAddress) if got == nil { t.Fatal("no location for an address geojs answered") } if got.Latitude != nil || got.Longitude != nil || got.City != nil || got.CountryCode != nil || got.AccuracyRadiusKm != nil || got.TimeZone != nil { t.Errorf("unknown parts are not nil: %+v", got) } if got.Source != sourceGeoJS { t.Errorf("source %s, want %s", got.Source, sourceGeoJS) } } // TestLookupRejectsNumbersJSONCannotCarry checks that a coordinate that is not // a finite number is nil. func TestLookupRejectsNumbersJSONCannotCarry(t *testing.T) { t.Parallel() for _, text := range []string{"NaN", "Inf", "-Inf", "not a number", ""} { if got := knownNumber(text); got != nil { t.Errorf("knownNumber(%q) = %v, want nil", text, *got) } } } // TestLookupFallsBackToTheDatabase makes geojs fail in each way the fallback // covers, and expects the database's answer. func TestLookupFallsBackToTheDatabase(t *testing.T) { t.Parallel() tests := []struct { name string body string status int hang bool }{ {name: "timeout", hang: true}, {name: "error status", status: http.StatusBadGateway}, {name: "unreadable answer", body: "not JSON"}, {name: "address left out", body: "[]"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, tt.body) fake.status = tt.status fake.hang = tt.hang dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) locator.timeLimit = shortTimeLimit if !tt.hang { locator.timeLimit = testTimeLimit } giveDatabase(t, locator) got := lookupOne(t, locator, testAddress) requireCity(t, got, databaseCity, sourceDBIP) if *got.CountryCode != "DE" || *got.Country != "Germany" || *got.ContinentCode != "EU" || *got.Region != "Brandenburg" || *got.Latitude != 52.5 || *got.Longitude != 13.5 { t.Errorf("wrong location %+v", got) } if got.TimeZone != nil || got.AccuracyRadiusKm != nil { t.Errorf("parts the database lacks are not nil: %+v", got) } // The fallback is local, so it is not cached: the next lookup // asks geojs again. lookupOne(t, locator, testAddress) if requests := fake.requestsSeen(); len(requests) != 2 { t.Errorf("%d requests, want 2", len(requests)) } }) } } // TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database // copy, and then with a copy that does not cover the address. func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, "") fake.status = http.StatusBadGateway dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) if got := lookupOne(t, locator, testAddress); got != nil { t.Errorf("location %+v before the first download, want nil", got) } giveDatabase(t, locator) if got := lookupOne(t, locator, thirdAddress); got != nil { t.Errorf("location %+v for an address outside the database, want nil", got) } } // TestAnsweredAddressesAreNotAskedAgain checks that an address held in memory // makes no request, and that only the others go to geojs. func TestAnsweredAddressesAreNotAskedAgain(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsTwoAddresses) dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) lookupOne(t, locator, testAddress) second := locator.Lookup(t.Context(), parseAddresses(t, testAddress, otherAddress)) requireCity(t, second[0], testCity, sourceGeoJS) requests := fake.requestsSeen() if len(requests) != 2 || requests[1] != otherAddress { t.Errorf("requests %v, want the second to name only %s", requests, otherAddress) } } // TestCacheSurvivesARestart writes an answer, then starts a new Locator on the // same cache.db. It answers with no request, each part as it was. func TestCacheSurvivesARestart(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsTwoAddresses) dir := t.TempDir() db := openCacheDB(t, dir) locator := newTestLocator(t, fake, db, dir) want := locator.Lookup(t.Context(), parseAddresses(t, testAddress, otherAddress)) locator.flush(t.Context()) restarted := newTestLocator(t, fake, db, dir) err := restarted.load(t.Context()) if err != nil { t.Fatalf("load: %v", err) } got := restarted.Lookup(t.Context(), parseAddresses(t, testAddress, otherAddress)) if requests := fake.requestsSeen(); len(requests) != 1 { t.Errorf("%d requests, want only the first", len(requests)) } requireCity(t, got[0], testCity, sourceGeoJS) if *got[0].Latitude != *want[0].Latitude || *got[0].AccuracyRadiusKm != *want[0].AccuracyRadiusKm || !got[0].Updated.Equal(want[0].Updated) { t.Errorf("reloaded %+v, want %+v", got[0], want[0]) } if got[1].Latitude != nil || got[1].City != nil { t.Errorf("unknown parts came back set: %+v", got[1]) } } // TestStopWritesWhatIsPending starts a Locator, takes an answer and stops it. // The answer is in cache.db afterwards. func TestStopWritesWhatIsPending(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsTwoAddresses) dir := t.TempDir() db := openCacheDB(t, dir) locator := newTestLocator(t, fake, db, dir) err := locator.Start(t.Context()) if err != nil { t.Fatalf("Start: %v", err) } lookupOne(t, locator, testAddress) locator.Stop(t.Context()) stored, err := locator.readRow(t.Context(), testAddress) if err != nil { t.Fatalf("readRow: %v", err) } requireCity(t, stored, testCity, sourceGeoJS) } // TestAnswersExpireAfterSevenDays checks an answer just before and at the end // of its lifetime, and that the sweep deletes its row and memory entry. func TestAnswersExpireAfterSevenDays(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, geojsOneAddress) dir := t.TempDir() db := openCacheDB(t, dir) locator := newTestLocator(t, fake, db, dir) lookupOne(t, locator, testAddress) locator.flush(t.Context()) locator.now = func() time.Time { return fixedTime().Add(entryLifetime - time.Second) } lookupOne(t, locator, testAddress) if requests := fake.requestsSeen(); len(requests) != 1 { t.Fatalf("%d requests just before expiry, want 1", len(requests)) } expiry := fixedTime().Add(entryLifetime) locator.now = func() time.Time { return expiry } locator.sweep(t.Context()) if locator.entries.Len() != 0 { t.Errorf("%d entries in memory after the sweep, want 0", locator.entries.Len()) } var rows int err := db.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM geolocations").Scan(&rows) if err != nil { t.Fatalf("counting rows: %v", err) } if rows != 0 { t.Errorf("%d rows after the sweep, want 0", rows) } lookupOne(t, locator, testAddress) if requests := fake.requestsSeen(); len(requests) != 2 { t.Errorf("%d requests after expiry, want 2", len(requests)) } } // limitMemory makes the locator's memory hold at most size answers. func limitMemory(t *testing.T, locator *Locator, size int) { t.Helper() entries, err := lru.New[string, *Location](size) if err != nil { t.Fatalf("lru.New: %v", err) } locator.entries = entries locator.maxEntries = size } // answerFor returns a location for store to keep, fetched at the time given. func answerFor(fetched time.Time) *Location { return &Location{City: new(testCity), Source: sourceGeoJS, Updated: fetched} } // TestMemoryDropsTheLeastRecentlyUsedAnswer fills memory with two answers, // uses the older one and stores a third. The unused one leaves memory, and its // row still answers without a request. func TestMemoryDropsTheLeastRecentlyUsedAnswer(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, "[]") dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) limitMemory(t, locator, 2) locator.store(testAddress, answerFor(fixedTime())) locator.store(otherAddress, answerFor(fixedTime())) locator.flush(t.Context()) lookupOne(t, locator, testAddress) locator.store(thirdAddress, answerFor(fixedTime())) if locator.entries.Contains(otherAddress) { t.Fatal("the least recently used answer stayed in memory") } if !locator.entries.Contains(testAddress) { t.Fatal("the recently used answer left memory") } requireCity(t, lookupOne(t, locator, otherAddress), testCity, sourceGeoJS) if requests := fake.requestsSeen(); len(requests) != 0 { t.Errorf("requests %v, want the row to answer", requests) } } // TestLoadKeepsTheNewestAnswers loads more rows than memory holds. The newest // are in memory, and the newest of all is the last to be dropped. func TestLoadKeepsTheNewestAnswers(t *testing.T) { t.Parallel() fake := newFakeGeoJS(t, "[]") dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) locator.store(testAddress, answerFor(fixedTime().Add(-3*time.Hour))) locator.store(otherAddress, answerFor(fixedTime().Add(-2*time.Hour))) locator.store(thirdAddress, answerFor(fixedTime().Add(-time.Hour))) locator.flush(t.Context()) limitMemory(t, locator, 2) err := locator.load(t.Context()) if err != nil { t.Fatalf("load: %v", err) } if locator.entries.Contains(testAddress) { t.Error("the oldest answer was loaded") } locator.entries.Add("192.0.2.99", answerFor(fixedTime())) if !locator.entries.Contains(thirdAddress) { t.Error("the newest answer was the first to be dropped") } } // fakeDownload serves file for the January 2026 database and counts requests. func fakeDownload(t *testing.T, file []byte) *httptest.Server { t.Helper() server := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != databasePath { http.NotFound(w, r) return } _, _ = w.Write(file) })) t.Cleanup(server.Close) return server } // newDownloadLocator returns a locator that downloads file. func newDownloadLocator(t *testing.T, file []byte) *Locator { t.Helper() fake := newFakeGeoJS(t, "[]") fake.status = http.StatusBadGateway dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) locator.databaseURLFormat = fakeDownload(t, file).URL + "/dbip-city-lite-%s.mmdb.gz" return locator } // requireNoDownloadFile checks that a download left no temporary file. func requireNoDownloadFile(t *testing.T, locator *Locator) { t.Helper() _, err := os.Stat(filepath.Join(locator.dir, databaseFileName+databaseDownloadSuffix)) if !errors.Is(err, fs.ErrNotExist) { t.Errorf("the download's file is still there: %v", err) } } // TestDownloadPutsTheDatabaseInUse downloads this month's file, unpacks it, and // answers from it. A restart opens the copy. func TestDownloadPutsTheDatabaseInUse(t *testing.T) { t.Parallel() file := gzipped(t, databaseFile(t)) locator := newDownloadLocator(t, file) err := locator.downloadDatabase(t.Context(), fixedTime()) if err != nil { t.Fatalf("downloadDatabase: %v", err) } requireNoDownloadFile(t, locator) got := lookupOne(t, locator, testAddress) requireCity(t, got, databaseCity, sourceDBIP) if !got.Updated.Equal(fixedTime().Truncate(time.Second)) { t.Errorf("updated %v, want the download's time", got.Updated) } restarted := newDownloadLocator(t, file) restarted.dir = locator.dir restarted.openDatabaseCopy() requireCity(t, lookupOne(t, restarted, testAddress), databaseCity, sourceDBIP) } // TestDownloadIsDueOnceAMonth checks when a copy needs replacing: with no copy, // not again in the month it was fetched, and in the next month. func TestDownloadIsDueOnceAMonth(t *testing.T) { t.Parallel() locator := newDownloadLocator(t, gzipped(t, databaseFile(t))) if !locator.databaseDue(fixedTime()) { t.Error("no download due with no copy") } err := locator.downloadDatabase(t.Context(), fixedTime()) if err != nil { t.Fatalf("downloadDatabase: %v", err) } if locator.databaseDue(fixedTime().AddDate(0, 0, 20)) { t.Error("a download is due in the same month") } if !locator.databaseDue(fixedTime().AddDate(0, 1, 0)) { t.Error("no download due in the next month") } } // TestFailedDownloadKeepsTheCopy serves a copy that cannot replace the one in // use, in each way, and expects the old copy to go on answering. func TestFailedDownloadKeepsTheCopy(t *testing.T) { t.Parallel() tests := []struct { name string file func(t *testing.T) []byte max int64 }{ { name: "not gzip", file: func(*testing.T) []byte { return []byte("not gzip") }, }, { name: "not a database", file: func(t *testing.T) []byte { t.Helper() return gzipped(t, []byte("not a database")) }, }, { name: "cut off", file: func(t *testing.T) []byte { t.Helper() packed := gzipped(t, databaseFile(t)) return packed[:len(packed)/2] }, }, { name: "over the size limit", file: func(t *testing.T) []byte { t.Helper() return gzipped(t, databaseFile(t)) }, max: 10, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() locator := newDownloadLocator(t, tt.file(t)) if tt.max != 0 { locator.maxDatabaseBytes = tt.max } giveDatabase(t, locator) err := locator.downloadDatabase(t.Context(), fixedTime()) if err == nil { t.Fatal("the download succeeded") } requireNoDownloadFile(t, locator) requireCity(t, lookupOne(t, locator, testAddress), databaseCity, sourceDBIP) }) } } // TestMissingFileIsAFailedDownload checks that a month whose file is not // published yet is an error and leaves no copy. func TestMissingFileIsAFailedDownload(t *testing.T) { t.Parallel() locator := newDownloadLocator(t, nil) err := locator.downloadDatabase(t.Context(), fixedTime().AddDate(0, 1, 0)) if !errors.Is(err, errDatabaseStatus) { t.Fatalf("error %v, want %v", err, errDatabaseStatus) } if !locator.databaseDue(fixedTime()) { t.Error("a copy is in use after a failed download") } } // TestUnopenableCopyIsDownloadedAgain leaves a damaged copy from this month. It // counts as no copy, so the download is due. func TestUnopenableCopyIsDownloadedAgain(t *testing.T) { t.Parallel() locator := newDownloadLocator(t, nil) err := os.WriteFile(filepath.Join(locator.dir, databaseFileName), []byte("damaged"), 0o600) if err != nil { t.Fatalf("writing the copy: %v", err) } locator.openDatabaseCopy() if !locator.databaseDue(fixedTime()) { t.Error("no download due with a copy that does not open") } } // TestLookupDuringReplaceIsSafe looks up addresses while the copy is replaced. // The race detector and the mapped file both catch a lookup of a closed copy. func TestLookupDuringReplaceIsSafe(t *testing.T) { t.Parallel() file := gzipped(t, databaseFile(t)) locator := newDownloadLocator(t, file) giveDatabase(t, locator) addrs := parseAddresses(t, testAddress) ctx, cancel := context.WithCancel(t.Context()) defer cancel() var readers sync.WaitGroup readers.Go(func() { for ctx.Err() == nil { locator.fromDatabase(addrs[0]) } }) for range 5 { err := locator.downloadDatabase(t.Context(), fixedTime()) if err != nil { t.Fatalf("downloadDatabase: %v", err) } } cancel() readers.Wait() } // TestRequestNamesRouteWatch checks the header each outside request carries, // which tells geojs and DB-IP who is asking. func TestRequestNamesRouteWatch(t *testing.T) { t.Parallel() var seen atomic.Pointer[string] server := httptest.NewServer(http.HandlerFunc( func(_ http.ResponseWriter, r *http.Request) { seen.Store(new(r.Header.Get(userAgentHeader))) })) t.Cleanup(server.Close) fake := newFakeGeoJS(t, "[]") dir := t.TempDir() locator := newTestLocator(t, fake, openCacheDB(t, dir), dir) locator.geojsURL = server.URL + geojsPath lookupOne(t, locator, testAddress) got := seen.Load() if got == nil || !strings.HasPrefix(*got, "routewatch") { t.Errorf("User-Agent %v", got) } }