Files
smallwebwaf/internal/geojs/geojs.go
T
clawbot e7cc0adbdb
check / check (push) Waiting to run
waf: end the asks to GeoJS, the DNSBL zones, AbuseIPDB and routewatch at the stop (closes #439)
Each of the four keeps the cancel function of every ask it started, and a
wait group of their goroutines. Its new method (GeoJS.StopAsking,
DNSBL.StopAsking, AbuseIPDB.StopChecking, routewatch Client.StopAsking)
refuses further asks, ends those under way, and returns once their
goroutines have. An ask ended that way is no failure: no alert, no failure
count, no wait before the next. Each provider in internal/sww adds the
method as a stop hook, so the last write of the state files follows it.

Model: opus-5-5
2026-11-19 19:02:12 +01:00

569 lines
15 KiB
Go

// Package geojs looks up clients' AS numbers and countries through the GeoJS
// web service. Its answers are kept in memory, for at most 100,000 clients and
// for 7 days each. A caller waits for an answer as long as it says. A request to
// GeoJS goes on after its callers stop waiting, and its answer is kept. The
// package imports no other package of this repository.
package geojs
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/netip"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/hashicorp/golang-lru/v2/simplelru"
)
// URL is GeoJS's endpoint for an address's place and network. Asked about
// several addresses at once, comma separated in its ip parameter, it
// answers with a list.
const URL = "https://get.geojs.io/v1/ip/geo.json"
const (
// keepFor is how long an answer is used instead of asking GeoJS again.
keepFor = 7 * 24 * time.Hour
// maxAnswers is how many answers are kept. Past it, the one used
// longest ago is dropped.
maxAnswers = 100000
// maxWaiting is how many clients may wait to be asked about. Past it,
// a new client counts as not found and is not asked about until there
// is room, so that a swarm of new addresses while GeoJS is down cannot
// fill the memory.
maxWaiting = 10000
// maxPerRequest is how many addresses one request to GeoJS asks about.
maxPerRequest = 200
// requestTimeout is how long a request to GeoJS may take before it is
// abandoned, which is a failure. It is apart from how long callers wait.
requestTimeout = 10 * time.Second
// unknownASN is the AS number GeoJS gives when it knows none.
unknownASN = 64512
// After a failure GeoJS is not asked again for a second, and for
// retryDelayFactor times as long after each further failure in a row,
// up to five minutes.
firstRetryDelay = time.Second
retryDelayFactor = 2
maxRetryDelay = 5 * time.Minute
// maxResponseBytes is the most of GeoJS's answer that is read.
maxResponseBytes = 1 << 20
)
var (
errStatus = errors.New("GeoJS answered")
errLeftOut = errors.New("GeoJS's answer left out")
)
// Params are what New needs.
type Params struct {
// URL is where GeoJS is asked, normally URL.
URL string
// Answered, unless nil, is given each answer GeoJS gives, once it is
// kept.
Answered func(Answer)
// Now tells the time, normally time.Now in UTC.
Now func() time.Time
// Asked is called for each request to GeoJS.
Asked func()
// Failed is called for each failure of GeoJS, with what went wrong and
// how long GeoJS is left alone after it. The error never names the
// addresses asked about.
Failed func(err error, askingAgainIn time.Duration)
// WentWithoutAnswer is called each time a caller that waits for an
// answer goes on without one.
WentWithoutAnswer func()
}
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure.
httpClient *http.Client
mu sync.Mutex
answers *simplelru.LRU[netip.Prefix, *Answer]
// waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about.
waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way, and endAsking
// ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration
retryAt time.Time
}
// Answer is what GeoJS said about a client: its AS number, such as AS64496,
// and the AS's name, both "" when GeoJS knows no AS number for it; its
// country, "" when GeoJS cannot place it; when GeoJS said so; and when the
// answer was last used. The zero Answer is that of a client with no answer.
//
//nolint:tagliatelle // the state files use snake_case, as the request log does
type Answer struct {
Client netip.Prefix `json:"client"`
ASN string `json:"asn"`
ASName string `json:"as_name"`
Country string `json:"country"`
Answered time.Time `json:"answered"`
Used time.Time `json:"used"`
}
// wait is a client waiting for its answer.
type wait struct {
// asked is closed when the client gets its answer, and closed and
// replaced each time GeoJS fails before then.
asked chan struct{}
// late is true once the client has gone without an answer, for a
// whole wait or because GeoJS failed: its callers no longer wait.
late bool
}
// New returns a GeoJS with no answer kept yet.
func New(params Params) *GeoJS {
answers, err := simplelru.NewLRU[netip.Prefix, *Answer](maxAnswers, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
return &GeoJS{
url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
},
answers: answers,
waiting: map[netip.Prefix]*wait{},
}
}
// LookUp returns the answer GeoJS gave about client, with its country as
// a two-letter code in capitals, or the zero Answer when there is none
// yet. An answer is kept for 7 days. Without one, the client is asked
// about in the background, and LookUp waits up to wait for the answer,
// unless the client has gone without one before. The wait ends early when
// ctx ends. Neither ends the request to GeoJS.
//
// GeoJS is asked about the client's first address, which is the client's
// own address for IPv4, and an address in the same place for an IPv6
// netblock.
func (g *GeoJS) LookUp(
ctx context.Context, client netip.Prefix, wait time.Duration,
) Answer {
answer, asked := g.answerOrWait(ctx, client, true)
if asked == nil {
return answer
}
timer := time.NewTimer(wait)
defer timer.Stop()
select {
case <-asked:
case <-timer.C:
case <-ctx.Done():
}
g.mu.Lock()
defer g.mu.Unlock()
answer, found := g.kept(client)
if !found {
g.wentWithoutAnswer()
}
w, waiting := g.waiting[client]
if !found && waiting {
w.late = true
}
return answer
}
// LookUpWithoutWaiting is LookUp for a caller that does not need the answer:
// it returns the zero Answer at once when none is kept.
func (g *GeoJS) LookUpWithoutWaiting(
ctx context.Context,
client netip.Prefix,
) Answer {
answer, _ := g.answerOrWait(ctx, client, false)
return answer
}
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock()
defer g.mu.Unlock()
return g.kept(client)
}
// Snapshot returns every answer kept, sorted by client.
func (g *GeoJS) Snapshot() []Answer {
g.mu.Lock()
answers := make([]Answer, 0, g.answers.Len())
for _, kept := range g.answers.Values() {
answers = append(answers, *kept)
}
g.mu.Unlock()
slices.SortFunc(answers, func(a, b Answer) int {
return a.Client.Compare(b.Client)
})
return answers
}
// Load keeps answers, such as those of an earlier Snapshot, in place of the
// answers it keeps, in the order they were last used, so that the one used
// longest ago is dropped first. Answers GeoJS gave keepFor ago or more are
// dropped.
func (g *GeoJS) Load(answers []Answer) {
answers = slices.Clone(answers)
slices.SortStableFunc(answers, func(a, b Answer) int {
return a.Used.Compare(b.Used)
})
g.mu.Lock()
defer g.mu.Unlock()
g.answers.Purge()
now := g.now()
for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor {
g.answers.Add(answer.Client, &answer)
}
}
}
// answerOrWait returns client's kept answer if it has one. Otherwise it
// puts the client among those waiting if there is room, has GeoJS asked
// about them if it can be, and returns what to wait on for the answer, or
// nil when there is nothing to wait for or mustWait is false.
func (g *GeoJS) answerOrWait(
ctx context.Context, client netip.Prefix, mustWait bool,
) (Answer, <-chan struct{}) {
g.mu.Lock()
defer g.mu.Unlock()
answer, found := g.kept(client)
if found {
return answer, nil
}
w, waiting := g.waiting[client]
if !waiting && len(g.waiting) < maxWaiting {
w = &wait{asked: make(chan struct{})}
g.waiting[client] = w
}
g.ask(ctx)
if !mustWait {
return Answer{}, nil // the answer is not needed before the caller goes on
}
if w == nil {
g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already
}
if !g.asking {
// GeoJS is left alone after a failure, or asked nothing after
// StopAsking, so no answer can come.
w.late = true
}
if w.late {
g.wentWithoutAnswer()
return Answer{}, nil
}
return Answer{}, w.asked
}
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.now()
kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor {
return Answer{}, false
}
kept.Used = now
return *kept, true
}
// ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way, it is left alone after a failure, or StopAsking has
// been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.now().Before(g.retryAt) {
return
}
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
g.asking, g.endAsking = true, end
g.askGoroutines.Go(func() {
defer end()
g.askAboutWaiting(askCtx)
})
}
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
// answer kept is given to Answered, outside the lock, since Answered takes
// locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for {
clients := g.nextClients()
if len(clients) == 0 {
return
}
given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err)
if g.answered != nil {
for _, answer := range kept {
g.answered(answer)
}
}
if !answered {
return
}
}
}
// nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, or StopAsking has been called, it returns none and
// notes that no request to GeoJS is under way.
func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.waiting) == 0 || g.stopped {
g.asking = false
return nil
}
clients := make([]netip.Prefix, 0, min(len(g.waiting), maxPerRequest))
for client := range g.waiting {
if len(clients) == maxPerRequest {
break
}
clients = append(clients, client)
}
return clients
}
// keep notes how a request to GeoJS about clients ended, given being the
// answer for each address GeoJS's answer names. It returns the answers it
// kept, and reports whether GeoJS answered about all of the clients. Each
// client whose address GeoJS's answer names gets its answer. An answer
// that leaves an address out is a failure. After a failure GeoJS is left
// alone for a while, and every client still waiting stops waiting and is
// asked about once GeoJS is asked again.
func (g *GeoJS) keep(
clients []netip.Prefix, given map[netip.Addr]Answer, err error,
) ([]Answer, bool) {
g.mu.Lock()
defer g.mu.Unlock()
now := g.now()
kept := make([]Answer, 0, len(clients))
leftOut := 0
for _, client := range clients {
answer, named := given[client.Addr()]
if !named {
leftOut++
continue
}
answer.Client, answer.Answered, answer.Used = client, now, now
g.answers.Add(client, &answer)
kept = append(kept, answer)
close(g.waiting[client].asked)
delete(g.waiting, client)
}
if err == nil && leftOut > 0 {
err = fmt.Errorf(
"%w %d of %d addresses",
errLeftOut,
leftOut,
len(clients),
)
}
if err != nil {
g.retryDelay = min(
max(retryDelayFactor*g.retryDelay, firstRetryDelay),
maxRetryDelay,
)
g.retryAt = now.Add(g.retryDelay)
g.asking = false
for _, w := range g.waiting {
close(w.asked)
w.asked = make(chan struct{})
w.late = true
}
g.failed(err, g.retryDelay)
return kept, false
}
g.retryDelay = 0
return kept, true
}
// request asks GeoJS about clients in one request, and returns the answer
// for each address GeoJS's answer names: its AS number and the AS's name,
// both "" for the AS number 64512, which GeoJS gives when it knows none,
// and its country, in capitals. The request is abandoned after
// requestTimeout.
func (g *GeoJS) request(
ctx context.Context, clients []netip.Prefix,
) (map[netip.Addr]Answer, error) {
addrs := make([]string, 0, len(clients))
for _, client := range clients {
addrs = append(addrs, client.Addr().String())
}
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
defer cancel()
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
g.url,
http.NoBody,
)
if err != nil {
return nil, fmt.Errorf("make the request to GeoJS: %w", err)
}
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.asked()
res, err := g.httpClient.Do(req)
if err != nil {
// Do's error names the URL, and so the visitors' addresses, which
// are not to be logged: only what went wrong is kept.
return nil, fmt.Errorf("ask GeoJS: %w", errors.Unwrap(err))
}
defer func() {
_ = res.Body.Close()
}()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
}
//nolint:tagliatelle // GeoJS's own names
var answers []struct {
IP string `json:"ip"`
ASN int64 `json:"asn"`
ASName string `json:"organization_name"`
CountryCode string `json:"country_code"`
}
err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).
Decode(&answers)
if err != nil {
return nil, fmt.Errorf("read GeoJS's answer: %w", err)
}
given := make(map[netip.Addr]Answer, len(answers))
for _, item := range answers {
addr, err := netip.ParseAddr(item.IP)
if err != nil {
continue
}
answer := Answer{Country: strings.ToUpper(item.CountryCode)}
if item.ASN != 0 && item.ASN != unknownASN {
answer.ASN = "AS" + strconv.FormatInt(item.ASN, 10)
answer.ASName = item.ASName
}
given[addr] = answer
}
return given, nil
}