Files
smallwebwaf/internal/reputation/dnsbl.go
T
clawbot e7cc0adbdb
check / check (push) Waiting to run
waf: end the asks to GeoJS, the DNSBL zones, AbuseIPDB and routewatch at the stop (closes #439)
Each of the four keeps the cancel function of every ask it started, and a
wait group of their goroutines. Its new method (GeoJS.StopAsking,
DNSBL.StopAsking, AbuseIPDB.StopChecking, routewatch Client.StopAsking)
refuses further asks, ends those under way, and returns once their
goroutines have. An ask ended that way is no failure: no alert, no failure
count, no wait before the next. Each provider in internal/sww adds the
method as a stop hook, so the last write of the state files follows it.

Model: opus-5-5
2026-11-19 19:02:12 +01:00

371 lines
11 KiB
Go

package reputation
import (
"cmp"
"context"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
"slices"
"strings"
"sync"
"time"
"github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
)
const (
// maxVerdicts is how many verdicts of the DNSBL zones are kept, and how
// many scores of AbuseIPDB. Past it, the one fetched longest ago is
// dropped.
maxVerdicts = 100000
// maxQueries is how many queries may be under way at once. Past it, a
// zone is not asked about a client until the client's next request, so
// that a swarm of new addresses cannot fill the memory.
maxQueries = 1000
// failureDelay is how long a zone is not asked again after a query to
// it fails, and no client is checked with AbuseIPDB after a check
// fails, so that a source refusing them is not asked on every request.
failureDelay = time.Minute
)
var (
errAsk = errors.New("ask the zone")
errRefused = errors.New("the zone refused the query")
errNotListing = errors.New("the answer is outside 127.0.0.0/8")
)
// Verdict is what a zone said about a client, as dnsbl.json holds
// it: the zone, the client's address, whether the zone lists it, and when
// the zone answered.
type Verdict struct {
Zone string `json:"zone"`
Client netip.Addr `json:"client"`
Listed bool `json:"listed"`
Fetched time.Time `json:"fetched"`
}
// DNSBLParams are what NewDNSBL needs.
type DNSBLParams struct {
// Zones are the DNSBL zones clients are asked about in
// (SWW_WAF_DNSBL_ZONES).
Zones []string
// Resolver is the resolver they are asked through
// (SWW_WAF_DNSBL_RESOLVER), or, while it is the zero AddrPort, the
// host's, as /etc/resolv.conf names it.
Resolver netip.AddrPort
// CacheTTL is how long a verdict is used after it was fetched
// (SWW_WAF_IP_REPUTATION_CACHE_TTL), and Timeout how long a query may
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
CacheTTL time.Duration
Timeout time.Duration
// Now tells the time, normally time.Now in UTC.
Now func() time.Time
// ProcessLog receives each query that fails, and why.
ProcessLog *slog.Logger
// Alerts receive a source_failure alert for each query that fails.
Alerts *alerts.Queue
}
// DNSBL asks the DNSBL zones about clients, in the background, and keeps
// their verdicts. It is safe for concurrent use.
type DNSBL struct {
params DNSBLParams
resolver *net.Resolver
mu sync.Mutex
// verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way, each with the function that ends
// it. stopped is true once StopAsking has been called, and
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be
// asked again.
queries map[string]int
failures map[string]int
retryAt map[string]time.Time
}
// query is a client's address, to ask a zone about.
type query struct {
zone string
client netip.Addr
}
// NewDNSBL returns a DNSBL with no verdict yet.
func NewDNSBL(params DNSBLParams) *DNSBL {
verdicts, err := simplelru.NewLRU[query, Verdict](maxVerdicts, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
resolver := &net.Resolver{}
if params.Resolver.IsValid() {
// Dial is used by Go's own resolver alone.
resolver.PreferGo = true
resolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
var dialer net.Dialer
return dialer.DialContext(ctx, network, params.Resolver.String())
}
}
return &DNSBL{
params: params,
resolver: resolver,
verdicts: verdicts,
asking: map[query]context.CancelFunc{},
queries: map[string]int{},
failures: map[string]int{},
retryAt: map[string]time.Time{},
}
}
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
func (d *DNSBL) Zones() []string {
return slices.Clone(d.params.Zones)
}
// ListedBy returns the zones whose verdict on addr, a client's address,
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
// masked, as config.MaskZoneKey masks it, since they go to the request
// log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, maxQueries are under way, or
// StopAsking has been called; ListedBy never waits for a query. ctx is the
// context of the client's request, and a query goes on after the request
// ends, until StopAsking.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock()
defer d.mu.Unlock()
now := d.params.Now()
var listedBy []string
for _, zone := range d.params.Zones {
q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed {
listedBy = append(listedBy, config.MaskZoneKey(zone))
}
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
len(d.asking) < maxQueries:
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
d.queries[zone]++
d.queryGoroutines.Go(func() {
defer end()
d.ask(queryCtx, q)
})
}
}
return listedBy
}
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int {
d.mu.Lock()
defer d.mu.Unlock()
return d.queries[zone]
}
// Failures returns how many queries to zone failed.
func (d *DNSBL) Failures(zone string) int {
d.mu.Lock()
defer d.mu.Unlock()
return d.failures[zone]
}
// Snapshot returns every verdict still in use, sorted by client, then by
// zone, as dnsbl.json lists them.
func (d *DNSBL) Snapshot() []Verdict {
d.mu.Lock()
now := d.params.Now()
verdicts := make([]Verdict, 0, d.verdicts.Len())
for _, kept := range d.verdicts.Values() {
if now.Sub(kept.Fetched) < d.params.CacheTTL {
verdicts = append(verdicts, kept)
}
}
d.mu.Unlock()
slices.SortFunc(verdicts, func(a, b Verdict) int {
return cmp.Or(
a.Client.Compare(b.Client),
strings.Compare(a.Zone, b.Zone),
)
})
return verdicts
}
// Load keeps verdicts, read from dnsbl.json, in place of those it
// keeps, but for those of a zone SWW_WAF_DNSBL_ZONES does not name, and,
// past maxVerdicts, those fetched longest ago. One fetched CacheTTL ago or
// more is neither used nor written, as for any verdict.
func (d *DNSBL) Load(verdicts []Verdict) {
verdicts = slices.Clone(verdicts)
slices.SortStableFunc(verdicts, func(a, b Verdict) int {
return a.Fetched.Compare(b.Fetched)
})
d.mu.Lock()
defer d.mu.Unlock()
d.verdicts.Purge()
for _, kept := range verdicts {
if slices.Contains(d.params.Zones, kept.Zone) {
d.verdicts.Add(query{zone: kept.Zone, client: kept.Client}, kept)
}
}
}
// ask asks q's zone about q's client, keeps the verdict, and notes the
// query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q)
now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock()
delete(d.asking, q)
switch {
case err == nil:
d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
})
case isFailure:
d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay)
}
d.mu.Unlock()
if isFailure {
const failed = "asking a DNSBL zone failed"
shown := config.MaskZoneKey(q.zone)
// Raised before it is logged, so that the alert is there once the
// log line is.
raiseFailure(d.params.Alerts, failed, shown, err)
d.params.ProcessLog.Warn(failed, "zone", shown, "error", err.Error())
}
}
// lookUp asks q's zone about q's client through the resolver, and returns
// whether the zone lists it, as readAnswer reads the answer. No such name
// is a client the zone does not list. A query not answered within Timeout
// fails.
func (d *DNSBL) lookUp(ctx context.Context, q query) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, d.params.Timeout)
defer cancel()
answer, err := d.resolver.LookupNetIP(
ctx,
"ip4",
queryName(q.zone, q.client),
)
var dnsErr *net.DNSError
switch {
case err == nil:
return readAnswer(answer)
case errors.As(err, &dnsErr) && dnsErr.IsNotFound:
return false, nil
case errors.As(err, &dnsErr):
// The error names the name asked about, which holds the client's
// address, which is not to be logged: only what went wrong is kept.
return false, fmt.Errorf("%w: %s", errAsk, dnsErr.Err)
default:
return false, fmt.Errorf("%w: %w", errAsk, err)
}
}
// queryName returns the name a zone is asked about addr by, as RFC 5782
// builds it: the four numbers of an IPv4 address, or the 32 hex digits of
// an IPv6 address, in reverse order, each followed by a dot, then the zone
// and a dot, which makes it a full name, to which the resolver adds no
// search domain of /etc/resolv.conf.
func queryName(zone string, addr netip.Addr) string {
parts := strings.Split(addr.String(), ".")
if addr.Is6() {
parts = strings.Split(hex.EncodeToString(addr.AsSlice()), "")
}
slices.Reverse(parts)
return strings.Join(parts, ".") + "." + zone + "."
}
// readAnswer reads the addresses a zone answered with. An address in
// 127.0.0.0/8 lists the client, as RFC 5782 has zones answer, but one in
// 127.255.255.0/24 is how Spamhaus refuses a query, such as one sent
// through a public resolver or one past its limit, and is a failure. So is
// an address outside 127.0.0.0/8, such as a resolver gives that answers
// even for names that do not exist.
func readAnswer(answer []netip.Addr) (bool, error) {
listing := netip.MustParsePrefix("127.0.0.0/8")
refusal := netip.MustParsePrefix("127.255.255.0/24")
for _, addr := range answer {
switch {
case refusal.Contains(addr):
return false, fmt.Errorf("%w: %s", errRefused, addr)
case !listing.Contains(addr):
return false, fmt.Errorf("%w: %s", errNotListing, addr)
}
}
return len(answer) > 0, nil
}