check / check (push) Waiting to run
Each of the four keeps the cancel function of every ask it started, and a wait group of their goroutines. Its new method (GeoJS.StopAsking, DNSBL.StopAsking, AbuseIPDB.StopChecking, routewatch Client.StopAsking) refuses further asks, ends those under way, and returns once their goroutines have. An ask ended that way is no failure: no alert, no failure count, no wait before the next. Each provider in internal/sww adds the method as a stop hook, so the last write of the state files follows it. Model: opus-5-5
537 lines
14 KiB
Go
537 lines
14 KiB
Go
package reputation
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/netip"
|
|
"net/url"
|
|
"slices"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"cloud.google.com/go/civil"
|
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
)
|
|
|
|
const (
|
|
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
|
// AbuseIPDB's API.
|
|
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
|
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
|
|
// the blacklist endpoint of AbuseIPDB's API.
|
|
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
|
|
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
|
// name AbuseIPDB.
|
|
AbuseIPDBSource = "abuseipdb"
|
|
// maxAnswerBytes is the most of an answer of AbuseIPDB that is read.
|
|
maxAnswerBytes = 64 << 10
|
|
)
|
|
|
|
var (
|
|
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
|
errNoBlacklist = errors.New("the answer gives no data")
|
|
errNotAddress = errors.New("is not an address")
|
|
errBudgetUsedUp = errors.New(
|
|
"checks spent; none is made until the day ends at 00:00 UTC")
|
|
)
|
|
|
|
// Score is what AbuseIPDB said about a client, as abuseipdb-scores.json
|
|
// holds it: the client, an IPv4 address or an IPv6 group, its abuse
|
|
// confidence score, from 0 to 100, and when AbuseIPDB answered.
|
|
type Score struct {
|
|
Client netip.Prefix `json:"client"`
|
|
Score int64 `json:"score"`
|
|
Fetched time.Time `json:"fetched"`
|
|
}
|
|
|
|
// ChecksSpent is what abuseipdb-checks-spent.json keeps of the checks of
|
|
// clients with AbuseIPDB: the day, in UTC, of the checks Spent counts,
|
|
// zero before the first.
|
|
type ChecksSpent struct {
|
|
Day civil.Date
|
|
Spent int
|
|
}
|
|
|
|
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
|
|
// abuse confidence score.
|
|
type BlacklistEntry struct {
|
|
Addr netip.Addr
|
|
Score int64
|
|
}
|
|
|
|
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
|
type AbuseIPDBParams struct {
|
|
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
|
// the account's key (SWW_WAF_ABUSEIPDB_KEY).
|
|
URL string
|
|
Key string
|
|
// MinScore is the least score that is a hit (SWW_WAF_ABUSEIPDB_MIN_SCORE),
|
|
// and DailyBudget the most checks made in a day, in UTC
|
|
// (SWW_WAF_ABUSEIPDB_DAILY_BUDGET).
|
|
MinScore int64
|
|
DailyBudget int
|
|
// CacheTTL is how long a score is used after it was fetched
|
|
// (SWW_WAF_IP_REPUTATION_CACHE_TTL), and Timeout how long a check may
|
|
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
|
CacheTTL time.Duration
|
|
Timeout time.Duration
|
|
// BlacklistURL is where the blacklist is fetched from, normally
|
|
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
|
|
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
|
|
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
|
|
// receives the clients on each blacklist fetched.
|
|
BlacklistURL string
|
|
Threshold int64
|
|
BlacklistRefresh time.Duration
|
|
Blacklisted func([]BlacklistEntry)
|
|
// Now tells the time, normally time.Now in UTC.
|
|
Now func() time.Time
|
|
// ProcessLog receives each check or fetch of the blacklist that fails,
|
|
// and why, and the day's budget used up.
|
|
ProcessLog *slog.Logger
|
|
// Alerts receive a source_failure alert for each.
|
|
Alerts *alerts.Queue
|
|
}
|
|
|
|
// AbuseIPDB checks clients with AbuseIPDB, in the background, and keeps
|
|
// their scores. It is safe for concurrent use.
|
|
type AbuseIPDB struct {
|
|
params AbuseIPDBParams
|
|
httpClient *http.Client
|
|
|
|
mu sync.Mutex
|
|
// scores are by client. Each is added as it is fetched and never moved
|
|
// up, so that the one fetched longest ago is the first dropped.
|
|
scores *simplelru.LRU[netip.Prefix, Score]
|
|
// checking are the clients whose check is under way, each with the
|
|
// function that ends it. stopped is true once StopChecking has been
|
|
// called, and checkGoroutines waits for the goroutine of each check.
|
|
checking map[netip.Prefix]context.CancelFunc
|
|
stopped bool
|
|
checkGoroutines sync.WaitGroup
|
|
// day is the day, in UTC, of the checks spent counts.
|
|
day civil.Date
|
|
spent int
|
|
// checks and failures count the checks made and those that failed,
|
|
// and retryAt is when a client may be checked again after the last
|
|
// check failed.
|
|
checks int
|
|
failures int
|
|
retryAt time.Time
|
|
}
|
|
|
|
// NewAbuseIPDB returns an AbuseIPDB with no score yet, and no check spent.
|
|
func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|
scores, err := simplelru.NewLRU[netip.Prefix, Score](maxVerdicts, nil)
|
|
if err != nil {
|
|
panic(err) // NewLRU fails only for a size below one
|
|
}
|
|
|
|
return &AbuseIPDB{
|
|
params: params,
|
|
httpClient: &http.Client{},
|
|
scores: scores,
|
|
checking: map[netip.Prefix]context.CancelFunc{},
|
|
}
|
|
}
|
|
|
|
// Hit returns AbuseIPDB's score of client, an IPv4 address or an IPv6
|
|
// group, and whether it is a hit: MinScore or more. A score is used until
|
|
// CacheTTL has passed since it was fetched, whichever of the client's
|
|
// addresses its request comes from. A client without one is checked in
|
|
// the background, by addr, the address its request came from, if
|
|
// offender, if it has committed an offence, unless its check is under
|
|
// way, a check failed less than failureDelay ago, the day's checks have
|
|
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
|
// for a check. The check that uses the budget up is logged and raised as a
|
|
// source_failure alert. ctx is the context of the client's request, and a
|
|
// check goes on after the request ends, until StopChecking.
|
|
func (a *AbuseIPDB) Hit(
|
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
|
) (int64, bool) {
|
|
a.mu.Lock()
|
|
|
|
now := a.params.Now()
|
|
|
|
kept, found := a.scores.Peek(client)
|
|
if found && now.Sub(kept.Fetched) < a.params.CacheTTL {
|
|
a.mu.Unlock()
|
|
|
|
return kept.Score, kept.Score >= a.params.MinScore
|
|
}
|
|
|
|
if today := civil.DateOf(now.UTC()); a.day != today {
|
|
a.day, a.spent = today, 0
|
|
}
|
|
|
|
_, checking := a.checking[client]
|
|
|
|
check := offender && !checking && !a.stopped &&
|
|
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
|
if check {
|
|
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
a.checking[client] = end
|
|
a.checks++
|
|
a.spent++
|
|
|
|
a.checkGoroutines.Go(func() {
|
|
defer end()
|
|
|
|
a.check(checkCtx, client, addr)
|
|
})
|
|
}
|
|
|
|
usedUp := check && a.spent == a.params.DailyBudget
|
|
|
|
a.mu.Unlock()
|
|
|
|
if usedUp {
|
|
a.alert("the daily budget of AbuseIPDB checks is used up",
|
|
fmt.Errorf("%d %w", a.params.DailyBudget, errBudgetUsedUp))
|
|
}
|
|
|
|
return 0, false
|
|
}
|
|
|
|
// StopChecking ends the checks under way, and returns once their goroutines
|
|
// have. A check it ends gives no score and is no failure: it is neither
|
|
// counted, logged nor raised as an alert, and checks are not left alone
|
|
// after it. The ended check stays spent from the day's budget. After
|
|
// StopChecking, Hit checks no client.
|
|
func (a *AbuseIPDB) StopChecking() {
|
|
a.mu.Lock()
|
|
|
|
a.stopped = true
|
|
for _, end := range a.checking {
|
|
end()
|
|
}
|
|
|
|
a.mu.Unlock()
|
|
|
|
a.checkGoroutines.Wait()
|
|
}
|
|
|
|
// Checked returns how many checks were made.
|
|
func (a *AbuseIPDB) Checked() int {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
return a.checks
|
|
}
|
|
|
|
// Failures returns how many checks failed.
|
|
func (a *AbuseIPDB) Failures() int {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
return a.failures
|
|
}
|
|
|
|
// BudgetLeft returns how many checks the day's budget has left.
|
|
func (a *AbuseIPDB) BudgetLeft() int {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
if a.day != civil.DateOf(a.params.Now().UTC()) {
|
|
return a.params.DailyBudget
|
|
}
|
|
|
|
return max(a.params.DailyBudget-a.spent, 0)
|
|
}
|
|
|
|
// Snapshot returns every score still in use, sorted by client, as
|
|
// abuseipdb-scores.json keeps them.
|
|
func (a *AbuseIPDB) Snapshot() []Score {
|
|
a.mu.Lock()
|
|
|
|
now := a.params.Now()
|
|
scores := make([]Score, 0, a.scores.Len())
|
|
|
|
for _, kept := range a.scores.Values() {
|
|
if now.Sub(kept.Fetched) < a.params.CacheTTL {
|
|
scores = append(scores, kept)
|
|
}
|
|
}
|
|
|
|
a.mu.Unlock()
|
|
|
|
slices.SortFunc(scores, func(x, y Score) int {
|
|
return x.Client.Compare(y.Client)
|
|
})
|
|
|
|
return scores
|
|
}
|
|
|
|
// Load keeps scores, read from abuseipdb-scores.json, in place of those
|
|
// it keeps, but for the scores past maxVerdicts, those fetched longest
|
|
// ago. One fetched CacheTTL ago or more is neither used nor written, as
|
|
// for any score.
|
|
func (a *AbuseIPDB) Load(scores []Score) {
|
|
scores = slices.Clone(scores)
|
|
slices.SortStableFunc(scores, func(x, y Score) int {
|
|
return x.Fetched.Compare(y.Fetched)
|
|
})
|
|
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
a.scores.Purge()
|
|
|
|
for _, kept := range scores {
|
|
a.scores.Add(kept.Client, kept)
|
|
}
|
|
}
|
|
|
|
// ChecksSpent returns the checks spent, as abuseipdb-checks-spent.json
|
|
// keeps them.
|
|
func (a *AbuseIPDB) ChecksSpent() ChecksSpent {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
return ChecksSpent{Day: a.day, Spent: a.spent}
|
|
}
|
|
|
|
// LoadChecksSpent keeps checks, read from abuseipdb-checks-spent.json, in
|
|
// place of the checks spent.
|
|
func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
|
|
a.day, a.spent = checks.Day, checks.Spent
|
|
}
|
|
|
|
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
|
|
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
|
|
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
|
|
// fails is logged and raised as a source_failure alert, unless it was cut
|
|
// off as ctx ended.
|
|
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|
if a.params.Key == "" {
|
|
return
|
|
}
|
|
|
|
for {
|
|
listed, err := a.fetchBlacklist(ctx)
|
|
|
|
switch {
|
|
case ctx.Err() != nil:
|
|
return
|
|
case err != nil:
|
|
a.alert("fetching AbuseIPDB's blacklist failed", err)
|
|
default:
|
|
a.params.Blacklisted(listed)
|
|
}
|
|
|
|
timer := time.NewTimer(a.params.BlacklistRefresh)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
|
// the score as client's, and notes the check as no longer under way. A
|
|
// check that fails gives no score: it is counted, logged and raised as a
|
|
// source_failure alert, and no client is checked for failureDelay. A check
|
|
// ended as ctx ends, by StopChecking, is no failure.
|
|
func (a *AbuseIPDB) check(
|
|
ctx context.Context,
|
|
client netip.Prefix,
|
|
addr netip.Addr,
|
|
) {
|
|
score, err := a.ask(ctx, addr)
|
|
now := a.params.Now()
|
|
isFailure := err != nil && ctx.Err() == nil
|
|
|
|
a.mu.Lock()
|
|
|
|
delete(a.checking, client)
|
|
|
|
switch {
|
|
case err == nil:
|
|
a.scores.Add(
|
|
client,
|
|
Score{Client: client, Score: score, Fetched: now},
|
|
)
|
|
case isFailure:
|
|
a.failures++
|
|
a.retryAt = now.Add(failureDelay)
|
|
}
|
|
|
|
a.mu.Unlock()
|
|
|
|
if isFailure {
|
|
a.alert("checking a client with AbuseIPDB failed", err)
|
|
}
|
|
}
|
|
|
|
// ask asks AbuseIPDB for addr's abuse confidence score, sending the key
|
|
// in the header Key. An answer other than 200, one that gives no score,
|
|
// and none within Timeout, fail.
|
|
func (a *AbuseIPDB) ask(
|
|
ctx context.Context,
|
|
addr netip.Addr,
|
|
) (int64, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, a.params.Timeout)
|
|
defer cancel()
|
|
|
|
query := url.Values{"ipAddress": {addr.String()}}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
|
a.params.URL+"?"+query.Encode(), http.NoBody)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("make the request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("Key", a.params.Key)
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
res, err := a.httpClient.Do(req)
|
|
if err != nil {
|
|
// Do's error names the URL, which holds the client's address, which
|
|
// is not to be logged: only what went wrong is kept.
|
|
return 0, fmt.Errorf("check the client: %w", errors.Unwrap(err))
|
|
}
|
|
|
|
defer func() {
|
|
_ = res.Body.Close()
|
|
}()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
return 0, fmt.Errorf("%w %s", errStatus, res.Status)
|
|
}
|
|
|
|
var answer struct {
|
|
Data struct {
|
|
AbuseConfidenceScore *int64 `json:"abuseConfidenceScore"`
|
|
} `json:"data"`
|
|
}
|
|
|
|
err = json.NewDecoder(io.LimitReader(res.Body, maxAnswerBytes)).
|
|
Decode(&answer)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("read the answer: %w", err)
|
|
}
|
|
|
|
if answer.Data.AbuseConfidenceScore == nil {
|
|
return 0, errNoScore
|
|
}
|
|
|
|
return *answer.Data.AbuseConfidenceScore, nil
|
|
}
|
|
|
|
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
|
|
// Threshold or more, sending the key in the header Key. An answer other
|
|
// than 200, one longer than maxListBytes or without the list, an entry
|
|
// whose address does not read, and none within fetchTimeout, fail.
|
|
func (a *AbuseIPDB) fetchBlacklist(
|
|
ctx context.Context,
|
|
) ([]BlacklistEntry, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
|
defer cancel()
|
|
|
|
query := url.Values{
|
|
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
|
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("make the request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("Key", a.params.Key)
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
res, err := a.httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
|
|
}
|
|
|
|
defer func() {
|
|
_ = res.Body.Close()
|
|
}()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
}
|
|
|
|
if len(body) > maxListBytes {
|
|
return nil, errTooLong
|
|
}
|
|
|
|
return parseBlacklist(body)
|
|
}
|
|
|
|
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
|
|
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
|
|
var answer struct {
|
|
Data *[]struct {
|
|
IPAddress string `json:"ipAddress"`
|
|
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
|
|
} `json:"data"`
|
|
}
|
|
|
|
err := json.Unmarshal(body, &answer)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
}
|
|
|
|
if answer.Data == nil {
|
|
return nil, errNoBlacklist
|
|
}
|
|
|
|
listed := make([]BlacklistEntry, 0, len(*answer.Data))
|
|
|
|
for i, entry := range *answer.Data {
|
|
addr, err := netip.ParseAddr(entry.IPAddress)
|
|
if err != nil || addr.Zone() != "" {
|
|
// The error leaves out what the entry holds, as it does a
|
|
// client's address.
|
|
return nil, fmt.Errorf(
|
|
"entry %d's ipAddress %w",
|
|
i+1,
|
|
errNotAddress,
|
|
)
|
|
}
|
|
|
|
listed = append(listed, BlacklistEntry{
|
|
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
|
|
})
|
|
}
|
|
|
|
return listed, nil
|
|
}
|
|
|
|
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
|
// and logs them.
|
|
func (a *AbuseIPDB) alert(reason string, err error) {
|
|
// Raised before it is logged, so that the alert is there once the log
|
|
// line is.
|
|
raiseFailure(a.params.Alerts, reason, AbuseIPDBSource, err)
|
|
a.params.ProcessLog.Warn(
|
|
reason,
|
|
"source",
|
|
AbuseIPDBSource,
|
|
"error",
|
|
err.Error(),
|
|
)
|
|
}
|