CrowdSec decision list fetched, kept, and its clients banned until the decision ends (closes #106)

SWWAF_CROWDSEC_LAPI_URL and SWWAF_CROWDSEC_LAPI_KEY name an engine whose
decision list, <url>/v1/decisions, is fetched every minute with the key in
X-Api-Key, following no redirect, and kept as a blocklist is: used while a
fetch fails, and across restarts through reputation.json. Ban decisions on an
Ip or a Range end at the fetch time plus their duration. A listed client's
request is refused and bans its netblock with the cause crowdsec until the
decision ends; bans.json, ban notes and metrics take the cause.

Judgement call: fetched every minute, not a setting.
Judgement call: a crowdsec ban never lengthens a limit ban.
Judgement call: a lifted crowdsec ban is remade while its decision lasts.

Model: opus-5-5

internal/reputation/reputation.go

@@ -1,8 +1,9 @@
 // Package reputation fetches the lists the settings name by URL: the
-// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines
-// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
-// whole, comment lines included, which is used while a fetch fails, and
-// when each was last tried. It also asks the DNSBL zones of
+// blocklists of SWWAF_BLOCKLIST_URLS, the file of AS:percent lines
+// SWWAF_ASN_LIMIT_PERCENT_URL names, and the decision list of the CrowdSec
+// engine SWWAF_CROWDSEC_LAPI_URL names. It keeps the last good copy of
+// each, whole, comment lines included, which is used while a fetch fails,
+// and when each was last tried. It also asks the DNSBL zones of
 // SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
 // clients with AbuseIPDB, and keeps their scores and the checks spent
 // today. The state package writes all of these to reputation.json and
@@ -11,6 +12,7 @@ package reputation
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -32,6 +34,10 @@ const (
 	maxListBytes = 16 << 20
 	// fetchTimeout bounds one fetch of a list.
 	fetchTimeout = time.Minute
+	// crowdSecRefresh is how long after the CrowdSec decision list was last
+	// fetched or tried it is fetched again: the engine is the operator's
+	// own, and makes and ends decisions all the time.
+	crowdSecRefresh = time.Minute
 	// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
 	// IPv4-mapped address.
 	mappedBits = 96
@@ -43,6 +49,8 @@ var (
 	errNotNetblock   = errors.New("is not an address or a netblock, such as 192.0.2.0/24")
 	errNotASNPercent = errors.New(
 		"is not an AS number, : and a percentage, such as AS64496:50")
+	errNotDecision = errors.New(
+		"does not give an address or a netblock and a duration, such as 4h0m0s")
 )
 
 // List is a list as reputation.json holds it: the URL it is fetched from,
@@ -63,8 +71,14 @@ type Params struct {
 	// (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset.
 	BlocklistURLs      []string
 	ASNLimitPercentURL string
+	// CrowdSecDecisionsURL is the CrowdSec decision list, "" while
+	// SWWAF_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey
+	// (SWWAF_CROWDSEC_LAPI_KEY).
+	CrowdSecDecisionsURL string
+	CrowdSecKey          string
 	// Refresh is how long after a list was last fetched or tried it is
-	// fetched again (SWWAF_BLOCKLIST_REFRESH).
+	// fetched again (SWWAF_BLOCKLIST_REFRESH), but for the CrowdSec decision
+	// list, which is fetched again crowdSecRefresh after.
 	Refresh time.Duration
 	// Now tells the time, normally time.Now in UTC.
 	Now func() time.Time
@@ -79,6 +93,10 @@ type Params struct {
 type Lists struct {
 	params     Params
 	httpClient *http.Client
+	// crowdSecClient fetches the CrowdSec decision list. It follows no
+	// redirect, so that the key goes to the engine alone: a redirect is a
+	// failure.
+	crowdSecClient *http.Client
 
 	mu sync.Mutex
 	// lists are by URL, one for each URL Params names.
@@ -95,17 +113,36 @@ type list struct {
 }
 
 // entries are what the lines of a copy say: for a blocklist, the netblocks
-// it names, with the lengths among them, and for the file of AS:percent
-// lines, the percentage it gives each AS number.
+// it names, with the lengths among them, for the file of AS:percent lines,
+// the percentage it gives each AS number, and for the CrowdSec decision
+// list, the decision on each netblock that ends last, with the lengths
+// among them.
 type entries struct {
 	netblocks map[netip.Prefix]bool
 	lengths   []int
 	percents  map[string]int64
+	decisions map[netip.Prefix]Decision
+}
+
+// Decision is a decision of the CrowdSec engine to ban a netblock: when
+// it ends, and the scenario that made it, such as crowdsecurity/ssh-bf.
+type Decision struct {
+	Expires  time.Time
+	Scenario string
 }
 
 // New returns the lists, without a copy of any yet.
 func New(params Params) *Lists {
-	l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}}
+	l := &Lists{
+		params:     params,
+		httpClient: &http.Client{},
+		crowdSecClient: &http.Client{
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+		lists: map[string]*list{},
+	}
 
 	for _, listURL := range l.URLs() {
 		l.lists[listURL] = &list{kept: List{URL: listURL}}
@@ -115,13 +152,18 @@ func New(params Params) *Lists {
 }
 
 // URLs returns the URL of every list: the blocklists' in the order
-// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL.
+// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL, then
+// the CrowdSec decision list's.
 func (l *Lists) URLs() []string {
 	urls := slices.Clone(l.params.BlocklistURLs)
 	if l.params.ASNLimitPercentURL != "" {
 		urls = append(urls, l.params.ASNLimitPercentURL)
 	}
 
+	if l.params.CrowdSecDecisionsURL != "" {
+		urls = append(urls, l.params.CrowdSecDecisionsURL)
+	}
+
 	return urls
 }
 
@@ -157,6 +199,37 @@ func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
 	return percent, listed
 }
 
+// CrowdSecDecision returns the decision of the copy of the CrowdSec
+// decision list on a netblock that holds addr and that ends last, and
+// whether it is still in force at now. A decision that has ended no
+// longer bans, even before the next fetch drops it.
+func (l *Lists) CrowdSecDecision(addr netip.Addr, now time.Time) (Decision, bool) {
+	if l.params.CrowdSecDecisionsURL == "" {
+		return Decision{}, false
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	kept := l.lists[l.params.CrowdSecDecisionsURL].entries
+
+	var last Decision
+
+	for _, length := range kept.lengths {
+		netblock, err := addr.Prefix(length)
+		if err != nil {
+			continue // an IPv6 netblock's length, past an IPv4 address's 32 bits
+		}
+
+		decision := kept.decisions[netblock]
+		if decision.Expires.After(last.Expires) {
+			last = decision
+		}
+	}
+
+	return last, now.Before(last.Expires)
+}
+
 // Fetched returns when the copy in use of the list at listURL was
 // fetched, or zero while there is none.
 func (l *Lists) Fetched(listURL string) time.Time {
@@ -174,10 +247,9 @@ func (l *Lists) Failures(listURL string) int {
 	return l.lists[listURL].failures
 }
 
-// Run fetches each list once Refresh has passed since it was last fetched
-// or tried, the later of the two, until ctx is done. A list never tried is
-// fetched at once, and so is one whose last try or copy, read from
-// reputation.json, is that old.
+// Run fetches each list once it is due, as due tells, until ctx is done. A
+// list never tried is fetched at once, and so is one that is due by its
+// last try or copy read from reputation.json.
 func (l *Lists) Run(ctx context.Context) {
 	if len(l.lists) == 0 {
 		return
@@ -225,11 +297,12 @@ func (l *Lists) Load(lists []List) error {
 	found := make(map[string]entries, len(lists))
 
 	for _, kept := range lists {
-		if _, named := l.lists[kept.URL]; !named {
-			continue
+		_, named := l.lists[kept.URL]
+		if !named || kept.Fetched.IsZero() {
+			continue // dropped, or a list tried but never fetched, without a copy
 		}
 
-		read, err := l.parse(kept.URL, kept.Lines)
+		read, err := l.parse(kept.URL, kept.Lines, kept.Fetched)
 		if err != nil {
 			return fmt.Errorf("the copy of %s: %w", kept.URL, err)
 		}
@@ -245,9 +318,8 @@ func (l *Lists) Load(lists []List) error {
 	}
 
 	for _, kept := range lists {
-		read, named := found[kept.URL]
-		if named {
-			l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read
+		if _, named := l.lists[kept.URL]; named {
+			l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, found[kept.URL]
 		}
 	}
 
@@ -276,7 +348,8 @@ func (l *Lists) fetchDue(ctx context.Context) time.Time {
 }
 
 // due returns when the list at listURL is to be fetched: Refresh after it
-// was last fetched or tried, the later of the two.
+// was last fetched or tried, the later of the two, or crowdSecRefresh
+// after for the CrowdSec decision list.
 func (l *Lists) due(listURL string) time.Time {
 	l.mu.Lock()
 	defer l.mu.Unlock()
@@ -288,6 +361,10 @@ func (l *Lists) due(listURL string) time.Time {
 		last = held.kept.Tried
 	}
 
+	if listURL == l.params.CrowdSecDecisionsURL {
+		return last.Add(crowdSecRefresh)
+	}
+
 	return last.Add(l.params.Refresh)
 }
 
@@ -298,14 +375,14 @@ func (l *Lists) due(listURL string) time.Time {
 // so that a restart waits for it: the server may have had its request.
 func (l *Lists) fetch(ctx context.Context, listURL string) {
 	lines, err := l.get(ctx, listURL)
+	now := l.params.Now()
 
 	var found entries
 	if err == nil {
-		found, err = l.parse(listURL, lines)
+		found, err = l.parse(listURL, lines, now)
 	}
 
 	cutOff := err != nil && ctx.Err() != nil
-	now := l.params.Now()
 
 	l.mu.Lock()
 
@@ -350,8 +427,11 @@ func raiseFailure(queue *alerts.Queue, reason, source string, err error) {
 	})
 }
 
-// get fetches the list at listURL, and returns its lines. An answer other
-// than 200, or a list longer than maxListBytes, is a failure.
+// get fetches the list at listURL, and returns its lines. The CrowdSec
+// decision list is fetched with CrowdSecKey in the header X-Api-Key, where
+// the engine looks for it, by crowdSecClient, which follows no redirect.
+// An answer other than 200, or a list longer than maxListBytes, is a
+// failure.
 func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
 	ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
 	defer cancel()
@@ -361,7 +441,14 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
 		return nil, fmt.Errorf("make the request: %w", err)
 	}
 
-	res, err := l.httpClient.Do(req)
+	client := l.httpClient
+
+	if listURL == l.params.CrowdSecDecisionsURL {
+		req.Header.Set("X-Api-Key", l.params.CrowdSecKey)
+		client = l.crowdSecClient
+	}
+
+	res, err := client.Do(req)
 	if err != nil {
 		// Do's error names the URL, which the log line and the alert name
 		// already: only what went wrong is kept.
@@ -393,16 +480,22 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
 	return lines, nil
 }
 
-// parse reads the lines of the list at listURL: those of a blocklist, or
-// of the file of AS:percent lines. Anything after a ; or a # on a line is
+// parse reads the lines of the list at listURL, fetched at fetched: those
+// of a blocklist, of the file of AS:percent lines, or of the CrowdSec
+// decision list. In the first two, anything after a ; or a # on a line is
 // left out, and so is a line left blank. Any other line that does not read
 // is an error naming it by its number.
-func (l *Lists) parse(listURL string, lines []string) (entries, error) {
-	if listURL == l.params.ASNLimitPercentURL {
+func (l *Lists) parse(
+	listURL string, lines []string, fetched time.Time,
+) (entries, error) {
+	switch listURL {
+	case l.params.ASNLimitPercentURL:
 		return parsePercents(lines)
+	case l.params.CrowdSecDecisionsURL:
+		return parseDecisions(lines, fetched)
+	default:
+		return parseNetblocks(lines)
 	}
-
-	return parseNetblocks(lines)
 }
 
 // parseNetblocks reads a blocklist's lines, each an address or a netblock
@@ -486,6 +579,56 @@ func parsePercents(lines []string) (entries, error) {
 	return found, nil
 }
 
+// parseDecisions reads the lines of the CrowdSec decision list fetched at
+// fetched: the engine's answer, a JSON list of its decisions in force,
+// null while it has none. A decision of the type ban whose scope is Ip or
+// Range, as CrowdSec names them, bans its value, an address or a netblock
+// as parseNetblock reads it, until its duration, the time it had left as
+// the engine answered, has passed since fetched. Any other decision, such
+// as one to show a captcha or one on a country, is left out. A decision
+// to ban whose value or duration does not read is an error naming it by
+// its number.
+func parseDecisions(lines []string, fetched time.Time) (entries, error) {
+	var answer []struct {
+		Duration string `json:"duration"`
+		Scenario string `json:"scenario"`
+		Scope    string `json:"scope"`
+		Type     string `json:"type"`
+		Value    string `json:"value"`
+	}
+
+	err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &answer)
+	if err != nil {
+		return entries{}, fmt.Errorf("read the answer: %w", err)
+	}
+
+	found := entries{decisions: map[netip.Prefix]Decision{}}
+
+	for i, decision := range answer {
+		if decision.Type != "ban" || (decision.Scope != "Ip" && decision.Scope != "Range") {
+			continue
+		}
+
+		netblock, ok := parseNetblock(decision.Value)
+
+		duration, err := time.ParseDuration(decision.Duration)
+		if !ok || err != nil {
+			return entries{}, fmt.Errorf("decision %d %w", i+1, errNotDecision)
+		}
+
+		expires := fetched.Add(duration)
+		if expires.After(found.decisions[netblock].Expires) {
+			found.decisions[netblock] = Decision{Expires: expires, Scenario: decision.Scenario}
+		}
+
+		if !slices.Contains(found.lengths, netblock.Bits()) {
+			found.lengths = append(found.lengths, netblock.Bits())
+		}
+	}
+
+	return found, nil
+}
+
 // withoutComment returns line without anything after a ; or a #, and
 // without the spaces around what is left.
 func withoutComment(line string) string {