CrowdSec decision list fetched, kept, and its clients banned until the decision ends (closes #106)

SWWAF_CROWDSEC_LAPI_URL and SWWAF_CROWDSEC_LAPI_KEY name an engine whose decision list, <url>/v1/decisions, is fetched every minute with the key in X-Api-Key, following no redirect, and kept as a blocklist is: used while a fetch fails, and across restarts through reputation.json. Ban decisions on an Ip or a Range end at the fetch time plus their duration. A listed client's request is refused and bans its netblock with the cause crowdsec until the decision ends; bans.json, ban notes and metrics take the cause. Judgement call: fetched every minute, not a setting. Judgement call: a crowdsec ban never lengthens a limit ban. Judgement call: a lifted crowdsec ban is remade while its decision lasts. Model: opus-5-5
2026-10-08 05:15:06 +02:00
parent 04e66d2069
commit 5f3fb48809
20 changed files with 1700 additions and 292 deletions
@@ -181,6 +181,13 @@ type Config struct {
 	ReputationLimitPercent int64
 	ReputationCacheTTL     time.Duration
 	ReputationTimeout      time.Duration
+	// CrowdSecDecisionsURL is where the decision list of the CrowdSec
+	// engine whose local API SWWAF_CROWDSEC_LAPI_URL names is fetched from:
+	// that URL with v1/decisions added to its path, "" while it is unset and
+	// none is. CrowdSecKey is the key the engine is asked with
+	// (SWWAF_CROWDSEC_LAPI_KEY).
+	CrowdSecDecisionsURL string
+	CrowdSecKey          string
 	// BanResponse is the status a refused client is answered with, 403
 	// or 429, or 0 to close the connection without an answer
 	// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -410,6 +417,13 @@ var (
 		"names IPv6 clients are asked about by")
 	errNotResolver = errors.New("is not an IP address with an optional port, " +
 		"such as 192.0.2.53 or [2001:db8::53]:5353")
+	errNotLAPIURL = errors.New(
+		"is not an http or https URL without a user or a fragment, " +
+			"such as http://172.17.0.1:8080")
+	errNeedsLAPIKey  = errors.New("the engine answers no request without it")
+	errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL")
+	errAnotherList   = errors.New(
+		"is in SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too")
 )

 // FromEnvironment reads the settings with lookupEnv, normally
@@ -471,6 +485,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		AbuseIPDBDailyBudget: env.numberNotOff("SWWAF_ABUSEIPDB_DAILY_BUDGET", "900"),
 		ReputationCacheTTL:   env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"),
 		ReputationTimeout:    env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"),
+		CrowdSecDecisionsURL: env.crowdSecDecisionsURL("SWWAF_CROWDSEC_LAPI_URL"),
+		CrowdSecKey:          env.secret("SWWAF_CROWDSEC_LAPI_KEY"),
 		BanResponse:          env.banResponse("SWWAF_BAN_RESPONSE", "403"),
 		LimitBanDuration:     env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
 		LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
@@ -523,6 +539,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 	env.checkLookupDBPath(cfg)
 	env.checkCountriesAndLookups(cfg)
 	env.checkASNLimitPercentURL(cfg)
+	env.checkCrowdSec(cfg)

 	if env.err != nil {
 		return nil, env.err
@@ -835,6 +852,26 @@ func (e *environment) resolver(name string) netip.AddrPort {
 	return resolver
 }

+// crowdSecDecisionsURL reads the setting that is the URL of the CrowdSec
+// engine's local API, such as http://172.17.0.1:8080, and returns the URL
+// its decision list is fetched from, that URL with v1/decisions added to
+// its path, "" while it is unset or empty.
+func (e *environment) crowdSecDecisionsURL(name string) string {
+	value := e.value(name, "")
+	if value == "" {
+		return ""
+	}
+
+	lapi, err := url.Parse(value)
+	if err != nil || !isHTTPURL(lapi) {
+		e.check(name, fmt.Errorf("%q %w", value, errNotLAPIURL))
+
+		return ""
+	}
+
+	return lapi.JoinPath("v1", "decisions").String()
+}
+
 // lookupSource reads the setting that is where clients are looked up:
 // geojs, file, or off.
 func (e *environment) lookupSource(name, defaultValue string) string {
@@ -912,6 +949,26 @@ func (e *environment) checkASNLimitPercentURL(cfg *Config) {
 	}
 }

+// checkCrowdSec refuses SWWAF_CROWDSEC_LAPI_URL without
+// SWWAF_CROWDSEC_LAPI_KEY, the key without the URL, and a decision list
+// that is fetched as another list too.
+func (e *environment) checkCrowdSec(cfg *Config) {
+	decisionsURL := cfg.CrowdSecDecisionsURL
+
+	switch {
+	case decisionsURL != "" && cfg.CrowdSecKey == "":
+		e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf(
+			"is set while SWWAF_CROWDSEC_LAPI_KEY is unset; %w", errNeedsLAPIKey))
+	case decisionsURL == "" && cfg.CrowdSecKey != "":
+		e.check("SWWAF_CROWDSEC_LAPI_KEY", fmt.Errorf(
+			"is set while SWWAF_CROWDSEC_LAPI_URL is unset; %w", errLAPIKeyUnused))
+	case decisionsURL != "" && (slices.Contains(cfg.BlocklistURLs, decisionsURL) ||
+		decisionsURL == cfg.ASNLimitPercentURL):
+		e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf("gives the decision list %q, which %w",
+			decisionsURL, errAnotherList))
+	}
+}
+
 // headerNames reads a setting that is a list of header names, and
 // returns them in lower case.
 func (e *environment) headerNames(name, defaultValue string) []string {
@@ -71,6 +71,8 @@ const (
 	reputationAction        = "SWWAF_REPUTATION_ACTION"
 	reputationCacheTTL      = "SWWAF_REPUTATION_CACHE_TTL"
 	reputationTimeout       = "SWWAF_REPUTATION_TIMEOUT"
+	crowdSecURL             = "SWWAF_CROWDSEC_LAPI_URL"
+	crowdSecKey             = "SWWAF_CROWDSEC_LAPI_KEY"
 	banResponse             = "SWWAF_BAN_RESPONSE"
 	limitBanDuration        = "SWWAF_LIMIT_BAN_DURATION"
 	limitBanRepeatWindow    = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
@@ -1653,6 +1655,121 @@ func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) {
 	}
 }

+func TestCrowdSecSettingsGiveTheDecisionListAndTheKey(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+	if cfg.CrowdSecDecisionsURL != "" || cfg.CrowdSecKey != "" {
+		t.Errorf("by default, the decision list %q and the key %q, want neither",
+			cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey)
+	}
+
+	for lapi, want := range map[string]string{
+		"http://172.17.0.1:8080":          "http://172.17.0.1:8080/v1/decisions",
+		"http://172.17.0.1:8080/":         "http://172.17.0.1:8080/v1/decisions",
+		"https://crowdsec.example/lapi/":  "https://crowdsec.example/lapi/v1/decisions",
+		"https://crowdsec.example:8443/x": "https://crowdsec.example:8443/x/v1/decisions",
+	} {
+		cfg := fromEnvironment(t, environment{crowdSecURL: lapi, crowdSecKey: token})
+		if cfg.CrowdSecDecisionsURL != want || cfg.CrowdSecKey != token {
+			t.Errorf("%s=%s gave the decision list %q and the key %q, want %s and %s",
+				crowdSecURL, lapi, cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey, want, token)
+		}
+	}
+}
+
+func TestInvalidCrowdSecSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
+	t.Parallel()
+
+	const (
+		lapi       = "http://172.17.0.1:8080"
+		notLAPIURL = " is not an http or https URL without a user or a fragment, " +
+			"such as http://172.17.0.1:8080"
+		anotherList = `gives the decision list "` + lapi + `/v1/decisions", which is in ` +
+			`SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too`
+	)
+
+	for _, tc := range []struct {
+		name string
+		env  environment
+		want string
+	}{
+		{
+			"a URL that is not http",
+			environment{crowdSecURL: "ftp://172.17.0.1", crowdSecKey: token},
+			crowdSecURL + `: "ftp://172.17.0.1"` + notLAPIURL,
+		},
+		{
+			"a URL with a user",
+			environment{crowdSecURL: "http://bouncer@172.17.0.1:8080", crowdSecKey: token},
+			crowdSecURL + `: "http://bouncer@172.17.0.1:8080"` + notLAPIURL,
+		},
+		{
+			"the URL without the key",
+			environment{crowdSecURL: lapi},
+			crowdSecURL + ": is set while " + crowdSecKey + " is unset; the engine " +
+				"answers no request without it",
+		},
+		{
+			"the key without the URL",
+			environment{crowdSecKey: token},
+			crowdSecKey + ": is set while " + crowdSecURL + " is unset; it is sent only " +
+				"to the engine at that URL",
+		},
+		{
+			"the decision list as a blocklist too",
+			environment{
+				crowdSecURL: lapi, crowdSecKey: token,
+				blocklistURLs: "https://lists.example/drop.txt," + lapi + "/v1/decisions",
+			},
+			crowdSecURL + ": " + anotherList,
+		},
+		{
+			"the decision list as the file of AS:percent lines too",
+			environment{
+				crowdSecURL: lapi + "/", crowdSecKey: token,
+				asnLimitPercentURL: lapi + "/v1/decisions",
+			},
+			crowdSecURL + ": " + anotherList,
+		},
+		// The key itself is never shown.
+		{
+			"a key with a control character",
+			environment{crowdSecURL: lapi, crowdSecKey: token + "\r"},
+			crowdSecKey + ": holds a control character, such as the carriage return " +
+				"of a Windows line end",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := config.FromEnvironment(tc.env.lookupEnv)
+			if err == nil || err.Error() != tc.want {
+				t.Errorf("error %v, want %s", err, tc.want)
+			}
+		})
+	}
+}
+
+func TestCrowdSecKeyIsLoggedMasked(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{
+		crowdSecURL: "http://172.17.0.1:8080", crowdSecKey: token,
+	})
+
+	var out bytes.Buffer
+
+	slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+	logged := out.String()
+	if strings.Contains(logged, token) ||
+		!strings.Contains(logged, `"`+crowdSecKey+`":"********"`) ||
+		!strings.Contains(logged, `"`+crowdSecURL+`":"http://172.17.0.1:8080"`) {
+		t.Errorf("the key is not logged masked beside the URL: %s", logged)
+	}
+}
+
 func TestSizesAndOff(t *testing.T) {
 	t.Parallel()

@@ -2096,6 +2213,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		reputationAction:        "limit:25",
 		reputationCacheTTL:      defaultReputationCacheTTL,
 		reputationTimeout:       "2s",
+		crowdSecURL:             "",
+		crowdSecKey:             "",
 		banResponse:             "403",
 		limitBanDuration:        "1h",
 		limitBanRepeatWindow:    "24h",