sneak/smallwebwaf
1
0 Fork 0
Code Issues 12 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Remove the CrowdSec decision feed (closes #315)
check / check (push) Failing after 2m23s
Details

Browse Source 
sneak ruled to remove it. Gone: the heuristic crowdsec, the ban cause
crowdsec and its earlier_bans count, the tag crowdsec, the settings
SWW_WAF_CROWDSEC_LAPI_URL and SWW_WAF_CROWDSEC_LAPI_KEY, the decision
list fetch and parse in internal/reputation, their tests, and every
mention in the README, SPEC.md and docs/. EVALUATION.md keeps its
comparison of tools.

The ban ledger's internal ban function loses its expires parameter,
which only CrowdSec bans used.

Model: opus-5-5
This commit is contained in:
clawbot committed 2026-11-13 19:08:21 +00:00
1 parent 50c50bfcae
commit 478c3affad
27 files changed
+125 -1851
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/config/waf.go
+3 -79
View File
@@ -157,13 +157,6 @@ type WAF struct {
DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration
// CrowdSecDecisionsURL is where the decision list of the CrowdSec
// engine whose local API SWW_WAF_CROWDSEC_LAPI_URL names is fetched from:
// that URL with v1/decisions added to its path, "" while it is unset and
// none is. CrowdSecKey is the key the engine is asked with
// (SWW_WAF_CROWDSEC_LAPI_KEY).
CrowdSecDecisionsURL string
CrowdSecKey string
// BanResponse is the status a refused client is answered with, 403
// or 429, or 0 to close the connection without an answer
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -473,21 +466,12 @@ var (
"is not an IP address with an optional port, " +
"such as 192.0.2.53 or [2001:db8::53]:5353",
)
errNotLAPIURL = errors.New(
"is not an http or https URL without a user or a fragment, " +
"such as http://172.17.0.1:8080")
errNotRouteWatchURL = errors.New(
"is not an http or https URL without a user or a fragment, " +
"such as https://routewatch.example")
errNeedsLAPIKey = errors.New("the engine answers no request without it")
errLAPIKeyUnused = errors.New(
"it is sent only to the engine at that URL",
)
errAnotherList = errors.New(
errFetchedOtherwise = errors.New(
"is in SWW_WAF_BLOCKLIST_URLS or is SWW_WAF_ASN_LIMIT_PERCENT_URL too",
)
errFetchedOtherwise = errors.New("is in SWW_WAF_BLOCKLIST_URLS, is " +
"SWW_WAF_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too")
errNotTagList = errors.New(
"is not a name, = and an http or https URL, such as " +
"torexit=https://example.com/tor.txt")
@@ -502,8 +486,6 @@ var (
errExploitTag = errors.New(
"cannot lower limits: the rate limits count a request before its " +
"rules are matched")
errCrowdSecTag = errors.New(
"cannot be used: a CrowdSec decision refuses the request first")
errNoRequestGetsTag = errors.New(
"is a tag no request can get: it is not a name in SWW_WAF_TAG_LISTS, an " +
"asn= or country= tag, a dnsbl= tag of a zone in SWW_WAF_DNSBL_ZONES, " +
@@ -605,10 +587,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
"2000",
),
CrowdSecDecisionsURL: env.crowdSecDecisionsURL(
"SWW_WAF_CROWDSEC_LAPI_URL",
),
CrowdSecKey: env.secret("SWW_WAF_CROWDSEC_LAPI_KEY"),
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
LimitBanDuration: env.durationNotOff(
"SWW_WAF_LIMIT_BAN_DURATION",
@@ -755,7 +733,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
env.checkLookupDBPath(cfg)
env.checkCountriesAndLookups(cfg)
env.checkASNLimitPercentURL(cfg)
env.checkCrowdSec(cfg)
env.checkTagLists(cfg)
env.checkTags(cfg)
@@ -1061,26 +1038,6 @@ func (e *environment) resolver(name string) netip.AddrPort {
return resolver
}
// crowdSecDecisionsURL reads the setting that is the URL of the CrowdSec
// engine's local API, such as http://172.17.0.1:8080, and returns the URL
// its decision list is fetched from, that URL with v1/decisions added to
// its path, "" while it is unset or empty.
func (e *environment) crowdSecDecisionsURL(name string) string {
value := e.value(name, "")
if value == "" {
return ""
}
lapi, err := url.Parse(value)
if err != nil || !isHTTPURL(lapi) {
e.check(name, fmt.Errorf("%q %w", value, errNotLAPIURL))
return ""
}
return lapi.JoinPath("v1", "decisions").String()
}
// routeWatchURL reads the setting that is the URL of a routewatch instance,
// "" while it is unset or empty.
func (e *environment) routeWatchURL(name string) string {
@@ -1190,39 +1147,12 @@ func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
}
}
// checkCrowdSec refuses SWW_WAF_CROWDSEC_LAPI_URL without
// SWW_WAF_CROWDSEC_LAPI_KEY, the key without the URL, and a decision list
// that is fetched as another list too.
func (e *environment) checkCrowdSec(cfg *WAF) {
decisionsURL := cfg.CrowdSecDecisionsURL
switch {
case decisionsURL != "" && cfg.CrowdSecKey == "":
e.check("SWW_WAF_CROWDSEC_LAPI_URL", fmt.Errorf(
"is set while SWW_WAF_CROWDSEC_LAPI_KEY is unset; %w",
errNeedsLAPIKey,
))
case decisionsURL == "" && cfg.CrowdSecKey != "":
e.check("SWW_WAF_CROWDSEC_LAPI_KEY", fmt.Errorf(
"is set while SWW_WAF_CROWDSEC_LAPI_URL is unset; %w",
errLAPIKeyUnused,
))
case decisionsURL != "" && (slices.Contains(cfg.BlocklistURLs, decisionsURL) ||
decisionsURL == cfg.ASNLimitPercentURL):
e.check(
"SWW_WAF_CROWDSEC_LAPI_URL",
fmt.Errorf("gives the decision list %q, which %w",
decisionsURL, errAnotherList),
)
}
}
// checkTagLists refuses a list of SWW_WAF_TAG_LISTS whose URL is fetched as
// another list too: the file at a URL is read as one kind of list.
func (e *environment) checkTagLists(cfg *WAF) {
for _, list := range cfg.TagLists {
if slices.Contains(cfg.BlocklistURLs, list.URL) ||
list.URL == cfg.ASNLimitPercentURL || list.URL == cfg.CrowdSecDecisionsURL {
list.URL == cfg.ASNLimitPercentURL {
e.check(
"SWW_WAF_TAG_LISTS",
fmt.Errorf("%q %w", list.URL, errFetchedOtherwise),
@@ -1232,8 +1162,7 @@ func (e *environment) checkTagLists(cfg *WAF) {
}
// checkTags refuses a tag of SWW_WAF_TAG_DENY or SWW_WAF_TAG_LIMIT_PERCENT that would
// never apply: crowdsec, whose request is refused before tags are acted on, and
// any tag these settings cannot give a request.
// never apply: any tag these settings cannot give a request.
func (e *environment) checkTags(cfg *WAF) {
for _, tag := range cfg.TagDeny {
e.check("SWW_WAF_TAG_DENY", checkTag(cfg, tag))
@@ -1246,10 +1175,6 @@ func (e *environment) checkTags(cfg *WAF) {
// checkTag returns an error when tag would never apply, as checkTags says.
func checkTag(cfg *WAF, tag string) error {
if tag == CrowdSecTag {
return fmt.Errorf("%q %w", tag, errCrowdSecTag)
}
if !requestCanGetTag(cfg, tag) {
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
}
@@ -2010,7 +1935,6 @@ const (
DNSBLTagPrefix = "dnsbl="
PrefixTagPrefix = "prefix="
AbuseIPDBTag = "abuseipdb"
CrowdSecTag = "crowdsec"
// ExploitSignatureTag is the tag of a request a rule file or the Core
// Rule Set matched.
ExploitSignatureTag = "exploitsignature"
internal/config/waf_test.go
+4 -149
View File
@@ -65,8 +65,6 @@ const (
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
routeWatchURL = "SWW_WAF_ROUTEWATCH_URL"
crowdSecURL = "SWW_WAF_CROWDSEC_LAPI_URL"
crowdSecKey = "SWW_WAF_CROWDSEC_LAPI_KEY"
banResponse = "SWW_WAF_BAN_RESPONSE"
limitBanDuration = "SWW_WAF_LIMIT_BAN_DURATION"
limitBanRepeatWindow = "SWW_WAF_LIMIT_BAN_REPEAT_WINDOW"
@@ -1813,9 +1811,6 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
"abuseipdb with a key",
environment{abuseIPDBKey: "k", tagLimitPercent: "abuseipdb:50"}, "",
},
{"crowdsec, though a list has its name", environment{
tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec",
}, `"crowdsec" cannot be used`},
{exploitTag, environment{tagDeny: exploitTag}, ""},
{
exploitTag + " with rule files off",
@@ -1887,26 +1882,20 @@ func TestErrorOfATagSettingLeavesOutAZoneKey(t *testing.T) {
func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) {
t.Parallel()
const lapi = "http://172.17.0.1:8080"
for _, tc := range []struct {
name, listURL string
env environment
}{
{"a blocklist", dropURL, environment{blocklistURLs: dropURL}},
{"the file of AS:percent lines", asnURL, environment{asnLimitPercentURL: asnURL}},
{
"the CrowdSec decision list", lapi + "/v1/decisions",
environment{"SWW_WAF_CROWDSEC_LAPI_URL": lapi, "SWW_WAF_CROWDSEC_LAPI_KEY": "key"},
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
tc.env[tagLists] = "dup=" + tc.listURL
want := tagLists + `: "` + tc.listURL + `" is in SWW_WAF_BLOCKLIST_URLS, is ` +
"SWW_WAF_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too"
want := tagLists + `: "` + tc.listURL + `" is in SWW_WAF_BLOCKLIST_URLS ` +
"or is SWW_WAF_ASN_LIMIT_PERCENT_URL too"
_, err := config.WAFFromEnvironment(tc.env.viper())
if err == nil || err.Error() != want {
@@ -2456,138 +2445,6 @@ func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) {
}
}
func TestCrowdSecSettingsGiveTheDecisionListAndTheKey(t *testing.T) {
t.Parallel()
cfg := wafFromEnvironment(t, environment{})
if cfg.CrowdSecDecisionsURL != "" || cfg.CrowdSecKey != "" {
t.Errorf(
"by default, the decision list %q and the key %q, want neither",
cfg.CrowdSecDecisionsURL,
cfg.CrowdSecKey,
)
}
for lapi, want := range map[string]string{
"http://172.17.0.1:8080": "http://172.17.0.1:8080/v1/decisions",
"http://172.17.0.1:8080/": "http://172.17.0.1:8080/v1/decisions",
"https://crowdsec.example/lapi/": "https://crowdsec.example/lapi/v1/decisions",
"https://crowdsec.example:8443/x": "https://crowdsec.example:8443/x/v1/decisions",
} {
cfg := wafFromEnvironment(
t,
environment{crowdSecURL: lapi, crowdSecKey: token},
)
if cfg.CrowdSecDecisionsURL != want || cfg.CrowdSecKey != token {
t.Errorf(
"%s=%s gave the decision list %q and the key %q, want %s and %s",
crowdSecURL,
lapi,
cfg.CrowdSecDecisionsURL,
cfg.CrowdSecKey,
want,
token,
)
}
}
}
func TestInvalidCrowdSecSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
t.Parallel()
const (
lapi = "http://172.17.0.1:8080"
notLAPIURL = " is not an http or https URL without a user or a fragment, " +
"such as http://172.17.0.1:8080"
anotherList = `gives the decision list "` + lapi + `/v1/decisions", which is in ` +
`SWW_WAF_BLOCKLIST_URLS or is SWW_WAF_ASN_LIMIT_PERCENT_URL too`
)
for _, tc := range []struct {
name string
env environment
want string
}{
{
"a URL that is not http",
environment{crowdSecURL: "ftp://172.17.0.1", crowdSecKey: token},
crowdSecURL + `: "ftp://172.17.0.1"` + notLAPIURL,
},
{
"a URL with a user",
environment{crowdSecURL: "http://bouncer@172.17.0.1:8080", crowdSecKey: token},
crowdSecURL + `: "http://bouncer@172.17.0.1:8080"` + notLAPIURL,
},
{
"the URL without the key",
environment{crowdSecURL: lapi},
crowdSecURL + ": is set while " + crowdSecKey + " is unset; the engine " +
"answers no request without it",
},
{
"the key without the URL",
environment{crowdSecKey: token},
crowdSecKey + ": is set while " + crowdSecURL + " is unset; it is sent only " +
"to the engine at that URL",
},
{
"the decision list as a blocklist too",
environment{
crowdSecURL: lapi, crowdSecKey: token,
blocklistURLs: "https://lists.example/drop.txt," + lapi + "/v1/decisions",
},
crowdSecURL + ": " + anotherList,
},
{
"the decision list as the file of AS:percent lines too",
environment{
crowdSecURL: lapi + "/", crowdSecKey: token,
asnLimitPercentURL: lapi + "/v1/decisions",
},
crowdSecURL + ": " + anotherList,
},
// The key itself is never shown.
{
"a key with a control character",
environment{crowdSecURL: lapi, crowdSecKey: token + "\r"},
crowdSecKey + ": holds a control character, such as the carriage return " +
"of a Windows line end",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := config.WAFFromEnvironment(tc.env.viper())
if err == nil || err.Error() != tc.want {
t.Errorf("error %v, want %s", err, tc.want)
}
})
}
}
func TestCrowdSecKeyIsLoggedMasked(t *testing.T) {
t.Parallel()
cfg := wafFromEnvironment(t, environment{
crowdSecURL: "http://172.17.0.1:8080", crowdSecKey: token,
})
var out bytes.Buffer
slog.New(slog.NewJSONHandler(&out, nil)).
Info("starting", "settings", cfg)
logged := out.String()
if strings.Contains(logged, token) ||
!strings.Contains(logged, `"`+crowdSecKey+`":"********"`) ||
!strings.Contains(
logged,
`"`+crowdSecURL+`":"http://172.17.0.1:8080"`,
) {
t.Errorf("the key is not logged masked beside the URL: %s", logged)
}
}
func TestSizesAndOff(t *testing.T) {
t.Parallel()
@@ -2714,10 +2571,10 @@ func TestInvalidLookupTagOrLogValueStopsTheStart(t *testing.T) {
{tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"},
{tagDeny, "asn="},
{tagDeny, "torexit,TOREXIT"}, {tagDeny, "torexti"},
{tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
{tagDeny, zenTag}, {tagDeny, "abuseipdb"},
{tagLimitPercent, "torexti:50"},
{tagLimitPercent, "dnsbl=zen.example:50"},
{tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"},
{tagLimitPercent, "abuseipdb:50"},
{tagLimitPercent, "vpn :50"},
{tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"},
{tagLimitPercent, "vpn:"},
@@ -3091,8 +2948,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
ipReputationCacheTTL: defaultIPReputationCacheTTL,
dnsblAbuseIPDBTimeout: "2000",
routeWatchURL: "",
crowdSecURL: "",
crowdSecKey: "",
banResponse: "403",
limitBanDuration: "1h",
limitBanRepeatWindow: "24h",