Request tags, with refusing and limits per tag (closes #175)

Each request gets sorted tags from the lookup, new address lists, the
reputation sources and the rule matches, logged as `tags`. New settings:
`SWW_TAG_LISTS`, `SWW_TAG_DENY`, `SWW_TAG_LIMIT_PERCENT`. They replace
`SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT` and
`SWW_DENIED_COUNTRIES`. New heuristics `tag_lists` and `tag_deny`; the
latter runs before the rate limits and again after the Core Rule Set.

Model: opus-5-5

docs/settings.md

@@ -94,7 +94,7 @@ between the two (see "The image's command" below).
 - `SWW_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
   exempt from rate limiting and the byte limits.
 - `SWW_DENY_NETS` (default empty): netblocks whose clients are refused (see step
-  3 in [`SPEC.md`](../SPEC.md)).
+  4 in [`SPEC.md`](../SPEC.md)).
 
 ## Request rate limits, per client
 
@@ -181,7 +181,9 @@ that no request can get stops the start. A request can get only these:
 - a `dnsbl=` tag of a zone in `SWW_DNSBL_ZONES`, in lower case, with a key
   masked as the `reputation` field shows it;
 - `abuseipdb`, while `SWW_ABUSEIPDB_KEY` is set;
-- `exploitsignature`, in `SWW_TAG_DENY` only.
+- `exploitsignature`, in `SWW_TAG_DENY` only, while `SWW_RULES_ENABLED` is
+  `true`, or while neither `SWW_WAF_MODE` nor `SWW_WAF_ANOMALY_THRESHOLD` is
+  `off`.
 
 `crowdsec` stops the start in both settings, because a CrowdSec decision refuses
 the request first.

internal/config/config.go

@@ -483,7 +483,8 @@ var (
 	errNoRequestGetsTag = errors.New(
 		"is a tag no request can get: it is not a name in SWW_TAG_LISTS, an " +
 			"asn= or country= tag, a dnsbl= tag of a zone in SWW_DNSBL_ZONES, " +
-			"or abuseipdb with SWW_ABUSEIPDB_KEY set")
+			"abuseipdb with SWW_ABUSEIPDB_KEY set, or exploitsignature with " +
+			"rule files or the Core Rule Set on")
 )
 
 // FromEnvironment reads the settings with lookupEnv, normally
@@ -1156,11 +1157,16 @@ func checkTag(cfg *Config, tag string) error {
 // requestCanGetTag reports whether anything cfg sets up can give a request tag.
 func requestCanGetTag(cfg *Config, tag string) bool {
 	isListName := func(list TagList) bool { return list.Name == tag }
-	if isLookupTag(tag) || tag == ExploitSignatureTag ||
+	if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
-		slices.ContainsFunc(cfg.TagLists, isListName) {
 		return true
 	}
 
+	if tag == ExploitSignatureTag {
+		coreRuleSetOn := cfg.WAFMode != WAFModeOff && cfg.WAFAnomalyThreshold != 0
+
+		return cfg.RulesEnabled || coreRuleSetOn
+	}
+
 	if tag == AbuseIPDBTag {
 		return cfg.AbuseIPDBKey != ""
 	}
@@ -1964,11 +1970,13 @@ func parseTagLists(value string) ([]TagList, error) {
 }
 
 // parseTags reads a comma-separated list of tags, which parseTag reads. A
-// tag listed twice is an error.
+// tag listed twice is an error, which shows the tag as MaskZoneKey does.
 func parseTags(value string) ([]string, error) {
 	items, err := parseList(value)
 	if err != nil {
-		return nil, err
+		// parseList's error, for an empty item, shows the whole value, keys
+		// included.
+		return nil, errEmptyItem
 	}
 
 	tags := make([]string, 0, len(items))
@@ -1980,7 +1988,7 @@ func parseTags(value string) ([]string, error) {
 		}
 
 		if slices.Contains(tags, tag) {
-			return nil, fmt.Errorf("%q %w", item, errListedTwice)
+			return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errListedTwice)
 		}
 
 		tags = append(tags, tag)
@@ -2025,7 +2033,9 @@ func parseTag(value string) (string, error) {
 func parseTagPercents(value string) (map[string]int64, error) {
 	items, err := parseList(value)
 	if err != nil {
-		return nil, err
+		// parseList's error, for an empty item, shows the whole value, keys
+		// included.
+		return nil, errEmptyItem
 	}
 
 	percents := make(map[string]int64, len(items))
@@ -2033,7 +2043,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
 	for _, item := range items {
 		colon := strings.LastIndex(item, ":")
 		if colon <= 0 {
-			return nil, fmt.Errorf("%q %w", item, errNotTagPercentItem)
+			return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errNotTagPercentItem)
 		}
 
 		tag, err := parseTag(item[:colon])
@@ -2051,7 +2061,9 @@ func parseTagPercents(value string) (map[string]int64, error) {
 		}
 
 		if _, listed := percents[tag]; listed {
-			return nil, fmt.Errorf("%q %w", item[:colon], errListedTwice)
+			// tag, since item[:colon] may end in a space, which MaskZoneKey
+			// would not mask.
+			return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
 		}
 
 		percents[tag] = percent

internal/config/config_test.go

@@ -208,6 +208,9 @@ const off = "off"
 // enabled is true, as a setting's value.
 const enabled = "true"
 
+// disabled is false, as a setting's value.
+const disabled = "false"
+
 // defaultLookupSource is the default of SWW_LOOKUP_SOURCE, and
 // fileSource the source that is the lookup database.
 const (
@@ -345,7 +348,7 @@ func TestValuesAsSet(t *testing.T) {
 		metricsToken:         token,
 		metricsTopN:          "10",
 		rulesDir:             "/srv/waf-rules",
-		rulesEnabled:         "false",
+		rulesEnabled:         disabled,
 	})
 
 	wantSettings(t, cfg, config.Config{
@@ -1099,7 +1102,7 @@ func TestInstanceNameNotUTF8StopsTheStart(t *testing.T) {
 	}
 }
 
-func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
+func TestCodeInAllowedCountriesAndTagDenyStopsTheStart(t *testing.T) {
 	t.Parallel()
 
 	_, err := config.FromEnvironment(environment{
@@ -1107,7 +1110,7 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 		allowedCountries: "de,RU",
 	}.lookupEnv)
 	if err == nil {
-		t.Fatal("ru on both country lists was accepted")
+		t.Fatal("ru in SWW_EXCLUSIVELY_ALLOWED_COUNTRIES and SWW_TAG_DENY was accepted")
 	}
 
 	if !strings.HasPrefix(err.Error(), allowedCountries+": ") ||
@@ -1357,10 +1360,11 @@ func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
 	}
 }
 
-// torExitTag and zenTag are tags these tests name.
+// torExitTag, zenTag and exploitTag are tags these tests name.
 const (
 	torExitTag = "torexit"
 	zenTag     = "dnsbl=zen.example"
+	exploitTag = "exploitsignature"
 )
 
 func TestTagSettingsAsSet(t *testing.T) {
@@ -1432,7 +1436,15 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
 		{"crowdsec, though a list has its name", environment{
 			tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec",
 		}, `"crowdsec" cannot be used`},
-		{"exploitsignature", environment{tagDeny: "exploitsignature"}, ""},
+		{exploitTag, environment{tagDeny: exploitTag}, ""},
+		{
+			exploitTag + " with rule files off",
+			environment{tagDeny: exploitTag, rulesEnabled: disabled}, "",
+		},
+		{
+			exploitTag + " with the Core Rule Set off",
+			environment{tagDeny: exploitTag, wafMode: off}, "",
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -1451,6 +1463,35 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
 	}
 }
 
+func TestErrorOfATagSettingLeavesOutAZoneKey(t *testing.T) {
+	t.Parallel()
+
+	const (
+		tag       = "dnsbl=" + spamhaus
+		emptyItem = "has an empty item in its list"
+		twice     = `"` + spamhausMasked + `" is listed twice`
+	)
+
+	for _, tc := range []struct{ name, value, want string }{
+		{tagDeny, tag + "," + tag, twice},
+		{tagDeny, tag + ",,", emptyItem},
+		{
+			tagLimitPercent, tag,
+			`"` + spamhausMasked + `" is not a tag, : and a percentage, ` +
+				"such as vpn:50 or asn=14061:25",
+		},
+		{tagLimitPercent, tag + ":50," + tag + " :25", twice},
+		{tagLimitPercent, tag + ":50,,", emptyItem},
+	} {
+		_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
+
+		want := tc.name + ": " + tc.want
+		if err == nil || err.Error() != want {
+			t.Errorf("%s=%s gave the error %v, want %s", tc.name, tc.value, err, want)
+		}
+	}
+}
+
 func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -2141,14 +2182,12 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{countryBytesPercent, "cn"}, {countryBytesPercent, "cn:"},
 		{countryBytesPercent, "cn:25:50"}, {countryBytesPercent, "china:25"},
 		{tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"},
-		{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"},
+		{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"}, {tagDeny, "torexti"},
-		{tagDeny, "torexti"}, {tagDeny, zenTag},
+		{tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
-		{tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
 		{tagLimitPercent, "torexti:50"}, {tagLimitPercent, "dnsbl=zen.example:50"},
 		{tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"},
-		{tagLimitPercent, "vpn :50"},
+		{tagLimitPercent, "vpn :50"}, {tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"},
-		{tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, {tagLimitPercent, "vpn:"},
+		{tagLimitPercent, "vpn:"}, {tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
-		{tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
 		{tagLimitPercent, "vpn:1.5"}, {tagLimitPercent, "vpn:50,VPN:25"},
 		{tagLimitPercent, "exploitsignature:50"}, {tagLimitPercent, "asn=abc:50"},
 		{tagLists, torExitTag}, {tagLists, "=https://example.com/a.txt"},
@@ -2167,6 +2206,25 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{logLevel, "INFO"}, {logLevel, "warning"}, {logLevel, "trace"}, {logLevel, ""},
 		{rulesEnabled, "yes"}, {rulesEnabled, "True"},
 	})
+
+	wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t)
+}
+
+// wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff checks that with rule
+// files off, and either setting turning off the Core Rule Set, the tag
+// exploitsignature in SWW_TAG_DENY stops the start.
+func wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t *testing.T) {
+	t.Helper()
+
+	for _, name := range []string{wafMode, wafAnomalyThreshold} {
+		_, err := config.FromEnvironment(environment{
+			tagDeny: exploitTag, rulesEnabled: disabled, name: off,
+		}.lookupEnv)
+		if err == nil || !strings.HasPrefix(err.Error(), tagDeny+": ") {
+			t.Errorf("%s=off with rule files off gave the error %v, want one naming %s",
+				name, err, tagDeny)
+		}
+	}
 }
 
 func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) {
@@ -2421,7 +2479,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		lookupSource:         defaultLookupSource,
 		lookupDBPath:         "",
 		lookupTimeout:        "800ms",
-		addLookupHeaders:     "false",
+		addLookupHeaders:     disabled,
 		allowedCountries:     "",
 		tagLists:             "",
 		tagDeny:              "",

internal/discriminator/reputation_test.go

@@ -168,7 +168,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(t *testing.T)
 	}
 }
 
-func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
+func TestBlocklistsComeAfterAllowedCountriesAndSkipAllowNets(t *testing.T) {
 	t.Parallel()
 
 	s, server, queue := startWithLookups(t, map[string]string{

internal/discriminator/tagdeny_test.go

@@ -46,6 +46,22 @@ func TestTagDenyRefusesAListTagBeforeTheRateLimitsCount(t *testing.T) {
 	}
 }
 
+func TestTagDenyRefusesAMixedCaseZoneByItsLowerCaseTag(t *testing.T) {
+	t.Parallel()
+
+	const mixedCaseZone = "DNSBL.example"
+
+	// The zone's tag is in lower case, however SWW_DNSBL_ZONES writes it.
+	s, server, _ := startWithLookups(t, map[string]string{
+		dnsblZones: mixedCaseZone, dnsblResolver: noResolver,
+		reputationAction: actionLog, tagDeny: "dnsbl=" + dnsblZone,
+	})
+	loadVerdicts(server, map[string][]string{fromDE: {mixedCaseZone}})
+
+	wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied),
+		"tag_deny", `{"tag": "dnsbl=`+dnsblZone+`"}`)
+}
+
 func TestTagDenyRefusesAnExploitSignatureOfARuleThatOnlyLogs(t *testing.T) {
 	t.Parallel()