Request tags, with refusing and limits per tag (closes #175)

Each request gets sorted tags from the lookup, new address lists, the
reputation sources and the rule matches, logged as `tags`. New settings:
`SWW_TAG_LISTS`, `SWW_TAG_DENY`, `SWW_TAG_LIMIT_PERCENT`. They replace
`SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT` and
`SWW_DENIED_COUNTRIES`. New heuristics `tag_lists` and `tag_deny`; the
latter runs before the rate limits and again after the Core Rule Set.

Model: opus-5-5

docs/settings.md

@@ -94,7 +94,7 @@ between the two (see "The image's command" below).
 - `SWW_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
   exempt from rate limiting and the byte limits.
 - `SWW_DENY_NETS` (default empty): netblocks whose clients are refused (see step
-  4 in [`SPEC.md`](../SPEC.md)).
+  3 in [`SPEC.md`](../SPEC.md)).
 
 ## Request rate limits, per client
 
@@ -181,9 +181,7 @@ that no request can get stops the start. A request can get only these:
 - a `dnsbl=` tag of a zone in `SWW_DNSBL_ZONES`, in lower case, with a key
   masked as the `reputation` field shows it;
 - `abuseipdb`, while `SWW_ABUSEIPDB_KEY` is set;
-- `exploitsignature`, in `SWW_TAG_DENY` only, while `SWW_RULES_ENABLED` is
-  `true`, or while neither `SWW_WAF_MODE` nor `SWW_WAF_ANOMALY_THRESHOLD` is
-  `off`.
+- `exploitsignature`, in `SWW_TAG_DENY` only.
 
 `crowdsec` stops the start in both settings, because a CrowdSec decision refuses
 the request first.

internal/config/config.go

@@ -483,8 +483,7 @@ var (
 	errNoRequestGetsTag = errors.New(
 		"is a tag no request can get: it is not a name in SWW_TAG_LISTS, an " +
 			"asn= or country= tag, a dnsbl= tag of a zone in SWW_DNSBL_ZONES, " +
-			"abuseipdb with SWW_ABUSEIPDB_KEY set, or exploitsignature with " +
-			"rule files or the Core Rule Set on")
+			"or abuseipdb with SWW_ABUSEIPDB_KEY set")
 )
 
 // FromEnvironment reads the settings with lookupEnv, normally
@@ -1157,16 +1156,11 @@ func checkTag(cfg *Config, tag string) error {
 // requestCanGetTag reports whether anything cfg sets up can give a request tag.
 func requestCanGetTag(cfg *Config, tag string) bool {
 	isListName := func(list TagList) bool { return list.Name == tag }
-	if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
+	if isLookupTag(tag) || tag == ExploitSignatureTag ||
+		slices.ContainsFunc(cfg.TagLists, isListName) {
 		return true
 	}
 
-	if tag == ExploitSignatureTag {
-		coreRuleSetOn := cfg.WAFMode != WAFModeOff && cfg.WAFAnomalyThreshold != 0
-
-		return cfg.RulesEnabled || coreRuleSetOn
-	}
-
 	if tag == AbuseIPDBTag {
 		return cfg.AbuseIPDBKey != ""
 	}
@@ -1970,13 +1964,11 @@ func parseTagLists(value string) ([]TagList, error) {
 }
 
 // parseTags reads a comma-separated list of tags, which parseTag reads. A
-// tag listed twice is an error, which shows the tag as MaskZoneKey does.
+// tag listed twice is an error.
 func parseTags(value string) ([]string, error) {
 	items, err := parseList(value)
 	if err != nil {
-		// parseList's error, for an empty item, shows the whole value, keys
-		// included.
-		return nil, errEmptyItem
+		return nil, err
 	}
 
 	tags := make([]string, 0, len(items))
@@ -1988,7 +1980,7 @@ func parseTags(value string) ([]string, error) {
 		}
 
 		if slices.Contains(tags, tag) {
-			return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errListedTwice)
+			return nil, fmt.Errorf("%q %w", item, errListedTwice)
 		}
 
 		tags = append(tags, tag)
@@ -2033,9 +2025,7 @@ func parseTag(value string) (string, error) {
 func parseTagPercents(value string) (map[string]int64, error) {
 	items, err := parseList(value)
 	if err != nil {
-		// parseList's error, for an empty item, shows the whole value, keys
-		// included.
-		return nil, errEmptyItem
+		return nil, err
 	}
 
 	percents := make(map[string]int64, len(items))
@@ -2043,7 +2033,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
 	for _, item := range items {
 		colon := strings.LastIndex(item, ":")
 		if colon <= 0 {
-			return nil, fmt.Errorf("%q %w", MaskZoneKey(item), errNotTagPercentItem)
+			return nil, fmt.Errorf("%q %w", item, errNotTagPercentItem)
 		}
 
 		tag, err := parseTag(item[:colon])
@@ -2061,9 +2051,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
 		}
 
 		if _, listed := percents[tag]; listed {
-			// tag, since item[:colon] may end in a space, which MaskZoneKey
-			// would not mask.
-			return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
+			return nil, fmt.Errorf("%q %w", item[:colon], errListedTwice)
 		}
 
 		percents[tag] = percent

internal/config/config_test.go

@@ -208,9 +208,6 @@ const off = "off"
 // enabled is true, as a setting's value.
 const enabled = "true"
 
-// disabled is false, as a setting's value.
-const disabled = "false"
-
 // defaultLookupSource is the default of SWW_LOOKUP_SOURCE, and
 // fileSource the source that is the lookup database.
 const (
@@ -348,7 +345,7 @@ func TestValuesAsSet(t *testing.T) {
 		metricsToken:         token,
 		metricsTopN:          "10",
 		rulesDir:             "/srv/waf-rules",
-		rulesEnabled:         disabled,
+		rulesEnabled:         "false",
 	})
 
 	wantSettings(t, cfg, config.Config{
@@ -1102,7 +1099,7 @@ func TestInstanceNameNotUTF8StopsTheStart(t *testing.T) {
 	}
 }
 
-func TestCodeInAllowedCountriesAndTagDenyStopsTheStart(t *testing.T) {
+func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
 	_, err := config.FromEnvironment(environment{
@@ -1110,7 +1107,7 @@ func TestCodeInAllowedCountriesAndTagDenyStopsTheStart(t *testing.T) {
 		allowedCountries: "de,RU",
 	}.lookupEnv)
 	if err == nil {
-		t.Fatal("ru in SWW_EXCLUSIVELY_ALLOWED_COUNTRIES and SWW_TAG_DENY was accepted")
+		t.Fatal("ru on both country lists was accepted")
 	}
 
 	if !strings.HasPrefix(err.Error(), allowedCountries+": ") ||
@@ -1360,11 +1357,10 @@ func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
 	}
 }
 
-// torExitTag, zenTag and exploitTag are tags these tests name.
+// torExitTag and zenTag are tags these tests name.
 const (
 	torExitTag = "torexit"
 	zenTag     = "dnsbl=zen.example"
-	exploitTag = "exploitsignature"
 )
 
 func TestTagSettingsAsSet(t *testing.T) {
@@ -1436,15 +1432,7 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
 		{"crowdsec, though a list has its name", environment{
 			tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec",
 		}, `"crowdsec" cannot be used`},
-		{exploitTag, environment{tagDeny: exploitTag}, ""},
-		{
-			exploitTag + " with rule files off",
-			environment{tagDeny: exploitTag, rulesEnabled: disabled}, "",
-		},
-		{
-			exploitTag + " with the Core Rule Set off",
-			environment{tagDeny: exploitTag, wafMode: off}, "",
-		},
+		{"exploitsignature", environment{tagDeny: "exploitsignature"}, ""},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -1463,35 +1451,6 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
 	}
 }
 
-func TestErrorOfATagSettingLeavesOutAZoneKey(t *testing.T) {
-	t.Parallel()
-
-	const (
-		tag       = "dnsbl=" + spamhaus
-		emptyItem = "has an empty item in its list"
-		twice     = `"` + spamhausMasked + `" is listed twice`
-	)
-
-	for _, tc := range []struct{ name, value, want string }{
-		{tagDeny, tag + "," + tag, twice},
-		{tagDeny, tag + ",,", emptyItem},
-		{
-			tagLimitPercent, tag,
-			`"` + spamhausMasked + `" is not a tag, : and a percentage, ` +
-				"such as vpn:50 or asn=14061:25",
-		},
-		{tagLimitPercent, tag + ":50," + tag + " :25", twice},
-		{tagLimitPercent, tag + ":50,,", emptyItem},
-	} {
-		_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
-
-		want := tc.name + ": " + tc.want
-		if err == nil || err.Error() != want {
-			t.Errorf("%s=%s gave the error %v, want %s", tc.name, tc.value, err, want)
-		}
-	}
-}
-
 func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -2182,12 +2141,14 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{countryBytesPercent, "cn"}, {countryBytesPercent, "cn:"},
 		{countryBytesPercent, "cn:25:50"}, {countryBytesPercent, "china:25"},
 		{tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"},
-		{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"}, {tagDeny, "torexti"},
-		{tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
+		{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"},
+		{tagDeny, "torexti"}, {tagDeny, zenTag},
+		{tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
 		{tagLimitPercent, "torexti:50"}, {tagLimitPercent, "dnsbl=zen.example:50"},
 		{tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"},
-		{tagLimitPercent, "vpn :50"}, {tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"},
-		{tagLimitPercent, "vpn:"}, {tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
+		{tagLimitPercent, "vpn :50"},
+		{tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, {tagLimitPercent, "vpn:"},
+		{tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
 		{tagLimitPercent, "vpn:1.5"}, {tagLimitPercent, "vpn:50,VPN:25"},
 		{tagLimitPercent, "exploitsignature:50"}, {tagLimitPercent, "asn=abc:50"},
 		{tagLists, torExitTag}, {tagLists, "=https://example.com/a.txt"},
@@ -2206,25 +2167,6 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{logLevel, "INFO"}, {logLevel, "warning"}, {logLevel, "trace"}, {logLevel, ""},
 		{rulesEnabled, "yes"}, {rulesEnabled, "True"},
 	})
-
-	wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t)
-}
-
-// wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff checks that with rule
-// files off, and either setting turning off the Core Rule Set, the tag
-// exploitsignature in SWW_TAG_DENY stops the start.
-func wantExploitTagStopsTheStartWithRulesAndCoreRuleSetOff(t *testing.T) {
-	t.Helper()
-
-	for _, name := range []string{wafMode, wafAnomalyThreshold} {
-		_, err := config.FromEnvironment(environment{
-			tagDeny: exploitTag, rulesEnabled: disabled, name: off,
-		}.lookupEnv)
-		if err == nil || !strings.HasPrefix(err.Error(), tagDeny+": ") {
-			t.Errorf("%s=off with rule files off gave the error %v, want one naming %s",
-				name, err, tagDeny)
-		}
-	}
 }
 
 func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) {
@@ -2479,7 +2421,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		lookupSource:         defaultLookupSource,
 		lookupDBPath:         "",
 		lookupTimeout:        "800ms",
-		addLookupHeaders:     disabled,
+		addLookupHeaders:     "false",
 		allowedCountries:     "",
 		tagLists:             "",
 		tagDeny:              "",

internal/discriminator/reputation_test.go

@@ -168,7 +168,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(t *testing.T)
 	}
 }
 
-func TestBlocklistsComeAfterAllowedCountriesAndSkipAllowNets(t *testing.T) {
+func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
 	t.Parallel()
 
 	s, server, queue := startWithLookups(t, map[string]string{

internal/discriminator/tagdeny_test.go

@@ -46,22 +46,6 @@ func TestTagDenyRefusesAListTagBeforeTheRateLimitsCount(t *testing.T) {
 	}
 }
 
-func TestTagDenyRefusesAMixedCaseZoneByItsLowerCaseTag(t *testing.T) {
-	t.Parallel()
-
-	const mixedCaseZone = "DNSBL.example"
-
-	// The zone's tag is in lower case, however SWW_DNSBL_ZONES writes it.
-	s, server, _ := startWithLookups(t, map[string]string{
-		dnsblZones: mixedCaseZone, dnsblResolver: noResolver,
-		reputationAction: actionLog, tagDeny: "dnsbl=" + dnsblZone,
-	})
-	loadVerdicts(server, map[string][]string{fromDE: {mixedCaseZone}})
-
-	wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied),
-		"tag_deny", `{"tag": "dnsbl=`+dnsblZone+`"}`)
-}
-
 func TestTagDenyRefusesAnExploitSignatureOfARuleThatOnlyLogs(t *testing.T) {
 	t.Parallel()