DNS blocklists asked in the background, verdicts kept (closes #104)

Zones in SWWAF_DNSBL_ZONES are asked about each client in the background, through SWWAF_DNSBL_RESOLVER or the host's resolver; no request waits. Verdicts last SWWAF_REPUTATION_CACHE_TTL, kept in reputation.json. SWWAF_REPUTATION_ACTION (limit:25) denies, limits or logs a listed client; each zone listing it raises reputation_hit. A failed query gives no verdict, raises source_failure, and pauses the zone a minute. A zone's key, its first label under dq.spamhaus.net, is masked everywhere but reputation.json. Zones compare without regard to case. Judgement call: answers in 127.255.255.0/24 or outside 127.0.0.0/8 are failures. Judgement call: the minute's pause after a failure; at most 1,000 queries at once. Judgement call: one zone given with two keys stops the start as listed twice. Rule suppressed: paralleltest on the DNSBL tests (Go's resolver shares state across synctest bubbles), funlen on the test of every logged setting. Model: opus-5-5
2026-10-07 21:09:51 +02:00
parent 2b8c98ba1f
commit 0b0f207423
18 changed files with 2330 additions and 236 deletions
@@ -152,6 +152,21 @@ type Config struct {
 	BlocklistRefresh      time.Duration
 	BlocklistAction       string
 	BlocklistLimitPercent int64
+	// DNSBLZones are the DNSBL zones clients are asked about
+	// (SWWAF_DNSBL_ZONES), through DNSBLResolver (SWWAF_DNSBL_RESOLVER), or
+	// the host's resolver while that is the zero AddrPort.
+	// ReputationAction is what is done with a client a zone's verdict lists
+	// (SWWAF_REPUTATION_ACTION): deny, limit or log; for limit,
+	// ReputationLimitPercent is the percentage of every limit it gets. A
+	// verdict is used for ReputationCacheTTL after it was fetched
+	// (SWWAF_REPUTATION_CACHE_TTL), and a query may take ReputationTimeout
+	// (SWWAF_REPUTATION_TIMEOUT). Neither can be off.
+	DNSBLZones             []string
+	DNSBLResolver          netip.AddrPort
+	ReputationAction       string
+	ReputationLimitPercent int64
+	ReputationCacheTTL     time.Duration
+	ReputationTimeout      time.Duration
 	// BanResponse is the status a refused client is answered with, 403
 	// or 429, or 0 to close the connection without an answer
 	// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -366,6 +381,11 @@ var (
 	errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h")
 	errNotAction       = errors.New(
 		"is not deny, limit:<percent> such as limit:25, or log")
+	errNotZone     = errors.New("is not a DNS zone such as dnsbl.dronebl.org")
+	errZoneTooLong = errors.New("is longer than 189 characters, too long for the " +
+		"names IPv6 clients are asked about by")
+	errNotResolver = errors.New("is not an IP address with an optional port, " +
+		"such as 192.0.2.53 or [2001:db8::53]:5353")
 )

 // FromEnvironment reads the settings with lookupEnv, normally
@@ -418,6 +438,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		ASNLimitPercentURL:   env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"),
 		BlocklistURLs:        env.listURLs("SWWAF_BLOCKLIST_URLS"),
 		BlocklistRefresh:     env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"),
+		DNSBLZones:           env.zones("SWWAF_DNSBL_ZONES"),
+		DNSBLResolver:        env.resolver("SWWAF_DNSBL_RESOLVER"),
+		ReputationCacheTTL:   env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"),
+		ReputationTimeout:    env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"),
 		BanResponse:          env.banResponse("SWWAF_BAN_RESPONSE", "403"),
 		LimitBanDuration:     env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
 		LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
@@ -462,6 +486,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		cfg.InstanceName, cfg.LogRemoteURL != nil)
 	cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
 		"SWWAF_BLOCKLIST_ACTION", "deny")
+	cfg.ReputationAction, cfg.ReputationLimitPercent = env.action(
+		"SWWAF_REPUTATION_ACTION", "limit:25")

 	env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
 	env.checkLookupDBPath(cfg)
@@ -733,9 +759,9 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
 	return duration
 }

-// action reads a setting that is what is done with a client a list names:
-// deny, log, or limit:<percent>, which it returns as limit and the
-// percentage.
+// action reads a setting that is what is done with a client a blocklist
+// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
+// as limit and the percentage.
 func (e *environment) action(name, defaultValue string) (string, int64) {
 	value := e.value(name, defaultValue)
 	if value == "deny" || value == "log" {
@@ -752,6 +778,33 @@ func (e *environment) action(name, defaultValue string) (string, int64) {
 	return "limit", percent
 }

+// zones reads the setting that is the list of DNSBL zones. It is empty by
+// default. The log shows each zone with its key masked, as MaskZoneKey
+// masks it.
+func (e *environment) zones(name string) []string {
+	value, _ := e.lookup(name)
+	zones, err := parseZones(value)
+	e.check(name, err)
+
+	logged := make([]string, len(zones))
+	for i, zone := range zones {
+		logged[i] = MaskZoneKey(zone)
+	}
+
+	e.settings = append(e.settings, slog.String(name, strings.Join(logged, ",")))
+
+	return zones
+}
+
+// resolver reads the setting that is the resolver the DNSBL zones are
+// asked through, the zero AddrPort while it is unset or empty.
+func (e *environment) resolver(name string) netip.AddrPort {
+	resolver, err := parseResolver(e.value(name, ""))
+	e.check(name, err)
+
+	return resolver
+}
+
 // lookupSource reads the setting that is where clients are looked up:
 // geojs, file, or off.
 func (e *environment) lookupSource(name, defaultValue string) string {
@@ -1697,6 +1750,113 @@ func parseListURLs(value string) ([]string, error) {
 	return urls, nil
 }

+const (
+	// maxZoneLength is the most characters a DNSBL zone may have: 253, the
+	// most a DNS name may have, less the 64 that come before the zone in
+	// the name an IPv6 client is asked about by, its 32 hex digits each
+	// followed by a dot.
+	maxZoneLength = 189
+	// maxLabelLength is the most characters a label of a DNS name may have.
+	maxLabelLength = 63
+	// labelChars are the characters a label of a DNS zone may hold.
+	labelChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-"
+	// dnsPort is the port a resolver is asked on when SWWAF_DNSBL_RESOLVER
+	// gives none.
+	dnsPort = 53
+)
+
+// parseZones reads a comma-separated list of DNSBL zones, each a DNS name
+// such as dnsbl.dronebl.org: labels separated by dots, each of 1 to 63
+// letters, digits and hyphens, neither starting nor ending with a hyphen,
+// and at most maxZoneLength characters in all. Go's resolver takes any
+// other name for one that does not exist, so that the zone would list no
+// client. A zone listed twice is an error, whatever the case of its
+// letters, which DNS names ignore, and whatever its key, since
+// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
+// MaskZoneKey does.
+func parseZones(value string) ([]string, error) {
+	zones, err := parseList(value)
+	if err != nil {
+		// parseList's error, for an empty item, shows the whole value, keys
+		// included.
+		return nil, errEmptyItem
+	}
+
+	for i, zone := range zones {
+		shown := MaskZoneKey(zone)
+		listedBefore := slices.ContainsFunc(zones[:i], func(earlier string) bool {
+			return strings.EqualFold(MaskZoneKey(earlier), shown)
+		})
+
+		switch {
+		case len(zone) > maxZoneLength:
+			return nil, fmt.Errorf("%q %w", shown, errZoneTooLong)
+		case !isZone(zone):
+			return nil, fmt.Errorf("%q %w", shown, errNotZone)
+		case listedBefore:
+			return nil, fmt.Errorf("%q %w", shown, errListedTwice)
+		}
+	}
+
+	return zones, nil
+}
+
+// MaskZoneKey returns zone with ******** in place of its key, if it is a
+// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
+// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
+// other zone it returns as it is. A zone is shown so wherever it leaves
+// the process: in the log, the alerts and the metrics.
+func MaskZoneKey(zone string) string {
+	// DNS names ignore case, and a name may be written with a dot at its
+	// end.
+	name := strings.TrimSuffix(strings.ToLower(zone), ".")
+	if !strings.HasSuffix(name, ".dq.spamhaus.net") {
+		return zone
+	}
+
+	_, rest, _ := strings.Cut(zone, ".")
+
+	return masked + "." + rest
+}
+
+// isZone reports whether each label of zone is as parseZones takes it.
+func isZone(zone string) bool {
+	for label := range strings.SplitSeq(zone, ".") {
+		badChar := strings.ContainsFunc(label, func(char rune) bool {
+			return !strings.ContainsRune(labelChars, char)
+		})
+
+		if label == "" || len(label) > maxLabelLength || badChar ||
+			strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
+			return false
+		}
+	}
+
+	return true
+}
+
+// parseResolver reads the resolver the DNSBL zones are asked through: an
+// IP address with a port from 1 to 65535, such as 192.0.2.53:5353 or
+// [2001:db8::53]:5353, or without one, such as 192.0.2.53 or 2001:db8::53,
+// for port 53. An empty value is none, the zero AddrPort.
+func parseResolver(value string) (netip.AddrPort, error) {
+	if value == "" {
+		return netip.AddrPort{}, nil
+	}
+
+	resolver, err := netip.ParseAddrPort(value)
+	if err == nil && resolver.Port() != 0 {
+		return resolver, nil
+	}
+
+	addr, err := netip.ParseAddr(value)
+	if err != nil {
+		return netip.AddrPort{}, fmt.Errorf("%q %w", value, errNotResolver)
+	}
+
+	return netip.AddrPortFrom(addr, dnsPort), nil
+}
+
 // parseWebhookHeaders reads a comma-separated list of headers, each its
 // name, :, and its value, and returns them, and how the log shows them,
 // with each value as ********. An error names the item by its place in
@@ -61,6 +61,11 @@ const (
 	blocklistURLs           = "SWWAF_BLOCKLIST_URLS"
 	blocklistRefresh        = "SWWAF_BLOCKLIST_REFRESH"
 	blocklistAction         = "SWWAF_BLOCKLIST_ACTION"
+	dnsblZones              = "SWWAF_DNSBL_ZONES"
+	dnsblResolver           = "SWWAF_DNSBL_RESOLVER"
+	reputationAction        = "SWWAF_REPUTATION_ACTION"
+	reputationCacheTTL      = "SWWAF_REPUTATION_CACHE_TTL"
+	reputationTimeout       = "SWWAF_REPUTATION_TIMEOUT"
 	banResponse             = "SWWAF_BAN_RESPONSE"
 	limitBanDuration        = "SWWAF_LIMIT_BAN_DURATION"
 	limitBanRepeatWindow    = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
@@ -152,6 +157,9 @@ const (
 	defaultAlertCooldown = "15m"
 )

+// defaultReputationCacheTTL is the default of SWWAF_REPUTATION_CACHE_TTL.
+const defaultReputationCacheTTL = "24h"
+
 // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS.
 const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," +
 	"content-type,origin,range"
@@ -1322,6 +1330,202 @@ func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) {
 	}
 }

+// dronebl is a DNSBL zone, and spamhaus one of Spamhaus's, a name
+// containing spamhausKey, the key of its keyed query service, which the
+// log shows as spamhausMasked.
+const (
+	dronebl        = "dnsbl.dronebl.org"
+	spamhausKey    = "abcdefghijklmnopqrstuvwxyz"
+	spamhaus       = spamhausKey + ".xbl.dq.spamhaus.net"
+	spamhausMasked = "********.xbl.dq.spamhaus.net"
+)
+
+func TestDNSBLSettingsAsSet(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+	if len(cfg.DNSBLZones) != 0 || cfg.DNSBLResolver.IsValid() ||
+		cfg.ReputationAction != actionLimit || cfg.ReputationLimitPercent != 25 ||
+		cfg.ReputationCacheTTL != 24*time.Hour || cfg.ReputationTimeout != 2*time.Second {
+		t.Errorf("by default, the zones %v, the resolver %s, the action %s:%d, the TTL %s "+
+			"and the timeout %s, want no zone, no resolver, limit:25, 24h and 2s",
+			cfg.DNSBLZones,
+			cfg.DNSBLResolver, cfg.ReputationAction, cfg.ReputationLimitPercent,
+			cfg.ReputationCacheTTL, cfg.ReputationTimeout)
+	}
+
+	// The longest zone allowed, of 189 characters, with labels of 63, the
+	// longest allowed.
+	longest := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." +
+		strings.Repeat("c", 61)
+
+	for _, tc := range []struct {
+		zones, resolver, action string
+		// want are the zones, resolver, action and percent Config gives.
+		want         []string
+		wantResolver string
+		wantAction   string
+		wantPercent  int64
+	}{
+		{
+			dronebl + ", " + spamhaus, "192.0.2.53", actionDeny,
+			[]string{dronebl, spamhaus}, "192.0.2.53:53", actionDeny, 0,
+		},
+		{
+			longest, "192.0.2.53:5353", actionLog,
+			[]string{longest}, "192.0.2.53:5353", actionLog, 0,
+		},
+		{
+			"Dnsbl-1.Example", "2001:db8::53", "limit:10",
+			[]string{"Dnsbl-1.Example"}, "[2001:db8::53]:53", actionLimit, 10,
+		},
+		{
+			dronebl, "[2001:db8::53]:5353", "limit:0",
+			[]string{dronebl}, "[2001:db8::53]:5353", actionLimit, 0,
+		},
+	} {
+		cfg := fromEnvironment(t, environment{
+			dnsblZones: tc.zones, dnsblResolver: tc.resolver, reputationAction: tc.action,
+			reputationCacheTTL: "12h", reputationTimeout: "3s",
+		})
+
+		if !slices.Equal(cfg.DNSBLZones, tc.want) ||
+			cfg.DNSBLResolver.String() != tc.wantResolver ||
+			cfg.ReputationAction != tc.wantAction ||
+			cfg.ReputationLimitPercent != tc.wantPercent ||
+			cfg.ReputationCacheTTL != 12*time.Hour ||
+			cfg.ReputationTimeout != 3*time.Second {
+			t.Errorf("%s=%s, %s=%s and %s=%s gave %v, %s, %s:%d, %s and %s", dnsblZones,
+				tc.zones, dnsblResolver, tc.resolver, reputationAction, tc.action,
+				cfg.DNSBLZones, cfg.DNSBLResolver, cfg.ReputationAction,
+				cfg.ReputationLimitPercent, cfg.ReputationCacheTTL, cfg.ReputationTimeout)
+		}
+	}
+}
+
+func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
+	t.Parallel()
+
+	const (
+		notZone     = " is not a DNS zone such as dnsbl.dronebl.org"
+		notResolver = " is not an IP address with an optional port, such as 192.0.2.53 " +
+			"or [2001:db8::53]:5353"
+		notAboveZero = " is not a duration above zero, such as 1h or 7d"
+	)
+
+	label64 := strings.Repeat("a", 64) + ".example"
+	tooLong := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." +
+		strings.Repeat("c", 62)
+
+	for _, tc := range []struct{ name, value, want string }{
+		{dnsblZones, "dnsbl..example", `"dnsbl..example"` + notZone},
+		{dnsblZones, "dnsbl.example.", `"dnsbl.example."` + notZone},
+		{dnsblZones, "-dnsbl.example", `"-dnsbl.example"` + notZone},
+		{dnsblZones, "dnsbl-.example", `"dnsbl-.example"` + notZone},
+		{dnsblZones, "dns_bl.example", `"dns_bl.example"` + notZone},
+		{dnsblZones, label64, `"` + label64 + `"` + notZone},
+		{
+			dnsblZones, tooLong,
+			`"` + tooLong + `" is longer than 189 characters, too long for the names ` +
+				`IPv6 clients are asked about by`,
+		},
+		{
+			dnsblZones, dronebl + "," + spamhaus + "," + dronebl,
+			`"` + dronebl + `" is listed twice`,
+		},
+		// DNS names ignore case.
+		{dnsblZones, "dnsbl.example,DNSBL.example", `"DNSBL.example" is listed twice`},
+		{dnsblResolver, "resolver.example", `"resolver.example"` + notResolver},
+		{dnsblResolver, "192.0.2.53:0", `"192.0.2.53:0"` + notResolver},
+		{dnsblResolver, "192.0.2.53:65536", `"192.0.2.53:65536"` + notResolver},
+		{dnsblResolver, "[2001:db8::53]", `"[2001:db8::53]"` + notResolver},
+		{
+			reputationAction, "ban",
+			`"ban" is not deny, limit:<percent> such as limit:25, or log`,
+		},
+		{reputationCacheTTL, off, `"off"` + notAboveZero},
+		{reputationTimeout, "0s", `"0s"` + notAboveZero},
+	} {
+		t.Run(tc.name+"="+tc.value, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
+
+			want := tc.name + ": " + tc.want
+			if err == nil || err.Error() != want {
+				t.Errorf("error %v, want %s", err, want)
+			}
+		})
+	}
+}
+
+func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(t *testing.T) {
+	t.Parallel()
+
+	for zone, want := range map[string]string{
+		spamhaus:                  spamhausMasked,
+		spamhaus + ".":            spamhausMasked + ".",
+		"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
+		dronebl:                   dronebl,
+		"dq.spamhaus.net":         "dq.spamhaus.net",
+		spamhaus + ".example":     spamhaus + ".example",
+	} {
+		if got := config.MaskZoneKey(zone); got != want {
+			t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
+		}
+	}
+}
+
+func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{dnsblZones: dronebl + ", " + spamhaus})
+
+	var out bytes.Buffer
+
+	slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+	logged := out.String()
+	if strings.Contains(logged, spamhausKey) ||
+		!strings.Contains(logged, `"`+dnsblZones+`":"`+dronebl+","+spamhausMasked+`"`) {
+		t.Errorf("the zones are not logged with the key masked: %s", logged)
+	}
+
+	// Nor does an error that stops the start show a key, in any case.
+	const (
+		notZone   = " is not a DNS zone such as dnsbl.dronebl.org"
+		otherKey  = "zyxwvutsrqponmlkjihgfedcba"
+		otherZone = otherKey + ".xbl.dq.spamhaus.net"
+	)
+
+	// 205 characters, 187 with the key masked.
+	labels := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." +
+		strings.Repeat("c", 30) + ".xbl.dq.spamhaus.net"
+
+	for _, tc := range []struct{ value, want string }{
+		{spamhaus + ".", `"` + spamhausMasked + `."` + notZone},
+		{spamhausKey + "_.xbl.dq.spamhaus.net", `"` + spamhausMasked + `"` + notZone},
+		{
+			spamhausKey + "." + labels,
+			`"********.` + labels + `" is longer than 189 characters, too long ` +
+				`for the names IPv6 clients are asked about by`,
+		},
+		{
+			spamhaus + "," + strings.ToUpper(spamhaus),
+			`"********.XBL.DQ.SPAMHAUS.NET" is listed twice`,
+		},
+		{spamhaus + "," + otherZone, `"` + spamhausMasked + `" is listed twice`},
+		{spamhaus + ",,", "has an empty item in its list"},
+	} {
+		_, err := config.FromEnvironment(environment{dnsblZones: tc.value}.lookupEnv)
+
+		want := dnsblZones + ": " + tc.want
+		if err == nil || err.Error() != want {
+			t.Errorf("%s=%s gave the error %v, want %s", dnsblZones, tc.value, err, want)
+		}
+	}
+}
+
 func TestSizesAndOff(t *testing.T) {
 	t.Parallel()

@@ -1704,6 +1908,7 @@ func writeFile(t *testing.T, contents string) string {
 	return path
 }

+//nolint:funlen // one line for each setting, a list that grows with them
 func TestLogsEachSettingWithItsValue(t *testing.T) {
 	t.Parallel()

@@ -1749,6 +1954,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		blocklistURLs:           "",
 		blocklistRefresh:        "24h",
 		blocklistAction:         actionDeny,
+		dnsblZones:              "",
+		dnsblResolver:           "",
+		reputationAction:        "limit:25",
+		reputationCacheTTL:      defaultReputationCacheTTL,
+		reputationTimeout:       "2s",
 		banResponse:             "403",
 		limitBanDuration:        "1h",
 		limitBanRepeatWindow:    "24h",