DNS blocklists asked in the background, verdicts kept (closes #104)
check / check (push) Waiting to run
check / check (push) Waiting to run
Work in progress. Model: opus-5-5
This commit is contained in:
@@ -46,10 +46,11 @@ const (
|
||||
EventAnomaly = "anomaly"
|
||||
// EventWAFBlock comes with the Core Rule Set; nothing raises it yet.
|
||||
EventWAFBlock = "waf_block"
|
||||
// EventReputationHit is a request whose client a blocklist lists.
|
||||
// EventReputationHit is a request whose client a blocklist or a DNSBL
|
||||
// zone lists.
|
||||
EventReputationHit = "reputation_hit"
|
||||
// EventSourceFailure is GeoJS failing or refusing smallwebwaf, or a
|
||||
// fetch of a list failing.
|
||||
// EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch
|
||||
// of a list failing, or a query to a DNSBL zone failing or refused.
|
||||
EventSourceFailure = "source_failure"
|
||||
// EventFileError is a rule file or state file edited while smallwebwaf
|
||||
// runs that does not parse, a replacement of the lookup database that
|
||||
|
||||
+126
-3
@@ -152,6 +152,21 @@ type Config struct {
|
||||
BlocklistRefresh time.Duration
|
||||
BlocklistAction string
|
||||
BlocklistLimitPercent int64
|
||||
// DNSBLZones are the DNSBL zones clients are asked about
|
||||
// (SWWAF_DNSBL_ZONES), through DNSBLResolver (SWWAF_DNSBL_RESOLVER), or
|
||||
// the host's resolver while that is the zero AddrPort.
|
||||
// ReputationAction is what is done with a client a zone's verdict lists
|
||||
// (SWWAF_REPUTATION_ACTION): deny, limit or log; for limit,
|
||||
// ReputationLimitPercent is the percentage of every limit it gets. A
|
||||
// verdict is used for ReputationCacheTTL after it was fetched
|
||||
// (SWWAF_REPUTATION_CACHE_TTL), and a query may take ReputationTimeout
|
||||
// (SWWAF_REPUTATION_TIMEOUT). Neither can be off.
|
||||
DNSBLZones []string
|
||||
DNSBLResolver netip.AddrPort
|
||||
ReputationAction string
|
||||
ReputationLimitPercent int64
|
||||
ReputationCacheTTL time.Duration
|
||||
ReputationTimeout time.Duration
|
||||
// BanResponse is the status a refused client is answered with, 403
|
||||
// or 429, or 0 to close the connection without an answer
|
||||
// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
|
||||
@@ -366,6 +381,11 @@ var (
|
||||
errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h")
|
||||
errNotAction = errors.New(
|
||||
"is not deny, limit:<percent> such as limit:25, or log")
|
||||
errNotZone = errors.New("is not a DNS zone such as dnsbl.dronebl.org")
|
||||
errZoneTooLong = errors.New("is longer than 189 characters, too long for the " +
|
||||
"names IPv6 clients are asked about by")
|
||||
errNotResolver = errors.New("is not an IP address with an optional port, " +
|
||||
"such as 192.0.2.53 or [2001:db8::53]:5353")
|
||||
)
|
||||
|
||||
// FromEnvironment reads the settings with lookupEnv, normally
|
||||
@@ -418,6 +438,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
ASNLimitPercentURL: env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"),
|
||||
BlocklistURLs: env.listURLs("SWWAF_BLOCKLIST_URLS"),
|
||||
BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"),
|
||||
DNSBLZones: env.zones("SWWAF_DNSBL_ZONES"),
|
||||
DNSBLResolver: env.resolver("SWWAF_DNSBL_RESOLVER"),
|
||||
ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"),
|
||||
ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"),
|
||||
BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"),
|
||||
LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
|
||||
LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
|
||||
@@ -462,6 +486,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||
"SWWAF_BLOCKLIST_ACTION", "deny")
|
||||
cfg.ReputationAction, cfg.ReputationLimitPercent = env.action(
|
||||
"SWWAF_REPUTATION_ACTION", "limit:25")
|
||||
|
||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||
env.checkLookupDBPath(cfg)
|
||||
@@ -733,9 +759,9 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
||||
return duration
|
||||
}
|
||||
|
||||
// action reads a setting that is what is done with a client a list names:
|
||||
// deny, log, or limit:<percent>, which it returns as limit and the
|
||||
// percentage.
|
||||
// action reads a setting that is what is done with a client a blocklist
|
||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||
// as limit and the percentage.
|
||||
func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||
value := e.value(name, defaultValue)
|
||||
if value == "deny" || value == "log" {
|
||||
@@ -752,6 +778,24 @@ func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||
return "limit", percent
|
||||
}
|
||||
|
||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||
// default.
|
||||
func (e *environment) zones(name string) []string {
|
||||
zones, err := parseZones(e.value(name, ""))
|
||||
e.check(name, err)
|
||||
|
||||
return zones
|
||||
}
|
||||
|
||||
// resolver reads the setting that is the resolver the DNSBL zones are
|
||||
// asked through, the zero AddrPort while it is unset or empty.
|
||||
func (e *environment) resolver(name string) netip.AddrPort {
|
||||
resolver, err := parseResolver(e.value(name, ""))
|
||||
e.check(name, err)
|
||||
|
||||
return resolver
|
||||
}
|
||||
|
||||
// lookupSource reads the setting that is where clients are looked up:
|
||||
// geojs, file, or off.
|
||||
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||
@@ -1697,6 +1741,85 @@ func parseListURLs(value string) ([]string, error) {
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
const (
|
||||
// maxZoneLength is the most characters a DNSBL zone may have: 253, the
|
||||
// most a DNS name may have, less the 64 that come before the zone in
|
||||
// the name an IPv6 client is asked about by, its 32 hex digits each
|
||||
// followed by a dot.
|
||||
maxZoneLength = 189
|
||||
// maxLabelLength is the most characters a label of a DNS name may have.
|
||||
maxLabelLength = 63
|
||||
// labelChars are the characters a label of a DNS zone may hold.
|
||||
labelChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-"
|
||||
// dnsPort is the port a resolver is asked on when SWWAF_DNSBL_RESOLVER
|
||||
// gives none.
|
||||
dnsPort = 53
|
||||
)
|
||||
|
||||
// parseZones reads a comma-separated list of DNSBL zones, each a DNS name
|
||||
// such as dnsbl.dronebl.org: labels separated by dots, each of 1 to 63
|
||||
// letters, digits and hyphens, neither starting nor ending with a hyphen,
|
||||
// and at most maxZoneLength characters in all. Go's resolver takes any
|
||||
// other name for one that does not exist, so that the zone would list no
|
||||
// client. A zone listed twice is an error.
|
||||
func parseZones(value string) ([]string, error) {
|
||||
zones, err := parseList(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, zone := range zones {
|
||||
switch {
|
||||
case len(zone) > maxZoneLength:
|
||||
return nil, fmt.Errorf("%q %w", zone, errZoneTooLong)
|
||||
case !isZone(zone):
|
||||
return nil, fmt.Errorf("%q %w", zone, errNotZone)
|
||||
case slices.Contains(zones[:i], zone):
|
||||
return nil, fmt.Errorf("%q %w", zone, errListedTwice)
|
||||
}
|
||||
}
|
||||
|
||||
return zones, nil
|
||||
}
|
||||
|
||||
// isZone reports whether each label of zone is as parseZones takes it.
|
||||
func isZone(zone string) bool {
|
||||
for label := range strings.SplitSeq(zone, ".") {
|
||||
badChar := strings.ContainsFunc(label, func(char rune) bool {
|
||||
return !strings.ContainsRune(labelChars, char)
|
||||
})
|
||||
|
||||
if label == "" || len(label) > maxLabelLength || badChar ||
|
||||
strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// parseResolver reads the resolver the DNSBL zones are asked through: an
|
||||
// IP address with a port from 1 to 65535, such as 192.0.2.53:5353 or
|
||||
// [2001:db8::53]:5353, or without one, such as 192.0.2.53 or 2001:db8::53,
|
||||
// for port 53. An empty value is none, the zero AddrPort.
|
||||
func parseResolver(value string) (netip.AddrPort, error) {
|
||||
if value == "" {
|
||||
return netip.AddrPort{}, nil
|
||||
}
|
||||
|
||||
resolver, err := netip.ParseAddrPort(value)
|
||||
if err == nil && resolver.Port() != 0 {
|
||||
return resolver, nil
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(value)
|
||||
if err != nil {
|
||||
return netip.AddrPort{}, fmt.Errorf("%q %w", value, errNotResolver)
|
||||
}
|
||||
|
||||
return netip.AddrPortFrom(addr, dnsPort), nil
|
||||
}
|
||||
|
||||
// parseWebhookHeaders reads a comma-separated list of headers, each its
|
||||
// name, :, and its value, and returns them, and how the log shows them,
|
||||
// with each value as ********. An error names the item by its place in
|
||||
|
||||
@@ -61,6 +61,11 @@ const (
|
||||
blocklistURLs = "SWWAF_BLOCKLIST_URLS"
|
||||
blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH"
|
||||
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
|
||||
dnsblZones = "SWWAF_DNSBL_ZONES"
|
||||
dnsblResolver = "SWWAF_DNSBL_RESOLVER"
|
||||
reputationAction = "SWWAF_REPUTATION_ACTION"
|
||||
reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL"
|
||||
reputationTimeout = "SWWAF_REPUTATION_TIMEOUT"
|
||||
banResponse = "SWWAF_BAN_RESPONSE"
|
||||
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
||||
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
||||
@@ -152,6 +157,9 @@ const (
|
||||
defaultAlertCooldown = "15m"
|
||||
)
|
||||
|
||||
// defaultReputationCacheTTL is the default of SWWAF_REPUTATION_CACHE_TTL.
|
||||
const defaultReputationCacheTTL = "24h"
|
||||
|
||||
// defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS.
|
||||
const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," +
|
||||
"content-type,origin,range"
|
||||
@@ -1322,6 +1330,130 @@ func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// dronebl is a DNSBL zone, and spamhaus one of Spamhaus's, a name
|
||||
// containing the key of its keyed query service.
|
||||
const (
|
||||
dronebl = "dnsbl.dronebl.org"
|
||||
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
||||
)
|
||||
|
||||
func TestDNSBLSettingsAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := fromEnvironment(t, environment{})
|
||||
if len(cfg.DNSBLZones) != 0 || cfg.DNSBLResolver.IsValid() ||
|
||||
cfg.ReputationAction != actionLimit || cfg.ReputationLimitPercent != 25 ||
|
||||
cfg.ReputationCacheTTL != 24*time.Hour || cfg.ReputationTimeout != 2*time.Second {
|
||||
t.Errorf("by default, the zones %v, the resolver %s, the action %s:%d, the TTL %s "+
|
||||
"and the timeout %s, want no zone, no resolver, limit:25, 24h and 2s",
|
||||
cfg.DNSBLZones,
|
||||
cfg.DNSBLResolver, cfg.ReputationAction, cfg.ReputationLimitPercent,
|
||||
cfg.ReputationCacheTTL, cfg.ReputationTimeout)
|
||||
}
|
||||
|
||||
// The longest zone allowed, of 189 characters, with labels of 63, the
|
||||
// longest allowed.
|
||||
longest := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." +
|
||||
strings.Repeat("c", 61)
|
||||
|
||||
for _, tc := range []struct {
|
||||
zones, resolver, action string
|
||||
// want are the zones, resolver, action and percent Config gives.
|
||||
want []string
|
||||
wantResolver string
|
||||
wantAction string
|
||||
wantPercent int64
|
||||
}{
|
||||
{
|
||||
dronebl + ", " + spamhaus, "192.0.2.53", actionDeny,
|
||||
[]string{dronebl, spamhaus}, "192.0.2.53:53", actionDeny, 0,
|
||||
},
|
||||
{
|
||||
longest, "192.0.2.53:5353", actionLog,
|
||||
[]string{longest}, "192.0.2.53:5353", actionLog, 0,
|
||||
},
|
||||
{
|
||||
"Dnsbl-1.Example", "2001:db8::53", "limit:10",
|
||||
[]string{"Dnsbl-1.Example"}, "[2001:db8::53]:53", actionLimit, 10,
|
||||
},
|
||||
{
|
||||
dronebl, "[2001:db8::53]:5353", "limit:0",
|
||||
[]string{dronebl}, "[2001:db8::53]:5353", actionLimit, 0,
|
||||
},
|
||||
} {
|
||||
cfg := fromEnvironment(t, environment{
|
||||
dnsblZones: tc.zones, dnsblResolver: tc.resolver, reputationAction: tc.action,
|
||||
reputationCacheTTL: "12h", reputationTimeout: "3s",
|
||||
})
|
||||
|
||||
if !slices.Equal(cfg.DNSBLZones, tc.want) ||
|
||||
cfg.DNSBLResolver.String() != tc.wantResolver ||
|
||||
cfg.ReputationAction != tc.wantAction ||
|
||||
cfg.ReputationLimitPercent != tc.wantPercent ||
|
||||
cfg.ReputationCacheTTL != 12*time.Hour ||
|
||||
cfg.ReputationTimeout != 3*time.Second {
|
||||
t.Errorf("%s=%s, %s=%s and %s=%s gave %v, %s, %s:%d, %s and %s", dnsblZones,
|
||||
tc.zones, dnsblResolver, tc.resolver, reputationAction, tc.action,
|
||||
cfg.DNSBLZones, cfg.DNSBLResolver, cfg.ReputationAction,
|
||||
cfg.ReputationLimitPercent, cfg.ReputationCacheTTL, cfg.ReputationTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
notZone = " is not a DNS zone such as dnsbl.dronebl.org"
|
||||
notResolver = " is not an IP address with an optional port, such as 192.0.2.53 " +
|
||||
"or [2001:db8::53]:5353"
|
||||
notAboveZero = " is not a duration above zero, such as 1h or 7d"
|
||||
)
|
||||
|
||||
label64 := strings.Repeat("a", 64) + ".example"
|
||||
tooLong := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." +
|
||||
strings.Repeat("c", 62)
|
||||
|
||||
for _, tc := range []struct{ name, value, want string }{
|
||||
{dnsblZones, "dnsbl..example", `"dnsbl..example"` + notZone},
|
||||
{dnsblZones, "dnsbl.example.", `"dnsbl.example."` + notZone},
|
||||
{dnsblZones, "-dnsbl.example", `"-dnsbl.example"` + notZone},
|
||||
{dnsblZones, "dnsbl-.example", `"dnsbl-.example"` + notZone},
|
||||
{dnsblZones, "dns_bl.example", `"dns_bl.example"` + notZone},
|
||||
{dnsblZones, label64, `"` + label64 + `"` + notZone},
|
||||
{
|
||||
dnsblZones, tooLong,
|
||||
`"` + tooLong + `" is longer than 189 characters, too long for the names ` +
|
||||
`IPv6 clients are asked about by`,
|
||||
},
|
||||
{
|
||||
dnsblZones, dronebl + "," + spamhaus + "," + dronebl,
|
||||
`"` + dronebl + `" is listed twice`,
|
||||
},
|
||||
{dnsblResolver, "resolver.example", `"resolver.example"` + notResolver},
|
||||
{dnsblResolver, "192.0.2.53:0", `"192.0.2.53:0"` + notResolver},
|
||||
{dnsblResolver, "192.0.2.53:65536", `"192.0.2.53:65536"` + notResolver},
|
||||
{dnsblResolver, "[2001:db8::53]", `"[2001:db8::53]"` + notResolver},
|
||||
{
|
||||
reputationAction, "ban",
|
||||
`"ban" is not deny, limit:<percent> such as limit:25, or log`,
|
||||
},
|
||||
{reputationCacheTTL, off, `"off"` + notAboveZero},
|
||||
{reputationTimeout, "0s", `"0s"` + notAboveZero},
|
||||
} {
|
||||
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
|
||||
|
||||
want := tc.name + ": " + tc.want
|
||||
if err == nil || err.Error() != want {
|
||||
t.Errorf("error %v, want %s", err, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizesAndOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1704,6 +1836,7 @@ func writeFile(t *testing.T, contents string) string {
|
||||
return path
|
||||
}
|
||||
|
||||
//nolint:funlen // one line for each setting, a list that grows with them
|
||||
func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1749,6 +1882,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
blocklistURLs: "",
|
||||
blocklistRefresh: "24h",
|
||||
blocklistAction: actionDeny,
|
||||
dnsblZones: "",
|
||||
dnsblResolver: "",
|
||||
reputationAction: "limit:25",
|
||||
reputationCacheTTL: defaultReputationCacheTTL,
|
||||
reputationTimeout: "2s",
|
||||
banResponse: "403",
|
||||
limitBanDuration: "1h",
|
||||
limitBanRepeatWindow: "24h",
|
||||
|
||||
+41
-12
@@ -256,24 +256,53 @@ func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() i
|
||||
)
|
||||
}
|
||||
|
||||
// AddReputation adds the metrics of the lists fetched from URLs, by
|
||||
// source, each list's URL: the requests whose client a blocklist lists,
|
||||
// which ReputationHit counts, and, read from lists as the metrics are
|
||||
// asked for, the fetches that failed and when the copy in use was fetched.
|
||||
// It is called once, before ReputationHit.
|
||||
func (m *Metrics) AddReputation(lists *reputation.Lists) {
|
||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||
// DNSBL zones, by source, each list's URL or each zone: the requests whose
|
||||
// client a blocklist or a zone's verdict lists, which ReputationHit
|
||||
// counts, and, read from lists and dnsbl as the metrics are asked for, for
|
||||
// a list, the fetches that failed and when the copy in use was fetched,
|
||||
// and for a zone, the queries made and those that failed. It is called
|
||||
// once, before ReputationHit.
|
||||
func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) {
|
||||
const (
|
||||
sourceLabel = "source"
|
||||
failuresHelp = "Fetches of the list, or queries to the DNSBL zone, that failed."
|
||||
)
|
||||
|
||||
m.reputationHits = counterVec("smallwebwaf_reputation_hits_total",
|
||||
"Requests whose client a blocklist lists, by the blocklist's URL.",
|
||||
[]string{"source"})
|
||||
"Requests whose client a blocklist or a DNSBL zone lists, by the "+
|
||||
"blocklist's URL or the zone.",
|
||||
[]string{sourceLabel})
|
||||
m.registry.MustRegister(m.reputationHits)
|
||||
|
||||
for _, zone := range dnsbl.Zones() {
|
||||
source := prometheus.Labels{sourceLabel: zone}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_reputation_queries_total",
|
||||
Help: "Queries to the DNSBL zone.",
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
return float64(dnsbl.Queries(zone))
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_reputation_failures_total",
|
||||
Help: failuresHelp,
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
return float64(dnsbl.Failures(zone))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
for _, listURL := range lists.URLs() {
|
||||
source := prometheus.Labels{"source": listURL}
|
||||
source := prometheus.Labels{sourceLabel: listURL}
|
||||
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_reputation_failures_total",
|
||||
Help: "Fetches of the list that failed.",
|
||||
Help: failuresHelp,
|
||||
ConstLabels: source,
|
||||
}, func() float64 {
|
||||
return float64(lists.Failures(listURL))
|
||||
@@ -295,8 +324,8 @@ func (m *Metrics) AddReputation(lists *reputation.Lists) {
|
||||
}
|
||||
}
|
||||
|
||||
// ReputationHit counts a request whose client the blocklist at source, its
|
||||
// URL, lists.
|
||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||
// by its URL, or a DNSBL zone.
|
||||
func (m *Metrics) ReputationHit(source string) {
|
||||
m.reputationHits.WithLabelValues(source).Inc()
|
||||
}
|
||||
|
||||
+22
-15
@@ -28,16 +28,18 @@ func biasedThresholdsSet(cfg *config.Config) bool {
|
||||
|
||||
// limitPercentages returns the client's limit percentages, for the rate
|
||||
// limits and for the byte limits, by its AS number and country as looked
|
||||
// up, each "" when unknown, and the blocklists that list it. Each is the
|
||||
// lowest of those the settings give it, the first of them in the order
|
||||
// below when several are lowest: the percentage SWWAF_ASN_LIMIT_PERCENT
|
||||
// gives its AS number, the one the file SWWAF_ASN_LIMIT_PERCENT_URL names
|
||||
// gives it, the one SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a
|
||||
// client without a country, SWWAF_UNKNOWN_LIMIT_PERCENT, and for a client
|
||||
// a blocklist lists, the percentage of SWWAF_BLOCKLIST_ACTION while it is
|
||||
// limit. For the byte limits, SWWAF_ASN_BYTES_PERCENT and
|
||||
// SWWAF_COUNTRY_BYTES_PERCENT take the place of the first three for an AS
|
||||
// number or a country they list.
|
||||
// up, each "" when unknown, and the blocklists and DNSBL zones that list
|
||||
// it. Each is the lowest of those the settings give it, the first of them
|
||||
// in the order below when several are lowest: the percentage
|
||||
// SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
||||
// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one
|
||||
// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
||||
// country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists,
|
||||
// the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a
|
||||
// client a DNSBL zone's verdict lists, the percentage of
|
||||
// SWWAF_REPUTATION_ACTION while it is limit. For the byte limits,
|
||||
// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place
|
||||
// of the first three for an AS number or a country they list.
|
||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg := rq.h.config
|
||||
asn, country := rq.line.ASN, rq.line.Country
|
||||
@@ -52,9 +54,14 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
fetched = percentage{percent, "SWWAF_ASN_LIMIT_PERCENT_URL"}
|
||||
}
|
||||
|
||||
listed := percentage{percent: whole}
|
||||
if len(rq.line.Reputation) > 0 && cfg.BlocklistAction == "limit" {
|
||||
listed = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"}
|
||||
blocklisted := percentage{percent: whole}
|
||||
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||
blocklisted = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"}
|
||||
}
|
||||
|
||||
dnsblListed := percentage{percent: whole}
|
||||
if rq.dnsblListed && cfg.ReputationAction == "limit" {
|
||||
dnsblListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"}
|
||||
}
|
||||
|
||||
asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"),
|
||||
@@ -71,8 +78,8 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT")
|
||||
}
|
||||
|
||||
return lowest(asnRequests, countryRequests, unknown, listed),
|
||||
lowest(asnBytes, countryBytes, unknown, listed)
|
||||
return lowest(asnRequests, countryRequests, unknown, blocklisted, dnsblListed),
|
||||
lowest(asnBytes, countryBytes, unknown, blocklisted, dnsblListed)
|
||||
}
|
||||
|
||||
// given returns the percentage percents, the setting named setting, gives
|
||||
|
||||
+31
-11
@@ -71,15 +71,15 @@ type Params struct {
|
||||
Rules *rules.Files
|
||||
// Alerts receive the alert for each ban the proxy makes or makes
|
||||
// permanent, for each count over an anomaly threshold, for each request
|
||||
// whose client a blocklist lists, and for GeoJS failing or a fetch of a
|
||||
// list failing.
|
||||
// whose client a blocklist or a DNSBL zone lists, and for GeoJS failing,
|
||||
// a fetch of a list failing or a query to a DNSBL zone failing.
|
||||
Alerts *alerts.Queue
|
||||
}
|
||||
|
||||
// Server is the server smallwebwaf runs, with the parts of the proxy
|
||||
// whose state the state files keep, the lookup database, nil unless
|
||||
// SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run
|
||||
// fetches, and the metrics.
|
||||
// fetches, the DNSBL zones' verdicts, and the metrics.
|
||||
type Server struct {
|
||||
*http.Server
|
||||
|
||||
@@ -89,6 +89,7 @@ type Server struct {
|
||||
Anomalies *anomaly.Counters
|
||||
LookupFile *lookup.File
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
@@ -101,6 +102,7 @@ type Server struct {
|
||||
func New(params Params) *Server {
|
||||
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
|
||||
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
|
||||
lists, dnsbl := newReputation(params)
|
||||
h := &handler{
|
||||
config: params.Config,
|
||||
requestLog: params.RequestLog,
|
||||
@@ -136,13 +138,10 @@ func New(params Params) *Server {
|
||||
Alerts: params.Alerts,
|
||||
}),
|
||||
lookupFile: params.LookupFile,
|
||||
lists: reputation.New(reputation.Params{
|
||||
BlocklistURLs: params.Config.BlocklistURLs, Refresh: params.Config.BlocklistRefresh,
|
||||
ASNLimitPercentURL: params.Config.ASNLimitPercentURL, Now: params.Now,
|
||||
ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||
}),
|
||||
rules: params.Rules,
|
||||
alerts: params.Alerts,
|
||||
lists: lists,
|
||||
dnsbl: dnsbl,
|
||||
rules: params.Rules,
|
||||
alerts: params.Alerts,
|
||||
}
|
||||
h.geojs = lookup.New(lookup.Params{
|
||||
URL: params.GeoJSURL,
|
||||
@@ -160,7 +159,7 @@ func New(params Params) *Server {
|
||||
})
|
||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
||||
m.AddRules(params.Rules)
|
||||
m.AddReputation(h.lists)
|
||||
m.AddReputation(h.lists, h.dnsbl)
|
||||
|
||||
return &Server{
|
||||
Server: &http.Server{
|
||||
@@ -181,10 +180,30 @@ func New(params Params) *Server {
|
||||
Anomalies: h.anomalies,
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: h.lists,
|
||||
DNSBL: h.dnsbl,
|
||||
Metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// newReputation returns the lists fetched from URLs and the DNSBL zones'
|
||||
// verdicts, as the settings in params name them, with none fetched or
|
||||
// asked for yet.
|
||||
func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) {
|
||||
cfg := params.Config
|
||||
lists := reputation.New(reputation.Params{
|
||||
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL, Now: params.Now,
|
||||
ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||
})
|
||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.ReputationCacheTTL,
|
||||
Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
|
||||
return lists, dnsbl
|
||||
}
|
||||
|
||||
// handler is the proxy. It holds what every request shares; what belongs
|
||||
// to one request is in a request.
|
||||
type handler struct {
|
||||
@@ -201,6 +220,7 @@ type handler struct {
|
||||
anomalies *anomaly.Counters
|
||||
lookupFile *lookup.File
|
||||
lists *reputation.Lists
|
||||
dnsbl *reputation.DNSBL
|
||||
rules *rules.Files
|
||||
alerts *alerts.Queue
|
||||
}
|
||||
|
||||
@@ -1,21 +1,47 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
)
|
||||
|
||||
// blocklistDenied notes in the log line the URLs of the blocklists that
|
||||
// list the client, counts each of them in the metrics and raises a
|
||||
// reputation_hit alert for it, and reports whether SWWAF_BLOCKLIST_ACTION,
|
||||
// being deny, refuses the request. Being limit, it lowers the client's
|
||||
// limits instead (see limitPercentages), and being log, it does nothing
|
||||
// more.
|
||||
// blocklistDenied notes the blocklists that list the client, as
|
||||
// noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny,
|
||||
// refuses the request. Being limit, it lowers the client's limits instead
|
||||
// (see limitPercentages), and being log, it does nothing more.
|
||||
func (rq *request) blocklistDenied() bool {
|
||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
||||
rq.line.Reputation = listedBy
|
||||
rq.blocklisted = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a blocklist")
|
||||
|
||||
for _, listURL := range listedBy {
|
||||
rq.h.metrics.ReputationHit(listURL)
|
||||
return rq.blocklisted && rq.h.config.BlocklistAction == "deny"
|
||||
}
|
||||
|
||||
// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
|
||||
// noteListed does, and reports whether SWWAF_REPUTATION_ACTION, being
|
||||
// deny, refuses the request. Being limit, it lowers the client's limits
|
||||
// instead (see limitPercentages), and being log, it does nothing more. A
|
||||
// zone without a verdict on the client is asked about it in the
|
||||
// background, and the request does not wait for the answer. ctx is the
|
||||
// request's own context.
|
||||
func (rq *request) dnsblDenied(ctx context.Context) bool {
|
||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
return rq.dnsblListed && rq.h.config.ReputationAction == "deny"
|
||||
}
|
||||
|
||||
// noteListed adds sources, the URLs of the blocklists or the DNSBL zones
|
||||
// that list the client, to the log line's reputation, counts each of them
|
||||
// in the metrics, and raises a reputation_hit alert, with reason, for
|
||||
// each.
|
||||
func (rq *request) noteListed(sources []string, reason string) {
|
||||
rq.line.Reputation = append(rq.line.Reputation, sources...)
|
||||
|
||||
for _, source := range sources {
|
||||
rq.h.metrics.ReputationHit(source)
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventReputationHit,
|
||||
Client: rq.client,
|
||||
@@ -23,10 +49,8 @@ func (rq *request) blocklistDenied() bool {
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reason: "listed by a blocklist",
|
||||
Detail: map[string]any{"source": listURL},
|
||||
Reason: reason,
|
||||
Detail: map[string]any{"source": source},
|
||||
})
|
||||
}
|
||||
|
||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction == "deny"
|
||||
}
|
||||
|
||||
@@ -21,11 +21,14 @@ const (
|
||||
asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
|
||||
)
|
||||
|
||||
// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a
|
||||
// percentage.
|
||||
// The actions of SWWAF_BLOCKLIST_ACTION and SWWAF_REPUTATION_ACTION:
|
||||
// limitHalf gives a listed client half of every limit, and limitQuarter a
|
||||
// quarter.
|
||||
const (
|
||||
actionDeny = "deny"
|
||||
actionLog = "log"
|
||||
actionDeny = "deny"
|
||||
actionLog = "log"
|
||||
limitHalf = "limit:50"
|
||||
limitQuarter = "limit:25"
|
||||
)
|
||||
|
||||
// The lists these tests name, which are never fetched: each test puts in
|
||||
@@ -55,7 +58,7 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
|
||||
},
|
||||
{
|
||||
// Half of 4 requests a minute: the third breaks the limit.
|
||||
"limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
|
||||
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
|
||||
[]string{forward, forward, requestlog.ActionRateLimited},
|
||||
"50 from " + blocklistAction,
|
||||
},
|
||||
@@ -151,10 +154,10 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
|
||||
// percentText gives them, and limitHit its limit_hit.
|
||||
want, limitHit string
|
||||
}{
|
||||
{"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes},
|
||||
{limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes},
|
||||
// The AS number's, the first of two alike.
|
||||
{"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
||||
{actionLog, asnDE + ":100", none, ""},
|
||||
} {
|
||||
t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
|
||||
@@ -299,11 +302,279 @@ func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
||||
}
|
||||
}
|
||||
|
||||
// The DNSBL settings.
|
||||
const (
|
||||
dnsblZones = "SWWAF_DNSBL_ZONES"
|
||||
dnsblResolver = "SWWAF_DNSBL_RESOLVER"
|
||||
reputationAction = "SWWAF_REPUTATION_ACTION"
|
||||
)
|
||||
|
||||
// The DNSBL zones these tests name, which are never asked about the
|
||||
// clients the tests send requests from: each test puts in the verdicts it
|
||||
// needs, as reputation.json would at start. A query a test does start is
|
||||
// sent to noResolver, where nothing listens, so that none leaves the host.
|
||||
const (
|
||||
dnsblZone = "dnsbl.example"
|
||||
otherZone = "other.example"
|
||||
noResolver = "127.0.0.1:9"
|
||||
)
|
||||
|
||||
func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forward, denied := requestlog.ActionForward, requestlog.ActionDenied
|
||||
|
||||
for _, tc := range []struct {
|
||||
action string
|
||||
// statuses and actions are those of a listed client's three
|
||||
// requests, and percent their limit_percent, as percentText gives it.
|
||||
statuses []int
|
||||
actions []string
|
||||
percent string
|
||||
}{
|
||||
{
|
||||
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
|
||||
[]string{denied, denied, denied}, none,
|
||||
},
|
||||
{
|
||||
// Half of 4 requests a minute: the third breaks the limit.
|
||||
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
|
||||
[]string{forward, forward, requestlog.ActionRateLimited},
|
||||
"50 from " + reputationAction,
|
||||
},
|
||||
{
|
||||
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
|
||||
[]string{forward, forward, forward}, none,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.action, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
rateLimitPerMinute: fourAMinute, dnsblZones: dnsblZone + "," + otherZone,
|
||||
dnsblResolver: noResolver, reputationAction: tc.action,
|
||||
})
|
||||
listedBy := map[string][]string{
|
||||
fromDE: {dnsblZone, otherZone}, fromKP: {otherZone}, unplaced: nil,
|
||||
}
|
||||
loadVerdicts(server, listedBy)
|
||||
|
||||
for _, from := range []string{fromDE, fromKP} {
|
||||
for i := range 3 {
|
||||
line := s.get(from, tc.statuses[i], tc.actions[i])
|
||||
// In the order SWWAF_DNSBL_ZONES names them.
|
||||
wantReputation(t, line, listedBy[from]...)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent,
|
||||
line.LimitPercentSetting, tc.percent)
|
||||
|
||||
// A request refused for the verdict is not counted.
|
||||
counted := line.fields["counts"] != nil
|
||||
if counted != (tc.actions[i] != denied) {
|
||||
t.Errorf("request from %s counted %t, logged %s", from, counted,
|
||||
tc.actions[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A client no zone lists has the whole limit.
|
||||
for range 3 {
|
||||
line := s.get(unplaced, http.StatusOK, forward)
|
||||
wantReputation(t, line)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||
none)
|
||||
}
|
||||
|
||||
// A refusal for the verdict makes no ban, and every client had its
|
||||
// verdicts, so no zone was asked.
|
||||
if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
|
||||
t.Errorf("bans %+v, want none", held)
|
||||
}
|
||||
|
||||
if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
|
||||
t.Errorf("%d queries, want none", queries)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, queue := startWithLookups(t, map[string]string{
|
||||
blocklistURLs: dropURL, dnsblZones: dnsblZone, dnsblResolver: noResolver,
|
||||
reputationAction: actionDeny, allowNets: fromDE,
|
||||
})
|
||||
loadLists(t, server, map[string][]string{dropURL: {fromKP}})
|
||||
loadVerdicts(server, map[string][]string{fromKP: {dnsblZone}, fromDE: {dnsblZone}})
|
||||
|
||||
// The blocklist refuses fromKP before its verdict is looked at, and
|
||||
// fromDE, in SWWAF_ALLOW_NETS, is not checked at all: neither is noted
|
||||
// for the zone, nor alerted, nor asked about.
|
||||
wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied),
|
||||
dropURL)
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))
|
||||
|
||||
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 1 || waiting[0].Detail["source"] != dropURL {
|
||||
t.Errorf("alerts waiting %+v, want the blocklist's reputation_hit alone", waiting)
|
||||
}
|
||||
|
||||
if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
|
||||
t.Errorf("%d queries, want none", queries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
blocklistAction, reputationAction string
|
||||
// want is the request's limit_percent and bytes_percent, as
|
||||
// percentText gives them.
|
||||
want string
|
||||
}{
|
||||
{limitHalf, limitQuarter, "25 from " + reputationAction},
|
||||
{limitQuarter, limitHalf, "25 from " + blocklistAction},
|
||||
// The blocklist's, the first of two alike.
|
||||
{limitQuarter, limitQuarter, "25 from " + blocklistAction},
|
||||
{actionLog, limitQuarter, "25 from " + reputationAction},
|
||||
{actionLog, actionLog, none},
|
||||
} {
|
||||
t.Run(tc.blocklistAction+" "+tc.reputationAction, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
blocklistURLs: dropURL, blocklistAction: tc.blocklistAction,
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver,
|
||||
reputationAction: tc.reputationAction,
|
||||
})
|
||||
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
||||
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
|
||||
|
||||
// Named by the blocklist, then by the zone.
|
||||
line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||
wantReputation(t, line, dropURL, dnsblZone)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||
tc.want)
|
||||
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
|
||||
tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
|
||||
dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
|
||||
reputationAction: actionLog, metricsToken: token,
|
||||
})
|
||||
loadVerdicts(server, map[string][]string{
|
||||
fromDE: {dnsblZone, otherZone}, unplaced: nil,
|
||||
})
|
||||
|
||||
// The second request's alerts are repeats, which the cooldown holds
|
||||
// back.
|
||||
for range 2 {
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
dnsblZone, otherZone)
|
||||
}
|
||||
|
||||
hit := func(zone string) alerts.Alert {
|
||||
return alerts.Alert{
|
||||
Instance: alertInstance,
|
||||
Time: clk.Now(),
|
||||
Event: alerts.EventReputationHit,
|
||||
Client: netip.MustParseAddr(fromDE),
|
||||
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
||||
ASN: asnDE,
|
||||
ASName: asNameDE,
|
||||
Country: "DE",
|
||||
Reason: "listed by a DNSBL zone",
|
||||
Detail: map[string]any{"source": zone},
|
||||
}
|
||||
}
|
||||
wantAlerts(t, queue, hit(dnsblZone), hit(otherZone))
|
||||
|
||||
if queue.Suppressed() != 2 {
|
||||
t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
|
||||
}
|
||||
|
||||
// Each zone's hits, and its queries and their failures, none, since
|
||||
// every client had its verdicts.
|
||||
metrics := s.scrape(unplaced)
|
||||
|
||||
for _, zone := range []string{dnsblZone, otherZone} {
|
||||
labels := `{instance="` + alertInstance + `",source="` + zone + `"}`
|
||||
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
|
||||
})
|
||||
server.DNSBL.Load([]reputation.Verdict{{
|
||||
Zone: otherZone, Client: netip.MustParseAddr(fromDE), Listed: true,
|
||||
Fetched: verdictsFetched(),
|
||||
}})
|
||||
|
||||
// The verdict of the other zone is used, and dnsbl.example, which has
|
||||
// none, is asked about the client in the background, once: the second
|
||||
// request finds the query under way, or the zone left alone after it
|
||||
// failed, since nothing answers at noResolver.
|
||||
for range 2 {
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), otherZone)
|
||||
}
|
||||
|
||||
if queries := server.DNSBL.Queries(dnsblZone); queries != 1 {
|
||||
t.Errorf("%d queries to %s, want 1", queries, dnsblZone)
|
||||
}
|
||||
|
||||
if queries := server.DNSBL.Queries(otherZone); queries != 0 {
|
||||
t.Errorf("%d queries to %s, want none", queries, otherZone)
|
||||
}
|
||||
}
|
||||
|
||||
// listsFetched is when loadLists has the copies fetched.
|
||||
func listsFetched() time.Time {
|
||||
return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
// verdictsFetched is when loadVerdicts has the verdicts fetched: half a
|
||||
// day before the time the tests' clock is set to, so that they are in use
|
||||
// until half a day later.
|
||||
func verdictsFetched() time.Time {
|
||||
return time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
// loadVerdicts puts into server's DNSBL, for each client listedBy names,
|
||||
// a verdict of each zone SWWAF_DNSBL_ZONES names, fetched at
|
||||
// verdictsFetched, as reputation.json would at start: one that lists the
|
||||
// client from each zone listedBy gives for it, and one that does not from
|
||||
// each other zone.
|
||||
func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
|
||||
verdicts := make([]reputation.Verdict, 0, len(listedBy)*len(server.DNSBL.Zones()))
|
||||
|
||||
for client, zones := range listedBy {
|
||||
for _, zone := range server.DNSBL.Zones() {
|
||||
verdicts = append(verdicts, reputation.Verdict{
|
||||
Zone: zone, Client: netip.MustParseAddr(client),
|
||||
Listed: slices.Contains(zones, zone), Fetched: verdictsFetched(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
server.DNSBL.Load(verdicts)
|
||||
}
|
||||
|
||||
// loadLists puts copies of lists into server's lists, by URL, each with
|
||||
// its lines, fetched at listsFetched, as reputation.json would at start.
|
||||
func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
|
||||
|
||||
@@ -63,7 +63,10 @@ type request struct {
|
||||
// limits and for the byte limits.
|
||||
counted bool
|
||||
limitPercent, bytesPercent percentage
|
||||
start time.Time
|
||||
// blocklisted is true once a blocklist is found to list the client,
|
||||
// and dnsblListed once a DNSBL zone's verdict is.
|
||||
blocklisted, dnsblListed bool
|
||||
start time.Time
|
||||
// checked is when the checks were done, and upstreamStart when the
|
||||
// request was handed to the app.
|
||||
checked time.Time
|
||||
@@ -213,14 +216,14 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
|
||||
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
|
||||
// so that a client either refuses is not looked up, then the lookup of
|
||||
// its AS number and country, then the country lists, and then the
|
||||
// blocklists; a request any of them refuses is not counted for the rate
|
||||
// limits. Then come the rate limits, unless the client is in
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted,
|
||||
// each of them by the client's limit percentages, and last the rule
|
||||
// files. A request exempt from the rate limits is exempt from the byte
|
||||
// limits too. ctx is the request's own context.
|
||||
// its AS number and country, then the country lists, then the blocklists,
|
||||
// and then the DNSBL zones' verdicts; a request any of them refuses is not
|
||||
// counted for the rate limits. Then come the rate limits, unless the
|
||||
// client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is
|
||||
// exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request
|
||||
// is counted, each of them by the client's limit percentages, and last the
|
||||
// rule files. A request exempt from the rate limits is exempt from the
|
||||
// byte limits too. ctx is the request's own context.
|
||||
func (rq *request) checkClient(ctx context.Context) string {
|
||||
cfg := rq.h.config
|
||||
if isInside(rq.client, cfg.AllowNets) {
|
||||
@@ -247,6 +250,10 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
if rq.dnsblDenied(ctx) {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||
!pathExempt(rq.in.URL, cfg.RateLimitExemptPaths)
|
||||
if rq.counted {
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
package reputation
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxVerdicts is how many verdicts are kept. Past it, the one fetched
|
||||
// longest ago is dropped.
|
||||
maxVerdicts = 100000
|
||||
// maxQueries is how many queries may be under way at once. Past it, a
|
||||
// zone is not asked about a client until the client's next request, so
|
||||
// that a swarm of new addresses cannot fill the memory.
|
||||
maxQueries = 1000
|
||||
// failureDelay is how long a zone is not asked again after a query to
|
||||
// it fails, so that a zone refusing queries is not asked on every
|
||||
// request.
|
||||
failureDelay = time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
errAsk = errors.New("ask the zone")
|
||||
errRefused = errors.New("the zone refused the query")
|
||||
errNotListing = errors.New("the answer is outside 127.0.0.0/8")
|
||||
)
|
||||
|
||||
// Verdict is what a zone said about a client, as reputation.json holds
|
||||
// it: the zone, the client's address, whether the zone lists it, and when
|
||||
// the zone answered.
|
||||
type Verdict struct {
|
||||
Zone string `json:"zone"`
|
||||
Client netip.Addr `json:"client"`
|
||||
Listed bool `json:"listed"`
|
||||
Fetched time.Time `json:"fetched"`
|
||||
}
|
||||
|
||||
// DNSBLParams are what NewDNSBL needs.
|
||||
type DNSBLParams struct {
|
||||
// Zones are the DNSBL zones clients are asked about in
|
||||
// (SWWAF_DNSBL_ZONES).
|
||||
Zones []string
|
||||
// Resolver is the resolver they are asked through
|
||||
// (SWWAF_DNSBL_RESOLVER), or, while it is the zero AddrPort, the
|
||||
// host's, as /etc/resolv.conf names it.
|
||||
Resolver netip.AddrPort
|
||||
// CacheTTL is how long a verdict is used after it was fetched
|
||||
// (SWWAF_REPUTATION_CACHE_TTL), and Timeout how long a query may take
|
||||
// (SWWAF_REPUTATION_TIMEOUT).
|
||||
CacheTTL time.Duration
|
||||
Timeout time.Duration
|
||||
// Now tells the time, normally time.Now in UTC.
|
||||
Now func() time.Time
|
||||
// ProcessLog receives each query that fails, and why.
|
||||
ProcessLog *slog.Logger
|
||||
// Alerts receive a source_failure alert for each query that fails.
|
||||
Alerts *alerts.Queue
|
||||
}
|
||||
|
||||
// DNSBL asks the DNSBL zones about clients, in the background, and keeps
|
||||
// their verdicts. It is safe for concurrent use.
|
||||
type DNSBL struct {
|
||||
params DNSBLParams
|
||||
resolver *net.Resolver
|
||||
|
||||
mu sync.Mutex
|
||||
// verdicts are by query. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
verdicts *simplelru.LRU[query, Verdict]
|
||||
// asking are the queries under way.
|
||||
asking map[query]bool
|
||||
// queries and failures count, by zone, the queries made and those that
|
||||
// failed, and retryAt is when a zone whose last query failed may be
|
||||
// asked again.
|
||||
queries map[string]int
|
||||
failures map[string]int
|
||||
retryAt map[string]time.Time
|
||||
}
|
||||
|
||||
// query is a client's address, to ask a zone about.
|
||||
type query struct {
|
||||
zone string
|
||||
client netip.Addr
|
||||
}
|
||||
|
||||
// NewDNSBL returns a DNSBL with no verdict yet.
|
||||
func NewDNSBL(params DNSBLParams) *DNSBL {
|
||||
verdicts, err := simplelru.NewLRU[query, Verdict](maxVerdicts, nil)
|
||||
if err != nil {
|
||||
panic(err) // NewLRU fails only for a size below one
|
||||
}
|
||||
|
||||
resolver := &net.Resolver{}
|
||||
if params.Resolver.IsValid() {
|
||||
// Dial is used by Go's own resolver alone.
|
||||
resolver.PreferGo = true
|
||||
resolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
|
||||
return dialer.DialContext(ctx, network, params.Resolver.String())
|
||||
}
|
||||
}
|
||||
|
||||
return &DNSBL{
|
||||
params: params,
|
||||
resolver: resolver,
|
||||
verdicts: verdicts,
|
||||
asking: map[query]bool{},
|
||||
queries: map[string]int{},
|
||||
failures: map[string]int{},
|
||||
retryAt: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// Zones returns the zones, in the order SWWAF_DNSBL_ZONES names them.
|
||||
func (d *DNSBL) Zones() []string {
|
||||
return slices.Clone(d.params.Zones)
|
||||
}
|
||||
|
||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||
// lists it, in the order SWWAF_DNSBL_ZONES names them. A verdict is used
|
||||
// until CacheTTL has passed since it was fetched. Each zone without one is
|
||||
// asked about addr in the background, unless a query about addr to it is
|
||||
// under way, the zone is left alone after a failure, or maxQueries are
|
||||
// under way; ListedBy never waits for a query. ctx is the context of the
|
||||
// client's request, and a query goes on after the request ends.
|
||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
now := d.params.Now()
|
||||
|
||||
var listedBy []string
|
||||
|
||||
for _, zone := range d.params.Zones {
|
||||
q := query{zone: zone, client: addr}
|
||||
|
||||
kept, found := d.verdicts.Peek(q)
|
||||
|
||||
switch {
|
||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||
if kept.Listed {
|
||||
listedBy = append(listedBy, zone)
|
||||
}
|
||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||
d.asking[q] = true
|
||||
d.queries[zone]++
|
||||
|
||||
go d.ask(context.WithoutCancel(ctx), q)
|
||||
}
|
||||
}
|
||||
|
||||
return listedBy
|
||||
}
|
||||
|
||||
// Queries returns how many queries were made to zone.
|
||||
func (d *DNSBL) Queries(zone string) int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
return d.queries[zone]
|
||||
}
|
||||
|
||||
// Failures returns how many queries to zone failed.
|
||||
func (d *DNSBL) Failures(zone string) int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
return d.failures[zone]
|
||||
}
|
||||
|
||||
// Snapshot returns every verdict still in use, sorted by client, then by
|
||||
// zone, as reputation.json lists them.
|
||||
func (d *DNSBL) Snapshot() []Verdict {
|
||||
d.mu.Lock()
|
||||
|
||||
now := d.params.Now()
|
||||
verdicts := make([]Verdict, 0, d.verdicts.Len())
|
||||
|
||||
for _, kept := range d.verdicts.Values() {
|
||||
if now.Sub(kept.Fetched) < d.params.CacheTTL {
|
||||
verdicts = append(verdicts, kept)
|
||||
}
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
slices.SortFunc(verdicts, func(a, b Verdict) int {
|
||||
return cmp.Or(a.Client.Compare(b.Client), strings.Compare(a.Zone, b.Zone))
|
||||
})
|
||||
|
||||
return verdicts
|
||||
}
|
||||
|
||||
// Load keeps verdicts, read from reputation.json, in place of those it
|
||||
// keeps. A verdict of a zone SWWAF_DNSBL_ZONES does not name, or one
|
||||
// fetched CacheTTL ago or more, is dropped, and past maxVerdicts, those
|
||||
// fetched longest ago.
|
||||
func (d *DNSBL) Load(verdicts []Verdict) {
|
||||
verdicts = slices.Clone(verdicts)
|
||||
slices.SortStableFunc(verdicts, func(a, b Verdict) int {
|
||||
return a.Fetched.Compare(b.Fetched)
|
||||
})
|
||||
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
d.verdicts.Purge()
|
||||
|
||||
now := d.params.Now()
|
||||
|
||||
for _, kept := range verdicts {
|
||||
if slices.Contains(d.params.Zones, kept.Zone) &&
|
||||
now.Sub(kept.Fetched) < d.params.CacheTTL {
|
||||
d.verdicts.Add(query{zone: kept.Zone, client: kept.Client}, kept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask asks q's zone about q's client, keeps the verdict, and notes the
|
||||
// query as no longer under way. A query that fails gives no verdict: it
|
||||
// is counted, logged and raised as a source_failure alert, and the zone is
|
||||
// not asked again for failureDelay.
|
||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||
listed, err := d.lookUp(ctx, q)
|
||||
now := d.params.Now()
|
||||
|
||||
d.mu.Lock()
|
||||
|
||||
delete(d.asking, q)
|
||||
|
||||
if err == nil {
|
||||
d.verdicts.Add(q, Verdict{
|
||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||
})
|
||||
} else {
|
||||
d.failures[q.zone]++
|
||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
const failed = "asking a DNSBL zone failed"
|
||||
|
||||
// Raised before it is logged, so that the alert is there once the
|
||||
// log line is.
|
||||
d.params.Alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventSourceFailure,
|
||||
Reason: failed,
|
||||
Detail: map[string]any{"source": q.zone, "error": err.Error()},
|
||||
})
|
||||
d.params.ProcessLog.Warn(failed, "zone", q.zone, "error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// lookUp asks q's zone about q's client through the resolver, and returns
|
||||
// whether the zone lists it, as readAnswer reads the answer. No such name
|
||||
// is a client the zone does not list. A query not answered within Timeout
|
||||
// fails.
|
||||
func (d *DNSBL) lookUp(ctx context.Context, q query) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, d.params.Timeout)
|
||||
defer cancel()
|
||||
|
||||
answer, err := d.resolver.LookupNetIP(ctx, "ip4", queryName(q.zone, q.client))
|
||||
|
||||
var dnsErr *net.DNSError
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
return readAnswer(answer)
|
||||
case errors.As(err, &dnsErr) && dnsErr.IsNotFound:
|
||||
return false, nil
|
||||
case errors.As(err, &dnsErr):
|
||||
// The error names the name asked about, which holds the client's
|
||||
// address, which is not to be logged: only what went wrong is kept.
|
||||
return false, fmt.Errorf("%w: %s", errAsk, dnsErr.Err)
|
||||
default:
|
||||
return false, fmt.Errorf("%w: %w", errAsk, err)
|
||||
}
|
||||
}
|
||||
|
||||
// queryName returns the name a zone is asked about addr by, as RFC 5782
|
||||
// builds it: the four numbers of an IPv4 address, or the 32 hex digits of
|
||||
// an IPv6 address, in reverse order, each followed by a dot, then the zone
|
||||
// and a dot, which makes it a full name, to which the resolver adds no
|
||||
// search domain of /etc/resolv.conf.
|
||||
func queryName(zone string, addr netip.Addr) string {
|
||||
parts := strings.Split(addr.String(), ".")
|
||||
if addr.Is6() {
|
||||
parts = strings.Split(hex.EncodeToString(addr.AsSlice()), "")
|
||||
}
|
||||
|
||||
slices.Reverse(parts)
|
||||
|
||||
return strings.Join(parts, ".") + "." + zone + "."
|
||||
}
|
||||
|
||||
// readAnswer reads the addresses a zone answered with. An address in
|
||||
// 127.0.0.0/8 lists the client, as RFC 5782 has zones answer, but one in
|
||||
// 127.255.255.0/24 is how Spamhaus refuses a query, such as one sent
|
||||
// through a public resolver or one past its limit, and is a failure. So is
|
||||
// an address outside 127.0.0.0/8, such as a resolver gives that answers
|
||||
// even for names that do not exist.
|
||||
func readAnswer(answer []netip.Addr) (bool, error) {
|
||||
listing := netip.MustParsePrefix("127.0.0.0/8")
|
||||
refusal := netip.MustParsePrefix("127.255.255.0/24")
|
||||
|
||||
for _, addr := range answer {
|
||||
switch {
|
||||
case refusal.Contains(addr):
|
||||
return false, fmt.Errorf("%w: %s", errRefused, addr)
|
||||
case !listing.Contains(addr):
|
||||
return false, fmt.Errorf("%w: %s", errNotListing, addr)
|
||||
}
|
||||
}
|
||||
|
||||
return len(answer) > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,675 @@
|
||||
package reputation_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// The tests of the DNSBL zones run in synctest bubbles, as those of the
|
||||
// lists do, and the resolver the zones are asked through is a stand-in
|
||||
// reached through an in-memory connection, net.Pipe's, for the same
|
||||
// reason. They run one at a time, none in parallel with another test of
|
||||
// this package: Go's resolver counts the queries under way in one
|
||||
// sync.WaitGroup for the whole process, and the process fails when
|
||||
// queries from two bubbles, or from a bubble and from outside one, are
|
||||
// under way at once. TestMain has the resolver make its configuration,
|
||||
// which it makes on its first query, outside every bubble, since the
|
||||
// configuration holds a channel, which the bubble it was made in would
|
||||
// keep to itself.
|
||||
|
||||
const (
|
||||
// zone and otherZone are the DNSBL zones the tests name.
|
||||
zone = "dnsbl.example"
|
||||
otherZone = "other.example"
|
||||
// cacheTTL is the tests' SWWAF_REPUTATION_CACHE_TTL, and timeout their
|
||||
// SWWAF_REPUTATION_TIMEOUT: a second, the least time /etc/resolv.conf
|
||||
// can have Go's resolver wait for one server, so that it is the
|
||||
// DNSBL's own timeout that ends a query, whatever that file says.
|
||||
cacheTTL = 24 * time.Hour
|
||||
timeout = time.Second
|
||||
// listed and unlisted are clients zone is asked about by the names
|
||||
// listedName and unlistedName, and most tests have zone list the first
|
||||
// alone, by answering with listing.
|
||||
listed = "192.0.2.99"
|
||||
unlisted = "192.0.2.100"
|
||||
listedName = "99.2.0.192." + zone + "."
|
||||
unlistedName = "100.2.0.192." + zone + "."
|
||||
listing = "127.0.0.2"
|
||||
)
|
||||
|
||||
// The DNS response codes the stand-in answers with, besides no error.
|
||||
const (
|
||||
serverFailure = 2
|
||||
noSuchName = 3
|
||||
refused = 5
|
||||
)
|
||||
|
||||
var errNoNetwork = errors.New("the test dials nothing")
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// A query that fails at once, as nothing is dialled for it.
|
||||
resolver := &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(context.Context, string, string) (net.Conn, error) {
|
||||
return nil, errNoNetwork
|
||||
},
|
||||
}
|
||||
_, _ = resolver.LookupNetIP(context.Background(), "ip4", "warm-up.invalid.")
|
||||
|
||||
m.Run()
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestZonesListOrNotClientsByTheirIPv4AndIPv6Addresses(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
// The addresses of the examples of RFC 5782, and the names it
|
||||
// gives for them.
|
||||
const (
|
||||
v4 = "192.0.2.99"
|
||||
v6 = "2001:db8:1:2:3:4:567:89ab"
|
||||
// v6Name is the hex digits of v6, in reverse order.
|
||||
v6Name = "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2."
|
||||
)
|
||||
|
||||
resolver := &resolverStandIn{answers: map[string]answer{
|
||||
"99.2.0.192." + zone + ".": {addrs: []string{listing}},
|
||||
v6Name + otherZone + ".": {addrs: []string{"127.0.0.4", "127.0.0.10"}},
|
||||
"99.2.0.192." + otherZone + ".": {},
|
||||
}}
|
||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||
|
||||
// Neither client has a verdict yet, so neither is listed, and each
|
||||
// zone is asked about each.
|
||||
wantZones(t, dnsbl, v4)
|
||||
wantZones(t, dnsbl, v6)
|
||||
synctest.Wait()
|
||||
|
||||
wantZones(t, dnsbl, v4, zone)
|
||||
wantZones(t, dnsbl, v6, otherZone)
|
||||
wantAsked(t, resolver,
|
||||
"99.2.0.192."+zone+".", "99.2.0.192."+otherZone+".",
|
||||
v6Name+zone+".", v6Name+otherZone+".")
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestListedByNeverWaitsForAQuery(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dnsbl := newDNSBL(&resolverStandIn{hanging: true}, dnsblParams(zone))
|
||||
began := time.Now()
|
||||
|
||||
// The second, while the first's query is under way, starts none.
|
||||
wantZones(t, dnsbl, listed)
|
||||
wantZones(t, dnsbl, listed)
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("waited %s for the query, want no wait", waited)
|
||||
}
|
||||
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestVerdictUsedUntilTheCacheTTLHasPassedSinceItWasFetched(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
resolver := &resolverStandIn{answers: map[string]answer{
|
||||
listedName: {addrs: []string{listing}},
|
||||
}}
|
||||
dnsbl := newDNSBL(resolver, dnsblParams(zone))
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
|
||||
// The zone lists the other client from now on, but the verdicts
|
||||
// kept are used, and the zone is not asked again, until the TTL
|
||||
// has passed.
|
||||
resolver.set(listedName, answer{rcode: noSuchName})
|
||||
resolver.set(unlistedName, answer{addrs: []string{listing}})
|
||||
time.Sleep(cacheTTL - time.Nanosecond)
|
||||
|
||||
wantZones(t, dnsbl, listed, zone)
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 2, 0)
|
||||
|
||||
// Then neither verdict is used, and both clients are asked about
|
||||
// again.
|
||||
time.Sleep(time.Nanosecond)
|
||||
wantZones(t, dnsbl, listed)
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
|
||||
wantQueries(t, dnsbl, 4, 0)
|
||||
wantZones(t, dnsbl, listed)
|
||||
wantZones(t, dnsbl, unlisted, zone)
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := dnsblParams(zone)
|
||||
p.Alerts = queue
|
||||
dnsbl := newDNSBL(&resolverStandIn{hanging: true}, p)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
time.Sleep(timeout - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 1)
|
||||
|
||||
if got := waiting(queue); len(got) != 1 ||
|
||||
got[0].Detail["error"] != "ask the zone: i/o timeout" {
|
||||
t.Errorf("alerts waiting %+v, want the timeout's", got)
|
||||
}
|
||||
|
||||
if verdicts := dnsbl.Snapshot(); len(verdicts) != 0 {
|
||||
t.Errorf("verdicts %+v, want none", verdicts)
|
||||
}
|
||||
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
answer answer
|
||||
error string
|
||||
}{
|
||||
{
|
||||
"a server failure", answer{rcode: serverFailure},
|
||||
"ask the zone: server misbehaving",
|
||||
},
|
||||
{"a refusal", answer{rcode: refused}, "ask the zone: server misbehaving"},
|
||||
{
|
||||
"an answer in 127.255.255.0/24, with which Spamhaus refuses a query",
|
||||
answer{addrs: []string{"127.255.255.254"}},
|
||||
"the zone refused the query: 127.255.255.254",
|
||||
},
|
||||
{
|
||||
"an answer outside 127.0.0.0/8, as for a name that does not exist",
|
||||
answer{addrs: []string{"192.0.2.1"}},
|
||||
"the answer is outside 127.0.0.0/8: 192.0.2.1",
|
||||
},
|
||||
} {
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var log bytes.Buffer
|
||||
|
||||
queue := newQueue()
|
||||
p := dnsblParams(zone)
|
||||
p.Alerts = queue
|
||||
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
|
||||
dnsbl := newDNSBL(&resolverStandIn{answers: map[string]answer{
|
||||
listedName: tc.answer,
|
||||
}}, p)
|
||||
|
||||
// The failure gives no verdict, and the zone is not asked
|
||||
// again within a minute of it.
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
time.Sleep(time.Minute - time.Nanosecond)
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 1)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 2, 2)
|
||||
|
||||
if verdicts := dnsbl.Snapshot(); len(verdicts) != 0 {
|
||||
t.Errorf("verdicts %+v, want none", verdicts)
|
||||
}
|
||||
|
||||
// One alert for the first failure; the cooldown holds back
|
||||
// the second.
|
||||
wantAlert(t, queue, alerts.Alert{
|
||||
Time: time.Now().Add(-time.Minute),
|
||||
Event: alerts.EventSourceFailure,
|
||||
Reason: "asking a DNSBL zone failed",
|
||||
Detail: map[string]any{"source": zone, "error": tc.error},
|
||||
})
|
||||
|
||||
if !strings.Contains(log.String(), `"msg":"asking a DNSBL zone failed",`+
|
||||
`"zone":"`+zone+`","error":"`+tc.error) {
|
||||
t.Errorf("logged\n%s\nwant the failures", log.String())
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestAtMost1000QueriesUnderWay(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dnsbl := newDNSBL(&resolverStandIn{hanging: true}, dnsblParams(zone))
|
||||
|
||||
client := netip.MustParseAddr("198.18.0.0")
|
||||
for range 1001 {
|
||||
dnsbl.ListedBy(t.Context(), client)
|
||||
client = client.Next()
|
||||
}
|
||||
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1000, 0)
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
resolver := &resolverStandIn{answers: map[string]answer{
|
||||
listedName: {addrs: []string{listing}},
|
||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||
}}
|
||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||
m := metrics.New(1, "app")
|
||||
m.AddReputation(reputation.New(params()), dnsbl)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
|
||||
scraped := httptest.NewRecorder()
|
||||
m.ServeHTTP(scraped, httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"/", http.NoBody))
|
||||
|
||||
for series, want := range map[string]string{
|
||||
"queries_total" + `{instance="app",source="` + zone + `"}`: "1",
|
||||
"failures_total" + `{instance="app",source="` + zone + `"}`: "0",
|
||||
"queries_total" + `{instance="app",source="` + otherZone + `"}`: "1",
|
||||
"failures_total" + `{instance="app",source="` + otherZone + `"}`: "1",
|
||||
} {
|
||||
line := "\nsmallwebwaf_reputation_" + series + " " + want + "\n"
|
||||
if !strings.Contains(scraped.Body.String(), line) {
|
||||
t.Errorf("metrics\n%s\nwant%s", scraped.Body.String(), line)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestVerdictsKeptAcrossARestart(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
resolver := &resolverStandIn{answers: map[string]answer{
|
||||
listedName: {addrs: []string{listing}},
|
||||
}}
|
||||
dnsbl := newDNSBL(resolver, dnsblParams(zone))
|
||||
fetched := time.Now()
|
||||
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
|
||||
kept := dnsbl.Snapshot()
|
||||
|
||||
want := []reputation.Verdict{
|
||||
{Zone: zone, Client: netip.MustParseAddr(listed), Listed: true, Fetched: fetched},
|
||||
{Zone: zone, Client: netip.MustParseAddr(unlisted), Fetched: fetched},
|
||||
}
|
||||
if !reflect.DeepEqual(kept, want) {
|
||||
t.Errorf("verdicts %+v, want %+v", kept, want)
|
||||
}
|
||||
|
||||
// Restarted an hour later with what reputation.json keeps, it uses
|
||||
// the verdicts, and asks the zone nothing, until the TTL has passed
|
||||
// since they were fetched.
|
||||
time.Sleep(time.Hour)
|
||||
|
||||
restarted := &resolverStandIn{}
|
||||
again := newDNSBL(restarted, dnsblParams(zone))
|
||||
again.Load(kept)
|
||||
|
||||
wantZones(t, again, listed, zone)
|
||||
wantZones(t, again, unlisted)
|
||||
synctest.Wait()
|
||||
wantAsked(t, restarted)
|
||||
|
||||
time.Sleep(cacheTTL - time.Hour)
|
||||
wantZones(t, again, listed)
|
||||
synctest.Wait()
|
||||
wantAsked(t, restarted, listedName)
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadKeepsTheNewestVerdictsOfTheZonesNamedWithinTheirTTL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
|
||||
p := dnsblParams(zone)
|
||||
p.Now = func() time.Time { return now }
|
||||
dnsbl := reputation.NewDNSBL(p)
|
||||
|
||||
// Of a zone not named, and fetched a TTL ago.
|
||||
notNamed := reputation.Verdict{
|
||||
Zone: otherZone, Client: netip.MustParseAddr(listed), Listed: true, Fetched: now,
|
||||
}
|
||||
stale := reputation.Verdict{
|
||||
Zone: zone, Client: netip.MustParseAddr(listed), Listed: true,
|
||||
Fetched: now.Add(-cacheTTL),
|
||||
}
|
||||
// Then 100,001 verdicts, each fetched a millisecond after the one
|
||||
// before, the last now: the first is one too many.
|
||||
const count = 100001
|
||||
|
||||
verdicts := make([]reputation.Verdict, 0, 2+count)
|
||||
verdicts = append(verdicts, notNamed, stale)
|
||||
|
||||
client := netip.MustParseAddr("198.18.0.0")
|
||||
for i := range count {
|
||||
verdicts = append(verdicts, reputation.Verdict{
|
||||
Zone: zone, Client: client,
|
||||
Fetched: now.Add(time.Duration(i-count+1) * time.Millisecond),
|
||||
})
|
||||
client = client.Next()
|
||||
}
|
||||
|
||||
dnsbl.Load(verdicts)
|
||||
|
||||
got := dnsbl.Snapshot()
|
||||
if len(got) != count-1 || slices.Contains(got, verdicts[2]) ||
|
||||
!slices.Contains(got, verdicts[3]) || !slices.Contains(got, verdicts[count+1]) {
|
||||
t.Errorf("%d verdicts kept, want all but the one fetched first", len(got))
|
||||
}
|
||||
|
||||
if slices.Contains(got, notNamed) || slices.Contains(got, stale) {
|
||||
t.Error("the verdict of a zone not named, or past its TTL, was kept")
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestQueriesGoToTheResolverSWWAFDNSBLResolverNames(t *testing.T) {
|
||||
resolver := &resolverStandIn{answers: map[string]answer{
|
||||
listedName: {addrs: []string{listing}},
|
||||
}}
|
||||
|
||||
conn, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
served := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
resolver.serveUDP(conn)
|
||||
close(served)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = conn.Close()
|
||||
|
||||
<-served
|
||||
})
|
||||
|
||||
p := dnsblParams(zone)
|
||||
p.Resolver = netip.MustParseAddrPort(conn.LocalAddr().String())
|
||||
// On the real clock: the stand-in answers at once, so only a test
|
||||
// process held up for a whole minute would see the query fail.
|
||||
p.Timeout = time.Minute
|
||||
|
||||
isListed, err := reputation.NewDNSBL(p).LookUp(zone, netip.MustParseAddr(listed))
|
||||
if err != nil || !isListed {
|
||||
t.Errorf("listed %t (%v), want true", isListed, err)
|
||||
}
|
||||
|
||||
wantAsked(t, resolver, listedName)
|
||||
}
|
||||
|
||||
// resolverStandIn is a stand-in for the resolver the zones are asked
|
||||
// through. It answers each query by the name asked about, as answers
|
||||
// gives, with no such name for a name answers does not give, and not at
|
||||
// all while hanging. It notes each name asked about.
|
||||
type resolverStandIn struct {
|
||||
mu sync.Mutex
|
||||
answers map[string]answer
|
||||
hanging bool
|
||||
names []string
|
||||
}
|
||||
|
||||
// answer is how the stand-in answers a name: with an A record of each of
|
||||
// addrs, or with the response code rcode, unless it is 0, for no error.
|
||||
type answer struct {
|
||||
addrs []string
|
||||
rcode uint16
|
||||
}
|
||||
|
||||
// What the stand-in reads of a query, and writes in its reply.
|
||||
const (
|
||||
// headerLength is the length of a DNS message's header, which the
|
||||
// question follows: its id, its flags, and how many questions,
|
||||
// answers and other records it holds, two bytes each.
|
||||
headerLength = 12
|
||||
// typeAndClass is the length of the type and the class that end a
|
||||
// question, after its name.
|
||||
typeAndClass = 4
|
||||
// replyFlags mark a reply to a query that asked for recursion, which
|
||||
// is available, with no error. The response code goes in their last
|
||||
// four bits.
|
||||
replyFlags = 0x8180
|
||||
// maxMessage is the longest query read over UDP.
|
||||
maxMessage = 1232
|
||||
)
|
||||
|
||||
// set has the stand-in answer name with given.
|
||||
func (s *resolverStandIn) set(name string, given answer) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.answers[name] = given
|
||||
}
|
||||
|
||||
// dial connects Go's resolver to the stand-in through an in-memory
|
||||
// connection, on which it sends each query, and reads each reply, after
|
||||
// its length, as over TCP.
|
||||
func (s *resolverStandIn) dial(context.Context, string, string) (net.Conn, error) {
|
||||
client, server := net.Pipe()
|
||||
|
||||
go s.serve(server)
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// serve answers the queries that come on conn until the resolver closes
|
||||
// it.
|
||||
func (s *resolverStandIn) serve(conn net.Conn) {
|
||||
defer func() {
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
for {
|
||||
var length [2]byte
|
||||
|
||||
_, err := io.ReadFull(conn, length[:])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
message := make([]byte, binary.BigEndian.Uint16(length[:]))
|
||||
|
||||
_, err = io.ReadFull(conn, message)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
reply, answered := s.reply(message)
|
||||
if !answered {
|
||||
continue // the resolver gives up, and closes conn
|
||||
}
|
||||
|
||||
//nolint:gosec // a reply of a few dozen bytes
|
||||
_, err = conn.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(reply))),
|
||||
reply...))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serveUDP answers the queries that come on conn, each in a datagram, as
|
||||
// a resolver does, until conn is closed.
|
||||
func (s *resolverStandIn) serveUDP(conn net.PacketConn) {
|
||||
message := make([]byte, maxMessage)
|
||||
|
||||
for {
|
||||
n, from, err := conn.ReadFrom(message)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
reply, answered := s.reply(message[:n])
|
||||
if answered {
|
||||
_, _ = conn.WriteTo(reply, from)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reply returns the stand-in's reply to message, a query, and false for
|
||||
// none, while it hangs. It notes the name asked about.
|
||||
func (s *resolverStandIn) reply(message []byte) ([]byte, bool) {
|
||||
// The name is labels, each after its length, ended by a length of 0.
|
||||
var labels []string
|
||||
|
||||
end := headerLength
|
||||
for message[end] != 0 {
|
||||
length := int(message[end])
|
||||
labels = append(labels, string(message[end+1:end+1+length]))
|
||||
end += 1 + length
|
||||
}
|
||||
|
||||
end += 1 + typeAndClass
|
||||
name := strings.Join(labels, ".") + "."
|
||||
|
||||
s.mu.Lock()
|
||||
s.names = append(s.names, name)
|
||||
given, found := s.answers[name]
|
||||
hanging := s.hanging
|
||||
s.mu.Unlock()
|
||||
|
||||
if hanging {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if !found {
|
||||
given = answer{rcode: noSuchName}
|
||||
}
|
||||
|
||||
// The query's id, the flags, one question, the answers, and no other
|
||||
// records, then the question, as asked.
|
||||
reply := slices.Clone(message[:2])
|
||||
reply = binary.BigEndian.AppendUint16(reply, replyFlags|given.rcode)
|
||||
reply = binary.BigEndian.AppendUint16(reply, 1)
|
||||
//nolint:gosec // a handful of answers
|
||||
reply = binary.BigEndian.AppendUint16(reply, uint16(len(given.addrs)))
|
||||
reply = append(reply, 0, 0, 0, 0)
|
||||
reply = append(reply, message[headerLength:end]...)
|
||||
|
||||
// An A record starts with the name asked about, by a pointer to it in
|
||||
// the question, then its type, A, its class, IN, how long it may be
|
||||
// kept, 60 seconds, and the length of its address, 4 bytes.
|
||||
record := []byte{0xc0, headerLength, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4}
|
||||
|
||||
for _, addr := range given.addrs {
|
||||
reply = append(reply, record...)
|
||||
reply = append(reply, netip.MustParseAddr(addr).AsSlice()...)
|
||||
}
|
||||
|
||||
return reply, true
|
||||
}
|
||||
|
||||
// dnsblParams returns the DNSBLParams of zones, with the tests' cache TTL
|
||||
// and timeout, by the bubble's clock, with alerts to a queue that sends
|
||||
// none.
|
||||
func dnsblParams(zones ...string) reputation.DNSBLParams {
|
||||
return reputation.DNSBLParams{
|
||||
Zones: zones,
|
||||
CacheTTL: cacheTTL,
|
||||
Timeout: timeout,
|
||||
Now: time.Now,
|
||||
ProcessLog: slog.New(slog.DiscardHandler),
|
||||
Alerts: newQueue(),
|
||||
}
|
||||
}
|
||||
|
||||
// waitForTheResolver waits, on the bubble's clock, an hour, until Go's
|
||||
// resolver has given up on every stand-in that does not answer: it waits
|
||||
// for a server as long as /etc/resolv.conf has it wait, a few seconds,
|
||||
// even after the query was given up, and a bubble cannot end before it.
|
||||
func waitForTheResolver() {
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
|
||||
// newDNSBL returns the DNSBL of p, asking resolver.
|
||||
func newDNSBL(resolver *resolverStandIn, p reputation.DNSBLParams) *reputation.DNSBL {
|
||||
dnsbl := reputation.NewDNSBL(p)
|
||||
dnsbl.SetDial(resolver.dial)
|
||||
|
||||
return dnsbl
|
||||
}
|
||||
|
||||
// wantZones checks the zones whose verdict dnsbl says lists client, as a
|
||||
// request from client finds them.
|
||||
func wantZones(t *testing.T, dnsbl *reputation.DNSBL, client string, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
got := dnsbl.ListedBy(t.Context(), netip.MustParseAddr(client))
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("%s is listed by %v, want %v", client, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// wantQueries checks how many queries dnsbl made to zone, and how many of
|
||||
// them failed.
|
||||
func wantQueries(t *testing.T, dnsbl *reputation.DNSBL, queries, failures int) {
|
||||
t.Helper()
|
||||
|
||||
if dnsbl.Queries(zone) != queries || dnsbl.Failures(zone) != failures {
|
||||
t.Errorf("%d queries and %d failures, want %d and %d", dnsbl.Queries(zone),
|
||||
dnsbl.Failures(zone), queries, failures)
|
||||
}
|
||||
}
|
||||
|
||||
// wantAsked checks the names the stand-in was asked about, in any order.
|
||||
func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
resolver.mu.Lock()
|
||||
got := slices.Sorted(slices.Values(resolver.names))
|
||||
resolver.mu.Unlock()
|
||||
|
||||
slices.Sort(want)
|
||||
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("asked about %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,27 @@
|
||||
package reputation
|
||||
|
||||
import "net/http"
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
)
|
||||
|
||||
// SetTransport has l's fetches go through transport instead of the
|
||||
// network.
|
||||
func (l *Lists) SetTransport(transport http.RoundTripper) {
|
||||
l.httpClient.Transport = transport
|
||||
}
|
||||
|
||||
// SetDial has d's queries go through dial instead of the network.
|
||||
func (d *DNSBL) SetDial(
|
||||
dial func(ctx context.Context, network, address string) (net.Conn, error),
|
||||
) {
|
||||
d.resolver = &net.Resolver{PreferGo: true, Dial: dial}
|
||||
}
|
||||
|
||||
// LookUp asks zone about addr at once, as a query in the background does,
|
||||
// and returns whether zone lists addr.
|
||||
func (d *DNSBL) LookUp(zone string, addr netip.Addr) (bool, error) {
|
||||
return d.lookUp(context.Background(), query{zone: zone, client: addr})
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines
|
||||
// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
|
||||
// whole, comment lines included, which is used while a fetch fails, and
|
||||
// when each was last tried, which the state package writes to
|
||||
// reputation.json and reads from it, so that a restart keeps them too.
|
||||
// when each was last tried. It also asks the DNSBL zones of
|
||||
// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts. The state
|
||||
// package writes all of these to reputation.json and reads them from it,
|
||||
// so that a restart keeps them too.
|
||||
package reputation
|
||||
|
||||
import (
|
||||
|
||||
@@ -35,8 +35,8 @@ const (
|
||||
// rule.
|
||||
ActionRuleBlocked = "rule_blocked"
|
||||
// ActionDenied is a request refused because its client is in
|
||||
// SWWAF_DENY_NETS, or in a blocklist while SWWAF_BLOCKLIST_ACTION is
|
||||
// deny.
|
||||
// SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny,
|
||||
// or listed by a DNSBL zone while SWWAF_REPUTATION_ACTION is deny.
|
||||
ActionDenied = "denied"
|
||||
// ActionCountryDenied is a request refused for its client's country.
|
||||
ActionCountryDenied = "country_denied"
|
||||
@@ -138,7 +138,8 @@ type Line struct {
|
||||
// Counts names its count: minute, hour or day for a rate limit, and
|
||||
// minute_bytes, hour_bytes or day_bytes for a byte limit.
|
||||
LimitHit string `json:"limit_hit,omitempty"`
|
||||
// Reputation are the URLs of the blocklists that list the client.
|
||||
// Reputation are the URLs of the blocklists that list the client, then
|
||||
// the DNSBL zones whose verdict lists it.
|
||||
Reputation []string `json:"reputation,omitempty"`
|
||||
// Offence is the offence the request was held as, OffenceLimit.
|
||||
Offence string `json:"offence,omitempty"`
|
||||
|
||||
@@ -201,6 +201,7 @@ func loadStateFiles(
|
||||
Limiter: server.Limiter,
|
||||
GeoJS: server.GeoJS,
|
||||
Lists: server.Lists,
|
||||
DNSBL: server.DNSBL,
|
||||
Alerts: alertQueue,
|
||||
Anomalies: server.Anomalies,
|
||||
Now: now,
|
||||
|
||||
+41
-8
@@ -2,7 +2,8 @@
|
||||
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
|
||||
// bans.json holds the bans, clients.json each client's counters and
|
||||
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
||||
// last good copy of each list fetched from a URL, and alerts.json the
|
||||
// last good copy of each list fetched from a URL and the DNSBL zones'
|
||||
// verdicts, and alerts.json the
|
||||
// cooldowns, the hour under way, the alerts waiting for each destination
|
||||
// and the anomaly counters. Load
|
||||
// reads them at start, Watch takes in an admin's edit of one while
|
||||
@@ -76,13 +77,14 @@ type Params struct {
|
||||
// is (SWWAF_STATE_COUNTER_INTERVAL).
|
||||
WriteDelay time.Duration
|
||||
CounterInterval time.Duration
|
||||
// Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state.
|
||||
// Alerts also receive a file_error alert for an edit set aside, and for
|
||||
// a write that fails while smallwebwaf runs.
|
||||
// Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the
|
||||
// state. Alerts also receive a file_error alert for an edit set aside,
|
||||
// and for a write that fails while smallwebwaf runs.
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *lookup.GeoJS
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
Alerts *alerts.Queue
|
||||
Anomalies *anomaly.Counters
|
||||
// Now tells the time by which the counters' buckets run out, normally
|
||||
@@ -145,8 +147,9 @@ type lookupsFile struct {
|
||||
// reputationFile is reputation.json, indented for an admin to read and
|
||||
// edit, so that each line of a list's copy is on a line of its own.
|
||||
type reputationFile struct {
|
||||
Version int `json:"version"`
|
||||
Lists []reputation.List `json:"lists"`
|
||||
Version int `json:"version"`
|
||||
Lists []reputation.List `json:"lists"`
|
||||
Verdicts []reputation.Verdict `json:"verdicts"`
|
||||
}
|
||||
|
||||
// alertsFile is alerts.json, indented for an admin to read and edit.
|
||||
@@ -432,6 +435,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
|
||||
return 0, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
|
||||
f.params.DNSBL.Load(file.Verdicts)
|
||||
entries = len(file.Lists)
|
||||
case alertsJSON:
|
||||
waiting, err := f.takeInAlerts(path, data)
|
||||
@@ -567,6 +571,7 @@ func (f *Files) encode(name string) ([]byte, error) {
|
||||
case reputationJSON:
|
||||
return encodeIndented(reputationFile{
|
||||
Version: version, Lists: f.params.Lists.Snapshot(),
|
||||
Verdicts: f.params.DNSBL.Snapshot(),
|
||||
})
|
||||
default: // alerts.json
|
||||
held := f.params.Alerts.Snapshot()
|
||||
@@ -729,8 +734,12 @@ func (f *lookupsFile) check(data []byte) error {
|
||||
// check refuses a list without its URL, which would name no list, or the
|
||||
// time it was last tried, which would have it fetched at once, and a copy
|
||||
// of it without the time it was fetched, or without its lines, which hold
|
||||
// the list.
|
||||
func (f *reputationFile) check([]byte) error {
|
||||
// the list. It refuses a verdict without its zone or its client, which
|
||||
// would be about no one, whether the zone lists the client, or the time
|
||||
// it was fetched, which would drop it. A verdict's listed is false for a
|
||||
// client the zone does not list, which Verdicts cannot tell from a
|
||||
// missing one, so each listed is read again as written.
|
||||
func (f *reputationFile) check(data []byte) error {
|
||||
for i, kept := range f.Lists {
|
||||
switch {
|
||||
case kept.URL == "":
|
||||
@@ -744,6 +753,30 @@ func (f *reputationFile) check([]byte) error {
|
||||
}
|
||||
}
|
||||
|
||||
var written struct {
|
||||
Verdicts []struct {
|
||||
Listed *bool `json:"listed"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(data, &written)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, verdict := range f.Verdicts {
|
||||
switch {
|
||||
case verdict.Zone == "":
|
||||
return fmt.Errorf("verdicts %w", missing(i, "zone"))
|
||||
case !verdict.Client.IsValid():
|
||||
return fmt.Errorf("verdicts %w", missing(i, "client"))
|
||||
case written.Verdicts[i].Listed == nil:
|
||||
return fmt.Errorf("verdicts %w", missing(i, "listed"))
|
||||
case verdict.Fetched.IsZero():
|
||||
return fmt.Errorf("verdicts %w", missing(i, "fetched"))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -38,9 +38,11 @@ const (
|
||||
lookupsJSON = "lookups.json"
|
||||
reputationJSON = "reputation.json"
|
||||
alertsJSON = "alerts.json"
|
||||
// blocklistURL and torURL are the blocklists the tests' lists name.
|
||||
// blocklistURL and torURL are the blocklists the tests' lists name, and
|
||||
// dnsblZone the DNSBL zone of the tests' verdicts.
|
||||
blocklistURL = "https://lists.example/drop.txt"
|
||||
torURL = "https://lists.example/tor.txt"
|
||||
dnsblZone = "dnsbl.example"
|
||||
// The AS number and AS name the tests' clients are looked up in.
|
||||
asn = "AS64496"
|
||||
asName = "Example Net"
|
||||
@@ -210,7 +212,8 @@ const filledAlertsJSON = `{
|
||||
`
|
||||
|
||||
// filledReputationJSON is reputation.json holding the blocklists' last
|
||||
// tries and the copy of one, with its comment line, as fill puts them in.
|
||||
// tries and the copy of one, with its comment line, and two verdicts of a
|
||||
// DNSBL zone, as fill puts them in.
|
||||
const filledReputationJSON = `{
|
||||
"version": 1,
|
||||
"lists": [
|
||||
@@ -228,6 +231,20 @@ const filledReputationJSON = `{
|
||||
"url": "https://lists.example/tor.txt",
|
||||
"tried": "2026-10-06T00:00:00Z"
|
||||
}
|
||||
],
|
||||
"verdicts": [
|
||||
{
|
||||
"zone": "dnsbl.example",
|
||||
"client": "203.0.113.9",
|
||||
"listed": true,
|
||||
"fetched": "2026-10-05T23:00:00Z"
|
||||
},
|
||||
{
|
||||
"zone": "dnsbl.example",
|
||||
"client": "2001:db8::1",
|
||||
"listed": false,
|
||||
"fetched": "2026-10-05T22:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
`
|
||||
@@ -263,6 +280,8 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
|
||||
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
|
||||
}
|
||||
|
||||
wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
|
||||
|
||||
if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
|
||||
got, want) {
|
||||
t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
|
||||
@@ -422,8 +441,8 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
|
||||
held := params.Alerts.Snapshot()
|
||||
if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
|
||||
len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
|
||||
len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
|
||||
held.Hour.Sent != 0 {
|
||||
len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
|
||||
len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
|
||||
t.Error("state from no files")
|
||||
}
|
||||
}
|
||||
@@ -592,6 +611,11 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||
drop = `"url": "` + blocklistURL + `", `
|
||||
tried = `"tried": "2026-10-06T00:00:00Z", `
|
||||
fetched = `"fetched": "2026-10-06T00:00:00Z"`
|
||||
// verdictZone, verdictClient and listed start a verdict, which
|
||||
// fetched ends.
|
||||
verdictZone = `"zone": "` + dnsblZone + `", `
|
||||
verdictClient = `"client": "198.51.100.7", `
|
||||
listed = `"listed": false, `
|
||||
)
|
||||
|
||||
for _, tc := range []struct {
|
||||
@@ -621,6 +645,30 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||
`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
|
||||
`: entry 2 has no "lines"`,
|
||||
},
|
||||
{
|
||||
"a verdict without its zone",
|
||||
`{"version": 1, "verdicts": [{` + verdictClient + listed + fetched + `}]}`,
|
||||
`: verdicts entry 1 has no "zone"`,
|
||||
},
|
||||
{
|
||||
"a verdict without its client",
|
||||
`{"version": 1, "verdicts": [{` + verdictZone + listed + fetched + `}]}`,
|
||||
`: verdicts entry 1 has no "client"`,
|
||||
},
|
||||
{
|
||||
// A client the zone does not list has a listed of false, which is
|
||||
// not having none.
|
||||
"a verdict without whether the zone lists the client",
|
||||
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient + listed +
|
||||
fetched + `}, {` + verdictZone + verdictClient + fetched + `}]}`,
|
||||
`: verdicts entry 2 has no "listed"`,
|
||||
},
|
||||
{
|
||||
"a verdict without the time it was fetched",
|
||||
`{"version": 1, "verdicts": [{` + verdictZone + verdictClient +
|
||||
`"listed": true}]}`,
|
||||
`: verdicts entry 1 has no "fetched"`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -1133,7 +1181,8 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
|
||||
|
||||
edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
|
||||
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
|
||||
`"lines": ["198.51.100.7"]}]}`)
|
||||
`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
|
||||
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
|
||||
wantTakenIn(t, lines, dir, reputationJSON)
|
||||
|
||||
listedBy := params.Lists.ListedBy(client.Addr())
|
||||
@@ -1142,6 +1191,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
|
||||
client.Addr(), listedBy)
|
||||
}
|
||||
|
||||
wantEqual(t, reputationJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
|
||||
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
|
||||
}})
|
||||
|
||||
// A netblock with bits past its length is read as the netblock it is
|
||||
// in.
|
||||
edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
|
||||
@@ -1558,6 +1611,10 @@ func newParams(dir string) state.Params {
|
||||
BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour,
|
||||
Now: midnight, ProcessLog: discard, Alerts: queue,
|
||||
}),
|
||||
DNSBL: reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second,
|
||||
Now: midnight, ProcessLog: discard, Alerts: queue,
|
||||
}),
|
||||
Alerts: queue,
|
||||
Anomalies: anomaly.New(anomaly.Params{
|
||||
Net: anomaly.Thresholds{RequestsPerMinute: 1000},
|
||||
@@ -1582,9 +1639,10 @@ func office() netip.Prefix {
|
||||
|
||||
// fill puts a permanent ban an admin made, a ban for a broken limit and
|
||||
// one for a clear sign of attack, clients with counts and histories,
|
||||
// GeoJS answers, the blocklists' last tries and the copy of one, as
|
||||
// filledReputationJSON holds them, and alerts and anomaly counters, as
|
||||
// filledAlertsJSON holds them, into the parts of params.
|
||||
// GeoJS answers, the blocklists' last tries and the copy of one, and two
|
||||
// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts
|
||||
// and anomaly counters, as filledAlertsJSON holds them, into the parts of
|
||||
// params.
|
||||
func fill(params state.Params) {
|
||||
now := midnight()
|
||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||
@@ -1631,6 +1689,14 @@ func fill(params state.Params) {
|
||||
panic(err) // the copy reads
|
||||
}
|
||||
|
||||
params.DNSBL.Load([]reputation.Verdict{
|
||||
{
|
||||
Zone: dnsblZone, Client: netip.MustParseAddr("2001:db8::1"),
|
||||
Fetched: now.Add(-2 * time.Hour),
|
||||
},
|
||||
{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
|
||||
})
|
||||
|
||||
// An alert waiting, a repeat of it the cooldown holds back, another
|
||||
// alert waiting, and one past the two an hour, for the hour's summary.
|
||||
ban := alerts.Alert{
|
||||
|
||||
Reference in New Issue
Block a user